diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts index c0cc666..2c49b12 100644 --- a/src/contracts/grokWorkerContract.ts +++ b/src/contracts/grokWorkerContract.ts @@ -14,15 +14,68 @@ * run directly and saves one `search_tool` round trip per tool (P0: 3 → 2 * requests). */ + +/** + * The atoms of that route, and its single definition. + * + * Both texts a Grok worker receives are rendered from them: the pinned system + * prompt below, and the caller's identity envelope + * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). + * They were worded independently once, and the envelope told the model to call + * the tools by their bare names - which Grok 1.0.34 refuses outright, before + * any HTTP: `'moltnet_read' is not a valid MCP tool name. Tool names must be + * qualified as \`server__tool\`` (local rig, real CLI, real rendered config). + * There is exactly one valid spelling, so two independently worded naming rules + * are one rule too many; this is the single definition both render from. + */ +export const DAIMON_GROK_MCP_SERVER = "daimon" as const; +/** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ +export const DAIMON_GROK_TOOL_PREFIX = `${DAIMON_GROK_MCP_SERVER}__` as const; +export const grokDaimonToolName = (tool: string): string => `${DAIMON_GROK_TOOL_PREFIX}${tool}`; +/** Grok's two MCP meta-tools, and the argument that names a tool for the first. */ +export const GROK_MCP_INVOKE_TOOL = "use_tool" as const; +export const GROK_MCP_SEARCH_TOOL = "search_tool" as const; +export const GROK_MCP_TOOL_NAME_ARGUMENT = "tool_name" as const; +/** Illustrative Daimon tools for the system prompt, which cannot know a wake's real mount. */ +const DAIMON_GROK_EXAMPLE_TOOLS = Object.freeze(["moltnet_read", "moltnet_send", "memory_search", "memory_register"] as const); + export const DAIMON_GROK_SYSTEM_PROMPT = [ "You are a headless Daimon agent; no human is present.", "Your identity, instructions and wake event are in the user prompt.", - "Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know.", + `Daimon tools are MCP tools on server ${DAIMON_GROK_MCP_SERVER}: call a known one directly with ${GROK_MCP_INVOKE_TOOL} (${GROK_MCP_TOOL_NAME_ARGUMENT} ${DAIMON_GROK_EXAMPLE_TOOLS.map(grokDaimonToolName).join(", ")}, or another ${DAIMON_GROK_TOOL_PREFIX} name you were given); use ${GROK_MCP_SEARCH_TOOL} only for a name you do not know.`, "If a tool result says output was saved to a file, read that path with read_file.", "If a tool fails, do not retry it in a loop: stop and report the failure.", "Your final answer is a private note to the runtime: one line, or empty." ].join(" "); +/** + * The same route, stated once for the caller's identity envelope, where a + * wake's real mounted tools are known. + * + * It contributes exactly what the pinned prompt cannot know - the wake's real + * mounted names - and the one rule that makes them callable. It asserts rather + * than corrects: one bare catalogue, one prefix rule, one example, and the + * prefixed form named as the *only* valid form, because that is the CLI's own + * verdict on a bare name rather than a preference. + * + * What it deliberately leaves out is as load bearing. It never claims the + * agent's own instructions spell a tool wrongly, never offers a shell or CLI + * route, never repeats the catalogue in prefixed form - and never restates the + * `search_tool` rule. A Grok worker already reads two authoritative sentences + * about `search_tool`: the pinned prompt's ("only for a name you do not know") + * and Grok's own injected notice, which says the model MUST call it before any + * MCP tool. Observed on the rig: that contradiction is not enforced, and + * `use_tool` works with no prior `search_tool`. A third wording would only add + * a voice, so this sentence stays out of that argument entirely. + */ +export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { + const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); + return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}, and its only valid tool name is ` + + `${DAIMON_GROK_TOOL_PREFIX}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${example}. ` + + "A bare name is not a valid MCP tool name and reaches nothing."; +}; + /** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ export const GROK_BROKER_MODELS = Object.freeze(["grok-4.6", "grok-4.5", "grok-build"] as const); export const GROK_BROKER_REASONING_EFFORTS = Object.freeze(["low", "medium", "high"] as const); diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index a9eb1be..7bed4e5 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -53,9 +53,9 @@ export const GROK_ENGINE_BROKER = { systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. configSha256: { - "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, - "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, - "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } + "grok-4.6": { low: "ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e", medium: "df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f", high: "65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3" }, + "grok-4.5": { low: "8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6", medium: "59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155", high: "c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f" }, + "grok-build": { low: "fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5", medium: "8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656", high: "a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3" } }, // Worker `GROK_HOME` layout the broker attests before every turn. The home and // its `sessions/` directory are root-owned, worker-group writable and sticky so @@ -64,10 +64,23 @@ export const GROK_ENGINE_BROKER = { directory: { uid: 0, group: "worker", mode: 0o1771 }, sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, - sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, + // The launcher exports TMPDIR=/tmp; Grok's strict profile grants TMPDIR read-write. + privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, + // Strict also grants shared /tmp and /var/tmp read-write and refuses to start if either is + // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker + // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. + sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // The organization runtime home of a brokered Grok agent: traverse-only for the + // worker group so the worker can reach `tool-output/` and nothing else (no group + // read, no group write, no world bits; `physicalReadiness.ts` refuses anything else). + organizationRuntimeHome: { owner: "organization", group: "worker", mode: 0o710 }, + // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, + // files written 0640 by the runtime, never other-readable. + spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, - bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 262_144 }, // Accounting and limits (P2). The broker is the single sealed usage writer. controlProtocolVersion: "noopolis.daimon.engine-broker.v2", turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], @@ -120,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", - x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", - arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" + sourceSha256: "dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24", + x64Sha256: "67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7", + arm64Sha256: "c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98" } } as const; export const AGY_SUBSCRIPTION_REALM = { @@ -172,5 +185,5 @@ export const RUNTIME_CONTRACT_MANIFEST = { ] }, healthResponseSchema: { type: "object", additionalProperties: false, required: ["version", "state", "agents"], properties: { version: { const: "noopolis.daimon.organization-runtime-health.v1" }, state: { enum: ["starting", "running", "stopping", "stopped"] }, agents: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agentId", "state"], properties: { agentId: text, state: { enum: ["starting", "running", "stopping", "stopped", "idle", "failed"] } } } } } }, activityResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: "noopolis.daimon.organization-runtime-activity.v1" }, items: { type: "array", maxItems: 100, items: activityItem }, nextCursor: { type: "string", minLength: 1, maxLength: 16, pattern: "^(0|[1-9][0-9]{0,15})$" } } }, - activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } + activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] } } } } } } } as const; diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 044b4ca..b594157 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -1,7 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; -import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; +import { appendFile, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -106,8 +107,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => }; const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { - const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true }); + const directory = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -182,7 +182,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true }); + await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry"); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -207,7 +207,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true }); + await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index c6dbd7f..bc55516 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -1,7 +1,8 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -206,8 +207,7 @@ export class OrgObserver { } async write(runtimeRoot: string): Promise { - const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true }); + const telemetryDir = await ensureRuntimeHomeDirectory(runtimeRoot, "telemetry"); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/AGENTS.md b/src/pi/AGENTS.md index eb26811..4685c74 100644 --- a/src/pi/AGENTS.md +++ b/src/pi/AGENTS.md @@ -9,3 +9,13 @@ removing redundant nested denies that cannot be mounted by its Linux sandbox. An intervening readable path or workspace root makes a deeper deny necessary. Verify changes with generated production arguments and real local sandbox commands; a model call is neither required nor permitted for this check. + +The per-wake MCP mount is torn down on the wake's own completion path, so that +teardown must be bounded. `Server.close()` waits for every open connection, and +a connection the MCP transport has no record of — a socket opened before +`initialize`, or an idle keep-alive socket a client's pool still holds, which is +what relaying a turn through the broker MCP facade leaves behind — is not the +transport's to end. Close the transport first, then end the remaining +connections; never wait for the client to release them. A finished turn that +parks here publishes nothing and dies to an outer deadline, which loses exactly +the terminal evidence the turn existed to produce. diff --git a/src/pi/cliChildOutput.ts b/src/pi/cliChildOutput.ts index 19b5ca2..78a87d4 100644 --- a/src/pi/cliChildOutput.ts +++ b/src/pi/cliChildOutput.ts @@ -1,13 +1,14 @@ import type { ChildProcess } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { headUtf8, tailUtf8 } from "../runtime/toolResultSpill.js"; import type { TurnUsageFailureReason } from "../runtime/turnUsageLedger.js"; import { decodeCodexTurnUsage, type CodexTurnUsage } from "./codexHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; /** Maximum assistant reply bytes retained from stdout. */ export const CLI_ENGINE_MAX_OUTPUT_BYTES = 64 * 1024; -/** Tail bytes retained from stderr only for a failed-child diagnostic. */ +/** Diagnostic bytes retained from stderr for a failed child: its head and its tail together. */ export const CLI_ENGINE_MAX_DIAGNOSTIC_BYTES = 768; const CLI_ENGINE_FAILURE_SCAN_CHARS = 256; @@ -39,18 +40,36 @@ const redactChildOutput = (value: string, secretValues: readonly string[]): stri return redactCredentialText(value, secretValues, CLI_ENGINE_MAX_OUTPUT_BYTES); }; -const utf8Tail = (value: string, maxBytes: number): string => { - const bytes = Buffer.from(value, "utf8"); - if (bytes.length <= maxBytes) return value; - let result = bytes.subarray(bytes.length - maxBytes).toString("utf8"); - while (result.startsWith("\uFFFD")) result = result.slice(1); - return result; +/** + * One bounded window over a failed child's own output: its head AND its tail, + * with an explicit marker naming the bytes elided between them. + * + * A pure tail is the wrong end for the process this exists for. A worker that + * dies early prints its error first and then echoes its own input, so the tail + * is the echo: one live brokered turn reported 512 bytes of its own prompt + * read back, with the actual error already off the front and discarded. Both + * ends cost the same window, and the marker is the one oversized tool results + * already use (`toolResultSpill.ts`), so a reader meets one shape everywhere. + * + * The marker is paid for out of the same budget — it is sized against the + * largest count it could carry — so the result never exceeds `maxBytes`, and + * output that fits is returned byte-identical with no marker at all. + */ +export const boundedDiagnosticWindow = (value: string, maxBytes: number): string => { + const total = Buffer.byteLength(value, "utf8"); + if (total <= maxBytes) return value; + const budget = Math.max(0, maxBytes - Buffer.byteLength(elisionMarker(total), "utf8")); + const head = headUtf8(value, Math.floor(budget / 2)); + const tail = tailUtf8(value, budget - Buffer.byteLength(head, "utf8")); + const elided = total - Buffer.byteLength(head, "utf8") - Buffer.byteLength(tail, "utf8"); + return `${head}${elisionMarker(elided)}${tail}`; }; +const elisionMarker = (elided: number): string => `[… ${elided} bytes elided …]`; const childDiagnostic = (stdout: string, stderr: string, secretValues: readonly string[]): string => { const output = stderr.trim().length > 0 ? stderr : stdout; const redacted = redactCredentialText(output, secretValues, Number.MAX_SAFE_INTEGER).trim(); - const bounded = utf8Tail(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + const bounded = boundedDiagnosticWindow(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return bounded.length > 0 ? `: ${bounded}` : ""; }; @@ -104,7 +123,13 @@ export const readChild = ( const stdout: Buffer[] = []; let stdoutTail = Buffer.alloc(0); let droppingStdoutLine = false; + // Both ends of stderr, retained as it streams: the head frozen once it is + // full, the tail sliding. A single sliding tail dropped the head at capture + // time, which is where the cause of an early death lives — no later window + // can recover what was never kept. + let stderrHead = Buffer.alloc(0); let stderrTail = Buffer.alloc(0); + let stderrBytes = 0; let stdoutBytes = 0; let stdoutRemainder = Buffer.alloc(0); let droppingNdjsonLine = false; @@ -113,8 +138,18 @@ export const readChild = ( let cleanupStarted = false; let classifiedFailure: Error | undefined; let failureScanTail = ""; - const stderrRetentionBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - + Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + /** + * Each retained end carries its reported share PLUS one whole secret, which + * is the invariant that keeps exact redaction exact: a secret that reaches + * the reported window can extend at most its own length past that window's + * cut, so retaining that much more on each side means the redactor always + * sees the secret whole. Halving one shared budget instead broke it — a + * 2000-byte secret was cut in the middle and its tail fragment + * (`…qqq-secret-end`) reached the diagnostic verbatim. + */ + const stderrSecretAllowance = Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + const stderrHeadBytes = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; + const stderrTailBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; const settle = (action: () => void): void => { if (settled) return; settled = true; @@ -221,18 +256,33 @@ export const readChild = ( } stdout.push(value); }; - const retainStderrTail = (chunk: Buffer): void => { + const retainStderrWindow = (chunk: Buffer): void => { const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); classifyFailure(value); - if (value.length >= stderrRetentionBytes) { - stderrTail = Buffer.from(value.subarray(value.length - stderrRetentionBytes)); + stderrBytes += value.length; + if (stderrHead.length < stderrHeadBytes) stderrHead = Buffer.concat([stderrHead, value.subarray(0, stderrHeadBytes - stderrHead.length)]); + if (value.length >= stderrTailBytes) { + stderrTail = Buffer.from(value.subarray(value.length - stderrTailBytes)); return; } - const overflow = stderrTail.length + value.length - stderrRetentionBytes; + const overflow = stderrTail.length + value.length - stderrTailBytes; stderrTail = Buffer.concat([overflow > 0 ? stderrTail.subarray(overflow) : stderrTail, value]); }; + /** + * The retained stderr, reassembled exactly. + * + * Nothing was elided while the whole output fitted the budget, and then the + * two ends overlap: they tile the stream, so dropping the overlap from the + * tail rebuilds it byte-identically. Above the budget the ends are joined by + * the marker, which names how many bytes never reached this process at all. + */ + const retainedStderr = (): string => { + const elided = stderrBytes - stderrHead.length - stderrTail.length; + if (elided <= 0) return Buffer.concat([stderrHead, stderrTail.subarray(stderrHead.length + stderrTail.length - stderrBytes)]).toString("utf8"); + return `${stderrHead.toString("utf8")}${elisionMarker(elided)}${stderrTail.toString("utf8")}`; + }; child.stdout?.on("data", retainStdout); - child.stderr?.on("data", retainStderrTail); + child.stderr?.on("data", retainStderrWindow); const timer = timeoutMs === undefined ? undefined : setTimeout(() => abort(tagCliChildFailure(new Error(options.timeoutErrorMessage ?? "CLI engine timed out"), "wake_timeout")), timeoutMs); child.once("error", abort); child.once("close", (code, signal) => { @@ -244,7 +294,7 @@ export const readChild = ( return; } settle(() => reject(tagCliChildFailure(classifiedFailure ?? new Error(`CLI engine exited ${code ?? signal}${childDiagnostic( - retainedStdout.toString("utf8"), stderrTail.toString("utf8"), secretValues + retainedStdout.toString("utf8"), retainedStderr(), secretValues )}`), "engine_exit"))); }); }); diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index 49f8af8..b045f4b 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; import { createServer, type Server } from "node:http"; import { spawn, type ChildProcess } from "node:child_process"; -import { mkdir } from "node:fs/promises"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; @@ -35,6 +34,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -138,14 +138,9 @@ type CliTurnEnd = Extract; export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined): Promise => { if (runtimeHomePath === undefined) return; - await Promise.all([ - runtimeHomePath, - `${runtimeHomePath}/.config`, - `${runtimeHomePath}/.local/share`, - `${runtimeHomePath}/.local/state`, - `${runtimeHomePath}/.cache`, - `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true }))); + await ensureRuntimeHome(runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp"] + .map((relative) => ensureRuntimeHomeDirectory(runtimeHomePath, relative))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => @@ -188,7 +183,17 @@ const startMcp = async ( await startupSettled; await transport.close().catch(() => undefined); await mcpServer.close().catch(() => undefined); - if (httpServer.listening) await new Promise((resolve) => httpServer.close(() => resolve())); + // `close` only stops accepting and then waits for every open connection, + // including the ones the transport has no record of and so cannot end (a + // socket opened before `initialize`, or a client pool's idle keep-alive + // socket, which relaying a turn through the broker MCP facade leaves + // behind). That wait is unbounded and sits on the wake's own completion + // path: measured, one such connection parked a finished broker turn with + // its result in hand and published nothing. By here this one wake's engine + // has returned, failed or been cancelled, so anything still connected is a + // leftover — see `AGENTS.md`, and the facade, which bounds itself the same + // way. + if (httpServer.listening) await new Promise((resolve) => { httpServer.close(() => resolve()); httpServer.closeAllConnections(); }); lifecycle = "closed"; })(); const mount = { get endpoint(): string { return endpoint; }, close }; diff --git a/src/pi/cliSessionMcpMountClose.test.ts b/src/pi/cliSessionMcpMountClose.test.ts new file mode 100644 index 0000000..69b74ed --- /dev/null +++ b/src/pi/cliSessionMcpMountClose.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { connect, type Socket } from "node:net"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; + +import { createCliSessionFactory } from "./cliSession.js"; + +type PublishedTurn = { readonly message: { readonly content: readonly unknown[] } }; + +/** + * A finished turn must not wait on a connection to its own per-wake MCP mount. + * + * The mount's HTTP server is torn down on the wake's completion path, and + * `Server.close()` only stops accepting: it waits for every open connection. + * The MCP transport ends the sessions it knows about, but a connection it has + * no record of — a socket opened before `initialize`, or an idle HTTP + * keep-alive socket a client's connection pool is still holding — is not its + * to end. The broker MCP facade is exactly such a client: relaying a turn's + * session leaves pooled connections to the mount behind it. + * + * Measured before this was bounded: the broker turn returned its result and + * the wake then never completed and never published — the terminal evidence + * that is the whole point of letting a finished turn finish. + */ +const TURN_COMPLETION_BOUND_MS = 5_000; + +test("a finished Grok broker turn publishes without waiting on an open connection to its own MCP mount", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-mount-close-")); + const peers: Socket[] = []; + const hangUp = (): void => { for (const peer of peers) peer.destroy(); }; + try { + const { session } = await createCliSessionFactory({ + engine: "grok", + command: "/nonexistent-engine", + grokBrokerTurn: async (_prompt, mcpEndpoint) => { + // A peer holding a connection the transport never saw an `initialize` + // on: none of this connection is the mount's own session state. + const peer = connect({ host: "127.0.0.1", port: Number(new URL(mcpEndpoint).port) }); + peers.push(peer); + peer.on("error", () => undefined); + await new Promise((resolve, reject) => { peer.once("connect", resolve); peer.once("error", reject); }); + return "Filed and delivered."; + } + })({ cwd: root }); + const published: PublishedTurn[] = []; + session.subscribe((event) => { published.push(event as unknown as PublishedTurn); }); + try { + const settled = session.prompt("wake").then(() => "completed" as const); + const outcome = await Promise.race([settled, delay(TURN_COMPLETION_BOUND_MS).then(() => "parked" as const)]); + // Hang the peer up before asserting, so a regression reports the parked + // wake instead of parking the suite's own teardown behind it. + hangUp(); + assert.equal(outcome, "completed", `the finished wake did not complete within ${TURN_COMPLETION_BOUND_MS}ms`); + assert.equal(published.length, 1); + assert.deepEqual(published[0]?.message.content, [{ type: "text", text: "Filed and delivered." }]); + } finally { hangUp(); await session.disposeAsync?.(); } + } finally { + hangUp(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/src/pi/cliSessionOutput.test.ts b/src/pi/cliSessionOutput.test.ts index 3e6ec4a..65d0b2c 100644 --- a/src/pi/cliSessionOutput.test.ts +++ b/src/pi/cliSessionOutput.test.ts @@ -34,11 +34,15 @@ test("verbose progress stderr is drained without invalidating a bounded successf } }); -test("failed verbose stderr retains only a redacted bounded diagnostic tail", async () => { +test("failed verbose stderr retains a redacted bounded window of its head AND its tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-stderr-diagnostic-")); const engine = path.join(root, "verbose-failure.mjs"); const secret = "bounded-diagnostic-secret-value"; await writeFile(engine, [ + // The real shape of an early death: the cause first, then a flood, then + // the last thing the child happened to print. A pure tail kept only the + // last of the three. + `process.stderr.write(${JSON.stringify("first-cause: profile refused\n")});`, `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, `process.stderr.write(${JSON.stringify(` final-error ${secret}`)});`, "process.exitCode = 7;" @@ -54,7 +58,9 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { assert.ok(error instanceof Error); assert.match(error.message, /CLI engine exited 7/); + assert.match(error.message, /first-cause: profile refused/, "the head of the output is the error, and must survive the bound"); assert.match(error.message, /final-error \[REDACTED\]/); + assert.match(error.message, /\[… \d+ bytes elided …\]/, "and what was dropped between the two ends is named"); assert.equal(error.message.includes(secret), false); assert.ok(Buffer.byteLength(error.message) <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES + 80); return true; @@ -64,6 +70,45 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as } }); +/** + * The mirror of the long-secret case, at the other cut. Retaining both ends + * only stays safe while each end carries a whole secret's worth beyond its + * reported share: sizing the two ends by halving one shared budget let a + * 2000-byte secret straddle the cut and leak its own tail verbatim. + */ +test("a secret straddling the head's own retention edge is still redacted whole", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-head-edge-secret-")); + const engine = path.join(root, "head-edge-secret.mjs"); + // The first ten bytes are a distinctive token, so any surviving prefix of + // this credential is visible to the assertion rather than merely shorter. + const secret = `CREDENTIAL-${"z".repeat(200)}-END`; + // Land it across the edge of the head's REPORTED share + // (`CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2`): 80 bytes inside the window, the + // rest past it. Only the extra secret-length the head retains beyond that + // share lets the redactor match it whole; without it those 80 bytes are a + // verbatim credential prefix in the diagnostic. + const pad = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) - 80; + await writeFile(engine, [ + `process.stderr.write("h".repeat(${pad}));`, + `process.stderr.write(${JSON.stringify(secret)});`, + `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, + `process.stderr.write(${JSON.stringify(" terminal-detail")});`, + "process.exitCode = 9;" + ].join("\n")); + try { + const child = spawnEngine({ + command: process.execPath, commandArgs: [engine], engine: "agy", + maxToolTurns: 1, timeoutMs: 10_000 + }, "verbose", { cwd: root }, undefined); + await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /terminal-detail/u, "the tail still reports the last words"); + assert.doesNotMatch(error.message, /CREDENTIAL|z{32}/u, "no fragment of the secret may survive the cut"); + return true; + }); + } finally { await rm(root, { recursive: true, force: true }); } +}); + test("redacts a 2000-byte exact secret before retaining a failed stderr tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-long-secret-")); const engine = path.join(root, "long-secret-failure.mjs"); diff --git a/src/pi/grokSandbox.ts b/src/pi/grokSandbox.ts index ddc9ee4..1c448ed 100644 --- a/src/pi/grokSandbox.ts +++ b/src/pi/grokSandbox.ts @@ -8,6 +8,7 @@ import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; +import { assertGrokWorkerDenyPathsPlaceable } from "../runtime/grokWorkerDenyPlacement.js"; import { GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH, GROK_WORKER_SANDBOX_PROFILE, renderGrokWorkerSandboxProfile } from "../runtime/grokWorkerSandboxProfile.js"; export const GROK_DAIMON_SANDBOX_PROFILE = GROK_WORKER_SANDBOX_PROFILE; @@ -36,6 +37,11 @@ export async function prepareAndVerifyGrokSandbox( if (denied.some((entry) => overlaps(entry, cwd) || overlaps(entry, engineHome))) { throw unavailable(); } + // Grok 1.0.34 materializes every deny target inside bubblewrap as the uid it + // runs Grok under — here, this process's own — and refuses the whole profile + // if one cannot be resolved. Refusing now names the path; letting it through + // would kill every turn with a bare `bwrap: Can't create file at …`. + await assertGrokWorkerDenyPathsPlaceable(denied, { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }); await writeProfile(engineHome, denied); const beforeBytes = await eventFileSize(path.join(engineHome, SANDBOX_EVENTS)); const child = trackCliChild(spawn(authority.command, [ diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 19a1ace..cc4e235 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,6 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -97,18 +98,12 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { async startAgent(input: AgentStartInput): Promise { validatePiRawTrainingCaptureOptions(this.options.rawTrainingCapture); - await Promise.all([ - input.runtimeHomePath, - `${input.runtimeHomePath}/.config`, - `${input.runtimeHomePath}/.local/share`, - `${input.runtimeHomePath}/.local/state`, - `${input.runtimeHomePath}/.cache`, - `${input.runtimeHomePath}/.tmp`, - `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true }))); + await ensureRuntimeHome(input.runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp", "tool-state"] + .map((relative) => ensureRuntimeHomeDirectory(input.runtimeHomePath, relative))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true }); + await ensureRuntimeHome(memoryRuntimeHomePath); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 0f0f45c..1833ed0 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -1,11 +1,12 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -268,8 +269,7 @@ export const writeTurnTraceRecord = async ( record: PiTurnTraceRecord ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); - const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true }); + const turnsPath = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry/turns"); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 93d59df..9ee6f8b 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -1,10 +1,11 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -188,8 +189,7 @@ export const persistPiWorldTrajectory = async ( } }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); - const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true }); + const trajectoriesPath = await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry/world-trajectories"); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 1628dc5..08250d9 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -77,6 +77,62 @@ model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected an is still metered. Control protocol v2 is refused-v1 on the wire because both ends ship in this package. +A failed brokered turn also carries the worker's own last words. The launcher +gives the worker one pipe for stdout and stderr and publishes no output for a +failure, so a `worker_failed` turn used to reach the host as nothing but +`exit=1` — the reason the worker printed died with the container's tmpfs. +`DBL_MAX_DIAGNOSTIC` (512 bytes) is now the launcher's bounded tail of that +pipe, sent beside the fixed result frame in `diagnostic_length` and kept only +for a worker that exited on its own account: an output-limit tail would be the +very payload the bound refused, a cancelled turn has no reader left, and a +prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail +exactly as the CLI child path redacts a failed engine child +(`redactCredentialText` with the turn's own provider/MCP capabilities as exact +secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to +one line as `diagnostic.reason`. + +Two rules that live capture taught, both cheap and both load bearing. The +bytes are **decoded**, never stringified: `Uint8Array.prototype.toString("utf8")` +ignores its argument and renders bytes as comma-separated decimals, and a +worker's last words reached an operator as +`reason=108,111,110,101,46,32,87,104,101,110,...` — a string, control-character +free, inside the bound, and passing every check on the way out. So the frame is +normalized to a `Buffer` once on entry and the diagnostic goes through an +explicit `TextDecoder`, which also replaces rather than throws on the +multi-byte sequence a byte-counted window can cut in half. And the window +keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early +prints its error before it echoes its input, so a pure tail is the echo. The +marker is paid out of the same budget, and output that fits is returned +byte-identical. The launcher's own 512-byte window keeps both ends too +(`diagnostic_window`, `native/AGENTS.md`), so the head of a large blob now +survives the one place it used to be erased. Its elision is a cut, and a cut +can split a capability in half into a fragment exact redaction cannot match, so +`scrubCutFragments` matches that fragment here, where the turn's capabilities +are known — on both sides of every marker and at the window's outer ends. A +margin reserved in the launcher could not do this: there, what is kept is +exactly what is sent. It is an optional, control-character-free +member of the sealed terminal response's closed diagnostic — admitted by +`engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — +so it replays with the sealed record and reaches the operator through +`engineBrokerControlClient.ts`'s failure message. Nothing new is written to +disk: the reason travels inside the response the broker already seals. + +A turn whose worker said nothing still records what it spent. The launcher can +refuse to publish a worker's output (`DBL_MAX_OUTPUT`, `native/AGENTS.md`) and +the native transport can fail outright, and in both cases `result.text` never +exists, so there are no stream frames to read usage from. `streamOrMeterUsage` +then falls to the proxy's own per-request measurements — the meter admitted and +settled every forwarded request, so the broker knows the spend even when the +worker never speaks — and `finishBrokerTurnWithUsage` seals and appends it with +`outcome: "failed"`. That is the whole of the guarantee and it is pinned by +"a worker whose work succeeded but whose output crossed the launcher bound" +(`grokEngineBrokerUsage.test.ts`), which builds the launcher's own +output-limit frame at the ABI offsets and decodes it with the shipped client. +Deleting the meter fallback, or refusing that frame shape in +`decodeNativeBrokerResult`, both turn it red. The one window that stays open is +the documented one: a crash before the turn record's rename, which the next +boot seals `usage: null`. + Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's Grok credential without holding it. `request_inference_grant {model, @@ -117,6 +173,22 @@ must be run with `--model daimon-inference-grok`. The inference ledger directory must be provisioned setgid to the organization group (e.g. `2100:2000 2750`) for uid 2000 to read rows the broker creates `0640`. +Every model block the worker can reach carries `max_retries = 0`. Grok 1.0.34's +default retries a refused or failed request with backoff **past 45 s**, blindly: +one live turn emitted the same refusal fifteen times over five minutes, spent +$0 and died with no account of why. The session-title sink and the evaluator +client (`grokInferenceClientConfig.ts`) always pinned it; the worker's own +model — the single path that spends money — was left on the default, so the one +place a stall costs a wake was the only one that could idle for minutes after +its work was done, silently, because a retried request that never reaches +upstream writes no ledger row and prints no proxy line. Daimon owns the retry +decision here because the thing being retried is Daimon's own proxy: a +genuinely transient fault is already answered 503 and is the broker's to +retry, and everything else is a refusal that repeating cannot fix. The worker +fails fast instead and the turn reaches the host with a status. These bytes are +manifest-pinned per model and effort, so changing them rotates +`GROK_ENGINE_BROKER.worker.configSha256` and every deployment must re-vendor. + `grokBrokerProjection.ts` is the public, I/O-free projection of one brokered Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt @@ -144,11 +216,198 @@ refuses a turn whose worker config does not hash to the declared one. Three capability reaches the model through `env_key = "DAIMON_PROVIDER_CAPABILITY"` set by the native launcher (as exposed as `DAIMON_MCP_CAPABILITY`); the per-turn `session_title` request cannot be disabled by any key, so -`[models] session_summary` points it at a hidden model on closed loopback port -9; and effort is only sent when the model declares it, so the declared effort is +`[models] session_summary` points it at a hidden model +(`GROK_SESSION_TITLE_SINK_MODEL_ID`) whose `base_url` is the broker's own +provider proxy and whose `api_key` is a placeholder too short to ever be a turn +capability — so the request does reach the proxy and is refused there, before +any capability lookup, isolation guard, credential read or upstream call, and +Grok falls back to the truncated prompt as the title. That refusal and a bare +unauthenticated `GET /` probe are the two requests a healthy turn always makes +and the proxy never forwards; neither prints a `refused:` line, because for as +long as they did, every healthy turn read as broken. Every *other* refused +request does name itself on the broker's stderr, and a fault that is not a +`GrokBrokerProxyRefusal` names its own class and message beside +`broker_unavailable` — `[grok-proxy] refused: broker_unavailable (TypeError: +…)` — because the bare word carries no diagnostic content and is answered 503, +which Grok blind-retries: one live turn emitted it fifteen times over five +minutes, spent $0, and died with no account of why. That cause is the error's +class and message, plus one level of its own `cause` — every failed provider +`fetch` is `TypeError: fetch failed` and names nothing without it, so the line +reads `broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)`, an +errno cause with no message named by its `code`. Nothing else: never a body, +bearer, capability, session id or +header. It is redacted through `redactCredentialText` with that request's own +capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, +flattened to one line, exactly as the failed CLI child and the launcher's +worker diagnostic are. It is a log line only: the 503 is unchanged, because a +genuinely transient fault is still transient. + +One fault is *not* transient and no longer wears that shape: a fenced +credential realm. `isStale()` is checked on the turn path before the +credential read, and the request that discovers the fence (the authority's own +generic error) is promoted to the same refusal, so a stale realm is a named +400 `auth_stale` instead of one 503 plus fourteen blind retries — the training +login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning +nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single +name behind the turn failure code, this refusal reason and the grant path's +401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, +and the title sink keeps its 503 on a fenced realm like everywhere else. + +The sink keeps that 503 +shape because every live capture was taken with it: forcing 400 and 503 there +were both observed to end the turn `exit=0, result: success`, so a hard 4xx on +that request does *not* end Grok's session. And effort is only sent when the +model declares it, so the declared effort is the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in the image even for a loopback `http://` URL ("Failed to build HTTP client"). +`engineBrokerMcpFacade.ts` is the worker's only route to its per-wake MCP mount +and rebuilds every header from a closed allowlist in both directions, so the +worker's bearer never reaches the mount and no mount header reaches the worker +uninvited. That allowlist must include the Streamable HTTP transport's own +routing headers or the route does not exist: forwarding only +`content-type`/`accept` destroyed `Mcp-Session-Id`, so `initialize` returned 200 +while every request after it — `notifications/initialized`, `tools/list`, +`tools/call` — came back HTTP 400 `Mcp-Session-Id header is required`, and the +model saw `search_tool` answer `{"results":[],"total_hidden_tools":0,"status": +"partial"}`. Client to mount: `content-type`, `accept`, `mcp-session-id`, +`mcp-protocol-version`, `last-event-id`. Mount to client: `content-type`, +`mcp-session-id`, `mcp-protocol-version`, plus the facade's own +`cache-control: no-store`. The session id is an opaque routing value and is +never logged or ledgered. The facade also carries the three methods the +transport uses — POST, the standalone `GET` SSE stream that is the only route a +server notification or progress frame can take, and the `DELETE` that ends a +session — and streams each body rather than buffering it, because a GET tunnel +stays open for the whole session. Streaming means backpressure, and a +backpressured tunnel must never park: `awaitMcpTunnelDrain` races the client's +`drain` against its `close`/`error` and the turn's abort, because a bare +`once("drain")` cannot fire for a client that hung up mid-write and left the +handler — and the upstream call it was relaying — awaiting for the life of the +process, with no status, no refusal and no line anywhere to read. Every +outcome but a real drain rejects, so the relay tears the tunnel down instead +of writing into a socket that is gone. Never widen it into a transparent proxy: the +whole point of the boundary is that the allowlist is closed. + +The facade is also the only place an MCP tool call is observable *while it is +still running*. Daimon writes a tool receipt on completion, so a call that +started and never returned is byte-identical, in every artifact, to a call that +was never made — and that was the last unlit path under a live hang where the +worker stopped acting after its eighth provider response, the per-request +ledger published `open: 0`, and the trial deadline killed it seven minutes +later. `engineBrokerMcpCallLog.ts` records each relayed `tools/call` POST and +whether the facade ever answered it, and the observation rides the *sealed +terminal response* of a failed turn (`mcpCalls`, optional and v2-only) — +the seam the worker's redacted last words and the sealed usage already take, +because the slot's control root is tmpfs that dies with the container. It +replays with the record and reaches the operator through +`engineBrokerControlClient.ts` as `mcp=/ answered` plus +`mcp_outstanding=@ms`. Its rules are the per-request ledger's: names +and timings only (never arguments, never a result, never a session id or +bearer; a name that is not a plain short identifier is ``, and the +list is bounded with a `` last entry); absence stays absence (a turn +the facade never registered observes as *nothing*, a turn that called nothing +observes `started: 0`, and a POST body the facade could not read counts in +`undecoded` rather than inventing a name); and it can never fail, delay or +refuse a turn. "Answered" means one thing and it is load bearing: the relay +reached its own `end()`. A tunnel torn down when the worker dies did not +answer, so the call it was blocked on stays outstanding with the elapsed time +it had reached — otherwise the turn's death would erase the evidence the +instrument exists to keep. + +The facade relays one more thing, for the whole session, and until now wrote +nothing about it. A `tools/call` is a POST that answers; the standalone `GET` +SSE tunnel is the route a server notification or progress frame takes, and it +stays open from `initialize` to the worker's own shutdown. A worker parked +reading it was, in every artifact the broker wrote, identical to a worker doing +nothing: every provider request closed, every tool call answered, idle to the +deadline. `EngineBrokerMcpCallLog.openTunnel` records that lifecycle on the same +observation — `tunnels: {opened, closed, delivered, open: [{openMs, delivered}]}` +— so a turn sealed with one still open says so and says how long it had been +open, and `delivered` separates a tunnel actively carrying frames from one held +open having received nothing, which is the difference that decides whether it is +the blocker. Bounded at `ENGINE_BROKER_MCP_TUNNEL_MAX` open entries (a session +opens one), counts and elapsed milliseconds only, never a frame, an event +payload or a session id. It is *observation only*: nothing here closes, times +out or refuses a tunnel, because an instrument that tore the stream down would +destroy the evidence it exists to gather. The member is optional on the wire for +one reason — a turn sealed before it existed must still replay — so its absence +means "not measured" and never zero, exactly as `mcp`'s own absence does. +A request the facade *refuses* is the sharpest form of the same silence, and +it used to observe as nothing at all: `route()` threw before `calls.begin`, so +a turn 403'd on every request sealed `answered == started, outstanding: []` — +byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts +each one by a closed reason class (`route`, `expired`, `exhausted`, +`unrouted`, `oversized`), because the classes call for opposite fixes: an +exhausted per-turn capability is a budget, an unserved route is a worker +asking for something that does not exist. That budget is *derived*, not +picked: `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` is `GROK_WORKER_MAX_TURNS` +times `ENGINE_BROKER_MCP_ROUND_REQUESTS` (3 — a round's `search_tool`, its +`use_tool`, and one spare for a retry or a second discovery) plus +`ENGINE_BROKER_MCP_SESSION_REQUESTS` (5 — `initialize`, +`notifications/initialized`, `tools/list`, the GET tunnel, the DELETE). It was +a literal 128 against a bound of 48 rounds whose legitimate traffic is ~101, so +the first round that also retried met a mid-turn 403 storm; the two numbers +that must agree now live in one place, and raising the turn bound can no longer +silently exhaust the budget. It stays a bound rather than a comfortable number +because the derivation is exact: the request *after* the worst-case legitimate +session is refused, so a compromised worker gets three MCP calls per round it +was compiled to take and not one more. `engineBrokerMcpObservation.test.ts` +drives that worst case through the real facade, computed from the turn bound +alone. Attribution comes from +`EngineBrokerCapabilities.classifyToken`, which names the token's turn and why +it would be refused *without spending its budget*; a bearer no grant matches +names no turn and stays unattributed, because guessing an owner would be +inventing the measurement. Counts only: never the token, the capability, the +URL or the body. The member is optional on the wire for `tunnels`' one reason, +and reaches the operator as `mcp_refused=exhausted:41` and the seal row's +`mcp.refusals`. + +Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the +tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and +**closes 16 ms before the worker exits** — the close is the worker's own +shutdown, not the facade's. A turn that never reaches that shutdown is the one +that seals with it open; a deliberately stalled `tools/call` sealed +`open: [{openMs: 14652, delivered: false}]` beside its outstanding call. + +That seam is enough for a turn that *fails with a reply* and not for the turn +the instrument was built for. A worker that crashes still produces a terminal +response; a worker that HANGS is cancelled by its client's deadline, and a +cancelled turn has no client left to answer, so the sealed response — with +`mcpCalls` and the worker's redacted last words riding on it — is sealed into a +turn record in the broker's own `0700` turn store and dies with the slot's +tmpfs. Six live runs reproduced that exactly. What *does* survive a slot is the +broker's ledger directory, which Paideia already recovers `usage.jsonl` and +`requests.jsonl` from on the failure path, so `engineBrokerSealLedger.ts` writes +a third stream beside them: one `noopolis.daimon.turn-seal.v1` row per sealed +terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from +the sealed response and nothing else. Its members are the accounting, the +failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with +the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the +facade's `mcp` observation — names, counts, refusals by reason class, +GET-tunnel lifecycle and elapsed milliseconds. That projection is a closed +allow-list and `engineBrokerSealLedger.test.ts` asserts the *exact key set* of +a written row for a completed turn: replacing it with `...terminal` writes +`usage`, `diagnostic`, `mcpCalls` and the model's entire reply into the +ledger, and that mutation is what the assertion exists to catch. Never a +prompt, body, reply, bearer, capability or session id; the terminal response +carries none of those in the first place, and the projection is an allow-list +rather than a spread, so a future additive member of the response cannot become +a ledger field by accident. + +Two invariants make it worth having. The row is rendered for *every* terminal +turn including one whose `usage` is `null` — a turn cancelled before any spend +could be attributed writes no usage row at all, and is precisely the turn whose +outstanding call has no other route out. And absence stays absence three ways: +no `mcp` member when the facade never observed the turn, `started: 0` when it +observed a turn that called nothing, and no row when nothing sealed. Reading +any of those three as another is the failure this stream exists to prevent. The +line is sealed into the turn record's ledger bytes with the other two and +appended last, so a replay completes an interrupted append the same way and +readers dedupe on `turn`; `seal` is optional in `parseBrokerTurnLedgerLines`, so +a record written before the stream existed still replays. It is advisory +throughout: `recordLedgerLines` swallows every I/O fault, and nothing here can +refuse, delay or fail a turn. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, @@ -160,6 +419,92 @@ every profile inside bubblewrap, where a non-empty `deny` list is enforced; `grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process can neither write, rename, nor unlink any of the root-owned files. +Deny-path placement (`grokWorkerDenyPlacement.ts`). Grok 1.0.34 materializes +every `deny` entry inside bubblewrap **as the worker uid**, bind-mounting +`$GROK_HOME/sandbox-blocked-{file,dir}` over the target, so an entry is +placeable only when every ancestor directory is searchable by that uid and the +target already exists and is not a symlink. One unplaceable entry makes Grok +refuse the *whole* profile (`bwrap: Can't create file at …: Permission +denied`), so every turn of that worker fails, not just that path. Matrix: +`.runtime/grok-deny-placement/EVIDENCE.md` in the ecosystem folder. The rule +therefore has two halves: +- shape, decidable without a filesystem and asserted by the renderer: canonical, + and strictly below every base-profile grant (`GROK_WORKER_BASE_PROFILE_GRANTS`); +- placement, asserted by whoever provisions the paths — root provisioning and + every slot recycle on the Spawnfile side, `prepareGrokWorkerAttestation` + before every brokered turn, and `prepareAndVerifyGrokSandbox` on the direct + path, which runs as the worker uid itself. The broker (uid 2100) cannot + descend into a `2000: 0710` runtime home, so an `EACCES` below an + ancestor the worker *can* search is left undecided there; root, which holds + `CAP_DAC_READ_SEARCH`, decides every entry. + +When a protected path is not placeable, the deny entry is **lifted** to the +nearest ancestor that is — never adding `o+x` to a private directory, because a +lift masks a superset and never widens the worker's reach. The durable +wake-acceptance store is exactly that case: it lives under the organization's +`state` directory, which the ownership guard secures `2000:2000 0700`, so the +mask goes on that directory (`acceptanceStoreDenyPath` in +`grokBrokerProjection.ts`, which refuses a mask that does not contain the +store). + +Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every +turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). +Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write +and refuses to start if either, or any path equal to or above a base grant, is +in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; +`/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: +- the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its + read-write grants; Python, Node and `mktemp` use it); provision it + `: 0700`. Every registered worker's private temp is attested + before any turn, so one misprovisioned sibling refuses all turns; +- `/tmp` and `/var/tmp` must be `root: 1774`: + Grok needs to open the directory, but without search or write a worker can + only list names — `cat`/`read_file` get EACCES and it cannot create files. + `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root + process outside that group that needs temp space must get its own `TMPDIR`; +- the organization runtime home of a brokered Grok agent is `2000: + 0710` — traverse-only, so the worker can reach `tool-output/` and nothing + else. `physicalReadiness.ts` accepts exactly that shape for a `grok` agent + (owner the runtime user, mode `0710`, group a worker group that is not the + runtime's own) and keeps the plain `0700` rule for every other engine; wider + (`0711`, `0730`, `0750`, `0770`, any world bit, setgid) is refused, and so is + a `0700` home for a Grok agent, because its worker could not read its own + spills. Everything Daimon creates inside a runtime home is `0700` + (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, + `tool-state`, the engine XDG directories, `.tmp`), so a traversable home + still exposes nothing but `tool-output/`. A deployment-provisioned memory + home under that runtime home must stay `0700` for the same reason. That mode + is *asserted and corrected*, not merely passed to `mkdir`, because `mkdir`'s + `mode` decides nothing for a directory that already exists: a `telemetry/` + left at `0755` by a pre-branch Daimon or pre-created by a deployment stayed + `0755` forever, and under a `0710` home that is the worker reading its own + agent's prompts, replies and causal history. `ensureRuntimeHomeDirectory` + walks every level below the home, opens each through + `O_DIRECTORY|O_NOFOLLOW` and `fchmod`s the directory it stat'd; one owned by + another uid is **refused**, never widened, and a symlink planted where a + directory belongs is refused rather than followed. The home itself is + create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok + agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The + mode constant lives only in that module, a test fails the build if any writer + imports it again, and the same test refuses any `mkdir` that names a runtime + home outside the layout — a `mode:` argument covers only the install where + the directory is new. `wakeAcceptanceFs.ts` is the one exception and closes + the hole the other way, by asserting the directory it found and refusing a + wider one; +- spills (`toolResultSpill.ts`) are written `0640`; provision + `/tool-output` as `2000: 2750` (setgid) under a + runtime home the worker can traverse, so each spill carries that agent's + worker group and no other worker can read it. The writer pins the directory + (`O_DIRECTORY|O_NOFOLLOW`, dev/ino re-checked before publishing) and refuses + one that is a symlink, not owned by the runtime, wider than `2750`, or + group-open without setgid or in the runtime's own group; it cannot tell + *which* worker gid belongs to the agent, so that mapping stays the + deployment's. A spill is published by rename, replacing any existing entry + (a planted symlink included) without following it. +- registered workspace and home paths must be canonical (no `.`, `..`, empty + components or trailing slash) in both `service.json` and `registrations.bin`; + the launcher refuses the slot otherwise. + `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. `agySubscriptionBootstrap.ts` owns only the interactive first-enrollment AGY @@ -254,6 +599,34 @@ is swallowed, because instrumentation must never fail a wake. The existing ledger's version, path, and field list are untouched, so Spawnfile's `v`-pinned reader is unaffected. +Each Grok row also carries `tool_calls`: the tool-call NAMES that request's +response carried, read by the proxy from the body it already buffers for usage +(`parseGrokResponseToolNames` in `grokBrokerTurnMeter.ts`). Timings and tokens +alone cannot answer "did the model ever *try* to call `use_tool` or +`search_tool`", which is exactly the question two live turns left open. Names +only — never arguments, never message content, never a bearer; a `name` that is +not a plain short identifier is recorded as `` rather than passed +through, and the list is bounded at `GROK_REQUEST_TOOL_CALLS_MAX` (16) entries +with a `` last entry, so a pathological response cannot write an +unbounded row. Absence stays absence, as everywhere in these ledgers: a decoded +response that called nothing records `[]`, and a response that could not be +decoded records *no field at all*, because a fabricated empty list is +byte-identical to a measured one. On the stream row path the names are attached +only when the proxy's timings and the worker's stream requests are aligned +request-for-request, since an unaligned index would credit one request's attempt +to another. The *usage* decode beside it is wrapped the same way, and for a +sharper reason: the upstream call has already succeeded, so a decoder fault +that failed the request would throw away a response the broker paid for and +have Grok buy it again. A fault there falls through to the documented estimate +(`ceil(bodyBytes/2) + 4096`, `usage_source: "estimated"`, counted in +`estimated_requests`) — never to zero and never to absence, because the +ceiling must still count what was spent. It is an additive field inside the unchanged +`noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: +Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia +only relocates this stream's path. The whole path is advisory — the parse is +wrapped, and nothing it does can refuse, delay, or fail a turn, or reach the +spend gate. + `testRuntimeSubprocess.ts` is an unexported, explicit-test-only JSONL process surface for exercising the real control, schedule, and acceptance paths with a controlled clock and deterministic scripted cognition. Its ephemeral loopback @@ -352,3 +725,55 @@ unmarked/deferred deliveries wait for new input without a self-wake loop. Live turn authority is `activity.executions`, independent of receipt completion; its execution id must equal the engine wake id. Budget pauses retain acceptance, and operator stop remains a hard latch. + +That authority has to outlive the host, because the caller who needs it reads it +last. `activityV2` used to answer `undefined` once `stop()` closed the acceptance +store — HTTP 503 `native_host_unavailable` through a caller's route — and the one +caller that must prove an execution closed asks *after* the runtime stopped: a +harness worker stops its host the moment a delivery closes its execution and +stays deferred awaiting external input. So a trial whose subject really ran, +spent its budget and simply did not do the work could not be told from a hung or +crashed one, and reported as an unscorable infrastructure failure. `stop()` now +seals the projection between the dispatcher's own shutdown — which awaits every +admitted turn, so `executions` is settled rather than momentary — and the store's +close, and `activityV2` serves that seal afterwards with `state: "stopped"`. A +stopped host has *more* certainty about quiescence than a live poll, not less, +because nothing can be admitted after the seal. Three things it is not: a bypass +of the control token, a fabricated idle runtime (a host that never started and +one whose seal could not be read both still answer nothing, because absence must +stay absence), and a claim about the store-backed routes beside it — +`availability` and `wakeReceipt` keep answering `undefined` after a stop, since +neither settles a closure proof. `state` is optional on the wire for the reason +every additive member here is: a projection published before the seal existed +must still parse, and its absence means "not stated", never "running". + +A delivery returned to the inbox for restart records the outcome that returned it, +and **only a wake outcome can return one**. `attentionDispatcher` reclaims an +undisposed delivery to `accepted` on exactly one condition — a wake result of +`stopped`, which is also the shape an aborted in-flight wake arrives in +(`organizationRuntimeHost.ts` settles a queued job `queued_wake_stopped` and the +in-flight one `active_wake_aborted`). The dispatcher's own `stopping` latch used to +share that condition, and it is a HOST-LIFECYCLE fact, not a wake outcome: a wake +that *completed* had its evidence discarded because the dispatcher happened to be +halting, and the delivery was recorded `accepted, deferred: false, execution id +retained, no code` — byte-identical to "never ran" and to "ran but forgotten". +Production tolerated that because a restart re-delivers and the agent redoes the +work; a one-shot isolated trial has no restart, so the information was simply lost +and a subject that ran and made a choice reported as an infrastructure failure. It +is the wrong record for production too: an agent that read a delivery and declined +to dispose of it is **deferred**, whichever way the host is heading, and a restart +must not re-deliver it as fresh work. So a completed or failed wake takes the +deferred path regardless of dispatcher state, and `stopping` guards only the +pre-wake path, which is where it belongs — it must never be restored to the +post-wake decision. `WakeReceiptCode` carries `queued_wake_stopped` and +`active_wake_aborted` beside the existing five, because those are the two shapes a +shutdown really gives a wake and neither had an honest name. The wake's own code is +recorded exactly; nothing else names a reclaim, because a plausible name for an +undetermined cause gets acted on and a missing one does not. Two consequences, both +load bearing: `accepted` is the one non-terminal state a record may carry a code in, +since it is the only one reached *from* an ended execution (`running` and +`completed` still refuse one), and `transitionClaimed` no longer carries a code +across a transition — it describes the transition that produced the current state, +and a reclaimed delivery is claimed again later. Widening the enum rotates the +contract manifest digest, so Spawnfile must re-vendor +`contract-manifest.json`/`.sha256` and its pinned constant. diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 4774ae2..6e1c0b6 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; -import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { AttentionDispatcher, inboxPrompt } from "./attentionDispatcher.js"; +import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; import { parseWakeAcceptanceRequest } from "./wakeAcceptanceTypes.js"; @@ -14,7 +15,7 @@ import type { OrganizationRuntimeHost, OrganizationRuntimeWakeRequest, Organizat const token = "attention-test"; const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; -const config = (maxExecutions = 20) => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: "codex" }, attention: { maxBatchMessages: 3, maxExecutions } })) }); +const config = (maxExecutions = 20, engine: "codex" | "grok" = "codex") => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: engine }, attention: { maxBatchMessages: 3, maxExecutions } })) }); const request = (id: string, agent_id = "alpha") => ({ token, agent_id, delivery_id: id, event: { version: "noopolis.daimon.wake.v2", kind: "message", text: `Handle ${id}`, occurred_at: "2026-09-11T00:00:00.000Z" } }); const pause = (ms = 5) => new Promise((resolve) => setTimeout(resolve, ms)); async function until(test: () => boolean | Promise): Promise { for (let n = 0; n < 200; n++) { if (await test()) return; await pause(); } throw new Error("expected side effect did not appear"); } @@ -29,13 +30,13 @@ class Core implements OrganizationRuntimeHost { async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; } async stop() { this.stops++; this.releases.forEach((release, index) => release({ version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: this.wakes[index]!.agentId, wakeId: this.wakes[index]!.event.id, code: "active_wake_aborted" })); return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } } -async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000) { +async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000, engine: "codex" | "grok" = "codex") { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-")); await chmod(root, 0o700); const usage = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-usage-")); await writeFile(path.join(usage, "usage.jsonl"), ""); const registry: AttentionRegistry = new Map(); const core = new Core(); const options = { acceptanceStorePath: root, controlToken: token, storeOptions: { ...storeOptions, claimTtlMs }, attentionRegistryForTest: registry, fuseEnvironment: { DAIMON_WAKE_FUSE_DIRECTORY: usage, DAIMON_WAKE_FUSE_EPOCH: "attention", DAIMON_WAKE_FUSE_MAX_WAKES: String(maxWakes), DAIMON_WAKE_FUSE_MAX_TOKENS: "10000", DAIMON_TURN_USAGE_LEDGER_PATH: path.join(usage, "usage.jsonl") } }; - const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit), core, options); await control.start(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit, engine), core, options); await control.start(); return { root, usage, registry, core, options, control, cleanup: async () => { await control.stop(); await rm(root, { recursive: true, force: true }); await rm(usage, { recursive: true, force: true }); } }; } @@ -200,3 +201,65 @@ test("claim-renewal failure revokes execution authority, stops cognition, and la const accepted = await f.control.accept(request("after-fence")); assert.equal(accepted.state, "stopped"); assert.equal(accepted.blocked!.reason, "ledger_unavailable"); } finally { WakeAcceptanceStore.prototype.renewClaim = original; await f.cleanup(); } }); + +test("an inbox turn leads with each delivery's own text and keeps the accounting after the work", async () => { + const f = await fixture(); + try { + await f.control.accept(request("d-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text; + // The task comes first: a delivery's text is the work, not a JSON payload to account for. + assert.match(text, /^Carry out this delivery\./u); + assert.match(text, //u); + const task = text.indexOf("Handle d-1"), accounting = text.indexOf("daimon_inbox_disposition"); + assert.ok(task >= 0 && accounting > task, "accounting must follow the delivery text"); + assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); + } finally { await f.cleanup(); } +}); + +/** + * `daimon_inbox_disposition` is the tool that records a finished wake as + * complete; an agent that cannot name it leaves its work recorded as deferred. + * On Grok the bare name reaches nothing, so the inbox prompt must name the + * `daimon__` form the engine can actually invoke. + */ +test("a Grok inbox turn names both inbox tools the way use_tool can call them", async () => { + const f = await fixture(20, 100, 240000, "grok"); + try { + await f.control.accept(request("g-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes(grokDaimonToolName("daimon_inbox_disposition")), "disposition tool carries the daimon__ prefix"); + assert.ok(text.includes(grokDaimonToolName("daimon_inbox")), "inbox tool carries the daimon__ prefix"); + // No bare occurrence survives: every mention is the prefixed one. + assert.equal(text.split("daimon_inbox").length - 1, text.split(DAIMON_GROK_TOOL_PREFIX).length - 1); + } finally { await f.cleanup(); } +}); + +test("every other engine's inbox turn keeps the bare tool names", async () => { + const f = await fixture(); + try { + await f.control.accept(request("c-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes("with daimon_inbox_disposition (complete)")); + assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); + } finally { await f.cleanup(); } +}); + +/** + * Every branch of the inbox prompt, not only the one a small delivery takes: + * the oversized-payload fallback is the branch a busy agent meets, and it names + * `daimon_inbox` too. + */ +test("every branch of the inbox prompt names its tools the way the engine can call them", () => { + const delivery = { acceptance_id: "a-1", delivery_id: "d-1", kind: "message", text: "Do the thing", occurred_at: "2026-09-11T00:00:00.000Z" }; + const oversized = { ...delivery, text: "x".repeat(2_000) }; + for (const [messages, budget] of [[[delivery], 12_000], [[oversized], 64], [[oversized], 8]] as const) { + const grok = inboxPrompt(messages, "grok", budget), codex = inboxPrompt(messages, "codex", budget); + // Mutation guard: an unprefixed branch leaves a bare name in the Grok text. + assert.equal(grok.split("daimon_inbox").length - 1, grok.split(DAIMON_GROK_TOOL_PREFIX).length - 1, grok); + assert.ok(grok.includes(grokDaimonToolName("daimon_inbox")), grok); + assert.equal(codex.includes(DAIMON_GROK_TOOL_PREFIX), false, codex); + assert.ok(codex.includes("daimon_inbox"), codex); + } + // The smallest budget is the fallback that only points at the tool. + assert.match(inboxPrompt([oversized], "grok", 8), new RegExp(`exceeds the prompt budget; read it with ${grokDaimonToolName("daimon_inbox")}\\.$`, "u")); +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index b1a8f77..a63c9eb 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -6,6 +6,7 @@ import { WakeAcceptanceStore, WakeExecutionClaimLostError, type WakeExecutionCla import type { StoredWakeAcceptanceRecord } from "./wakeAcceptanceRecord.js"; import { WakeFuse } from "./wakeFuse.js"; import { ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS, ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES } from "../contracts/organizationRuntimeContract.js"; +import { grokDaimonToolName } from "../contracts/grokWorkerContract.js"; type Claimed = { record: StoredWakeAcceptanceRecord; claim: WakeExecutionClaim; done: boolean }; type Options = Readonly<{ store: WakeAcceptanceStore; host: OrganizationRuntimeHost; fuse: WakeFuse; agents: readonly OrganizationRuntimeAgentConfig[]; registry: AttentionRegistry; token: string | undefined; onIdle(agentId: string): void }>; @@ -111,7 +112,7 @@ export class AttentionDispatcher { result = await host.wake({ token, agentId: agent.id, event: { version: "noopolis.daimon.wake.v1", id: agent.attention === undefined ? first.delivery_id : executionId, kind: first.event.kind, occurredAt: first.event.occurred_at, - text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.attention.maxBatchBytes) + text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.engine.kind, agent.attention.maxBatchBytes) } }); } catch (error) { result = { version: "noopolis.daimon.wake-result.v1", status: "failed", agentId: agent.id, wakeId: executionId, code: "engine_failed", detail: engineFailureDetail(error) }; @@ -121,7 +122,17 @@ export class AttentionDispatcher { const executionError = result.status === "rejected" ? `wake rejected: ${result.code}` : result.status === "failed" ? `engine_failed: ${result.detail ?? "engine execution failed"}` : null; for (const item of claimed.filter((value) => !value.done)) { - if (this.stopping || result.status === "stopped") await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted"); + // Returned to the inbox for restart, and recording WHY. Only a WAKE OUTCOME + // decides this: `stopped` — which an aborted in-flight wake also carries, with + // its own code — is the runtime reclaiming work nobody read. The dispatcher's + // own halt is not an outcome and must not stand in for one; it guards the + // pre-wake path, where it belongs. Keying on it here discarded a completed + // wake's evidence because the host happened to be halting, leaving a record + // byte-identical to "never ran". Production tolerated it because a restart + // re-delivers; a one-shot isolated trial has no restart and simply lost it. + if (result.status === "stopped") { + await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted", result.code); + } else if (agent.attention !== undefined) { // Successful reading is not completion. A failed execution also keeps // unfinished deliveries, and its execution id for idempotent retry. @@ -171,15 +182,56 @@ export function selectBatch(records: readonly StoredWakeAcceptanceRecord[], agen return selected.length ? selected : [first]; } -function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { +/** One claimed delivery, rendered as the task it is. */ +function deliveryBlock(message: unknown, index: number): string | undefined { + if (message === null || typeof message !== "object") return undefined; + const row = message as Record; + const text = typeof row.text === "string" ? row.text : undefined; + if (text === undefined) return undefined; + const from = typeof row.from === "string" ? row.from : undefined; + const kind = typeof row.kind === "string" ? row.kind : "delivery"; + const id = typeof row.delivery_id === "string" ? row.delivery_id : `#${index + 1}`; + return [``, text, ""].join("\n"); +} + +/** + * The inbox turn, task first. + * + * A delivery's own text *is* the work. Leading with bookkeeping and handing the + * model `JSON.stringify(messages)` buried the task: an agent read the JSON, did + * the accounting and deferred without doing the job (observed on Grok: nine + * model requests, no tool calls, nothing filed). The deliveries are therefore + * rendered as labelled blocks and the `daimon_inbox` accounting follows them as + * what to do *after* the work, with the machine-readable payload kept as a + * trailing appendix while it fits the same budget. + * + * Both tools are named the way the agent's own engine can call them. On Grok a + * Daimon tool is an MCP tool of server `daimon` and its bare name reaches + * nothing (`grokDaimonToolName`, the same contract module the worker's system + * prompt and identity envelope render from), so an agent handed the bare name + * cannot mark its work complete — and an unmarked, finished wake is recorded as + * deferred. + */ +export function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); - const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; + const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); + const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; + const accounting = `\nWhen the work above is done, record each delivery with ${tool("daimon_inbox_disposition")} (complete), or defer the ones you could not finish; use ${tool("daimon_inbox")} for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n`; + const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; + const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES + && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; + if (blocks.length > 0 && Buffer.byteLength(blocks.join("\n\n")) <= maxBytes) { + const task = header + blocks.join("\n\n") + accounting; + const withPayload = `${task}\nMachine-readable payload: ${body}`; + // The inbox budget bounds selection; the v1 execution boundary independently + // bounds the complete prompt, including metadata, escaping, and instructions. + if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; + if (fits(task)) return task; + } + const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; - // The inbox budget bounds selection; the v1 execution boundary independently - // bounds the complete prompt, including metadata, escaping, and instructions. - if (Buffer.byteLength(body) > maxBytes || Buffer.byteLength(prompt) > ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES - || [...prompt].length > ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS) { - return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; + if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { + return prefix + `The selected payload exceeds the prompt budget; read it with ${tool("daimon_inbox")}.`; } return prompt; } diff --git a/src/runtime/engineBrokerCapabilities.ts b/src/runtime/engineBrokerCapabilities.ts index 0bbd102..1f99f8a 100644 --- a/src/runtime/engineBrokerCapabilities.ts +++ b/src/runtime/engineBrokerCapabilities.ts @@ -23,6 +23,28 @@ export class EngineBrokerCapabilities { } return undefined; } + /** + * Which grant a token names and why it would be refused, *without* spending + * it. + * + * The facade needs this on its refusal path alone. A 403 it cannot attribute + * to a turn is a 403 that seals as nothing at all, and a turn whose every + * request was refused then reads exactly like a healthy one — the silence + * `engineBrokerMcpCallLog.ts` exists to end. A token no grant matches names + * no turn and stays unattributed; nothing here returns the token, the grant + * or the agent's capability, only the turn id and a closed reason. + */ + classifyToken(token: string): Readonly<{ turnId: string; state: "live" | "expired" | "exhausted" }> | undefined { + const candidate = hash(token); + for (const grant of this.grants.values()) { + if (!timingSafeEqual(grant.digest, candidate)) continue; + // Budget before expiry: the TTL outlives every declared turn limit, so an + // exhausted grant is the reachable refusal and the actionable answer. + if (grant.requests >= grant.maxRequests) return { turnId: grant.turnId, state: "exhausted" }; + return { turnId: grant.turnId, state: grant.expiresAt <= Date.now() ? "expired" : "live" }; + } + return undefined; + } inspectToken(token:string):Readonly<{agentId:string;turnId:string}>|undefined{const candidate=hash(token);for(const grant of this.grants.values()){if(grant.expiresAt>Date.now()&&grant.requestscalls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpRefusals(calls.refusals)}${renderMcpTunnels(calls.tunnels)}`; +/** The refusals the facade never relayed, by reason class, and only the classes that happened: a turn refused 403 must not read as a turn that was served. */ +const renderMcpRefusals=(refusals:EngineBrokerMcpRefusalObservation|undefined):string=>{if(refusals===undefined)return"";const named=ENGINE_BROKER_MCP_REFUSAL_REASONS.filter((reason)=>refusals[reason]>0);return named.length===0?"":`; mcp_refused=${named.map((reason)=>`${reason}:${refusals[reason]}`).join(",")}`;}; +/** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ +const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; + export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; /** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ export class EngineBrokerInferenceGrantRefused extends Error { @@ -45,6 +59,6 @@ export class EngineBrokerControlClient implements EngineBrokerTurnClient { const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} - if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""}${renderMcpCalls(response.mcpCalls)})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts new file mode 100644 index 0000000..dbf51ac --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; + +const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); +const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); + +/** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +/** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ +const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; +/** An observed turn the facade never refused — also a measurement, and not the same statement as a turn it never observed. */ +const NO_REFUSALS = { route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }; + +const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { + let at = 1_000; + return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; +}; + +test("an unanswered tool call is outstanding with its name and its elapsed time; an answered one is neither", () => { + const { log, advance } = clock(); + log.open("turn"); + const answered = log.begin("turn", call("daimon__moltnet_send")); + advance(20); answered.answer(); answered.close(); + const hung = log.begin("turn", call("daimon__moltnet_read", 2)); + advance(420_000); + const observed = log.observe("turn"); + assert.deepEqual(observed?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1); assert.equal(observed?.undecoded, 0); + // A relay torn down by the turn's death answered nothing, and the elapsed + // time freezes where it stopped rather than growing with the report. + hung.close(); advance(5_000); + assert.deepEqual(log.observe("turn")?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.ok(!JSON.stringify(log.observe("turn")).includes("argument-bytes"), "names and timings only"); +}); + +test("absence stays absence: an unopened turn observes undefined, a turn that called nothing observes zero", () => { + const { log } = clock(); + assert.equal(log.observe("turn"), undefined); + log.open("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + // Everything that is not a tool call records nothing at all, so `started` + // stays a count of tool calls and not of traffic. + for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); + log.begin("turn", undefined).answer(); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + log.close("turn"); + assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); +}); + +test("a body the facade could not read counts as undecoded, never as a call with a name", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", Buffer.from("{not json", "utf8")); + log.undecodable("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + log.undecodable("absent"); +}); + +test("a tool name that is not a plain identifier is recorded as invalid, and a batch names each of its calls", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", call("moltnet read\nBearer sk-live-000")); + log.begin("turn", body([])); + log.begin("turn", Buffer.from(`[${call("memory_recall", 2).toString("utf8")},${call("world_probe", 3).toString("utf8")}]`, "utf8")); + assert.deepEqual(log.observe("turn")?.outstanding.map((entry) => entry.name), [ENGINE_BROKER_MCP_CALL_INVALID, "memory_recall", "world_probe"]); +}); + +test("the outstanding list is bounded, and the earliest calls are the ones kept", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4; index += 1) { log.begin("turn", call(`tool_${index}`, index)); advance(1); } + const observed = log.observe("turn"); + assert.equal(observed?.started, ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4); + assert.equal(observed?.outstanding.length, ENGINE_BROKER_MCP_OUTSTANDING_MAX); + assert.equal(observed?.outstanding[0]?.name, "tool_0"); + assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); +}); + +/** + * The channel a brokered turn had no light on at all. + * + * Seven live runs closed every provider request, answered every tool call, and + * still sat idle to the deadline. The one thing none of them could say is + * whether the worker was parked on the session's standalone GET SSE tunnel, + * because the facade relayed it and recorded nothing. The boundary these + * assertions straddle is exactly that: a tunnel still open at observation + * against one that ended before it — one is a worker that may still be reading, + * the other is a channel already closed and therefore not the blocker. + * + * Mutation: drop `openTunnels.delete(record)` from `close`, and the closed + * tunnel keeps reporting itself open; drop `tunnelsOpened += 1`, and an open + * tunnel becomes indistinguishable from a turn that never opened one. + */ +test("a GET tunnel still open reports its age; one that closed first reports closed and nothing open", () => { + const { log, advance } = clock(); + log.open("turn"); + const parked = log.openTunnel("turn"); + advance(430_000); + const open = log.observe("turn")?.tunnels; + assert.deepEqual(open, { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 430_000, delivered: false }] }); + + parked.close(); + advance(5_000); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 1, closed: 1, delivered: 0, open: [] }); +}); + +test("a turn that never opened a GET tunnel is not a turn that opened one, and neither is an unobserved turn", () => { + const { log, advance } = clock(); + log.open("turn"); + // Observed, and it measured zero: every count is a measurement. + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }); + log.begin("turn", call("daimon__moltnet_read")).answer(); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, "a POST is not a tunnel"); + const tunnel = log.openTunnel("turn"); + advance(1_000); + assert.equal(log.observe("turn")?.tunnels?.open.length, 1); + tunnel.close(); + // And the third state, which is not zero: a turn the facade never registered. + assert.equal(log.observe("absent"), undefined); + log.openTunnel("absent").deliver(); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); +}); + +test("a tunnel the mount pushed through is a different fact from one held open in silence", () => { + const { log, advance } = clock(); + log.open("turn"); + const silent = log.openTunnel("turn"); + const pushing = log.openTunnel("turn"); + advance(90_000); + pushing.deliver(); pushing.deliver(); + assert.deepEqual(log.observe("turn")?.tunnels, { + opened: 2, closed: 0, delivered: 1, + open: [{ openMs: 90_000, delivered: false }, { openMs: 90_000, delivered: true }] + }); + silent.close(); silent.close(); + assert.deepEqual(log.observe("turn")?.tunnels?.closed, 1, "closing twice closes one tunnel"); +}); + +test("the open-tunnel list is bounded, and the counts still name every tunnel beyond it", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_TUNNEL_MAX + 3; index += 1) { log.openTunnel("turn"); advance(1); } + const tunnels = log.observe("turn")?.tunnels; + assert.equal(tunnels?.opened, ENGINE_BROKER_MCP_TUNNEL_MAX + 3); + assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); + assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); +}); + +/** + * The refusal is the sharpest form of the silence this log exists to end. + * + * A refused request never reaches the relay, so a turn every one of whose + * requests was 403'd observed as `started: 0, answered: 0, outstanding: []` — + * the same three numbers a turn that simply had nothing to call observes. The + * boundary these assertions straddle is that one: a turn refused against a + * turn served, and an exhausted capability budget against a route the facade + * does not serve, because the two call for opposite fixes. + * + * Mutation: drop the `refusals` member from `observe`, or make `refuse` a + * no-op, and a refused turn reads as an idle one again. + */ +test("a refused request is counted by reason class, and refusing is not calling", () => { + const { log } = clock(); + log.open("turn"); + assert.deepEqual(log.observe("turn")?.refusals, NO_REFUSALS, "an observed turn that was never refused measured zero"); + log.refuse("turn", "exhausted"); log.refuse("turn", "exhausted"); log.refuse("turn", "route"); + const observed = log.observe("turn"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + // The reading the instrument has to keep honest: a refused turn is not a + // turn that called nothing and answered everything. + assert.deepEqual([observed?.started, observed?.answered, observed?.outstanding], [0, 0, []]); + // Every reason class is its own count, and the observation is a copy: a + // later refusal cannot rewrite a report already handed out. + log.refuse("turn", "unrouted"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + assert.deepEqual(log.observe("turn")?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1, unrouted: 1 }); +}); + +test("a refusal the facade cannot attribute is recorded against no turn at all", () => { + const { log } = clock(); + log.refuse("absent", "expired"); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "a refusal before the turn existed is not this turn's"); + log.refuse("absent", "oversized"); + log.close("absent"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "re-opening an id resets its refusals with everything else"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts new file mode 100644 index 0000000..f8d8b8d --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -0,0 +1,234 @@ +/** + * The in-flight MCP tool calls of one brokered turn. + * + * Daimon writes a tool receipt only when a call *completes*, so a call that + * started and never returned is byte-identical, in every artifact, to a call + * that was never made. A live Grok turn stopped acting after its eighth + * provider response and was killed by the trial deadline seven minutes later + * with the proxy's per-request ledger reporting `open: 0` — every provider + * request closed — which leaves exactly one unlit path: a tool call the worker + * issued and the facade never answered. + * + * This is that light, and it follows the per-request ledger's rules rather + * than inventing its own: + * + * - **names and timings only.** The tool name off the JSON-RPC envelope and + * two clocks. Never arguments, never a result, never a session id, never a + * capability or bearer. A name that is not a plain short identifier is + * recorded as {@link ENGINE_BROKER_MCP_CALL_INVALID} rather than passed + * through, and the list is bounded at + * {@link ENGINE_BROKER_MCP_OUTSTANDING_MAX} with + * {@link ENGINE_BROKER_MCP_CALL_TRUNCATED} as its last entry. + * - **absence stays absence.** A turn the log never opened observes as + * `undefined`; a turn that made no call observes `started: 0`, which is not + * the same statement as an answered call. A POST whose body the facade could + * not read counts in `undecoded` and never as a call with a name, because a + * fabricated name is byte-identical to a measured one — and because "zero + * calls started" is exactly the reading this instrument exists to make + * trustworthy. + * - **it cannot fail a turn.** Every operation here is arithmetic over a map, + * the one parse is wrapped, and the facade treats a missing handle as a + * no-op. + * + * The same map carries the facade's *other* channel, and for the same reason. + * A `tools/call` is a POST that answers; the standalone `GET` SSE tunnel the + * Streamable HTTP transport opens once per session is the route a server + * notification or progress frame takes, and it stays open for the whole + * session by design. A worker parked reading that tunnel is, in every artifact + * the broker writes, indistinguishable from a worker doing nothing at all: + * every provider request closed, every tool call answered, and the turn idle + * until its deadline. {@link EngineBrokerMcpCallLog.openTunnel} records the + * lifecycle — how many the facade relayed, how many ended, and for the ones + * still open at seal time how long each has been open and whether the mount + * ever pushed a single byte through it. A tunnel held open having delivered + * nothing is a different fact from one actively carrying frames, and it is the + * difference that decides whether the tunnel is the blocker. + * + * Observing is all it does. The facade's behaviour is unchanged: nothing here + * closes, times out or refuses a tunnel, because an instrument that tore down + * the stream would destroy the evidence it exists to gather. + * + * "Answered" means the facade wrote a complete response back to the worker — + * the relay reached its own `end()`. A relay that was torn down (the worker + * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay + * outstanding with the elapsed time they had reached. That is the whole point: + * the turn's death must not retroactively mark the call it was blocked on as + * finished. + */ +/** + * The same map carries the facade's *refusals*, for the sharpest form of the + * same problem. A refused request never reaches the relay at all, so a turn + * every one of whose requests was 403'd sealed as `answered == started, + * outstanding: []` — byte-identical to a healthy turn, which is precisely the + * reading this instrument exists to make trustworthy. The reason class is what + * makes it actionable: an exhausted per-turn capability budget — the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`, derived from the compiled turn + * bound times what a round may spend — is a different fault from a mount that + * was never registered, and both differ from a worker asking for a route the + * facade does not serve. Counts only, keyed by a closed vocabulary: + * never the token, the capability, the URL or the body. A refusal the facade + * cannot attribute to a turn — a bearer no live grant matches — is recorded + * nowhere, because attributing it to a turn would be inventing the fact. + */ +export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; +export const ENGINE_BROKER_MCP_CALL_INVALID = ""; +export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; +/** A plain short identifier, or one of the two sentinels above. */ +export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; +const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ +export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; + +/** + * Why the facade refused one relayed request, as a closed vocabulary: + * - `route`: a path or method the facade does not serve. + * - `expired`: the turn capability's TTL had passed. + * - `exhausted`: the turn capability's request budget was spent. + * - `unrouted`: a live capability whose turn has no registered mount. + * - `oversized`: a POST body past the facade's own request bound. + */ +export const ENGINE_BROKER_MCP_REFUSAL_REASONS = ["route", "expired", "exhausted", "unrouted", "oversized"] as const; +export type EngineBrokerMcpRefusalReason = (typeof ENGINE_BROKER_MCP_REFUSAL_REASONS)[number]; +/** How many requests of this turn the facade refused, by reason class. Every member is a measurement; the whole member is absent only where it was never measured. */ +export type EngineBrokerMcpRefusalObservation = Readonly>; + +export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ +export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; +/** + * What the facade saw of one turn's standalone GET SSE tunnels: how many it + * relayed, how many ended, how many ever carried a byte from the mount, and + * the ones still open with the age of each. + */ +export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed: number; delivered: number; open: readonly EngineBrokerOpenMcpTunnel[] }>; +/** + * What the facade saw of one turn's tool calls: how many started, how many the + * facade answered, how many POST bodies it could not read, and the ones still + * unanswered with the time each has been outstanding. + */ +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation; refusals?: EngineBrokerMcpRefusalObservation }>; + +/** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ +export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } +const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; +/** One relayed GET tunnel. `deliver` marks the first byte the mount pushed; `close` ends it. Both are idempotent. */ +export interface EngineBrokerMcpTunnelHandle { deliver(): void; close(): void } +const INERT_TUNNEL: EngineBrokerMcpTunnelHandle = { deliver: () => undefined, close: () => undefined }; + +type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; +type TunnelRecord = { readonly openedAt: number; delivered: boolean }; +type TurnLog = { + started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; + tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; + readonly refusals: Record; +}; +const noRefusals = (): Record => ({ route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + +export class EngineBrokerMcpCallLog { + private readonly turns = new Map(); + constructor(private readonly now: () => number = Date.now) {} + + /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set(), refusals: noRefusals() }); } + close(turnId: string): void { this.turns.delete(turnId); } + + /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ + begin(turnId: string, body: Uint8Array | undefined): EngineBrokerMcpCallHandle { + const log = this.turns.get(turnId); + if (log === undefined || body === undefined || body.byteLength === 0) return INERT; + const names = toolCallNames(body); + if (names === undefined) { log.undecoded += 1; return INERT; } + if (names.length === 0) return INERT; + const startedAt = this.now(); + const records = names.map((name): CallRecord => ({ name, startedAt })); + log.started += records.length; + for (const record of records) log.live.add(record); + let settled = false; + return { + answer: (): void => { + if (settled) return; settled = true; + log.answered += records.length; + for (const record of records) log.live.delete(record); + }, + close: (): void => { + if (settled) return; settled = true; + const endedAt = this.now(); + for (const record of records) { + log.live.delete(record); record.endedAt = endedAt; + // Retain only as many as can be reported; the earliest are the ones + // a hang is about, so a later flood cannot displace them. + if (log.ended.length < ENGINE_BROKER_MCP_OUTSTANDING_MAX) log.ended.push(record); + } + } + }; + } + + /** + * A GET SSE tunnel the facade is about to relay. Counted when it opens, not + * when it succeeds: a tunnel the mount refused still ends, so `opened` and + * `closed` stay a pair and an open one is exactly `opened - closed`. + */ + openTunnel(turnId: string): EngineBrokerMcpTunnelHandle { + const log = this.turns.get(turnId); + if (log === undefined) return INERT_TUNNEL; + const record: TunnelRecord = { openedAt: this.now(), delivered: false }; + log.tunnelsOpened += 1; + log.openTunnels.add(record); + let ended = false; + return { + deliver: (): void => { if (record.delivered) return; record.delivered = true; log.tunnelsDelivered += 1; }, + close: (): void => { if (ended) return; ended = true; log.tunnelsClosed += 1; log.openTunnels.delete(record); } + }; + } + + /** + * A request the facade refused before it could ever be relayed. A refusal it + * cannot attribute to a turn is never recorded against one, so an unknown + * turn is a no-op here exactly as every other operation is. + */ + refuse(turnId: string, reason: EngineBrokerMcpRefusalReason): void { const log = this.turns.get(turnId); if (log !== undefined) log.refusals[reason] += 1; } + + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ + undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } + + observe(turnId: string): EngineBrokerMcpCallObservation | undefined { + const log = this.turns.get(turnId); + if (log === undefined) return undefined; + const at = this.now(); + const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); + const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + const open = [...log.openTunnels] + .sort((left, right) => left.openedAt - right.openedAt) + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); + return { + started: log.started, answered: log.answered, undecoded: log.undecoded, + refusals: { ...log.refusals }, + tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, + outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX + ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] + : outstanding + }; + } +} + +/** + * The tool names one JSON-RPC POST asks for: `undefined` when the body did not + * decode at all (which is a fact of its own, not zero calls), `[]` when it + * decoded and asked for no tool. A batch names each of its calls. + */ +function toolCallNames(body: Uint8Array): readonly string[] | undefined { + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8")); } catch { return undefined; } + const entries: readonly unknown[] = Array.isArray(parsed) ? parsed : [parsed]; + const names: string[] = []; + for (const entry of entries) { + if (!isRecord(entry) || entry.method !== "tools/call") continue; + const name = isRecord(entry.params) ? entry.params.name : undefined; + names.push(typeof name === "string" && TOOL_NAME.test(name) ? name : ENGINE_BROKER_MCP_CALL_INVALID); + } + return names; +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 5244b4f..6ad19ca 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,10 +1,193 @@ import assert from "node:assert/strict"; import { createServer } from "node:http"; + import test from "node:test"; -import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import { ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; + +import { connectClient, FACADE_URL, releaseShared, sharedFacade, startFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; test("MCP facade routes only valid active capabilities to the registered mount", async () => { + const facade=await sharedFacade(); let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch("http://127.0.0.1:43124/mcp",{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); - try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} + const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} +}); + +test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { + const rig = await startRig(); + try { + // connect() is initialize + notifications/initialized: before the session + // header was forwarded the notification came back HTTP 400. + const { client, transport } = await connectClient(rig.capability); + try { + assert.equal(typeof transport.sessionId, "string", "the mount's session id must reach the client"); + const listed = await client.listTools(); + assert.deepEqual(listed.tools.map((tool) => tool.name), ["moltnet_read"]); + const called = await client.callTool({ name: "moltnet_read", arguments: { target: "room:desk" } }); + assert.deepEqual(called.structuredContent, { target: "room:desk" }); + assert.equal(called.isError, undefined); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the mount's server-initiated SSE stream, which only the GET route provides", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + // The standalone GET stream is the only route a server notification can + // take; a POST-only facade answers it 403 and this never arrives. + await new Promise((resolve) => setTimeout(resolve, 150)); + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); + assert.equal(typeof transport.sessionId, "string"); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const sessionId = transport.sessionId; + assert.equal(typeof sessionId, "string"); + await transport.terminateSession(); + assert.equal(transport.sessionId, undefined, "DELETE must be accepted, not 403ed"); + assert.ok(rig.observed.some((request) => request.method === "DELETE"), "the mount must have seen the DELETE"); + const stale = await fetch(FACADE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${rig.capability}`, + "content-type": "application/json", + accept: "application/json, text/event-stream", + "mcp-session-id": sessionId! + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 9, method: "tools/list", params: {} }) + }); + assert.ok(stale.status >= 400, `a terminated session must not still route (got ${stale.status})`); + await stale.body?.cancel(); + await client.close().catch(() => undefined); + } finally { + await rig.close(); + } +}); + +test("the facade forwards a closed header allowlist and never the worker's bearer", async () => { + const rig = await startRig(); + try { + const { client } = await connectClient(rig.capability); + await client.listTools(); + await client.close(); + const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); + assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); + assert.equal(forwarded.includes("cookie"), false); + const allowed = new Set([ + "host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", + "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound header, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode" + ]); + const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); + assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); + } finally { + await rig.close(); + } +}); + +test("the facade withholds a mount response header that is not on the allowlist", async () => { + // The facade comes first: nothing must be listening while the fixed port is + // still in doubt, or a refused start leaks this mount and parks the runner. + const facade = await sharedFacade(); + const target = createServer((_request, response) => { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-from-mount", + "set-cookie": "leak=1", + "www-authenticate": "Bearer realm=\"mount\"", + "x-mount-internal": "private" + }); + response.end('{"ok":true}'); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); + if (address === null || typeof address === "string") throw new Error("target address unavailable"); + const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); + try { + const answered = await fetch(FACADE_URL, { + method: "POST", + headers: { authorization: `Bearer ${capability}`, "content-type": "application/json" }, + body: "{}" + }); + assert.equal(answered.status, 200); + assert.equal(answered.headers.get("mcp-session-id"), "session-from-mount"); + assert.equal(answered.headers.get("cache-control"), "no-store"); + assert.equal(answered.headers.get("set-cookie"), null); + assert.equal(answered.headers.get("www-authenticate"), null); + assert.equal(answered.headers.get("x-mount-internal"), null); + await answered.body?.cancel(); + } finally { + facade.revoke("turn-response-headers"); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +test("the facade still refuses every route and method outside the MCP surface", async () => { + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-refusals", "http://127.0.0.1:1/mcp"); + const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { + method, headers: { authorization: `Bearer ${capability}` } + }); + try { + assert.equal((await call("GET", "/")).status, 403); + assert.equal((await call("GET", "/mcp?probe=1")).status, 403); + assert.equal((await call("PUT", "/mcp")).status, 403); + assert.equal((await call("PATCH", "/mcp")).status, 403); + assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); + } finally { + facade.revoke("turn-refusals"); + } +}); + +/** + * Last, because both cases take the fixed port for themselves: an open GET + * tunnel must not survive its own capability, and must not stall shutdown + * either — a server-to-client stream stays open for the whole session, so + * before it existed nothing could hold the listener open. + */ +test("revoking a turn tears down its open server-to-client stream, and closing never stalls on one", async () => { + await releaseShared(); + const facade = await startFacade(); + const rig = await startRig(facade); + const { client } = await connectClient(rig.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the GET stream must be open before revoking"); + + // The client's stream survives its own capability unless the facade ends the + // tunnel: the mount's GET response is the side that has to close. + facade.revoke(rig.turnId); + await withDeadline(rig.getStreamClosed, 4_000, "a revoked capability left its SSE tunnel open"); + + // A second turn's tunnel, deliberately left open, is what shutdown must not wait on. + const second = await startRig(facade); + const held = await connectClient(second.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + await withDeadline(facade.close(), 3_000, "closing the facade stalled on an open SSE tunnel"); + + await held.client.close().catch(() => undefined); + await client.close().catch(() => undefined); + await second.close().catch(() => undefined); + await rig.close().catch(() => undefined); }); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 156506b..68cd0d3 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,10 +1,325 @@ -import { createServer } from "node:http"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { Readable } from "node:stream"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; + +/** + * The brokered worker's only route to its own per-wake Daimon MCP mount. The + * worker holds a turn capability and nothing else: it never learns the mount's + * address, and the mount never learns the capability. Every header crossing + * either way is rebuilt from a closed allowlist, so this stays a boundary and + * not a transparent proxy — a blanket passthrough would hand the mount the + * worker's bearer and hand the worker whatever the mount chose to say. + * + * The allowlists are exactly the Streamable HTTP transport's own routing + * headers. Anything outside them (authorization above all, cookies, auth + * challenges, forwarding and tracing headers) is dropped in both directions. + * + * Client -> mount: + * - `content-type`: the JSON-RPC body's media type; the mount refuses a POST + * without it. + * - `accept`: the transport negotiates `application/json, text/event-stream` + * per request and the mount answers 406 when a POST does not accept both. + * - `mcp-session-id`: the opaque session the mount issued on `initialize`. + * Dropping it made the mount answer every later request with HTTP 400 + * `Mcp-Session-Id header is required`, so no tool was ever reachable. It is + * a routing value, not a secret — and not a value to log either. + * - `mcp-protocol-version`: the version the handshake settled on. The mount + * validates it and otherwise assumes a default that can disagree with what + * the client negotiated. + * - `last-event-id`: SSE resumability. A reconnecting stream replays from the + * last event it saw; without it the mount cannot tell where to resume. + * + * Mount -> client: + * - `content-type`: tells the client whether it got JSON or an SSE stream. + * - `mcp-session-id`: the id minted on `initialize`. The client must learn it + * or it can never make a second request. + * - `mcp-protocol-version`: the version the mount confirms for the session. + * - `cache-control: no-store` is the facade's own, not the mount's. + * + * Methods are the three the transport uses: POST for JSON-RPC, GET for the + * server-to-client SSE stream (notifications and progress arrive only there), + * and DELETE to end a session. POST alone left the GET stream answering 403. + */ +const FORWARDED_REQUEST_HEADERS = ["content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id"] as const; +const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-protocol-version"] as const; +const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); +const MAX_REQUEST_BYTES = 1024 * 1024; +export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +/** Requests one worker round may legitimately make: its `search_tool`, its `use_tool`, and one spare for a retry or a second discovery. */ +export const ENGINE_BROKER_MCP_ROUND_REQUESTS = 3; +/** The session's fixed cost, once per turn: `initialize`, `notifications/initialized`, `tools/list`, the standalone GET tunnel, the closing DELETE. */ +export const ENGINE_BROKER_MCP_SESSION_REQUESTS = 5; +/** + * Requests one turn capability may spend, across all three methods — *derived* + * from the compiled turn bound rather than chosen. + * + * It was 128, and a legitimate 48-round wake needs ~101 of them: a worker's + * round is a `search_tool` and a `use_tool`, so the first round that also + * retries, or looks something up twice, eats the margin. A bound that can be + * predicted to bite mid-turn is not a bound, it is a 403 storm waiting for a + * real wake — and a number raised until it feels comfortable is not one + * either, because the budget exists to cap a *compromised* worker. + * + * So the two numbers that must agree are kept in one place: the launcher's + * `--max-turns` backstop ({@link GROK_WORKER_MAX_TURNS}) times what a round + * may legitimately spend, plus the session's fixed cost. Raising the turn + * bound raises this with it, and a compromised worker still gets exactly three + * MCP calls per round it was compiled to take and not one more. The arithmetic + * is spelled out rather than folded into a literal so it can be audited: the + * two multiplicands above each say what they are. + */ +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; +export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; + +class FacadeRefusal extends Error {} export async function startEngineBrokerMcpFacade() { - const capabilities=new EngineBrokerCapabilities();const targets=new Map(); - const server=createServer((request,response)=>{void(async()=>{try{if(request.url!=="/mcp"||request.method!=="POST")throw new Error();const match=request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u);if(!match)throw new Error();const scope=capabilities.authorizeToken(match[1]!);if(!scope)throw new Error();const target=targets.get(scope.turnId);if(!target)throw new Error();const body=await bounded(request);const payload=body.buffer.slice(body.byteOffset,body.byteOffset+body.byteLength) as ArrayBuffer;const upstream=await fetch(target,{method:"POST",headers:{"content-type":request.headers["content-type"]??"application/json","accept":request.headers.accept??"application/json, text/event-stream"},body:payload});response.writeHead(upstream.status,{"content-type":upstream.headers.get("content-type")??"application/json","cache-control":"no-store"});response.end(Buffer.from(await upstream.arrayBuffer()));}catch{response.writeHead(403,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"forbidden"}');}})();}); - await new Promise((resolve,reject)=>{server.once("error",reject);server.listen(43_124,"127.0.0.1",()=>{server.off("error",reject);resolve();});}); - return {register(agentId:string,turnId:string,endpoint:string){const url=new URL(endpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid scoped MCP mount");if(targets.has(turnId))throw new Error("MCP turn already registered");targets.set(turnId,url.href);return capabilities.issue(agentId,turnId,15*60_000,128);},revoke(turnId:string){targets.delete(turnId);capabilities.revoke(turnId);},close:()=>new Promise((resolve,reject)=>server.close((error)=>error?reject(error):resolve()))}; + const capabilities = new EngineBrokerCapabilities(); + const targets = new Map(); + /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ + const inflight = new Map>(); + /** + * What the facade saw of each turn's tool calls. A tool receipt is written + * only on completion, so without this a call that started and never returned + * and a call never made are the same absence (`engineBrokerMcpCallLog.ts`). + */ + const calls = new EngineBrokerMcpCallLog(); + + const server = createServer((request, response) => { + void route(request, response).catch((error: unknown) => { + if (response.headersSent || response.destroyed) { response.destroy(); return; } + const status = error instanceof FacadeRefusal ? 403 : 502; + const body = error instanceof FacadeRefusal ? '{"error":"forbidden"}' : '{"error":"bad_gateway"}'; + response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(body); + }); + }); + + /** + * A 403 the call log can read, whenever the bearer names a turn. + * + * The refusal itself is unchanged — same status, same body, same silence + * towards the worker — but it is attributed first, through a lookup that + * does not spend the capability's budget. A bearer no grant matches names no + * turn and stays unattributed, because guessing whose it was would be + * inventing the measurement. A capability that is *also* spent or expired + * reports that instead of the route it asked for: it is the fault the + * operator can act on. + */ + function refuse(bearer: string | undefined, live: EngineBrokerMcpRefusalReason): FacadeRefusal { + const classified = bearer === undefined ? undefined : capabilities.classifyToken(bearer); + if (classified !== undefined) calls.refuse(classified.turnId, classified.state === "live" ? live : classified.state); + return new FacadeRefusal(); + } + + async function route(request: IncomingMessage, response: ServerResponse): Promise { + const method = request.method ?? ""; + const bearer = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u)?.[1]; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw refuse(bearer, "route"); + if (bearer === undefined) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(bearer); + // A live grant that authorizes is the only way past here; anything else is + // classified so an exhausted budget and an expired TTL reach the turn's + // sealed row as themselves. `live` cannot be the answer on this path — + // `authorizeToken` read the same grant a moment ago, expiry only moves + // forward and a budget only spends — so it is the unreachable arm. + if (!scope) throw refuse(bearer, "expired"); + const target = targets.get(scope.turnId); + if (target === undefined) { calls.refuse(scope.turnId, "unrouted"); throw new FacadeRefusal(); } + + // Only POST carries a JSON-RPC body; drain anything else so the socket + // never stalls waiting for a body the facade will not forward. + let body: Buffer | undefined; + // A body refused for size is a call the log can never name, and counting + // it keeps "no tool call started" an honest reading rather than a gap. + try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } + catch (error) { + calls.undecodable(scope.turnId); + // A body past the bound is refused, not merely unreadable: the worker + // gets a 403 for it, so it is counted as one too. + if (error instanceof FacadeRefusal) calls.refuse(scope.turnId, "oversized"); + throw error; + } + const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + const call = calls.begin(scope.turnId, body); + // The GET SSE tunnel is the session's other channel and the one that + // outlives every request: it stays open until the worker or the mount ends + // it, so a turn sealed with one still open is a turn whose worker may be + // parked on it. Observed only — never closed, timed out or refused here. + const tunnel = method === "GET" ? calls.openTunnel(scope.turnId) : undefined; + + const controller = new AbortController(); + const open = inflight.get(scope.turnId) ?? new Set(); + open.add(controller); + inflight.set(scope.turnId, open); + const abort = (): void => controller.abort(); + response.on("close", abort); + try { + // Answered only on a relay that reached its own end: a tunnel torn down + // by the worker's death must not mark the call it was blocked on as + // finished. + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response, tunnel)) call.answer(); + } finally { + call.close(); + tunnel?.close(); + response.off("close", abort); + open.delete(controller); + if (open.size === 0) inflight.delete(scope.turnId); + } + } + + function endTurnStreams(turnId: string): void { + for (const controller of inflight.get(turnId) ?? []) controller.abort(); + inflight.delete(turnId); + } + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(ENGINE_BROKER_MCP_FACADE_PORT, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + + return { + register(agentId: string, turnId: string, endpoint: string): string { + const url = new URL(endpoint); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); + if (targets.has(turnId)) throw new Error("MCP turn already registered"); + targets.set(turnId, url.href); + calls.open(turnId); + return capabilities.issue(agentId, turnId, ENGINE_BROKER_MCP_CAPABILITY_TTL_MS, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + }, + revoke(turnId: string): void { + targets.delete(turnId); + capabilities.revoke(turnId); + endTurnStreams(turnId); + calls.close(turnId); + }, + /** + * What the facade saw of this turn's tool calls, GET tunnels and refusals, + * or `undefined` for a turn + * it never registered. Read on the failure path, before `revoke`. + */ + observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), + close: async (): Promise => { + for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + // A GET SSE tunnel keeps its socket open indefinitely, and `close` + // only stops accepting; without this a shutdown would hang on it. + server.closeAllConnections(); + }); + } + }; +} + +/** The client -> mount allowlist, with the two defaults the mount requires of a POST. */ +function headersFor(method: string, request: IncomingMessage): Record { + const headers: Record = {}; + for (const name of FORWARDED_REQUEST_HEADERS) { + const value = request.headers[name]; + if (typeof value === "string" && value.length > 0) headers[name] = value; + } + if (method === "POST") { + headers["content-type"] ??= "application/json"; + headers["accept"] ??= "application/json, text/event-stream"; + } else { + delete headers["content-type"]; + } + return headers; +} + +/** + * Streams the exchange rather than buffering it: a GET stream stays open for + * the whole session, and a buffered POST would withhold progress + * notifications until the call had already finished. + */ +async function forward( + target: string, + method: string, + headers: Record, + body: ArrayBuffer | undefined, + signal: AbortSignal, + response: ServerResponse, + tunnel?: EngineBrokerMcpTunnelHandle +): Promise { + const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); + // MCP never redirects, and following one would let the mount aim the facade + // at a host the capability was never scoped to. `manual` also reports an + // opaque redirect as status 0, which is not a status to relay at all. + const relayable = (upstream.status >= 200 && upstream.status < 300) || (upstream.status >= 400 && upstream.status <= 599); + if (!relayable) throw new Error("unexpected MCP mount status"); + const outbound: Record = { "cache-control": "no-store" }; + for (const name of FORWARDED_RESPONSE_HEADERS) { + const value = upstream.headers.get(name); + if (value !== null && value.length > 0) outbound[name] = value; + } + outbound["content-type"] ??= "application/json"; + response.writeHead(upstream.status, outbound); + if (upstream.body === null) { response.end(); return true; } + const stream = Readable.fromWeb(upstream.body as Parameters[0]); + let delivered = false; + try { + for await (const chunk of stream) { + // The first byte the mount pushes: a tunnel that carried frames is a + // different fact from one held open having delivered nothing. + if (!delivered) { delivered = true; tunnel?.deliver(); } + if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); + if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); + } + response.end(); + return true; + } catch { + // The client hung up or the mount's stream broke: tear the tunnel down + // rather than leaving a half-written response open. + response.destroy(); + return false; + } finally { + stream.destroy(); + } +} + +/** + * Waits for a backpressured tunnel to drain, or for the tunnel to end — + * whichever happens first, but always one of them. + * + * A bare `once("drain")` never settles for a client that hung up mid-write: + * `drain` cannot fire on a socket nobody is reading, and neither the + * response's own `close` nor the turn's abort woke that await. The GET SSE + * tunnel stays open for a whole session, so the handler — and the upstream + * call it was relaying — leaked for the life of the broker process, with no + * refusal, no status and no line anywhere to read. Every outcome settles this + * now, and every outcome but an actual drain *rejects*, so the caller tears + * the tunnel down instead of writing into a socket that is gone. + * + * Exported for its own test: a hang is only observable from inside. + */ +export function awaitMcpTunnelDrain(response: ServerResponse, signal: AbortSignal): Promise { + if (response.destroyed || response.writableEnded) return Promise.reject(new Error("MCP tunnel closed")); + if (signal.aborted) return Promise.reject(new Error("MCP tunnel aborted")); + return new Promise((resolve, reject) => { + const settle = (finish: () => void) => (): void => { + response.off("drain", onDrain); response.off("close", onClosed); response.off("error", onClosed); + signal.removeEventListener("abort", onAborted); + finish(); + }; + const onDrain = settle(resolve); + const onClosed = settle(() => reject(new Error("MCP tunnel closed"))); + const onAborted = settle(() => reject(new Error("MCP tunnel aborted"))); + response.once("drain", onDrain); response.once("close", onClosed); response.once("error", onClosed); + signal.addEventListener("abort", onAborted, { once: true }); + }); +} + +async function bounded(request: AsyncIterable): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const value = Buffer.from(chunk as Uint8Array); + bytes += value.length; + if (bytes > MAX_REQUEST_BYTES) throw new FacadeRefusal(); + chunks.push(value); + } + return Buffer.concat(chunks); } -async function bounded(request:AsyncIterable):Promise{const chunks:Buffer[]=[];let bytes=0;for await(const chunk of request){const value=Buffer.from(chunk as Uint8Array);bytes+=value.length;if(bytes>1024*1024)throw new Error();chunks.push(value);}return Buffer.concat(chunks);} diff --git a/src/runtime/engineBrokerMcpFacadeRig.test.ts b/src/runtime/engineBrokerMcpFacadeRig.test.ts new file mode 100644 index 0000000..fd336a4 --- /dev/null +++ b/src/runtime/engineBrokerMcpFacadeRig.test.ts @@ -0,0 +1,164 @@ +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; + +import { randomUUID } from "node:crypto"; +import test from "node:test"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +/** + * The facade's shared test rig, and not a suite of its own. + * + * Two suites drive the same boundary — the facade's routing and header + * contract, and the observations it records while relaying — and both need one + * facade on the protocol's fixed port plus a real Daimon MCP mount behind a + * real Streamable HTTP transport. Splitting them into one file each kept both + * readable; duplicating the rig into each would have left two copies of the + * thing every assertion depends on. It carries a `.test.ts` name so it never + * reaches production `dist` (`tsconfig.build.json` excludes exactly that), and + * running it on its own asserts nothing, which is what it is. + * + * Each suite is its own process, so each holds its own shared facade and each + * takes the fixed port for the length of its file. + */ +export const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +export type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +export const sharedFacade = async (): Promise => (shared ??= await startFacade()); +export const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +/** + * The facade's port is the control protocol's own, so the two suites that + * drive it cannot each hold one at the same time. Whichever binds first runs; + * the other waits for it to release the port rather than failing on the + * collision, which is the whole cost of splitting this boundary in two. + */ +const bindFacade = async (): Promise => { + for (let attempt = 0; attempt < 240; attempt += 1) { + try { return await startEngineBrokerMcpFacade(); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + throw new Error("the MCP facade port never came free"); +}; + +export const startFacade = async (): Promise => { + const facade = await bindFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +export type Rig = Readonly<{ + facade: Facade; + turnId: string; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; + close: () => Promise; +}>; + +export const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +let turns = 0; + +export const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); + const mount = createServer((request, response) => { + observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); + return { + facade: host, turnId, server, capability, observed, getStreamClosed, + close: async () => { + host.revoke(turnId); + await closeMount(mount, transport, server); + } + }; +}; + +export const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + +export const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +export const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts new file mode 100644 index 0000000..86fdfeb --- /dev/null +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; +import { createServer, type ServerResponse } from "node:http"; + +import test from "node:test"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, ENGINE_BROKER_MCP_ROUND_REQUESTS, ENGINE_BROKER_MCP_SESSION_REQUESTS } from "./engineBrokerMcpFacade.js"; +import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; + +/** + * What the facade saw while it was relaying, which is the only place a call or + * a stream is observable *while it is still running*. + * + * Daimon writes a tool receipt on completion and nothing at all for the + * session's GET tunnel, so a call that never returned, a worker parked on an + * open stream, and a worker doing nothing were one indistinguishable silence. + * Both instruments are driven here through the real facade. + */ +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); + +/** + * The refusal, which was the one relay outcome that observed as nothing. + * + * `route()` refuses before the call log is ever touched, so a request the + * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn + * whose every request was refused sealed as `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals as. + * + * The middle of this test is a second guarantee and it is a *relationship*, + * not a number. The demand side is computed from the launcher's compiled + * `--max-turns` backstop alone — every round spending its full MCP allowance, + * plus the session's fixed cost — and the whole of it must be served on one + * capability, with the very next request refused. Two numbers that must agree + * and live apart drift: the budget was a literal 128 against a bound of 48 + * rounds, ~101 requests of legitimate traffic, and the first round that also + * retried would have met a mid-turn 403 storm. Deriving one from the other is + * what makes raising `GROK_WORKER_MAX_TURNS` unable to silently exhaust the + * budget — and asserting the refusal one past the worst case is what keeps the + * derivation a bound rather than a comfortable number. + * + * The boundary these assertions straddle: a turn served against a turn + * refused; within the refusals, a spent capability against a route the facade + * does not serve; and, for the budget, legitimate worst-case traffic against + * the first request beyond it. + * + * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` + * call in `route()`, and a 403'd turn reads as an idle one again. Pin + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` back to a literal `128` and this is + * already red at today's turn bound; raise `GROK_WORKER_MAX_TURNS` beside that + * literal and it stays red, which is the drift the derivation removes. Raising + * `GROK_WORKER_MAX_TURNS` *with* the derivation keeps it green, which is the + * guarantee itself. + */ +test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { + const facade = await sharedFacade(); + let served = 0; + const target = createServer((request, response) => { + served += 1; + request.resume(); + request.on("end", () => { response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-refused", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const listed = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }); + const send = async (url: string, bearer: string, method = "POST"): Promise => { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, ...(method === "POST" ? { body: listed } : {}) }); + await response.text(); + return response.status; + }; + try { + // A route the facade does not serve, asked for with a live capability. + assert.equal(await send(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/nope`, token), 403); + assert.equal(await send(FACADE_URL, token, "PUT"), 403); + assert.deepEqual(facade.observe(turnId)?.refusals, { route: 2, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + + // A bearer no live grant matches names no turn, so it is recorded against + // none: inventing an owner would be worse than the silence. + assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); + assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); + + // Neither refusal spent the capability, so what follows is the whole of it. + // The demand is read off the compiled turn bound and nothing else: every + // round the launcher admits, each spending its full MCP allowance, plus the + // one-off session cost. All of it must be served. + const legitimate = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; + for (let spent = 0; spent < legitimate; spent += 1) assert.equal(await send(FACADE_URL, token), 200, `request ${spent + 1} of ${legitimate} was refused`); + assert.equal(served, legitimate, "every legitimate request must reach the mount"); + // And it is still a bound: the first request past the worst case is refused, + // so the budget caps a compromised worker at exactly the traffic the turn + // bound compiles for. + assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); + assert.equal(served, legitimate, "an exhausted capability never reaches the mount"); + + const observed = facade.observe(turnId); + assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); + // And the reading the seal row used to publish for all of it: nothing. + assert.deepEqual([observed?.started, observed?.answered, observed?.undecoded, observed?.outstanding], [0, 0, 0, []]); + assert.ok(!JSON.stringify(observed).includes(token), "counts and reason classes only: never the capability"); + } finally { + facade.revoke(turnId); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerMcpTunnel.test.ts b/src/runtime/engineBrokerMcpTunnel.test.ts new file mode 100644 index 0000000..b22bf36 --- /dev/null +++ b/src/runtime/engineBrokerMcpTunnel.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { connect, type Socket } from "node:net"; +import test from "node:test"; + +import { awaitMcpTunnelDrain } from "./engineBrokerMcpFacade.js"; + +const withDeadline = async (work: Promise, ms: number, message: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(message)), ms); })]); } + finally { if (timer) clearTimeout(timer); } +}; + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 754dcfa..104150d 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;reserved?:number;text?:string}>={}):Buffer{ - const text=Buffer.from(values.text??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.reserved??0,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);return out; +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string|Buffer;text?:string}>={}):Buffer{ + const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } test("encodes ABI v2 and decodes a closed successful result",()=>{ @@ -21,7 +22,111 @@ test("returns bounded typed diagnostics for closed native failures",()=>{ ])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.stage!=="none"); }); -test("rejects unknown, reserved, output-bearing, and cross-class failure frames",()=>{ - for(const value of [{status:9},{reserved:1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); +test("rejects unknown, diagnostic-bearing success, output-bearing, and cross-class failure frames",()=>{ + for(const value of [{status:9},{diagnostic:"late words"},{status:1,stage:4,failure:4,pid:0,uid:0,ticks:0n,exit:-1,diagnostic:"no worker ran"},{status:2,stage:6,failure:5,exit:1,diagnosticLength:ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES+1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); for(const offset of [28,31,105,107,124,127]){const hostile=frame({text:"done"});hostile[offset]=1;assert.throws(()=>decodeNativeBrokerResult(hostile,turnId),/^Error: engine broker turn failed$/u);} }); + +test("a failed worker's own last words cross as a redacted, bounded reason",()=>{ + const words=`{"type":"error","message":"session store unwritable"}\nBearer provider-cap-secret-value\ngrok: exiting 1\n`; + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1,diagnostic:words}),turnId,["provider-cap-secret-value"]),(error:unknown)=>{ + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason=error.diagnostic.reason; + assert.ok(reason!==undefined,"the worker's own reason must reach the diagnostic"); + assert.match(reason,/session store unwritable/u); + assert.doesNotMatch(reason,/provider-cap-secret-value/u,"the turn capability must never reach a diagnostic"); + assert.doesNotMatch(reason,/[\n\r\u0000-\u001f]/u,"the reason is one bounded line"); + assert.ok(Buffer.byteLength(reason,"utf8")<=CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + return true; + }); + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); +}); + +/** + * The assertion is the SENTENCE. A live turn reported its worker's last words + * as `reason=108,111,110,101,46,32,87,104,101,110,...` — the bytes of "lone. + * When ..." rendered as decimals, because a byte view that is not a Node + * `Buffer` answers `toString("utf8")` with a comma-separated list and every + * check the diagnostic passed on the way out (a string, no control bytes, + * under the bound) is satisfied by digits. Nothing weaker than the decoded + * text can catch that. + */ +const words = "lone. When assigned, read `room:assignment`, open my row in the desk index."; + +test("a worker's last words cross as decoded text, not as the decimals of their bytes", () => { + for (const [shape, view] of [["a Buffer", (bytes: Buffer): Uint8Array => bytes], ["a plain Uint8Array", (bytes: Buffer): Uint8Array => new Uint8Array(bytes)]] as const) { + assert.throws(() => decodeNativeBrokerResult(view(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words })), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, `${shape}: the reason must be the worker's sentence, byte-identical`); + assert.doesNotMatch(error.diagnostic.reason ?? "", /^[0-9,]+$/u, `${shape}: a decimal byte list is what this regressed to before`); + return true; + }); + } +}); + +test("a window that cut a multi-byte sequence in half decodes with replacement instead of throwing", () => { + // The launcher's window is a byte count: these first two bytes are the tail + // of a three-byte sequence whose leading byte the window already dropped. + const cut = Buffer.concat([Buffer.from([0x9c, 0xa8]), Buffer.from(" grok: exiting 1")]); + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.match(reason, /grok: exiting 1$/u, "the legible remainder survives the cut sequence"); + assert.match(reason, /\uFFFD/u, "the cut sequence is replaced, not thrown on"); + return true; + }); +}); + +/** + * The window keeps both ends. A pure tail is what turned the one diagnostic + * this project has ever got out of a failed worker into 512 bytes of the + * worker's own prompt echoed back, with the error itself off the front. + */ +test("the bounded diagnostic window keeps the head, the tail, and a marker naming what it dropped", () => { + const blob = `START-OF-ERROR ${"m".repeat(40_000)} END-OF-ECHO`; + const window = boundedDiagnosticWindow(blob, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + assert.ok(window.startsWith("START-OF-ERROR "), `the head must survive: ${window.slice(0, 40)}`); + assert.ok(window.endsWith(" END-OF-ECHO"), "the tail must survive too"); + const marker = /\[… (\d+) bytes elided …\]/u.exec(window); + assert.ok(marker !== null, "the elision is named, not silent"); + assert.equal(Number(marker[1]) + Buffer.byteLength(window.replace(marker[0], ""), "utf8"), Buffer.byteLength(blob, "utf8"), "the marker's count is exactly what was dropped"); + assert.ok(Buffer.byteLength(window, "utf8") <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES, `the marker is paid for out of the same budget: ${Buffer.byteLength(window, "utf8")} bytes`); +}); + +test("output that fits the window is returned byte-identical, with no marker", () => { + for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) + assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); +}); + +/** + * The cut the launcher makes is the one place exact redaction cannot reach on + * its own, and it is the cut this test straddles: the capability begins inside + * the retained head and ends inside the elided middle, so the redactor never + * sees it whole and, without the scrub, its first characters travel verbatim. + * The mirror case is the tail's leading edge, where the capability's last + * characters survive instead. + */ +test("a capability the launcher's window cut in half never crosses as a fragment", () => { + const provider = `provider-${"A".repeat(34)}`, mcp = `mcp-${"B".repeat(39)}`; + const elision = "[… 9000 bytes elided …]"; + const cut = `grok: refused ${provider.slice(0, 20)}${elision}${mcp.slice(mcp.length - 20)} exiting 1`; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, [provider, mcp]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.doesNotMatch(reason, /A{12}|B{12}|provider-A|BBB-?mcp/u, `no credential fragment may cross: ${reason}`); + assert.match(reason, /grok: refused \[REDACTED\]/u, "the head's cut fragment is marked where it was"); + assert.match(reason, /\[REDACTED\] exiting 1$/u, "and so is the tail's"); + assert.ok(reason.includes(elision), "the launcher's own elision marker is left alone"); + return true; + }); +}); + +test("ordinary words at a cut are not eaten by the fragment scrub", () => { + const provider = `provider-${"A".repeat(34)}`; + const words = "grok: profile refused, exiting 1"; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words }), turnId, [provider]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, "text that is not a credential fragment is untouched"); + return true; + }); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 8d905bd..6a612e1 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,14 +1,19 @@ import { spawn } from "node:child_process"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; -const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; +/** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ +export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; +/** `DBL_MAX_OUTPUT`: the launcher's bound on a whole turn's stdout, and the control protocol's own `text` bound, which is the next boundary this output crosses. */ +const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 262_144; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; -export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } +export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;reason?:string;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } export class NativeBrokerTurnFailure extends Error { constructor(readonly diagnostic:NativeBrokerDiagnostic){super("engine broker turn failed");} } export type NativeBrokerTurn = Readonly<{slot:number;requestId:string;turnId:string;agentId:string;wakeId:string;prompt:string;providerCapability:string;mcpCapability:string}>; export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid:number;startTicks:bigint;diagnostic:NativeBrokerDiagnostic} @@ -16,22 +21,99 @@ export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid: export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTurn,signal?:AbortSignal):Promise>{ const frame=encodeNativeBrokerTurn(input),child=trackCliChild(spawn(executable,["--client"],{detached:process.platform!=="win32",env:{LANG:"C",LC_ALL:"C",TZ:"UTC"},stdio:["pipe","pipe","ignore"],...(signal===undefined?{}:{signal})}));const chunks:Buffer[]=[];let bytes=0; child.stdout!.on("data",(chunk:Buffer)=>{bytes+=chunk.length;if(bytes<=ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)chunks.push(chunk);});child.stdin!.end(frame);frame.fill(0); - try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} + try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId,[input.providerCapability,input.mcpCapability]);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string):NativeBrokerTurnResult{ +/** + * `output` is accepted as any byte view, and normalized to a `Buffer` once + * here: `Uint8Array.prototype.toString("utf8")` ignores its argument and + * renders the bytes as comma-separated decimals, which is how a live worker's + * last words reached an operator as `reason=108,111,110,101,...` instead of a + * sentence. Decoding is explicit from here on, never a stringification. + */ +export function decodeNativeBrokerResult(input:Uint8Array,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ + const output=Buffer.isBuffer(input)?input:Buffer.from(input.buffer,input.byteOffset,input.byteLength); if(output.lengthbytes.every((byte)=>byte===0)); - if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||reserved!==0||!paddingZero||observed!==turnId||length>MAX_OUTPUT)throw new Error("engine broker turn failed"); - const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; - const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0; - const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n; + if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); + // Decoded from the caller's own view, not from the normalized frame: the + // decode is what must be correct for any byte view, and it is the step the + // live `reason=108,111,110,...` failure came from. + const reason=workerReason(input.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; + const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; + const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; const worker=status===2&&stage===6&&(failure===5||failure===6)&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const outputFailure=status===3&&stage===7&&failure===7&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const cancelled=status===4&&stage===6&&failure===8&&profile===0&&pid>0&&uid>=2200&&ticks>0n; if(!success){if(length!==0||(!prelaunch&&!worker&&!outputFailure&&!cancelled))throw new Error("engine broker turn failed");throw new NativeBrokerTurnFailure(diagnostic);} return{text:output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES).toString("utf8"),workerUid:uid,workerPid:pid,startTicks:ticks,diagnostic}; } +/** + * The worker's own last words, fit to cross a boundary. + * + * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher + * merges the worker's stdout and stderr into one pipe and publishes no output + * for a failure, so this bounded window is the only account of why it failed. + * It keeps both ends of what it is given (`boundedDiagnosticWindow`), because + * a worker that dies early prints its error before it echoes anything. + * It is worker-controlled text, so it is redacted exactly as the CLI child + * path redacts a failed engine child (`redactCredentialText` with the turn's + * own capabilities as exact secrets, the same diagnostic bound) and flattened + * to one line, because it travels inside a failure message. + */ +function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefined{ + if(tail.byteLength===0)return undefined; + // Decoded explicitly, and with replacement rather than a throw: the window + // is a byte count, so it can cut a multi-byte sequence in half at either + // end, and a worker's last words must not be lost to its own encoding. + const flattened=UTF8.decode(tail).replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + // Redact first, unbounded, then window: redaction can lengthen the text + // ([REDACTED] is longer than a short secret), so bounding before it could + // hand back more bytes than the boundary admits. + const redacted=redactCredentialText(scrubCutFragments(flattened,secrets),secrets,Number.MAX_SAFE_INTEGER).trim(); + const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return reason.length===0?undefined:reason; +} +/** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ +const UTF8=new TextDecoder("utf-8"); + +/** + * A credential the launcher's window cut in half, at either side of a cut. + * + * Exact redaction matches a secret whole, so a secret a cut split survives as + * a fragment it can never match: the piece before a cut can end with a + * secret's prefix, and the piece after it can begin with a secret's suffix. + * The trick that answers this where Daimon owns both ends — retain one whole + * secret more than is reported (`cliChildOutput.ts`) — cannot work at this + * boundary, because the launcher's window *is* what it sends: a margin + * reserved there would be reported along with everything else. So the fragment + * is matched here, where the turn's own capabilities are known, and every cut + * the window can make is covered: the two sides of each elision marker, and + * the outer ends, where the launcher's capture itself stopped reading. + * + * Only a fragment long enough to be a credential is scrubbed. Below + * {@link MIN_CREDENTIAL_FRAGMENT} characters a piece of a random token is + * indistinguishable from ordinary words and carries nothing usable, and + * scrubbing it would eat real text. + */ +const MIN_CREDENTIAL_FRAGMENT=12; +const ELISION_MARKER=/(\[… \d+ bytes elided …\])/u; +const scrubCutFragments=(value:string,secrets:readonly string[]):string=> + value.split(ELISION_MARKER).map((part)=>ELISION_MARKER.test(part)?part:scrubEnds(part,secrets)).join(""); +function scrubEnds(part:string,secrets:readonly string[]):string{ + let result=part; + for(const secret of secrets){ + if(secret.length<=MIN_CREDENTIAL_FRAGMENT)continue; + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.endsWith(secret.slice(0,length))){result=`${result.slice(0,result.length-length)}[REDACTED]`;break;} + } + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.startsWith(secret.slice(secret.length-length))){result=`[REDACTED]${result.slice(length)}`;break;} + } + } + return result; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index ae762bc..8d59256 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -62,3 +62,85 @@ test("start_turn limits are an optional closed subset inside their bounds", () = assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); } }); + +test("a failed worker's redacted reason is an optional bounded member of its diagnostic",()=>{ + const worker={status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9"} as const; + const failed={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"} as const; + const named={...failed,diagnostic:{...worker,reason:"grok: session store unwritable"}} as const; + assert.deepEqual(parseEngineBrokerResponse(named),named); + const prelaunch={status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"} as const; + for(const bad of [ + {...failed,diagnostic:{...worker,reason:""}}, + {...failed,diagnostic:{...worker,reason:"x".repeat(769)}}, + {...failed,diagnostic:{...worker,reason:"line\nbreak"}}, + {...failed,diagnostic:{...worker,reason:7}}, + {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} + ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); +}); + +/** + * The in-flight tool-call observation (`engineBrokerMcpCallLog.ts`) rides the + * sealed failed frame, so the seam that carries the worker's last words and + * its accounting carries this too — nothing new on a tmpfs that dies with the + * container. It is names and timings, bounded, and internally consistent: a + * frame that claims more answered than started, or more outstanding than + * started minus answered, is a fabrication and is refused rather than clamped. + */ +test("a failed frame carries the broker's in-flight MCP tool-call observation, bounded and consistent", () => { + const value = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", mcpCalls: { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }, outcome: "failed", usage: null, model: "grok-4.6", requests: 8, limitReason: "timeout" } as const; + assert.deepEqual(parseEngineBrokerResponse(value), value); + for (const mcpCalls of [ + { ...value.mcpCalls, answered: 4 }, + { ...value.mcpCalls, started: 2 }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1 }, { name: "memory_recall", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: -1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "moltnet read; Bearer sk-live", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1, arguments: { text: "secret" } }] }, + { ...value.mcpCalls, outstanding: Array.from({ length: 17 }, () => ({ name: "tool", outstandingMs: 1 })) }, + { started: 3, answered: 2, outstanding: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); + // A v1 record predates the instrument; a v1 frame that carries it is forged. + assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); + + /** + * The session's standalone GET SSE tunnel rides the same member, under the + * same rules. It is optional for exactly one reason — a turn sealed before + * the facade observed that channel replays without it — so its absence means + * "not measured" and never zero, and a record that carries it must still be + * a measurement: nothing closes before it opens, nothing delivers without + * opening, and no more can be open than `opened - closed`. + */ + const tunnels = { opened: 2, closed: 1, delivered: 1, open: [{ openMs: 428_004, delivered: false }] } as const; + const observed = { ...value, mcpCalls: { ...value.mcpCalls, tunnels } } as const; + assert.deepEqual(parseEngineBrokerResponse(observed), observed); + assert.deepEqual(parseEngineBrokerResponse(value), value, "a frame sealed before the tunnel was observed still replays, without the member"); + for (const forged of [ + { ...tunnels, closed: 3 }, + { ...tunnels, delivered: 3 }, + { ...tunnels, opened: 1, closed: 1, open: [{ openMs: 1, delivered: false }] }, + { ...tunnels, open: Array.from({ length: 9 }, () => ({ openMs: 1, delivered: false })), opened: 12, closed: 0 }, + { ...tunnels, open: [{ openMs: -1, delivered: false }] }, + { ...tunnels, open: [{ openMs: 1, delivered: "yes" }] }, + { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, + { opened: 1, closed: 0, open: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); + + /** + * The refusals ride the same member under the same rules, and they are the + * counts that make a 403'd turn readable: without them a turn the facade + * refused every request of publishes `started: 0, answered: 0` — an idle + * turn's numbers. Every reason class is its own measurement, so a partial + * member is refused rather than zero-filled, and the whole member is optional + * for the one reason `tunnels` is. + */ + const refusals = { route: 1, expired: 0, exhausted: 41, unrouted: 0, oversized: 2 } as const; + const refused = { ...value, mcpCalls: { ...value.mcpCalls, refusals } } as const; + assert.deepEqual(parseEngineBrokerResponse(refused), refused); + for (const forged of [ + { ...refusals, exhausted: -1 }, + { ...refusals, exhausted: 1.5 }, + { ...refusals, exhausted: "41" }, + { route: 1, expired: 0, unrouted: 0, oversized: 0 }, + { ...refusals, capability: 3 } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, refusals: forged } }), /invalid broker frame/u, JSON.stringify(forged)); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 9b9f644..f5e37b4 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,4 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -20,15 +21,30 @@ export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }> | EngineBrokerInferenceRequest; -export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +/** + * `reason` is the worker's own last words (`engineBrokerNativeClient.ts`), + * already redacted and flattened to one bounded line by the broker. It is the + * only field a failed turn carries that the worker itself wrote, so it is + * optional, bounded, control-character free, and admitted only for the + * statuses where a worker actually ran and spoke. + */ +export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;reason?:string;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +export const ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES = 768; export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) - | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic; mcpCalls?: EngineBrokerMcpCallObservation }> & EngineBrokerTurnAccounting) | EngineBrokerInferenceResponse; -export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +/** + * The one name for a fenced credential realm. The turn's failure code, the + * proxy's refusal reason on a worker request, and the grant path's 401 body + * (`GROK_INFERENCE_AUTH_STALE_BODY`) all say this same word, so an operator + * greps one string across every surface instead of three spellings of it. + */ +export const ENGINE_BROKER_AUTH_STALE = "auth_stale" as const; +export const ENGINE_BROKER_FAILURE_CODES = [ENGINE_BROKER_AUTH_STALE, "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; export type EngineBrokerTerminalResponse = Extract; type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; @@ -100,21 +116,97 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting]; - exact(input, input.diagnostic === undefined ? fields : [...fields, "diagnostic"]); + // `mcpCalls` is the broker's own observation of the worker's tool calls + // (`engineBrokerMcpCallLog.ts`), additive in v2 and never part of a v1 + // record, which predates the instrument entirely. + if (expected === V1 && input.mcpCalls !== undefined) throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting, ...(input.diagnostic === undefined ? [] : ["diagnostic"]), ...(input.mcpCalls === undefined ? [] : ["mcpCalls"])]; + exact(input, fields); const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + if(input.diagnostic!==undefined){const value=record(input.diagnostic);const fields=["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"];exact(value,value.reason===undefined?fields:[...fields,"reason"]);if(value.reason!==undefined&&(typeof value.reason!=="string"||value.reason.length===0||Buffer.byteLength(value.reason,"utf8")>ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES||/[\u0000-\u001f\u007f]/u.test(value.reason)))throw new TypeError("invalid broker frame");const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; if (expected === V1) return { version: V1, ...base } as V1Failed; const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); - return { version: VERSION, ...base, ...accountingValue }; + const mcpCalls = input.mcpCalls === undefined ? undefined : parseMcpCallObservation(input.mcpCalls); + return { version: VERSION, ...base, ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accountingValue }; +} + +/** + * Names and timings, bounded, and internally consistent: a report can never + * claim more answered calls than started ones, nor more outstanding ones than + * started minus answered. Every count is its own measurement, so a missing + * field is refused rather than defaulted — a zero the broker did not measure + * would read exactly like one it did. + */ +function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { + const input = record(value); + // `tunnels` is optional for one reason only: a turn sealed before the GET + // tunnel was observed carries no such member, and its record must still + // replay. Absence there means "the instrument did not exist", never zero. + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"]), ...(input.refusals === undefined ? [] : ["refusals"])]); + const started = input.started, answered = input.answered, undecoded = input.undecoded; + if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); + const outstanding = input.outstanding.map((entry) => { + const call = record(entry); + exact(call, ["name", "outstandingMs"]); + if (typeof call.name !== "string" || !ENGINE_BROKER_MCP_CALL_NAME.test(call.name)) throw new TypeError("invalid broker frame"); + if (!Number.isSafeInteger(call.outstandingMs) || (call.outstandingMs as number) < 0) throw new TypeError("invalid broker frame"); + return { name: call.name, outstandingMs: call.outstandingMs as number }; + }); + if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); + const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); + const refusals = input.refusals === undefined ? undefined : parseMcpRefusalObservation(input.refusals); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }), ...(refusals === undefined ? {} : { refusals }) }; +} + +/** + * The refused requests, by reason class: one count per closed reason, all of + * them required once the member is present. Optional for the same single + * reason `tunnels` is — a turn sealed before the facade counted its refusals + * must still replay — so its absence means "not measured" and never zero, and + * a partial member is refused rather than zero-filled. + */ +function parseMcpRefusalObservation(value: unknown): EngineBrokerMcpRefusalObservation { + const input = record(value); + exact(input, ENGINE_BROKER_MCP_REFUSAL_REASONS); + for (const reason of ENGINE_BROKER_MCP_REFUSAL_REASONS) { + if (!Number.isSafeInteger(input[reason]) || (input[reason] as number) < 0) throw new TypeError("invalid broker frame"); + } + return Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, input[reason] as number])) as EngineBrokerMcpRefusalObservation; +} + +/** + * The GET SSE tunnels, under the call observation's rules: counts and elapsed + * milliseconds, bounded, and internally consistent. A tunnel cannot close + * before it opened, cannot deliver without having opened, and no more can be + * reported open than `opened - closed` — a report claiming otherwise is a + * frame, not a measurement. + */ +function parseMcpTunnelObservation(value: unknown): EngineBrokerMcpTunnelObservation { + const input = record(value); + exact(input, ["opened", "closed", "delivered", "open"]); + const opened = input.opened, closed = input.closed, delivered = input.delivered; + if (![opened, closed, delivered].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if ((closed as number) > (opened as number) || (delivered as number) > (opened as number)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.open) || input.open.length > ENGINE_BROKER_MCP_TUNNEL_MAX || input.open.length > (opened as number) - (closed as number)) throw new TypeError("invalid broker frame"); + const open = input.open.map((entry) => { + const tunnel = record(entry); + exact(tunnel, ["openMs", "delivered"]); + if (!Number.isSafeInteger(tunnel.openMs) || (tunnel.openMs as number) < 0 || typeof tunnel.delivered !== "boolean") throw new TypeError("invalid broker frame"); + return { openMs: tunnel.openMs as number, delivered: tunnel.delivered }; + }); + return { opened: opened as number, closed: closed as number, delivered: delivered as number, open }; } function closedDiagnostic(value:JsonRecord):boolean{ if(value.profileApplied!==false||(value.workerPid as number)<0||(value.workerUid as number)<0)return false; + // Only a worker that ran and wrote something can have said why it failed: + // no prelaunch failure and no attestation refusal carries worker words. + if(value.reason!==undefined&&!((value.status==="worker_failed"&&value.stage==="wait")||value.status==="output_failed"||value.status==="cancelled"))return false; const noWorker=value.workerPid===0&&value.workerUid===0&&value.startTicks==="0"; const worker=(value.workerPid as number)>0&&(value.workerUid as number)>=2200&&value.startTicks!=="0"; if(value.status==="prelaunch_failed")return noWorker&&({peer:"peer",request:"protocol",registration:"registration",executable:"executable",exec:"exec"} as Record)[value.stage as string]===value.failureClass; diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts new file mode 100644 index 0000000..5639d94 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -0,0 +1,246 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; + +/** + * The hung turn's evidence, across the one boundary it has to cross. + * + * A cancelled or timed-out turn never answers its client, so the sealed + * terminal response — the only carrier of `mcpCalls` and the worker's redacted + * last words — dies with the slot. These tests pin the other route: the same + * sealed response, projected into the broker's own ledger directory, which + * Paideia already recovers `usage.jsonl` and `requests.jsonl` from. + * + * The turn runs through its real registry, meter, seal and ledger path; only + * the launcher, the proxy registrations and the MCP facade's observation are + * stubbed, exactly as the live hang presented them. + */ +const registration = (usageLedgerPath: string): EngineBrokerServiceRegistration => ({ + agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", + profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", + profileSha256: "a".repeat(64), usageLedgerPath, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + model: { model: "grok-4.6", reasoningEffort: "low" } +}); + +const proxy: GrokEngineBrokerTurnDependencies["proxy"] = { + capabilities: { issue: () => "provider-capability-0123456789ab", revoke: () => undefined }, + registerIsolationGuard: () => undefined, revokeIsolationGuard: () => undefined, + registerTurn: () => undefined, revokeTurn: () => undefined +}; + +/** A worker that does real work and then stops acting, until the deadline aborts it. */ +const hangs = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { + const fail = (): void => reject(new Error("engine broker turn failed")); + if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); +}); + +async function cancelledTurn(observe: () => EngineBrokerMcpCallObservation | undefined): Promise | undefined; usage: string; failure: EngineBrokerTurnFailure }>> { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe }, + prepareIsolation: async () => async () => undefined, + runNative: async (_input, signal) => hangs(signal) + }; + const controller = new AbortController(); + const running = runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-hung", "prompt", "http://127.0.0.1:43124/mcp", controller.signal); + setTimeout(() => controller.abort(), 5); + const failure = await running.then(() => { throw new Error("the hung turn resolved"); }, (error: unknown) => error as EngineBrokerTurnFailure); + const text = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8").catch(() => ""); + const lines = text.split("\n").filter((line) => line.length > 0); + assert.ok(lines.length <= 1, "one sealed turn writes at most one seal row"); + return { seal: lines[0] === undefined ? undefined : JSON.parse(lines[0]) as Record, usage: await readFile(usageLedgerPath, "utf8").catch(() => ""), failure }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +/** + * The finding this whole instrument exists for, on the host side of the seam. + * + * Live: ten provider requests, all closed, one tool receipt, then 430 s of + * silence and an abort at 489 s. The facade's log says the worker was blocked + * on `use_tool` the whole time, the sealed response carries it — and the + * sealed response never travels, because a cancelled turn has no client left to + * answer. The row in `turns.jsonl` is that evidence on a durable file the + * evaluator already reads. + * + * Mutation: drop the `seal` append from `appendBrokerTurnLedger`, or the `mcp` + * member from `renderBrokerTurnSealLine`, and this goes red. So does reverting + * `renderBrokerTurnLedger` to return `EMPTY_BROKER_TURN_LEDGER` for a turn with + * no attributable usage — which is exactly this turn. + */ +test("a cancelled turn's outstanding MCP call reaches the host on the broker's own ledger directory", async () => { + const { seal, usage, failure } = await cancelledTurn(() => ({ started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstandingMs: 430_112 }] })); + assert.equal(failure.code, "cancelled"); + // The usage ledger stays silent for a turn with nothing to attribute, so the + // seal row is the only host-visible account this turn has. + assert.equal(usage, ""); + assert.ok(seal, "a cancelled turn seals a row"); + assert.equal(seal.v, TURN_SEAL_LEDGER_VERSION); + assert.equal(seal.agent, "foreman"); + assert.equal(seal.wake, "wake-hung"); + assert.equal(seal.outcome, "failed"); + assert.equal(seal.code, "cancelled"); + assert.equal(seal.limit_reason, "none"); + assert.equal(seal.model, "grok-4.6"); + assert.deepEqual(seal.mcp, { started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstanding_ms: 430_112 }] }); +}); + +/** + * Absence stays absence, and the two absences are not the same statement. + * + * A turn that called no tool measured `started: 0`; a turn the facade never + * registered measured nothing at all and writes no `mcp` member; a turn that + * never sealed writes no row. Reading them as one another is precisely the + * mistake this session kept making. + * + * Mutation: render `mcp` unconditionally (as `{}` or as zeros) when the + * observation is `undefined`, and the second assertion goes red. + */ +test("a cancelled turn that called no tool is distinguishable from one the facade never observed, and both from no row at all", async () => { + const called = await cancelledTurn(() => ({ started: 0, answered: 0, undecoded: 0, outstanding: [] })); + assert.deepEqual(called.seal?.mcp, { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + + const unobserved = await cancelledTurn(() => undefined); + assert.ok(unobserved.seal, "an unobserved turn still seals its row"); + assert.equal(Object.hasOwn(unobserved.seal, "mcp"), false); + + // And the third state: no seal row at all, which is what every one of the six + // live runs published before this stream existed. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-none-")); + try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } + finally { await rm(root, { recursive: true, force: true }); } +}); + +/** + * The channel the seal row could not see, on the same durable route. + * + * Seven live runs sealed with every provider request closed and every tool call + * answered, and still went idle to the deadline. The facade relays one more + * thing for the whole session — the standalone GET SSE tunnel — and recorded + * nothing about it, so a worker parked reading that stream and a worker doing + * nothing wrote identical rows. These three seal the boundary that separates + * them: still open at seal time, closed before it, and never opened at all. + * + * Mutation: drop the `tunnels` member from `renderBrokerTurnSealLine` and the + * first three go red; render it unconditionally as zeros when the observation + * carries none, and the fourth does — a zero nobody measured reads exactly + * like a zero somebody did. + */ +test("a cancelled turn's GET tunnel is sealed open with its age, closed, or never opened — three distinct rows", async () => { + const mcp = (tunnels: Record): EngineBrokerMcpCallObservation => + ({ started: 1, answered: 1, undecoded: 0, outstanding: [], ...tunnels } as EngineBrokerMcpCallObservation); + + const parked = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 428_004, delivered: false }] } })); + assert.deepEqual((parked.seal?.mcp as Record).tunnels, { + opened: 1, closed: 0, delivered: 0, open: [{ open_ms: 428_004, delivered: false }] + }, "a turn sealed with a tunnel still open must say so, and say how long it had been open"); + + const ended = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 1, delivered: 2, open: [] } })); + assert.deepEqual((ended.seal?.mcp as Record).tunnels, { opened: 1, closed: 1, delivered: 2, open: [] }, + "a tunnel that closed before the seal is not an open one"); + + const never = await cancelledTurn(() => mcp({ tunnels: { opened: 0, closed: 0, delivered: 0, open: [] } })); + assert.deepEqual((never.seal?.mcp as Record).tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, + "a turn whose facade never relayed a GET measured zero, which is not the same as not having looked"); + + // And the fourth state, which is the absence: a turn sealed before this + // channel was observed at all carries no `tunnels` member. + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); +}); + +/** + * The refusal, on the same durable route as the hang it looks like. + * + * A request the facade 403'd never reached the relay, so before it was counted + * a turn whose capability was spent sealed `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals. The row has to carry the reason class, because an + * exhausted budget and an unserved route are opposite fixes. + * + * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the + * first assertion goes red; render it unconditionally as zeros for an + * observation that carries none, and the second does — a zero nobody measured + * reads exactly like a zero somebody did. + */ +test("a turn whose MCP requests were refused seals the refusals by reason, and a turn sealed before they were counted seals none", async () => { + const refused = await cancelledTurn(() => ({ + started: 0, answered: 0, undecoded: 0, outstanding: [], + refusals: { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 } + })); + assert.deepEqual((refused.seal?.mcp as Record).refusals, { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 }); + // Without it this row is `started: 0, answered: 0` — an idle turn's row. + assert.deepEqual([(refused.seal?.mcp as Record).started, (refused.seal?.mcp as Record).answered], [0, 0]); + + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); +}); + +/** + * The projection is an allow-list, and this is the assertion that makes it one. + * + * `renderBrokerTurnSealLine` copies a closed field set out of the sealed + * terminal response. Replacing that copy with `...terminal` passed every other + * test in this suite while writing `usage`, `diagnostic`, `mcpCalls` — and, for + * a completed turn, `text`: the model's entire reply, into a ledger whose whole + * rule is that it carries no prompt, body or reply. Nothing sealed a completed + * turn and read the file back, so nothing was watching the one row that + * carries a reply at all. + * + * The boundary: the exact key set of a written row, for the turn kind that has + * the most to leak. + * + * Mutation: spread the terminal into the row (`...terminal, v: ..., agent: ...`) + * and this goes red on both halves — the key set gains `text`, `kind`, + * `version`, `requestId`, `workerPid`, `workerUid`, `workerStartTime` and + * `usage`, and the reply itself appears in the file's bytes. + */ +const reply = "TANGERINE-7-IS-THE-MODELS-OWN-REPLY"; +const answered = (text: string): string => { + const session = "01a0ad21-a90f-7f71-8054-93fdb4334d6a"; + const usage = { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }; + return [ + { type: "system", subtype: "init", session_id: session }, + { type: "assistant", message: { id: "msg_0", type: "message", role: "assistant", model: "daimon-broker-grok", content: [{ type: "text", text }], stop_reason: "end_turn", usage }, parent_tool_use_id: null, session_id: session }, + { type: "result", subtype: "success", is_error: false, num_turns: 1, result: text, stop_reason: "end_turn", total_cost_usd: 0.0024, usage, modelUsage: { "grok-4.6-build": {} }, session_id: session } + ].map((frame) => JSON.stringify(frame)).join("\n"); +}; + +test("a completed turn's seal row carries exactly its declared fields, and never the model's reply", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-completed-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, + prepareIsolation: async () => async () => undefined, + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }) + }; + const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); + assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); + + const bytes = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8"); + const lines = bytes.split("\n").filter((line) => line.length > 0); + assert.equal(lines.length, 1); + const row = JSON.parse(lines[0]!) as Record; + assert.deepEqual(Object.keys(row).sort(), ["agent", "at", "engine", "limit_reason", "model", "outcome", "requests", "turn", "v", "wake"]); + assert.deepEqual([row.v, row.agent, row.wake, row.engine, row.outcome, row.model, row.limit_reason, row.requests], [TURN_SEAL_LEDGER_VERSION, "foreman", "wake-done", "grok", "completed", "grok-4.6", "none", 1]); + // The second half of the same guarantee, on the bytes rather than the keys: + // a reply that reached the ledger under any name is the failure. + assert.ok(!bytes.includes(reply), "the model's reply must never reach the ledger"); + // A completed turn carries no failure members at all, and the facade's + // observation is a failed turn's member: neither may appear here. + for (const absent of ["text", "code", "diagnostic", "mcp", "usage", "workerPid", "workerUid", "kind", "version"]) { + assert.equal(Object.hasOwn(row, absent), false, absent); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts new file mode 100644 index 0000000..4d64893 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.ts @@ -0,0 +1,131 @@ +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; + +/** + * The operator-visible half of a sealed terminal turn, as a durable row. + * + * The seal itself already carries everything an operator needs to read a turn + * that stopped acting — the failure code, the worker's own redacted last words, + * and `engineBrokerMcpCallLog.ts`'s observation of the tool call the worker was + * blocked on. All of it travels in the control-protocol terminal response, and + * that response is the one artifact a *hung* turn never produces: the client is + * gone before the broker answers, the turn registry record lives in the + * broker's own `0700` turn store, and a training slot's control root is tmpfs + * that dies with the container. Six live runs reproduced the same signature and + * none of them could read the instrument built for it. + * + * What does survive a slot is the broker's ledger directory: Paideia already + * recovers `usage.jsonl` and `requests.jsonl` from it on the failure path. So + * this is a third stream beside those two, written by the broker — still the + * single sealed usage writer — from the sealed response and nothing else, at + * the moment that response is sealed. + * + * Its rules are the two ledgers' rules: + * + * - **Numbers, names, timings and closed vocabularies only.** The failure code, + * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` + * and its already-redacted, already-bounded, control-character-free `reason` + * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), + * plus tool-call names, GET-tunnel counts and elapsed milliseconds. Never a + * prompt, a body, a + * reply, a bearer, a capability or a session id — none of which the terminal + * response carries in the first place. + * - **Absence stays absence.** `mcp` is written only when the facade actually + * observed the turn, `diagnostic` only when the sealed response carried one, + * and `code` only for a failure. A turn that called no tool publishes + * `started: 0`, which is a measurement; a turn the facade never registered + * publishes no `mcp` member at all, which is not. + * - **It can never fail a turn.** The row is rendered from an + * already-validated frame and appended through `recordLedgerLines`, which + * swallows every I/O fault. + * + * A separate stream and a separate `v`, for `turnRequestLedger.ts`'s reason: + * Spawnfile's reader pins `noopolis.daimon.turn-usage.v1` and drops any other + * `v` outright, and Paideia's request reader refuses a row it cannot type. A + * row in a new file is invisible to both. + */ +export const TURN_SEAL_LEDGER_VERSION = "noopolis.daimon.turn-seal.v1" as const; + +/** Default location: beside `usage.jsonl` and `requests.jsonl`, no new mount. */ +export const TURN_SEAL_LEDGER = { + version: TURN_SEAL_LEDGER_VERSION, + directoryPath: TURN_USAGE_LEDGER.directoryPath, + filePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl`, + rotatedFilePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl.1`, + fileMode: TURN_USAGE_LEDGER.fileMode +} as const; + +/** A rendered seal line is bounded by its own contents: a 768-byte reason, 16 bounded names and 8 tunnel timings. */ +export const TURN_SEAL_MAX_LINE_BYTES = 8_192; + +export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; + +const bounded = (value: string): string => [...value].slice(0, TURN_USAGE_MAX_IDENTIFIER_CHARS).join(""); + +/** + * One newline-terminated row for one sealed terminal turn. + * + * Every member is copied from the terminal response the protocol already + * validated, so this projection cannot widen what the response admits; it is an + * allow-list rather than a spread, so a future additive member of the response + * does not silently become a ledger field. + */ +export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, entry: BrokerTurnSealEntry): string => `${JSON.stringify({ + v: TURN_SEAL_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at: entry.at, + turn: terminal.turnId, + outcome: terminal.outcome, + requests: terminal.requests, + model: terminal.model, + limit_reason: terminal.limitReason, + ...(terminal.kind === "failed" ? { code: terminal.code } : {}), + ...(terminal.kind === "failed" && terminal.diagnostic !== undefined + ? { + diagnostic: { + status: terminal.diagnostic.status, + stage: terminal.diagnostic.stage, + failure_class: terminal.diagnostic.failureClass, + exit_code: terminal.diagnostic.exitCode, + term_signal: terminal.diagnostic.termSignal, + ...(terminal.diagnostic.reason === undefined ? {} : { reason: terminal.diagnostic.reason }) + } + } + : {}), + ...(terminal.kind === "failed" && terminal.mcpCalls !== undefined + ? { + mcp: { + started: terminal.mcpCalls.started, + answered: terminal.mcpCalls.answered, + undecoded: terminal.mcpCalls.undecoded, + outstanding: terminal.mcpCalls.outstanding + .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // Every request the facade refused before it could relay it, by reason + // class. Without it a turn whose capability was spent seals as + // `answered == started, outstanding: []`, which is what a healthy turn + // seals as. + ...(terminal.mcpCalls.refusals === undefined + ? {} + : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), + // The session's standalone GET tunnel, absent for a turn sealed before + // the facade observed that channel at all. + ...(terminal.mcpCalls.tunnels === undefined + ? {} + : { + tunnels: { + opened: terminal.mcpCalls.tunnels.opened, + closed: terminal.mcpCalls.tunnels.closed, + delivered: terminal.mcpCalls.tunnels.delivered, + open: terminal.mcpCalls.tunnels.open + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((tunnel) => ({ open_ms: tunnel.openMs, delivered: tunnel.delivered })) + } + }) + } + } + : {}) +})}\n`; diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index d33f2bf..1dac5d5 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -56,3 +56,20 @@ test("a limit failure reaches the client with its code and limit reason", async await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); }); + +test("a failed worker's own reason reaches the client instead of a bare exit code", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); + },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); +}); + +/** + * The end of the seam: an outstanding tool call has to be readable by whoever + * reads the failure, not just sealed. The live hang would have read + * `mcp=1/2 answered; mcp_outstanding=daimon__moltnet_read@419000ms`. + */ +test("an outstanding MCP tool call reaches the client by name, with how long it waited", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=timeout; mcp=1\/2 answered; mcp_undecoded=1; mcp_outstanding=daimon__moltnet_read@419000ms\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:null,model:"grok-4.6",requests:8,limitReason:"timeout"},{started:2,answered:1,undecoded:1,outstanding:[{name:"daimon__moltnet_read",outstandingMs:419_000}]});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index aef6e33..79f55f0 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -54,7 +54,7 @@ function failed(socket:Socket,request:Extract parseEngineBrokerServiceConfig({ ...config("v1", [reg("agent-a", 0)]), inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }), /invalid engine broker service config/u); }); + +test("registration paths must be canonical, matching the native launcher's registration check", () => { + const good = reg("agent-a", 0); + for (const [field, value] of [ + ["workspace", "/workspace/0/../1"], ["workspace", "/workspace//0"], ["workspace", "/workspace/0/"], ["workspace", "/workspace/./0"], + ["profilePath", "/workers/0/../1/.grok/sandbox.toml"], ["profilePath", "/workers//0/.grok/sandbox.toml"] + ] as const) { + const registration = { ...good, [field]: value, ...(field === "profilePath" ? { eventsPath: value.replace(/sandbox\.toml$/u, "sessions/sandbox-events.jsonl") } : {}) }; + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [registration])), /invalid engine broker service config/u, `${field}=${value}`); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [good]), turnStore: "/var/lib/turns/" })); + assert.doesNotThrow(() => parseEngineBrokerServiceConfig(config("v1", [good]))); +}); diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index 742e2a5..eb6d274 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -12,7 +12,7 @@ export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v /** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ export type EngineBrokerServiceRegistration = Readonly<{ agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; - /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` beside it. */ + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` and per-turn seals to `turns.jsonl` beside it. */ usageLedgerPath: string; limits: EngineBrokerTurnLimits; model: GrokBrokerModelPolicy; @@ -31,11 +31,23 @@ const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model const invalid = (): TypeError => new TypeError("invalid engine broker service config"); const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; -const absolute = (item: unknown): item is string => typeof item === "string" && item.startsWith("/") && !item.includes("/../") && !item.endsWith("/..") && !item.includes("\0"); +/** + * Absolute and canonical: no `.`/`..`/empty components and no trailing slash. + * The native launcher derives HOME, GROK_HOME and TMPDIR from the registered + * home and refuses a non-canonical one, so the broker's view must match it. + */ +const absolute = (item: unknown): item is string => typeof item === "string" && item.length > 1 && item.startsWith("/") && !item.endsWith("/") && path.posix.normalize(item) === item && !item.split("/").slice(1).some((part) => part === "." || part === "..") && !item.includes("\0"); /** The per-request stream written beside a registration's usage ledger. */ export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); +/** + * The per-turn seal stream written beside the other two + * (`engineBrokerSealLedger.ts`): the operator-visible half of a sealed terminal + * response, for the turn whose response never reaches a client. + */ +export const engineBrokerSealLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "turns.jsonl"); + /** * Strict `service.json` parser. * @@ -62,7 +74,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; const usageLedgerPath = entry.usageLedgerPath; - if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath)) throw invalid(); + if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || usageLedgerPath === engineBrokerSealLedgerPathFor(usageLedgerPath)) throw invalid(); let limits: EngineBrokerTurnLimits; try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; @@ -71,7 +83,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ if (!Object.hasOwn(value, "inferenceLedgerPath")) return base; const inferenceLedgerPath = value.inferenceLedgerPath; if (!ledgerPath(inferenceLedgerPath)) throw invalid(); - const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath)])]); + const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath), engineBrokerSealLedgerPathFor(entry.usageLedgerPath)])]); if (subject.has(inferenceLedgerPath)) throw invalid(); return { ...base, inferenceLedgerPath }; } diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 0dae9a8..ca1b5e1 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -290,7 +290,7 @@ test("Daimon frames one escaped identity envelope for every production engine", const envelope = JSON.stringify({ id: config.id, name: identity.name, instructions: identity.instructions }); assert.equal(result.text.split(envelope).length - 1, 1); assert.match(result.text, //); - assert.match(result.text, /Colleagues only hear you when you call moltnet_send/u); + assert.ok(result.text.includes(`Colleagues only hear you when you call ${kind === "grok" ? "daimon__moltnet_send" : "moltnet_send"};`), `${kind} must name the send tool the way it can call it`); assert.match(result.text, /Do not seek transport credentials or invoke a transport CLI/u); assert.match(result.text, /payload/); await handle.stop(); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 4666aea..9b72149 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,4 +1,5 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; +import { grokDaimonToolName, grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; @@ -56,7 +57,8 @@ export async function startOrganizationRuntimeEngine( readablePaths: codexSandboxReadablePaths(canonicalAgent) } : undefined; - const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])], wakeContext, grokSandbox,grokBroker,codexSandboxPaths); + const mountedTools = [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])]; + const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, mountedTools, wakeContext, grokSandbox,grokBroker,codexSandboxPaths, mountedTools.map((tool) => tool.name)); const handle = await adapter.startAgent({ id: canonicalAgent.id, name: canonicalAgent.name, @@ -118,16 +120,16 @@ export function codexSandboxReadablePaths( return [path.join(currentAgent.runtimeHomePath, "tool-output")]; } -function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }): PiHarnessAdapter { +function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }, mountedToolNames: readonly string[] = []): PiHarnessAdapter { const engine = agent.engine.kind; const sessionFactory = createCliSessionFactory( engine === "agy" - ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, // AGY has no broker to meter it, so the session hands its decoded // terminal-frame usage straight to the same ledger the Grok broker // appends to. `recordTurnUsage` is advisory and never rejects. onTurnUsage: (usage, outcome) => recordTurnUsage(resolveTurnUsageLedgerPath(), { agent: agent.id, wake: wakeEnvironmentContext.current ?? "wake", engine: "agy", usage, outcome }) } - : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, ...(engine === "codex" ? { // Codex has no broker to meter it, so publish terminal-frame usage // to the shared advisory ledger — on the wake that published and on @@ -176,17 +178,55 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig): string { +/** + * The caller-owned prompt preamble. + * + * It names the mounted tools explicitly, because a CLI engine reaches Daimon's + * tools over MCP and an agent whose instructions name another engine's tool + * spelling can finish a turn having called nothing. The declared names are the + * caller's own configuration, not engine-supplied text. + * + * On Grok the bare names are not the callable ones: every Daimon tool is a + * deferred MCP tool of server `daimon`, and Grok 1.0.34 refuses an unqualified + * name before any HTTP ("Tool names must be qualified as `server__tool`"). This + * envelope used to instruct exactly that refused form. The correct rule is not + * restated here — it is rendered by `grokMountedToolNamingRule` in the same + * contract module that renders the worker's pinned system prompt, so the two + * texts cannot contradict each other again. Every other engine's sentence is + * unchanged, byte for byte. + */ +export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", - "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + ...(mountedToolNames.length === 0 ? [] : [ + (agent.engine.kind === "grok" + ? grokMountedToolNamingRule(mountedToolNames) + : `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently.") + + " No other tool reaches the newsroom." + ]), + // The one tool that reaches colleagues is named the way this engine can + // call it. Meaning and prohibition are unchanged; only the spelling is. + `Colleagues only hear you when you call ${engineToolName(agent, "moltnet_send")}; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. ` + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." ].join("\n") + "\n"; } +/** + * One Daimon tool name, spelled the way this agent's engine accepts it. + * + * On Grok the bare form is refused as an invalid MCP tool name, so any + * engine-facing sentence that *names* a tool renders it through the contract's + * `grokDaimonToolName`; every other engine keeps the bare name byte for byte. + * Grok's own native tools (`read_file`, `search_tool`, `use_tool`) are not + * Daimon tools and never take the prefix. + */ +const engineToolName = (agent: OrganizationRuntimeAgentConfig, tool: string): string => + agent.engine.kind === "grok" ? grokDaimonToolName(tool) : tool; + function cliHarness( agent: OrganizationRuntimeAgentConfig, sessionFactory: ReturnType, diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts new file mode 100644 index 0000000..13b1607 --- /dev/null +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import test from "node:test"; + +import { identityEnvelope } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const rootConfig = (root: string, kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): OrganizationRuntimeAgentConfig => ({ + id: `${kind}-agent`, name: kind, instructions: "Reply.", + workspacePath: path.join(root, "workspace", kind), runtimeHomePath: path.join(root, "runtime", kind), + engine: { kind } +}); + +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope names the prefixed form as the only valid one, once", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +/** + * The transport sentence names the one tool an agent needs to reach its + * colleagues. A bare `moltnet_send` one line under "a bare name is not a valid + * MCP tool name and reaches nothing" is the same self-contradiction, on the + * tool that matters most. + */ +test("the transport sentence names the send tool the way Grok can call it, prohibition unchanged", () => { + const grok = identityEnvelope(rootConfig("/private/org", "grok"), mounted); + const transport = grok.split("\n").find((line) => line.startsWith("Colleagues only hear you"))!; + // Mutation guard: un-prefixing this name leaves the bare form the line above declares invalid. + assert.ok(transport.includes(`you call ${grokDaimonToolName("moltnet_send")};`), transport); + assert.equal(new RegExp(`(? { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); + // The transport sentence keeps its bare spelling on every other engine. + const transport = "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool."; + for (const kind of ["codex", "agy"] as const) { + assert.ok(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(`\n${transport}\n`), kind); + assert.equal(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(DAIMON_GROK_TOOL_PREFIX), false, kind); + } +}); diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index a1f00ca..b28307c 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json index 3ccf85c..fd1e2de 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json @@ -2,7 +2,7 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 0b57369..03a3c9d 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 19cb96f..c2f4908 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index 91774b2..d5978c9 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json index 94542eb..bf3e344 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index 10794bb..960e917 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -52,3 +52,20 @@ test("a provisioned registration must describe its projection exactly", () => { assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); } }); + +test("the acceptance store mask may be the directory that covers the store", () => { + // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a + // deployment that secures `/state` to `2000:2000 0700` declares that directory here. + const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; + const state = "/var/lib/spawnfile/instance/state"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: state }); + assert.equal(lifted.denyPaths.includes(state), true); + assert.equal(lifted.denyPaths.includes(store), false); + // The store itself still works where its parent is traversable, and produces a different digest — + // a recomputation that disagrees with the deployment fails closed rather than certifying the slot. + const leaf = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }); + assert.equal(leaf.denyPaths.includes(store), true); + assert.notEqual(grokBrokerProjectionSha256(leaf), grokBrokerProjectionSha256(lifted)); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: "/var" }), + /base profile grant/u); +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index 4052e41..298b433 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -53,7 +53,21 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ architecture: "arm64" | "x64"; usageLedgerPath: string; limits: EngineBrokerTurnLimits; - /** The wake-acceptance store, always denied like the Codex projection's. */ + /** + * The deny entry that protects the durable wake-acceptance store: the store + * itself, or a directory containing it. + * + * Grok 1.0.34 materializes every deny target inside bubblewrap **as the + * worker uid**, so a target whose parent directory the worker cannot search + * is unplaceable and makes Grok refuse the whole profile — every turn of that + * worker then fails, not just that path. Deployments that keep the store + * under a private `state` directory (`2000:2000 0700`) therefore declare that + * directory here: it covers the store, nothing else lives there, and lifting + * the mask adds the worker no reach, where opening the parent with `o+x` + * would. The caller is the one party that knows both the layout and the + * modes; whoever recomputes this projection must pass the same value or the + * digests will not match, which is the intended fail-closed outcome. + */ acceptanceStorePath: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; @@ -79,9 +93,10 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ * * The agent must be a Grok agent that *declares* its model and reasoning * effort; nothing is defaulted. The deny list is Daimon's own protected set for - * this agent (realm, bootstrap, peers, acceptance store) plus the caller's - * evaluator paths, sorted and deduplicated exactly as the profile renderer - * does. A supplied `profileSha256` that differs is refused. + * this agent (realm, bootstrap, peers, and the mask covering the acceptance + * store) plus the caller's evaluator paths, sorted and deduplicated exactly as + * the profile renderer does. A supplied `profileSha256` that differs is + * refused. */ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { const parsed = parseOrganizationRuntimeConfig(config); diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 0e6e6de..93d4a9f 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -1,9 +1,14 @@ import assert from "node:assert/strict"; import { request as httpRequest } from "node:http"; +import { connect } from "node:net"; import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; -import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "./grokBrokerModelPolicy.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; type Proxy = Awaited>; const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { @@ -46,7 +51,8 @@ test("proxy refuses a fail-open tool set or an undeclared effort without calling const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { - assert.equal(await post(proxy.port, token, payload), 503); + // A policy miss is non-retryable: 400, so Grok fails fast instead of retrying a 503. + assert.equal(await post(proxy.port, token, payload), 400); } assert.equal(calls, 0); assert.equal(await post(proxy.port, token, leanBody()), 200); assert.equal(calls, 1); assert.ok(accessed >= 1); @@ -68,6 +74,7 @@ test("the session-title sink is refused before capability, guard, credential, or try { const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); + // The title sink keeps its transient 503 shape: a 4xx there ends Grok's session. assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); // The turn capability (budget 1 request) is untouched and still serves the real request. @@ -85,7 +92,7 @@ test("the isolation guard is awaited before the first upstream call, and a faili order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); if (fail) throw new Error("no enforcement evidence"); }); - assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.equal(await post(proxy.port, token, leanBody()), 400); assert.equal(upstreamCalls, 0); assert.deepEqual(order, ["guard-start", "guard-end"]); fail = false; order.length = 0; @@ -93,3 +100,147 @@ test("the isolation guard is awaited before the first upstream call, and a faili assert.deepEqual(order, ["guard-start", "guard-end", "credential", "upstream"]); } finally { await proxy.close(); } }); + +test("the two requests every healthy turn makes are not named as refusals", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => { lines.push(String(chunk)); return original(chunk as string, ...rest as []); }) as typeof process.stderr.write; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + arm(proxy, async () => undefined); + const title = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(title.status, 503, "the title sink keeps its transient shape"); + const probe = await fetch(`http://127.0.0.1:${proxy.port}/`); + assert.equal(probe.status, 400, "the unauthenticated probe keeps its non-retryable shape"); + assert.deepEqual(lines, [], "expected per-turn traffic must not read as a refusal on the broker's stderr"); + const miss = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${"z".repeat(48)}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(miss.status, 400); + assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("a non-refusal fault names its own class and message on one bounded line, credentials withheld", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + const provider = "provider-vqmxdfhlzptgnbwc"; let capability = ""; + // The fault's own words carry both credentials verbatim — the worker's turn + // capability and the broker's provider bearer, neither in a shape any generic + // pattern recognises — plus a newline, a control character, and far more text + // than the bound admits. + const proxy = await startGrokBrokerProxy( + { accessToken: async () => provider, markRejected: async () => undefined }, + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider}\u0007 ${"pad ".repeat(400)}`); }); + try { + capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); + assert.equal(lines.length, 1, "one line per fault"); + const line = lines[0]!; + assert.match(line, /^\[grok-proxy\] refused: broker_unavailable \(RangeError: socket hang up forwarding /u, "the fault names its own class and message"); + assert.ok(!line.includes(capability), `the worker's own capability is withheld: ${line}`); + assert.ok(!line.includes(provider), `the broker's provider bearer is withheld: ${line}`); + assert.match(line, /\[REDACTED\]/u, "the withheld values are marked, not silently dropped"); + assert.match(line, /^[^\n]+\n$/u, "one line: newlines and control characters are flattened"); + assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("a fault's own cause is named too, because `fetch failed` on its own names nothing", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // Exactly the shape undici throws when the provider is unreachable. + const fault = new TypeError("fetch failed"); (fault as { cause?: unknown }).cause = Object.assign(new Error(""), { code: "ENOTFOUND" }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { throw fault; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +/** + * A fenced realm is the one fault this proxy answered worst: the credential is + * gone until an operator logs in again, and 503 made Grok retry it fifteen + * times over five minutes for nothing. It is a named, non-retryable refusal + * now — and it wears the same name as the turn failure code and the grant + * path's 401 body, so one word finds it on every surface. + */ +test("a fenced credential realm is a named 400 auth_stale, before any credential read or upstream call", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + let reads = 0, calls = 0, stale = true; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { reads += 1; return "provider-token"; }, markRejected: async () => undefined, isStale: () => stale }, + async () => { calls += 1; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400, "a stale realm is not transient, so it must not be retryable"); + assert.deepEqual({ reads, calls }, { reads: 0, calls: 0 }, "no credential is read and nothing is forwarded for a fenced realm"); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + // The title sink keeps the 503 it has always had, fenced realm or not. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, leanBody()), 503); + // And the same capability serves the real request once the realm is healthy. + stale = false; lines.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual({ reads, calls, lines }, { reads: 1, calls: 1, lines: [] }); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the request that discovers the fence is named auth_stale too, not one transient fault", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // The live shape: `accessToken` fences the realm and throws the authority's + // own generic error, which on its own reads as a transient fault. + let stale = false; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { stale = true; throw new Error("Grok broker credential authority unavailable"); }, markRejected: async () => undefined, isStale: () => stale }, + async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the turn path and the grant path name a fenced realm the same way", () => { + assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); +}); + +/** + * A response the usage decoder cannot read is the worst case to get wrong: the + * upstream call already succeeded, so the money is spent whatever happens + * next. It must reach the worker anyway, and it must be charged — a fabricated + * zero is byte-identical to a measured one, so the documented estimate is the + * only honest landing place. (The fault is synthetic: the real one is a + * response body past the maximum string length, which is not a thing to + * allocate in a test.) + */ +test("a response whose usage cannot be decoded is still delivered, and charged the documented estimate", async () => { + const contentType = { toString: () => "application/json" } as unknown as string; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => "provider-token", markRejected: async () => undefined }, + async () => ({ status: 200, headers: { "content-type": contentType }, body: Buffer.from('{"usage":{"prompt_tokens":11,"completion_tokens":3}}') })); + try { + const token = proxy.capabilities.issue("agent", "turn"); + const meter = arm(proxy, async () => undefined); + const payload = leanBody(); + assert.equal(await post(proxy.port, token, payload), 200, "a paid response must not be thrown away over its own instrumentation"); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 1); + assert.equal(snapshot.estimatedRequests, 1, "the row says the charge was estimated, not measured"); + assert.deepEqual(snapshot.usage, estimateGrokRequestUsage(Buffer.byteLength(payload)), "charged the documented conservative estimate, never zero and never nothing"); + assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); + } finally { await proxy.close(); } +}); + +test("proxy shutdown ends a worker's leftover keep-alive socket instead of waiting for it", async () => { + const proxy = await startGrokBrokerProxy({ accessToken: async () => "token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }), DEFAULT_GROK_BROKER_MODEL_POLICY, 0); + // A worker's HTTP client keeps its connection to the proxy pooled; nothing + // in the proxy's own state accounts for it, so a shutdown that waits for the + // client to release it has no bound. + const socket = connect({ host: "127.0.0.1", port: proxy.port }); + socket.on("error", () => undefined); + await new Promise((resolve, reject) => { socket.once("connect", resolve); socket.once("error", reject); }); + try { + const outcome = await Promise.race([proxy.close().then(() => "closed" as const), delay(5_000).then(() => "parked" as const)]); + assert.equal(outcome, "closed", "proxy shutdown parked on a socket the proxy does not track"); + } finally { socket.destroy(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4d1e6d0..7ee024e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -1,10 +1,14 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; -import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { parseGrokResponseToolNames, parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -31,28 +35,171 @@ export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthor const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared,grants); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => { server.close((error) => error === undefined ? resolve() : reject(error)); /* `close` waits for every open connection, and a worker keeps its client pool's socket to this proxy open with nothing here accounting for it — so that wait has no bound. Ending them is this listener's to do, exactly as the MCP facade does. */ server.closeAllConnections(); }) }; +} + +/** + * A refusal the caller must not retry. + * + * Every internal refusal used to collapse into one bare 503. Grok treats 503 as + * transient and blind-retries the same request (observed: 14 retries, ~141k + * estimated tokens, then `exit 1`), so a policy miss burned a turn's budget and + * reported itself as an engine crash. Policy refusals now answer 400 with a + * reason, and only genuinely transient faults keep 503. + */ +export class GrokBrokerProxyRefusal extends Error { + constructor(readonly reason: string) { super(`grok broker proxy refused: ${reason}`); } } async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { - let settle:((usage:ReturnType)=>void)|undefined; + let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; + let titleSink = false; + // Every credential this request holds, kept only for this request and only so + // that a fault's own words can be redacted against them exactly as the CLI + // child and launcher diagnostics are. Nothing reads them but {@link brokerFaultCause}. + const secrets: string[] = []; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); + titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); - if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new Error();return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} - const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + if(match)secrets.push(match[1]!); + if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} + const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); + try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + // A fenced realm is not a transient fault: the credential is gone until an + // operator re-logs in, and 503 made Grok blind-retry it (observed: fifteen + // retries over five minutes, $0 spent, nothing learned). Named, 400, and + // checked before the credential read, so the miss costs one round trip. + if (authority.isStale?.() === true) throw new GrokBrokerProxyRefusal(ENGINE_BROKER_AUTH_STALE); + let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} - settle=(usage)=>{turn.meter.settle(admission.index,usage,body.byteLength);settle=undefined;}; + settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; let result = await upstream(prepared,admission.signal); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } - settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);secrets.push(token);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } + // Names only, bounded, and never a reason to fail the request: the response + // is already buffered here for its usage block, so what the model tried to + // call is in hand. A decoder fault records no attempt rather than a false + // empty one, and never disturbs the turn. + settle?.(usageOrEstimate(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch (error) { + settle?.(undefined); + // Name the refusal on the broker's own stderr (reason code only, never a body + // or a token) so a failing turn is diagnosable without a stub harness — + // except for the two requests every healthy turn makes anyway. + // The request that *discovers* the fence throws an ordinary error from the + // credential authority, so it is promoted to the same named refusal: one + // stale realm must not read as one transient fault plus fourteen retries. + const fenced = !(error instanceof GrokBrokerProxyRefusal) && authority.isStale?.() === true; + const refused = error instanceof GrokBrokerProxyRefusal || fenced; + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : fenced ? ENGINE_BROKER_AUTH_STALE : "broker_unavailable"; + // A named refusal is its own account; anything else used to reach the log as + // the bare word `broker_unavailable`, which names nothing — so it carries the + // fault's own class and message, and nothing else, beside it. + const named = refused ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); + // Grok's own session-title call is refused by design, and keeps the transient + // 503 shape it has always had. Forcing 400 and 503 on it were both observed + // to end the turn `exit=0, result: success`, so the shape is kept because it + // is the one every live capture was taken with, not because a 4xx there ends + // Grok's session — it does not. + if (titleSink) { + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + return; + } + if (refused) { + response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); + return; + } + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + } finally { secrets.length = 0; } +} + +/** + * A non-refusal fault, named on one bounded line. + * + * `broker_unavailable` on its own carries no diagnostic content at all, and it + * is answered 503, which Grok blind-retries: one live turn emitted it fifteen + * times over five minutes, spent $0 — so no upstream call ever succeeded — and + * died with no account of why. The error's own class and message are the whole + * of what is logged: never a request body, bearer, capability, session id or + * header. It is redacted exactly as the failed CLI child and the launcher's + * worker diagnostic are — `redactCredentialText` with this request's own + * capabilities as exact secrets and the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` + * bound — and flattened to one line, because it travels on a log line. Naming + * a fault must never be able to fail the response that reports it, so a value + * that cannot even be described degrades to a marker. + * + * One level of `cause` is named too, because the fault this exists for names + * nothing without it: every failed `fetch` to the provider is `TypeError: + * fetch failed`, and which fault it was — `ENOTFOUND`, `ECONNREFUSED`, a TLS + * refusal, an abort — is only in the cause. An errno error whose message is + * empty is named by its `code`. + */ +const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { + try { + const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; + const flattened = described.replace(/[\u0000-\u001f\u007f]+/gu, " ").replace(/\s+/gu, " ").trim(); + const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return named.length === 0 ? "unnamed" : named; + } catch { return "unnameable"; } +}; + +/** One value's class and words: an error's own, or the type of whatever else was thrown. */ +const describeFault = (error: unknown): string => { + if (!(error instanceof Error)) return `${typeof error}: ${String(error)}`; + const code = (error as NodeJS.ErrnoException).code; + const words = error.message.length > 0 ? error.message : typeof code === "string" ? code : "(no message)"; + return `${error.constructor?.name ?? error.name}: ${words}`; +}; + +/** + * The unauthenticated connectivity probe Grok sends before its own requests: a + * bare `GET /` with no Authorization header, which has no capability to look up + * and answers 400. + * + * It and the per-turn `session_title` POST are the only two requests a healthy + * turn makes that this proxy does not forward, and both used to print the same + * `refused: unknown_capability` line as a real policy miss — so every healthy + * turn read as two refusals and cost a live investigation. They answer exactly + * as before; they simply stop claiming a refusal on the broker's stderr, which + * is left for the misses that are actually worth reading. + */ +const expectedWorkerProbe = (request: IncomingMessage): boolean => + request.headers.authorization === undefined && (request.method ?? "") === "GET" && + new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; + +/** + * Upstream-reported usage, or the documented conservative estimate. + * + * The decoder faulting must not fail the request that already cost real + * money: the upstream call succeeded, the worker is owed its answer, and a + * 503 here would throw away a paid response and have Grok buy it again. The + * `undefined` this returns is not "no usage" — `GrokBrokerTurnMeter.settle` + * charges it `ceil(bodyBytes/2) + 4096` and marks the row + * `usage_source: "estimated"`, so the token ceiling still counts it and no + * fabricated zero ever reaches a ledger. + */ +const usageOrEstimate = (body: Uint8Array, contentType: string | undefined): ReturnType => { + try { return parseGrokUpstreamUsage(body, contentType); } catch { return undefined; } +}; + +/** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ +const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { + try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } +}; + +/** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ +function authorizeRequestOrRefuse(...args: Parameters): ReturnType { + try { return authorizeGrokBrokerProxyRequest(...args); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 0f28a88..852b514 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; -import { GrokBrokerTurnMeter, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; +import { GROK_REQUEST_TOOL_CALLS_MAX, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_TRUNCATED, GrokBrokerTurnMeter, parseGrokResponseToolNames, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); @@ -85,13 +85,21 @@ test("a request after the elapsed deadline is refused, and every admitted reques assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); }); -test("a turn without a registered meter is never forwarded", async () => { +// A live capability with no registered meter means the turn is already over: the +// launcher registers a turn before it starts the worker, so nothing can arrive +// before the meter exists, and nothing can make a finished turn live again. The +// answer is therefore 400 (a named, non-retryable refusal) rather than the 503 it +// once was — a retryable shape here bought only Grok's blind retry storm, which +// spent ~141k tokens re-asking a question that could never start being answerable. +test("a turn without a registered meter is refused non-retryably and never forwarded", async () => { let calls = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); try { const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); - assert.equal((await post(proxy.port, token)).status, 503); + const answer = await post(proxy.port, token); + assert.equal(answer.status, 400); + assert.equal(JSON.parse(answer.text).reason, "no_active_turn"); assert.equal(calls, 0); } finally { await proxy.close(); } }); @@ -181,3 +189,61 @@ test("an implausible per-request usage block is never added, and missing usage s }); assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); }); + +const events = (chunks: readonly unknown[]): Uint8Array => + Buffer.from(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); +/** One streaming tool call: the name arrives in one delta, the arguments in the next. */ +const callDeltas = (names: readonly string[]): unknown[] => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"query":"secret"}' } }] } }] })) +]; + +/** + * Two live turns ended with every tool correctly mounted and no way to tell + * whether the model had tried to call anything. These names are that answer, + * and nothing more than that answer. + */ +test("a response's tool-call names are read, bounded, and stripped of everything but the names", async () => { + // Mutation guard: passing the response through instead of the names leaks arguments here. + const names = parseGrokResponseToolNames(events([...callDeltas(["use_tool", "search_tool"]), { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }]), "text/event-stream"); + assert.deepEqual(names, ["use_tool", "search_tool"]); + assert.equal(JSON.stringify(names).includes("secret"), false); + + // A non-streaming body carries its calls on the message; one tool called + // twice is two attempts, because neither carries a streaming call index. + assert.deepEqual(parseGrokResponseToolNames(Buffer.from(JSON.stringify({ choices: [{ message: { tool_calls: [{ function: { name: "use_tool" } }, { function: { name: "use_tool" } }] } }] })), "application/json"), ["use_tool", "use_tool"]); + + // Absence stays absence: a decoded response that called nothing is `[]`, and + // an undecodable one is nothing at all. A zero-length list must never be + // invented for a response nobody could read. + assert.deepEqual(parseGrokResponseToolNames(events([{ choices: [{ index: 0, delta: { content: "x" } }] }]), "text/event-stream"), []); + assert.equal(parseGrokResponseToolNames(Buffer.from("gateway"), "text/html"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from(""), "text/event-stream"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from("data: not-json\n\n"), "text/event-stream"), undefined); + + // A name that is not a plain short identifier is counted, never passed through. + assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + + // A hostile but decodable shape yields no attempt instead of throwing: + // instrumentation must never be able to fail the turn it observes. + assert.deepEqual(parseGrokResponseToolNames(events([ + { choices: "not-an-array" }, { choices: [null, 7, { delta: { tool_calls: "no" } }, { message: { tool_calls: [null, { function: null }, { function: { name: 42 } }, { function: { name: "" } }] } }] } + ]), "text/event-stream"), []); + + // Mutation guard: unbounded, a pathological response writes 400 names into one row. + const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); + assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); + assert.equal(many!.at(-1), GROK_TOOL_CALL_TRUNCATED); + assert.deepEqual(many!.slice(0, 2), ["tool_0", "tool_1"]); + // Exactly the bound is not truncated. + assert.equal(parseGrokResponseToolNames(events(callDeltas(Array.from({ length: GROK_REQUEST_TOOL_CALLS_MAX }, (_value, index) => `tool_${index}`))), "text/event-stream")!.includes(GROK_TOOL_CALL_TRUNCATED), false); +}); + +test("the meter carries each request's tool-call names without letting them touch the spend gate", async () => { + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 11, completion_tokens: 2 }, meter, async (send) => { assert.equal((await send()).status, 200); }); + const [timing] = meter.snapshot().timings; + // The stub response carries no tool call, and says so rather than staying silent. + assert.deepEqual(timing!.toolCalls, []); + assert.deepEqual([meter.snapshot().tokens, meter.snapshot().limitReason, meter.snapshot().estimatedRequests], [13, "none", 0]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 3c14b7c..f266a16 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -1,8 +1,13 @@ import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; -/** `estimated` marks a request whose response carried no valid usage and was charged {@link estimateGrokRequestUsage}. */ -export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }>; +/** + * `estimated` marks a request whose response carried no valid usage and was + * charged {@link estimateGrokRequestUsage}. `toolCalls` are the tool-call names + * that request's response carried, names only ({@link parseGrokResponseToolNames}); + * absent means the response could not be decoded, `[]` that it called nothing. + */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }>; export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; /** @@ -36,7 +41,7 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n */ export class GrokBrokerTurnMeter { private readonly startedAt: number; - private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }[] = []; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; private inFlight: { index: number; controller: AbortController } | undefined; @@ -60,18 +65,21 @@ export class GrokBrokerTurnMeter { } /** - * Records one admitted request's end and its usage. A response without valid - * usage (absent, malformed, implausible, or a failed/aborted call) is charged - * a conservative estimate from the request body size, so a missing `usage` - * can never silently disable the token ceiling. + * Records one admitted request's end, its usage, and the tool-call names its + * response carried. A response without valid usage (absent, malformed, + * implausible, or a failed/aborted call) is charged a conservative estimate + * from the request body size, so a missing `usage` can never silently disable + * the token ceiling. `toolCalls` is observation only: it never affects + * admission, the running total, or any limit. */ - settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number, toolCalls?: readonly string[]): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } else timing.usage = usage; + if (toolCalls !== undefined) timing.toolCalls = toolCalls; this.tokens += timing.usage.total; } @@ -118,6 +126,26 @@ const count = (value: unknown): number | undefined => typeof value === "number" * zero-filled and never added — the meter charges an estimate instead. */ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of decodeUpstreamResponse(body, contentType) ?? []) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); + } + return found; +} + +/** + * Every decodable JSON object of one upstream response: each `data:` event of + * an SSE stream, or the single body of a JSON response. + * + * `undefined` means *nothing* decoded — an unparseable or non-JSON response. + * Callers must keep that distinct from a decoded response that said nothing, + * because the ledger never fabricates an observation it did not make. + */ +function decodeUpstreamResponse(body: Uint8Array, contentType: string | undefined): unknown[] | undefined { const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); const candidates: unknown[] = []; if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { @@ -125,20 +153,68 @@ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | u if (!line.startsWith("data:")) continue; const payload = line.slice(5).trim(); if (payload === "[DONE]" || payload.length === 0) continue; - try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries no usage */ } + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries neither usage nor a tool call */ } } } else { try { candidates.push(JSON.parse(text)); } catch { return undefined; } } - let found: EngineBrokerTurnUsage | undefined; - for (const candidate of candidates) { - if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; - // Last usage block wins even when invalid: an implausible final report - // must not fall back to an earlier, smaller block (the request is then - // charged the estimate instead). - found = decodeOpenAiUsage(candidate.usage); + return candidates.length === 0 ? undefined : candidates; +} + +/** At most this many names per request row; a longer list ends in {@link GROK_TOOL_CALL_TRUNCATED}. */ +export const GROK_REQUEST_TOOL_CALLS_MAX = 16; +/** A `name` that is not a plain short identifier is counted, never passed through. */ +export const GROK_TOOL_CALL_INVALID = ""; +export const GROK_TOOL_CALL_TRUNCATED = ""; +const TOOL_CALL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** + * The tool-call NAMES one upstream response carried, and nothing else. + * + * Two live turns could not answer "did the model ever try `use_tool` or + * `search_tool`", because the per-request rows recorded timings and tokens but + * never an attempt. This is that answer, under four rules: + * + * - names only. No arguments, no message content, no tokens, no header. A + * `name` that is not a plain short identifier is recorded as + * {@link GROK_TOOL_CALL_INVALID} rather than passing provider bytes through; + * - bounded. At most {@link GROK_REQUEST_TOOL_CALLS_MAX} entries, the last being + * {@link GROK_TOOL_CALL_TRUNCATED} when the response carried more, so a + * pathological response cannot write an unbounded row; + * - absence stays absence. A decoded response that called nothing returns `[]`; + * a response that could not be decoded returns `undefined` and the row records + * no field at all; + * - one streaming call names itself in one delta and streams its arguments in + * the rest, so a repeat of the same `(choice, call)` index is that same call, + * not a second attempt. + */ +export function parseGrokResponseToolNames(body: Uint8Array, contentType: string | undefined): readonly string[] | undefined { + const candidates = decodeUpstreamResponse(body, contentType); + if (candidates === undefined) return undefined; + const names: string[] = [], seen = new Set(); + scan: for (const candidate of candidates) { + if (!isRecord(candidate) || !Array.isArray(candidate.choices)) continue; + for (const choice of candidate.choices) { + if (!isRecord(choice)) continue; + for (const source of [choice.delta, choice.message]) { + if (!isRecord(source) || !Array.isArray(source.tool_calls)) continue; + for (const call of source.tool_calls) { + if (!isRecord(call) || !isRecord(call.function)) continue; + const name = call.function.name; + // An arguments-only delta names nothing; it is not an attempt of its own. + if (typeof name !== "string" || name.length === 0) continue; + if (typeof choice.index === "number" && typeof call.index === "number") { + const key = `${choice.index}:${call.index}`; + if (seen.has(key)) continue; + seen.add(key); + } + names.push(TOOL_CALL_NAME.test(name) ? name : GROK_TOOL_CALL_INVALID); + if (names.length > GROK_REQUEST_TOOL_CALLS_MAX) break scan; + } + } + } } - return found; + return names.length > GROK_REQUEST_TOOL_CALLS_MAX ? [...names.slice(0, GROK_REQUEST_TOOL_CALLS_MAX - 1), GROK_TOOL_CALL_TRUNCATED] : names; } function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index 091418a..0642b89 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -35,6 +35,21 @@ test("worker config disables every bundled 1.0.34 skill, workflows, and the per- assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); }); +test("every model the worker can reach fails fast rather than retrying a refusal blindly", () => { + // Grok 1.0.34's default retries a refused request with backoff past 45 s. + // The sink and the evaluator client always pinned this; the worker's own + // model — the one path that spends money — did not, so a refusal there could + // stall a turn for minutes after its work was done with nothing logged. + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const config = renderGrokBrokerWorkerConfig({ model, reasoningEffort }); + for (const block of ["[model.daimon-broker-grok]", "[model.daimon-session-title-disabled]"]) { + assert.match(section(config, block), /\nmax_retries = 0\n/u, `${block} ${model}/${reasoningEffort}`); + } + } + } +}); + test("the declared model and effort reach the worker's only model as its sole allowed effort", () => { const config = renderGrokBrokerWorkerConfig({ model: "grok-build", reasoningEffort: "medium" }); assert.match(section(config, "[model.daimon-broker-grok]"), /\nmodel = "grok-build"\n/u); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index 5a2f88c..fde8952 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -79,6 +79,21 @@ export const renderGrokLeanBaseConfig = (): string => [ "[workflows]", "enabled = false", "" ].join("\n"); +/** + * `max_retries = 0` on the worker's own model, for the same reason its two + * siblings already carry it (the session-title sink above, + * `grokInferenceClientConfig.ts` for the evaluator): with Grok 1.0.34's + * default, a refused or failed request is retried with backoff **past 45 s** + * instead of failing in ~0.35 s, and the retries are blind — one live turn + * emitted the same refusal fifteen times over five minutes, spent $0, and died + * with no account of why. Only this model block was left on the default, so + * the one request path that spends money was also the only one that could + * stall a turn for minutes after its work was done. Daimon owns the retry + * decision here because the proxy is the thing being retried: a genuinely + * transient fault is already answered 503 and is the broker's to retry, and + * anything else is a refusal that repeating cannot fix. The worker instead + * fails fast and the turn reaches the host with a status. + */ /** * The only source of broker worker `config.toml` bytes. * @@ -107,7 +122,7 @@ export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", ...renderSessionTitleSink(proxyPort), `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, - 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" ].join("\n"); diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index aceb394..35fd7cc 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -9,7 +9,7 @@ import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; -import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createGrokWorkerIsolationGuard,grokBrokerAttestationInput,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; import { createLedgeredGrokInferenceGrants, GrokInferenceGrantRefused, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; @@ -30,7 +30,7 @@ export type GrokEngineBroker = Awaited> */ export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; inferenceLedgerPath?: string }>) { const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); + const attestationFor = (registration: GrokEngineBrokerRegistration) => grokBrokerAttestationInput(registration, [...registrations.values()], grokBrokerWorkerConfigSha256(registration.model)); const inferenceLedgerPath=options.inferenceLedgerPath;const grants=inferenceLedgerPath===undefined?undefined:createLedgeredGrokInferenceGrants(inferenceLedgerPath); const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,undefined,undefined,grants);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; const facade = mcp; diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts index ebf6073..3d21e47 100644 --- a/src/runtime/grokEngineBrokerLedger.ts +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -1,5 +1,6 @@ import { readFile } from "node:fs/promises"; +import { renderBrokerTurnSealLine, TURN_SEAL_LEDGER_VERSION, TURN_SEAL_MAX_LINE_BYTES } from "./engineBrokerSealLedger.js"; import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; @@ -16,15 +17,23 @@ import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; * normal append finds the row and writes nothing, and readers dedupe on `turn` * should two replays race. */ -export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string }>; +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string; seal?: string }>; +/** A v1 record, or a replay with no sealed bytes at all: nothing to append, including no seal. */ export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { - if (terminal.usage === null) return EMPTY_BROKER_TURN_LEDGER; - const { usage } = terminal, at = new Date().toISOString(); + const at = new Date().toISOString(); + // The seal row is rendered for *every* terminal turn, including one whose + // usage is null. That is the whole point: a turn cancelled before any usage + // could be attributed is exactly the turn whose outstanding MCP call and + // redacted last words have no other route to the host. + const seal = renderBrokerTurnSealLine(terminal, { agent: detail.agentId, wake: detail.wakeId, at }); + if (terminal.usage === null) return { usage: null, requests: "", seal }; + const { usage } = terminal; return { + seal, usage: renderTurnUsageLine({ agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, @@ -38,11 +47,20 @@ export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, d const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); -/** Strict check of stored ledger bytes: exactly the turn's own rows, newline-terminated, bounded. */ +/** + * Strict check of stored ledger bytes: exactly the turn's own rows, + * newline-terminated, bounded. + * + * `seal` is optional so a record sealed before this stream existed still + * replays; its absence means the turn owes no seal row, never that one was + * lost. + */ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { const invalid = () => new Error("broker turn registry unavailable"); - if (value === null || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== 2 || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); - const { usage, requests } = value as { usage: unknown; requests: unknown }; + if (value === null || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const sealed = Object.hasOwn(value, "seal"); + if (Object.keys(value).length !== (sealed ? 3 : 2)) throw invalid(); + const { usage, requests, seal } = value as { usage: unknown; requests: unknown; seal?: unknown }; try { if (usage !== null) { if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); @@ -51,21 +69,32 @@ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): Brok } if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + if (sealed) { + if (typeof seal !== "string" || !seal.endsWith("\n") || Buffer.byteLength(seal) > TURN_SEAL_MAX_LINE_BYTES) throw invalid(); + const parsed = rows(seal); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_SEAL_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } } catch { throw invalid(); } - return { usage: usage as string | null, requests }; + return { usage: usage as string | null, requests, ...(sealed ? { seal: seal as string } : {}) }; } +export type BrokerTurnLedgerPaths = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; sealLedgerPath: string }>; + /** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ -export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: BrokerTurnLedgerPaths): Promise { if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); await recordLedgerLines(paths.requestLedgerPath, lines.requests); + // Last, and never conditional on usage: a turn with no attributable spend is + // precisely the one whose seal row is its only account of itself. + if (lines.seal !== undefined) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } /** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ -export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: BrokerTurnLedgerPaths): Promise { try { if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + if (lines.seal !== undefined && !await ledgerHasTurn(paths.sealLedgerPath, turnId)) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } catch { /* advisory: a replay never fails on its ledger */ } } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index 8b8fb31..d0df740 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -7,6 +7,8 @@ import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; export type BrokerTurnMetering = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; + /** Where the sealed response's operator-visible projection is appended (`engineBrokerSealLedger.ts`). */ + sealLedgerPath: string; agentId: string; wakeId: string; }>; @@ -31,8 +33,9 @@ export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: * * Both terminal kinds meter: a failed turn spent real tokens, so its partial * usage is written with `outcome: failed` and its closed `limitReason`. A turn - * with no usage at all (`usage: null`) writes nothing — a zero row is - * byte-identical to a measured zero. + * with no usage at all (`usage: null`) writes no *usage* row — a zero row is + * byte-identical to a measured zero — but it still writes its seal row, which + * is a record of what the turn did rather than of what it spent. * * Appends never reject, so an append failure cannot escape into the caller's * `catch` and rewrite a completed turn as failed; the caller also refuses to diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 674b572..61dfaa1 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -3,10 +3,11 @@ import { createHash, randomUUID } from "node:crypto"; import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; import { decodeGrokStreamUsage, type GrokStreamUsage } from "../pi/grokStreamUsage.js"; import { ENGINE_BROKER_VERSION, type EngineBrokerFailureCode, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; @@ -15,14 +16,14 @@ import { GrokWorkerAttestationFailure } from "./grokWorkerAttestation.js"; export type GrokEngineBrokerTurnResult = Readonly<{ text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting; export class EngineBrokerTurnFailure extends Error { - constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting) { super("engine broker turn failed"); } + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting, readonly mcpCalls?: EngineBrokerMcpCallObservation) { super("engine broker turn failed"); } } /** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ export type GrokEngineBrokerTurnDependencies = Readonly<{ turns: EngineBrokerTurnRegistry; proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; - mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void; observe?(turnId: string): EngineBrokerMcpCallObservation | undefined }>; credentialStale(): boolean; prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; @@ -52,7 +53,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; const begun = await deps.turns.begin(request, declared); - const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), sealLedgerPath: engineBrokerSealLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } const controller = new AbortController(); const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); @@ -91,10 +92,15 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; - const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; + // Read before the `finally` revokes this turn's MCP registration, and + // swallowed like every other instrument here: it must never be the reason + // a turn reports something other than why it failed. + let mcpCalls: EngineBrokerMcpCallObservation | undefined; + try { mcpCalls = deps.mcp.observe?.(turnId); } catch { mcpCalls = undefined; } + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting }; const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); - throw new EngineBrokerTurnFailure(code, diagnostic, accounting); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls); } finally { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); @@ -105,7 +111,7 @@ function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnRes const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; - throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting); + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting, response.mcpCalls); } /** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ @@ -120,12 +126,22 @@ function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokB return snapshot.usage; } -/** Per-request rows: stream usage with proxy timing when both describe the same requests, else the proxy's own measured requests. */ +/** Per-request rows: stream usage with the proxy's own observation when both describe the same requests, else the proxy's measured requests. */ function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { if (stream !== undefined && stream.requests.length > 0) { const timed = snapshot.timings.length === stream.requests.length; - return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? observed(snapshot.timings[index]!) : {}) })); } - return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...clock(timing) }]); + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...observed(timing) }]); } -const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); +/** + * What only the proxy saw of one request: its clock, and the tool-call names the + * response carried. Both are attached on the stream path only when the two + * descriptions are request-for-request aligned, because an unaligned index would + * credit one request's attempt to another. + */ +const observed = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ + startedAt: timing.startedAt, + ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }), + ...(timing.toolCalls === undefined ? {} : { toolCalls: timing.toolCalls }) +}); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 9964bd9..b8d6701 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -7,10 +7,11 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; -import type { NativeBrokerTurn, NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; @@ -47,10 +48,19 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +/** The stub provider's own SSE response: usage only, and no tool call, unless a test says otherwise. */ +const upstreamResponse = (): string => `data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`; +/** One streaming tool call per name, arguments in a following delta, then the usage event. */ +const upstreamToolCallResponse = (names: readonly string[]): string => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"tool_name":"daimon__moltnet_read"}' } }] } }] })), + { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: upstreamUsage } +].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; + +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15, upstreamBody: (call: number) => string = upstreamResponse): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0, aborted = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(upstreamBody(calls)) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -60,7 +70,7 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; const deps: GrokEngineBrokerTurnDependencies = { turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, - mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => mcpObservation }, prepareIsolation: async () => async () => undefined, runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) }; @@ -74,6 +84,9 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; +/** What the facade would have seen of this turn's tool calls; the turn only reads it. */ +let mcpObservation: EngineBrokerMcpCallObservation | undefined; + const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { @@ -244,3 +257,126 @@ test("the broker meters only through the single sealing helper, on both terminal assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); }); + +/** + * The question two live turns could not answer: did the model ever *try* to + * call a tool? The rows carried timings and tokens and nothing about an + * attempt, so a turn with zero tool calls and a turn whose calls all failed + * read identically after the fact. + */ +test("each per-request row records the tool-call names that request's response carried, names only", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-tools", twoRequests); + const rows = await requestRows(); + // Mutation guard: without the field these are `[undefined, undefined]`. + assert.deepEqual(rows.map((row) => row.tool_calls), [["use_tool", "search_tool"], ["use_tool", "search_tool"]]); + const text = JSON.stringify(rows); + // Names only: no arguments, no message content, no bearer. + assert.equal(text.includes("daimon__moltnet_read"), false, "an argument value must never reach the ledger"); + assert.equal(text.includes("arguments"), false); + assert.equal(text.includes("provider-token"), false); + }, undefined, () => 1, () => upstreamToolCallResponse(["use_tool", "search_tool"])); +}); + +test("a response that called nothing records an empty list, and one that cannot be decoded records no field at all", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-silent", twoRequests); + // Decoded, and it called nothing: that is an observation, not a gap. + assert.deepEqual((await requestRows()).map((row) => row.tool_calls), [[], []]); + }, undefined, () => 1); + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-undecodable", twoRequests); + const rows = await requestRows(); + // Mutation guard: a fabricated `[]` here would be byte-identical to the + // measured empty list above, and the ledger would claim an observation the + // proxy never made. + assert.deepEqual(rows.map((row) => Object.hasOwn(row, "tool_calls")), [false, false]); + assert.deepEqual(rows.map((row) => row.request), [0, 1], "the rows themselves are still written"); + }, undefined, () => 1, () => "bad gateway"); +}); + + +/** + * The exact 128-byte frame `supervise()` emits when a worker crosses + * `DBL_MAX_OUTPUT`: it stops reading, SIGKILLs the process group, and publishes + * `output_length = 0` with `DBL_STATUS_OUTPUT_FAILED`. Built here at the wire + * offsets the header's `_Static_assert`s pin, so the test drives the real + * decoder rather than a hand-made exception. + */ +function outputLimitFrame(turnId: string): Buffer { + const frame = Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES); + frame.writeUInt32LE(2, 0); frame.writeUInt32LE(3, 4); frame.writeUInt32LE(2_200, 8); frame.writeUInt32LE(0, 12); + frame.writeInt32LE(4_242, 16); frame.writeInt32LE(0, 20); frame.writeInt32LE(9, 24); + frame.writeBigUInt64LE(99n, 32); frame.write(turnId, 40, "utf8"); + frame.writeUInt32LE(7, 108); frame.writeUInt32LE(7, 112); frame.writeUInt32LE(0, 116); frame.writeUInt32LE(0, 120); + return frame; +} + +test("a worker whose work succeeded but whose output crossed the launcher bound still seals the spend the proxy measured", async () => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamCalls }) => { + // The worker does its real work through the real proxy — two admitted, + // metered upstream requests — and only then loses its whole output: the + // launcher refused to publish it and the turn's text never exists. The + // frame is decoded by the shipped client, so the failure reaches the turn + // exactly as the native transport delivers it. + const worker: Worker = async (send) => { + assert.equal(await send(), 200); + assert.equal(await send(), 200); + throw decodeNativeBrokerResult(outputLimitFrame(turnIdFor("foreman", "wake-output-limit")), turnIdFor("foreman", "wake-output-limit"), []) as never; + }; + await assert.rejects(turn("wake-output-limit", worker), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + // No limit tripped and the credential is live: this is the worker's + // transport failing, not the turn being refused. + assert.equal(error.code, "engine_failed"); + assert.equal(error.diagnostic?.failureClass, "output_limit"); + // Mutation guard: the turn has no stream to read usage from, so this can + // only come from the proxy's own per-request measurements. Falling back + // to `null` here would report a fabricated zero for real spend. + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 5_136, cacheRead: 256, cacheWrite: 0, output: 158, total: 5_550 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + return true; + }); + assert.equal(upstreamCalls(), 2); + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.total, row?.calls, row?.complete], ["failed", "unknown", 5_550, 2, false]); + assert.notEqual(row?.total, 0, "a zero row would be byte-identical to a measured zero"); + assert.deepEqual((await requestRows()).map((value) => value.request), [0, 1], "every request the proxy answered keeps its own row"); + // The sealed record is the durable truth: a replay returns that spend and never meters again. + await assert.rejects(turn("wake-output-limit", twoRequests), (error: unknown) => + error instanceof EngineBrokerTurnFailure && error.accounting?.usage?.total === 5_550); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); + }); +}); + +/** + * The hang this instrument was built for: the worker stops acting with every + * provider request closed, the deadline kills it, and the only remaining + * question is whether it was waiting on a tool call. The answer has to reach + * the host, and the slot's control root is tmpfs that dies with the container — + * so it rides the seam the worker's last words and the sealed usage already + * ride: the sealed terminal response, which a replay hands back unchanged. + */ +test("a failed turn carries the facade's in-flight tool-call observation, and its replay still does", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }; + const worker: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); throw new Error("engine broker turn failed"); }; + const carried = (error: unknown): boolean => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.deepEqual(error.mcpCalls, { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }); + return true; + }; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + // The replay reads the durable record back through the frame parser, so + // this is the sealed bytes answering, not the live facade. + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + }); +}); + +test("a turn whose facade observed nothing seals no observation at all", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-absent", async (send) => { await send(); throw new Error("engine broker turn failed"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.mcpCalls === undefined); + }); +}); diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 8fcb20e..8e1f44c 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -60,8 +60,10 @@ test("a grant refuses any tools member, the session_title request, and undeclare judgeBody({ messages: [{ role: "tool", content: "x" }] }), judgeBody({ messages: [{ role: "assistant", content: null, tool_calls: [] }] }), judgeBody({ response_format: { type: "json_object" } }) ]; - for (const body of refused) assert.equal((await post(port, token, body)).status, 503, body.slice(0, 120)); - assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 503); + // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. + for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); + assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); + // The title sink keeps its transient 503 shape on the grant path too. assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); assert.equal(bodies.length, 0); assert.equal(rows.length, 0); }); @@ -70,8 +72,8 @@ test("a grant refuses any tools member, the session_title request, and undeclare test("an expired, released or unknown grant is refused", async () => { await withProxy(async ({ port, grants, bodies }) => { const released = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); grants.release(released.grantId); - assert.equal((await post(port, released.token, judgeBody())).status, 503); - assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 503); + assert.equal((await post(port, released.token, judgeBody())).status, 400); + assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 400); assert.equal(bodies.length, 0); }); let now = 5_000; @@ -81,7 +83,7 @@ test("an expired, released or unknown grant is refused", async () => { const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); assert.equal((await post(proxy.port, token, judgeBody())).status, 200); now += 600_000; - assert.equal((await post(proxy.port, token, judgeBody())).status, 503); + assert.equal((await post(proxy.port, token, judgeBody())).status, 400); } finally { grants.close(); await proxy.close(); } }); @@ -93,8 +95,9 @@ test("a grant token never authorizes a subject turn and a turn capability never proxy.registerTurn("turn-a", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }) }); const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); - assert.equal((await post(port, grantToken, leanBody)).status, 503); - assert.equal((await post(port, turnToken, judgeBody())).status, 503); + // Cross-use is a policy miss on both paths: refused 400, never a retryable 503. + assert.equal((await post(port, grantToken, leanBody)).status, 400); + assert.equal((await post(port, turnToken, judgeBody())).status, 400); assert.equal(bodies.length, 0); assert.equal((await post(port, turnToken, leanBody)).status, 200); assert.equal((await post(port, grantToken, judgeBody())).status, 200); diff --git a/src/runtime/grokInferenceProxy.ts b/src/runtime/grokInferenceProxy.ts index 95169c1..34acc99 100644 --- a/src/runtime/grokInferenceProxy.ts +++ b/src/runtime/grokInferenceProxy.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { ServerResponse } from "node:http"; +import { GrokBrokerProxyRefusal } from "./grokBrokerProxy.js"; import type { GrokBrokerCredentialAuthority, GrokBrokerUpstream } from "./grokBrokerProxy.js"; import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -30,8 +31,10 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re let settle: ((usage: ReturnType) => void) | undefined; try { const grant = grants.authorize(input.token); - if (grant === undefined) throw new Error("inference grant unavailable"); - let prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); + if (grant === undefined) throw new GrokBrokerProxyRefusal("unknown_or_expired_grant"); + let prepared: ReturnType; + try { prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } let token = await authority.accessToken(false); const rejectedDigest = createHash("sha256").update(token).digest("hex"); prepared = withBearer(prepared, token); token = ""; const admission = grant.meter.admit(); @@ -47,9 +50,14 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re } settle?.(parseGrokUpstreamUsage(result.body, result.headers["content-type"])); json(response, result.status, result.body, result.headers["content-type"]); - } catch { + } catch (error) { settle?.(undefined); + // Same rule as the subject path: a policy miss is non-retryable (400), because a + // retryable 503 makes the client re-send a request the broker will never accept, + // charging estimated usage for every attempt. 503 stays for transient faults only. + process.stderr.write(`[grok-proxy] inference refused: ${error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"}\n`); if (authority.isStale?.() === true) json(response, 401, GROK_INFERENCE_AUTH_STALE_BODY); + else if (error instanceof GrokBrokerProxyRefusal) json(response, 400, JSON.stringify({ error: "broker refused this request", reason: error.reason })); else json(response, 503, '{"error":"broker unavailable"}'); } } diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index adfa3f5..c441411 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -3,7 +3,9 @@ import { constants } from "node:fs"; import { lstat,open } from "node:fs/promises"; import path from "node:path"; +import { assertGrokWorkerDenyPathsPlaceable } from "./grokWorkerDenyPlacement.js"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; +import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** @@ -60,13 +62,31 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). + * + * It also refuses a deny list bubblewrap could not materialize + * (`grokWorkerDenyPlacement.ts`), naming the entry and the ancestor that stops + * it. Without this the worker dies with a bare `bwrap: Can't create file at + * …: Permission denied` on *every* turn, because one unplaceable entry makes + * Grok refuse the whole profile. The broker cannot descend into a directory + * opened to the worker's group alone (`/tool-state` under a + * `2000: 0710` home), so an `EACCES` there is left undecided; root + * provisioning, which holds `CAP_DAC_READ_SEARCH`, is the authority that + * decides every entry. */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;workerGid?:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. + // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + await assertGrokWorkerDenyPathsPlaceable(denyPaths,{uid:input.workerUid,gid:input.workerGid??input.workerUid}); + const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); + if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } +/** The per-turn attestation input for one registration, carrying every registered worker so sibling temp is attested too. */ +export const grokBrokerAttestationInput=>(registration:T,registrations:readonly Readonly<{profilePath:string;workerUid:number}>[],configSha256:string)=>({...registration,brokerGid:2100,configSha256,registeredWorkers:registrations.map((entry)=>({profilePath:entry.profilePath,workerUid:entry.workerUid}))}); /** * The accepted `ProfileApplied` line of one turn, as an absolute byte range of * the events file plus its digest. diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 694391c..45480ed 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -1,12 +1,13 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { appendFile, chmod, link, mkdir, mkdtemp, open, rm, stat, writeFile } from "node:fs/promises"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, realpath, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test, { mock } from "node:test"; import { GrokWorkerAttestationFailure, + grokBrokerAttestationInput, prepareGrokWorkerAttestation, verifyGrokWorkerAttestation, type GrokWorkerAttestationSnapshot @@ -61,17 +62,95 @@ test("refuses events that change while they are being read", async (t) => { assert.ok(reading.mock.callCount() >= 1); }); -test("prepare refuses a worker home that fails attestation even when profile and events are valid", async (t) => { - // Run as a non-root owner so the profile and events legs pass; the home leg - // (root-owned, read-only config) cannot, and must be what refuses. - const home = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); - t.after(() => rm(home, { recursive: true, force: true })); +test("prepare refuses a worker home that fails attestation even when profile, temp and events are valid", async (t) => { + // Run as a non-root owner so the profile, temp and events legs pass; the home + // leg (root-owned, read-only config) cannot, and must be what refuses. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = path.join(root, ".grok"); + await mkdir(path.join(home, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); const profile = path.join(home, "sandbox.toml"); const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; await writeFile(profile, text); await chmod(profile, 0o444); - await mkdir(path.join(home, "sessions")); const events = path.join(home, "sessions", "sandbox-events.jsonl"); await writeFile(events, ""); await chmod(events, 0o640); const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; - await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid) }), /attestation unavailable/u); + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); +}); + +test("prepare refuses a worker without a private temp directory before the home check", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + const owner = { uid: self.uid, gid: Number((await stat(profile)).gid) }; + // No /tmp: the temp leg refuses (the home leg would refuse too, with a different message). + await assert.rejects(prepareGrokWorkerAttestation(input, owner), /temp isolation attestation unavailable/u); + // A profile outside /.grok cannot name the launcher's TMPDIR home. + await mkdir(path.join(root, "elsewhere", "sessions"), { recursive: true }); + const stray = { ...input, profilePath: path.join(root, "elsewhere", "sandbox.toml"), eventsPath: path.join(root, "elsewhere", "sessions", "sandbox-events.jsonl") }; + await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); + await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); +}); + +test("prepare refuses the current turn when a sibling registered worker's private temp is 0777", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-sibling-")); + t.after(() => rm(root, { recursive: true, force: true })); + const make = async (name: string) => { + const home = path.join(root, name), grok = path.join(home, ".grok"); + await mkdir(path.join(grok, "sessions"), { recursive: true }); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + return { home, profile: path.join(grok, "sandbox.toml"), events: path.join(grok, "sessions", "sandbox-events.jsonl") }; + }; + const own = await make("own"), sibling = await make("sibling"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(own.profile, text); await chmod(own.profile, 0o444); + await writeFile(own.events, ""); await chmod(own.events, 0o640); + const registration = { profilePath: own.profile, eventsPath: own.events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, workspace: "/w" }; + const input = grokBrokerAttestationInput(registration, [registration, { profilePath: sibling.profile, workerUid: self.uid }], "0".repeat(64)); + assert.deepEqual(input.registeredWorkers.map((entry) => entry.profilePath), [own.profile, sibling.profile]); + const seams = { uid: self.uid, gid: Number((await stat(own.profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }; + await chmod(root, 0o700); + // Sibling well provisioned: temp passes and the (root-only) home leg is what refuses. + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await chmod(path.join(sibling.home, "tmp"), 0o777); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); +}); + +test("prepare refuses a deny entry bubblewrap could not materialize, before any other leg", async (t) => { + // The production defect: the wake-acceptance store sits under a `0700` organization state directory, + // so bubblewrap — which materializes every deny target as the worker uid — could not create it and + // Grok refused the whole profile, failing every turn with `bwrap: Can't create file at …`. + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-guard-deny-"))); + t.after(async () => { await chmod(path.join(root, "state"), 0o700); await rm(root, { recursive: true, force: true }); }); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); + await mkdir(path.join(root, "state", "wake-acceptance"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const owner = { uid: self.uid, gid: Number((await stat(path.join(root, "tmp"))).gid) }; + const withDeny = async (denied: string) => { + const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["${denied}"]\n`; + await chmod(profile, 0o644).catch(() => undefined); + await writeFile(profile, text); await chmod(profile, 0o444); + return { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + }; + await chmod(path.join(root, "state"), 0o600); + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state", "wake-acceptance")), owner), + (error: Error) => /is not placeable/u.test(error.message) && error.message.includes(`cannot search ${path.join(root, "state")}`) + ); + // The lift: the private directory itself is placeable, so this leg passes and a later one refuses. + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state")), owner), + (error: Error) => !/is not placeable/u.test(error.message) + ); }); diff --git a/src/runtime/grokWorkerDenyPlacement.test.ts b/src/runtime/grokWorkerDenyPlacement.test.ts new file mode 100644 index 0000000..f5013a0 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + assertGrokWorkerDenyPathPlacement, + assertGrokWorkerDenyPathShape, + assertGrokWorkerDenyPathsPlaceable, + GROK_WORKER_BASE_PROFILE_GRANTS, + grokWorkerCanSearch, + grokWorkerDenyPathChain, + GrokWorkerDenyPlacementError, + type GrokWorkerDenyPathEntry, + type GrokWorkerDenyPathStep +} from "./grokWorkerDenyPlacement.js"; +import { renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +const entry = (uid: number, gid: number, mode: number, kind: "dir" | "file" | "link" = "dir"): GrokWorkerDenyPathEntry => + ({ uid, gid, mode, isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" }); +const worker = { uid: 2200, gid: 2200 }; +const steps = (denyPath: string, entries: readonly (GrokWorkerDenyPathEntry | string)[]): GrokWorkerDenyPathStep[] => + grokWorkerDenyPathChain(denyPath).map((target, index) => { + const value = entries[index]; + return typeof value === "string" ? { path: target, code: value } : { path: target, entry: value }; + }); + +test("search permission follows owner, then group, then other — as the worker's cleared-group process does", () => { + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o700), worker), true); + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o677), worker), false, "owner bits win even when group and other would allow"); + assert.equal(grokWorkerCanSearch(entry(2000, 2200, 0o710), worker), true); + assert.equal(grokWorkerCanSearch(entry(2000, 2000, 0o700), worker), false); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o711), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o755), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 2000, 0o750), worker), false); +}); + +test("refuses a deny entry at or above any Grok 1.0.34 base-profile grant", () => { + for (const grant of GROK_WORKER_BASE_PROFILE_GRANTS) { + assert.throws(() => assertGrokWorkerDenyPathShape(grant), GrokWorkerDenyPlacementError, grant); + } + assert.throws(() => assertGrokWorkerDenyPathShape("/var"), /base profile grant \/var/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/run"]), /base profile grant \/run/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/var/lib/spawnfile/daimon/usage", "/tmp"]), /base profile grant \/tmp/u); + // Strictly below every grant is exactly what Grok accepts. + assertGrokWorkerDenyPathShape("/tmp/sub"); + assertGrokWorkerDenyPathShape("/var/lib/spawnfile/daimon/usage"); +}); + +test("refuses the wake-acceptance shape: a deny entry under a parent the worker cannot search", () => { + // `/state` is `2000:2000 0700`; the store beneath it is what production used to deny. + const denyPath = "/var/lib/spawnfile/instances/daimon/org/state/wake-acceptance"; + assert.throws( + () => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700), entry(2000, 2000, 0o700) + ]), worker), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && /cannot search \/var\/lib\/spawnfile\/instances\/daimon\/org\/state \(700 2000:2000\); deny that directory itself instead/u.test(error.message) + ); + // The lift production now emits: the private directory itself, whose own parent is traversable. + const lifted = "/var/lib/spawnfile/instances/daimon/org/state"; + assertGrokWorkerDenyPathPlacement(lifted, steps(lifted, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700) + ]), worker); +}); + +test("refuses a missing target, a symlink and a non-directory ancestor; leaves an undecidable EACCES alone", () => { + const denyPath = "/run/training/slot/state"; + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "ENOENT" + ]), worker), /does not exist; bubblewrap would have to create it as the worker uid/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o777, "link") + ]), worker), /is a symlink; bubblewrap refuses to bind over one/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o644, "file"), entry(0, 0, 0o755), entry(0, 0, 0o700) + ]), worker), /is not a directory/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "EPERM" + ]), worker), /could not be read \(EPERM\)/u); + // The broker (uid 2100) cannot descend into a `2000: 0710` runtime home the worker itself can + // search, so an EACCES *below a worker-searchable ancestor* is undecided here, never a refusal. + assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(2000, 2200, 0o710), "EACCES" + ]), worker); +}); + +test("walks the real filesystem and names the ancestor that stops a deny entry", async () => { + // realpath: macOS `/var` is a symlink, and a symlinked ancestor is refused on purpose — Daimon's + // registered paths are canonical, and bubblewrap must bind over the inode the deny entry names. + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "grok-deny-"))); + const state = path.join(root, "state"); + // This process stands in for root provisioning: it can stat every component, and judges searchability + // for the worker from the modes it reads. Here the worker is this uid, so only `state` blocks it. + const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; + try { + mkdirSync(path.join(state, "wake-acceptance"), { recursive: true }); + writeFileSync(path.join(state, "wake-acceptance", "store.jsonl"), "{}\n"); + chmodSync(state, 0o600); + await assert.rejects( + assertGrokWorkerDenyPathsPlaceable([path.join(state, "wake-acceptance")], self), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && error.message.includes(`cannot search ${state}`) && error.message.includes("deny that directory itself instead") + ); + // The lift: the unsearchable directory itself is placeable, and masks strictly more. + await assertGrokWorkerDenyPathsPlaceable([state], self); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "absent")], self), /does not exist/u); + symlinkSync(state, path.join(root, "link")); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "link")], self), /is a symlink/u); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable(["relative/path"], self), /not an absolute path/u); + } finally { + chmodSync(state, 0o700); + rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/src/runtime/grokWorkerDenyPlacement.ts b/src/runtime/grokWorkerDenyPlacement.ts new file mode 100644 index 0000000..f2995d2 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.ts @@ -0,0 +1,147 @@ +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +/** + * Paths Grok 1.0.34's strict base profile grants read or read-write. A `deny` + * entry that equals or contains one of them makes Grok refuse the profile + * outright (verified for `/tmp`, `/var/tmp`, `/run`, `/etc`, `/var` and + * `sessions`; `/tmp/sub` is accepted), so every entry must sit strictly below + * each grant it touches. + */ +export const GROK_WORKER_BASE_PROFILE_GRANTS = Object.freeze([ + "/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp" +] as const); + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +export class GrokWorkerDenyPlacementError extends Error { + constructor(readonly denyPath: string, readonly reason: string) { + super(`Grok worker sandbox deny path ${JSON.stringify(denyPath)} is not placeable: ${reason}`); + this.name = "GrokWorkerDenyPlacementError"; + } +} + +/** + * The shape half of the deny-placement policy: everything decidable without + * touching a filesystem, so the profile renderer can refuse a bad entry before + * its bytes are ever pinned or written. + * + * `renderGrokWorkerSandboxProfile` already refuses entries that are relative, + * non-canonical, `/`, trailing-slashed, or carrying a character TOML or Grok + * would reinterpret; this adds the base-profile grant rule. + */ +export function assertGrokWorkerDenyPathShape(denyPath: string): void { + const grant = GROK_WORKER_BASE_PROFILE_GRANTS.find((candidate) => within(candidate, denyPath)); + if (grant !== undefined) { + throw new GrokWorkerDenyPlacementError(denyPath, `it equals or contains the base profile grant ${grant}, which Grok refuses`); + } +} + +/** The subset of `lstat` the placement rules read; pure so every refusal is testable without root. */ +export type GrokWorkerDenyPathEntry = Readonly<{ + uid: number; + gid: number; + mode: number; + isDirectory: () => boolean; + isSymbolicLink: () => boolean; +}>; + +/** One resolved path component: the entry, or the errno that stopped the walk. */ +export type GrokWorkerDenyPathStep = Readonly<{ path: string; entry?: GrokWorkerDenyPathEntry; code?: string }>; + +export type GrokWorkerDenyPathWorker = Readonly<{ uid: number; gid: number }>; + +/** + * POSIX search permission: owner bits win, then group, then other. The worker + * runs with its supplementary groups cleared, so its primary gid is the only + * group that can apply. + */ +export const grokWorkerCanSearch = (entry: GrokWorkerDenyPathEntry, worker: GrokWorkerDenyPathWorker): boolean => + entry.uid === worker.uid ? (entry.mode & 0o100) !== 0 + : entry.gid === worker.gid ? (entry.mode & 0o010) !== 0 + : (entry.mode & 0o001) !== 0; + +/** The `/`-rooted ancestor chain of `denyPath`, deepest last, followed by the entry itself. */ +export const grokWorkerDenyPathChain = (denyPath: string): readonly string[] => { + const components = denyPath.split("/").slice(1); + return ["/", ...components.map((_, index) => `/${components.slice(0, index + 1).join("/")}`)]; +}; + +/** + * The placement half of the policy, as a pure function of an already-walked + * chain. + * + * Grok 1.0.34 materializes every `deny` entry inside bubblewrap **as the worker + * uid**, by bind-mounting `$GROK_HOME/sandbox-blocked-{file,dir}` over the + * target. So bwrap must be able to *resolve* the target as that uid: every + * ancestor directory needs the search bit for it, and the target must already + * exist — otherwise bwrap tries to create it and needs write on the parent, + * which a private parent never grants. A single unplaceable entry makes Grok + * refuse the whole profile, so every turn of that worker fails, not just that + * path. Verified matrix: `.runtime/grok-deny-placement/EVIDENCE.md`. + * + * The walk may legitimately stop early: a caller that is neither root nor the + * worker (the broker, uid 2100) cannot descend into a directory the worker's + * own group opens to it alone — `/tool-state` under a + * `2000: 0710` runtime home is exactly that. An `EACCES` below an + * ancestor the *worker* can search is therefore "not decidable from here", not + * a refusal; every decidable failure still refuses. + */ +export function assertGrokWorkerDenyPathPlacement( + denyPath: string, + steps: readonly GrokWorkerDenyPathStep[], + worker: GrokWorkerDenyPathWorker +): void { + assertGrokWorkerDenyPathShape(denyPath); + const chain = grokWorkerDenyPathChain(denyPath); + if (steps.length !== chain.length || steps.some((step, index) => step.path !== chain[index])) { + throw new GrokWorkerDenyPlacementError(denyPath, "its resolved path chain does not match the entry"); + } + for (const [index, step] of steps.entries()) { + const ancestor = index < steps.length - 1; + if (step.entry === undefined) { + if (step.code === "ENOENT") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} does not exist; bubblewrap would have to create it as the worker uid`); + if (step.code !== "EACCES") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} could not be read (${step.code ?? "unknown error"})`); + // Undecidable from here, and only after every shallower ancestor passed. + return; + } + if (step.entry.isSymbolicLink()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is a symlink; bubblewrap refuses to bind over one`); + if (!ancestor) return; + if (!step.entry.isDirectory()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is not a directory`); + if (!grokWorkerCanSearch(step.entry, worker)) { + throw new GrokWorkerDenyPlacementError(denyPath, `worker uid ${worker.uid} cannot search ${step.path} (${(step.entry.mode & 0o7777).toString(8)} ${step.entry.uid}:${step.entry.gid}); deny that directory itself instead`); + } + } +} + +/** Walks one deny path on the real filesystem, recording what stopped it rather than throwing. */ +export async function readGrokWorkerDenyPathChain(denyPath: string): Promise { + const steps: GrokWorkerDenyPathStep[] = []; + for (const target of grokWorkerDenyPathChain(denyPath)) { + try { + steps.push({ path: target, entry: await lstat(target) }); + } catch (error) { + steps.push({ path: target, code: (error as NodeJS.ErrnoException).code }); + break; + } + } + const chain = grokWorkerDenyPathChain(denyPath); + while (steps.length < chain.length) steps.push({ path: chain[steps.length]!, code: steps.at(-1)?.code ?? "EACCES" }); + return steps; +} + +/** + * Fails closed before a worker is ever launched with a profile Grok would + * refuse. Callers with the widest view run it: root provisioning at container + * start and on every slot recycle, and the direct (non-broker) path, which runs + * as the worker uid itself. + */ +export async function assertGrokWorkerDenyPathsPlaceable( + denyPaths: readonly string[], + worker: GrokWorkerDenyPathWorker +): Promise { + for (const denyPath of denyPaths) { + if (!path.posix.isAbsolute(denyPath)) throw new GrokWorkerDenyPlacementError(denyPath, "it is not an absolute path"); + assertGrokWorkerDenyPathPlacement(denyPath, await readGrokWorkerDenyPathChain(denyPath), worker); + } +} diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts index b6ff126..881ffb1 100644 --- a/src/runtime/grokWorkerSandboxProfile.ts +++ b/src/runtime/grokWorkerSandboxProfile.ts @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; import path from "node:path"; +import { assertGrokWorkerDenyPathShape } from "./grokWorkerDenyPlacement.js"; + export const GROK_WORKER_SANDBOX_PROFILE = "daimon-strict" as const; export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events.jsonl" as const; @@ -17,7 +19,12 @@ export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events * * Entries are sorted and deduplicated so equal sets render equal bytes (and the * same `profileSha256`). A path that is not absolute and canonical, or that - * carries a character TOML or Grok would reinterpret, is refused. + * carries a character TOML or Grok would reinterpret, is refused — and so is + * one that equals or contains a base-profile grant, the first half of the + * deny-placement policy (`grokWorkerDenyPlacement.ts`). The other half — + * the entry exists and every ancestor is searchable by the worker uid — needs + * a filesystem, so it is asserted by whoever provisions the paths and, on the + * direct path, before every turn. */ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = []): string { const denied = [...new Set(denyPaths)].sort(); @@ -25,6 +32,7 @@ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = [] if (!path.posix.isAbsolute(entry) || path.posix.normalize(entry) !== entry || entry === "/" || entry.endsWith("/") || /["\\\u0000-\u001f\u007f*?[\]]/u.test(entry)) { throw new TypeError("invalid Grok worker sandbox deny path"); } + assertGrokWorkerDenyPathShape(entry); } return [ `[profiles.${GROK_WORKER_SANDBOX_PROFILE}]`, diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts new file mode 100644 index 0000000..a5a728c --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp, type GrokWorkerTmpOptions } from "./grokWorkerTmpAttestation.js"; + +const worker = 2200, sibling = 2201; +type Entry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>; +const dir = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); +const file = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o100000 | mode, isDirectory: () => false }); +type Entries = { privateTmps: { uid: number; entry: Entry | undefined }[]; shared: (Entry | undefined)[] }; +const good = (): Entries => ({ privateTmps: [{ uid: worker, entry: dir(0o700, worker, worker) }, { uid: sibling, entry: dir(0o700, sibling, sibling) }], shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); +const withShared = (shared: Entry | undefined): Entries => ({ ...good(), shared: [shared, dir(0o1774, 0, 2000)] }); +const withOwn = (entry: Entry | undefined, uid = worker): Entries => ({ ...good(), privateTmps: [{ uid, entry }, good().privateTmps[1]!] }); +const withSibling = (entry: Entry | undefined): Entries => ({ ...good(), privateTmps: [good().privateTmps[0]!, { uid: sibling, entry }] }); +const refused = /temp isolation attestation unavailable/u; + +test("accepts private worker temps and shared temp roots the workers cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good())); + // Per contract the shared group only has to be below the worker range: the broker group 2100 is fine. + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1774, 0, 2100)))); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1770, 0, 2000)))); +}); + +test("refuses shared temp roots that are missing, not directories, not root-owned, worker-grouped, or open to others", () => { + const cases: Record = { + "missing": withShared(undefined), + "regular file": withShared(file(0o1774, 0, 2000)), + "owned by the org user": withShared(dir(0o1774, 2000, 2000)), + "group 2200 (a worker group)": withShared(dir(0o1774, 0, 2200)), + "other search (1775)": withShared(dir(0o1775, 0, 2000)), + "other write (1776)": withShared(dir(0o1776, 0, 2000)), + "default /tmp (1777)": withShared(dir(0o1777, 0, 0)), + "no shared roots at all": { ...good(), shared: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("refuses a private temp that is missing, not a directory, owned by someone else, or has any group or other bit", () => { + const cases: Record = { + "missing": withOwn(undefined), + "regular file": withOwn(file(0o600, worker, worker)), + "owned by another worker": withOwn(dir(0o700, sibling, sibling)), + "group read (0740)": withOwn(dir(0o740, worker, worker)), + "other execute (0701)": withOwn(dir(0o701, worker, worker)), + "other write (0702)": withOwn(dir(0o702, worker, worker)), + "other read (0704)": withOwn(dir(0o704, worker, worker)), + "worker uid below the worker range": withOwn(dir(0o700, 2100, 2100), 2100), + "no workers at all": { ...good(), privateTmps: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("a misprovisioned sibling worker's temp refuses the current worker's turn", () => { + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o777, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o770, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(undefined)), refused); +}); + +test("on a real filesystem: sibling 0777 temp, symlinked temps and symlinked shared roots are refused", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const uid = process.getuid?.() ?? 0; + const own = path.join(root, "own"), other = path.join(root, "other"), shared = path.join(root, "shared"), elsewhere = path.join(root, "elsewhere"); + for (const directory of [path.join(own, "tmp"), path.join(other, "tmp"), shared, elsewhere]) { await mkdir(directory, { recursive: true }); await chmod(directory, 0o700); } + // Seams: this test runs unprivileged, so the owner and worker-range floor are the test user. + const options: GrokWorkerTmpOptions = { sharedRoots: [shared], sharedOwnerUid: uid, firstWorkerUid: uid }; + const workers = [{ home: own, uid }, { home: other, uid }]; + await verifyGrokWorkerTmp(workers, options); + + await chmod(path.join(other, "tmp"), 0o777); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "sibling 0777"); + await chmod(path.join(other, "tmp"), 0o700); + + await rm(path.join(own, "tmp"), { recursive: true }); await symlink(elsewhere, path.join(own, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "private temp symlink to a valid directory"); + await rm(path.join(own, "tmp")); await mkdir(path.join(own, "tmp"), { mode: 0o700 }); + + const linkedShared = path.join(root, "linked-shared"); await symlink(elsewhere, linkedShared); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [linkedShared] }), refused, "shared root symlink to a valid directory"); + const regular = path.join(root, "regular"); await writeFile(regular, ""); await chmod(regular, 0o600); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [regular] }), refused, "shared root regular file"); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [] }), refused, "empty shared roots"); + await verifyGrokWorkerTmp(workers, options); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts new file mode 100644 index 0000000..863a9ab --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -0,0 +1,68 @@ +import type { Stats } from "node:fs"; +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Readonly<{ isDirectory(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; +const FIRST_WORKER_UID = GROK_ENGINE_BROKER.identities.firstWorkerUid; + +export type GrokWorkerTmpWorker = Readonly<{ home: string; uid: number }>; +/** Test seams only; production uses the manifest defaults. */ +export type GrokWorkerTmpOptions = Readonly<{ sharedRoots?: readonly string[]; sharedOwnerUid?: number; firstWorkerUid?: number }>; + +/** + * Temp-directory isolation, checked before every turn for *every* registered + * worker, not only the one about to run: a misprovisioned sibling temp + * directory (group- or world-writable) would be a place this worker could + * write into and that sibling would read from. + * + * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` + * read-write, and it refuses to start when either (or any ancestor of a + * granted path) is in `deny` — verified live: `deny = ["/tmp"]`, + * `["/var/tmp"]`, `["/run"]`, `["/etc"]` all fail with "could not apply the + * sandbox profile", while `["/tmp/sub"]` works. So the kernel profile cannot + * keep evaluator temp files away from the worker. Unix modes can, because the + * launcher drops the worker to its own uid/gid with no supplementary groups: + * + * - shared temp roots are root-owned, owned by a group below the worker range, + * and give "other" at most read (Grok opens the directory; without search + * the worker can list names but cannot open, stat, or create anything); a + * deployment that lets workers traverse or write them is refused; + * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which + * strict grants read-write) is a real directory owned by the worker with no + * group or other access, and every registered worker's is checked. + * + * + * Entries come from `lstat`, so a symlink is never a directory here: a + * symlinked temp root or private temp is refused by the directory check. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmps: readonly Readonly<{ uid: number; entry: Entry | undefined }>[]; shared: readonly (Entry | undefined)[] }>, options: GrokWorkerTmpOptions = {}): void { + const shared = HOME.sharedTmp; + const firstWorkerUid = options.firstWorkerUid ?? FIRST_WORKER_UID; + if (entries.shared.length === 0 || entries.privateTmps.length === 0) throw unavailable(); + for (const entry of entries.shared) { + if (entry === undefined || !entry.isDirectory() || entry.uid !== (options.sharedOwnerUid ?? shared.uid) || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + for (const { uid, entry } of entries.privateTmps) { + if (!Number.isSafeInteger(uid) || uid < firstWorkerUid || entry === undefined || !entry.isDirectory() || entry.uid !== uid || (Number(entry.mode) & 0o077) !== 0) throw unavailable(); + } +} + +/** `workers` must list every registered worker (the running one included). */ +export async function verifyGrokWorkerTmp(workers: readonly GrokWorkerTmpWorker[], options: GrokWorkerTmpOptions = {}): Promise { + const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; + assertGrokWorkerTmpEntries({ + privateTmps: await Promise.all(workers.map(async (worker) => ({ uid: worker.uid, entry: await inspect(path.join(worker.home, HOME.privateTmp.relativeToWorkerHome)) }))), + shared: await Promise.all((options.sharedRoots ?? HOME.sharedTmp.paths).map(inspect)) + }, options); +} + +/** A registration's worker home is the parent of its `/.grok/sandbox.toml`. */ +export const grokWorkerHomeForProfile = (profilePath: string): string | undefined => + path.basename(path.dirname(profilePath)) === ".grok" ? path.dirname(path.dirname(profilePath)) : undefined; + +const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 24195c5..e16c59c 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -14,8 +14,11 @@ export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInference export { GROK_INFERENCE_PROXY_BASE_URL, ENGINE_BROKER_INFERENCE_FAILURE_CODES, type EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; export { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION, GROK_INFERENCE_PURPOSES, type GrokInferencePurpose } from "./inferenceUsageLedger.js"; export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; +export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; -export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { TURN_SEAL_LEDGER, TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 3e7f8ae..be65878 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -25,6 +25,9 @@ environment: `DAIMON_MCP_CAPABILITY` and `DAIMON_PROVIDER_CAPABILITY` (Grok config reads the proxy capability through `env_key`). `--auth-provider` mode remains for callers of the older contract. +It also exports `TMPDIR=/tmp`, the worker's private temp +directory, derived only from the root-owned registration. + Received descriptors carry `MSG_CMSG_CLOEXEC` and can already occupy fds 3-5, so `launch()` lifts prompt, capability, output, executable and status fds above 16 before `dup2`-ing them into place; a `dup2` onto itself keeps close-on-exec @@ -36,3 +39,131 @@ Holding one verified descriptor and `execveat`-ing it would not make a replaced binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path (`/usr/local/bin/grok`), so the image path's root ownership, not the launcher descriptor, is what protects the sandboxed process. + +**The worker's end of that pipe is a blocking pipe.** `O_NONBLOCK` is a +property of the open file description, not of a descriptor, so creating the +merged stdout/stderr pipe with `pipe2(..., O_NONBLOCK)` handed non-blocking +writes to the worker along with `pipes[1]`: Grok 1.0.34 makes the first EAGAIN +from a headless stdout write fatal (`stdout write failed: Resource temporarily +unavailable (os error 11)`) and exits 1 before it issues a single model +request, so the turn burns a wake and buys nothing. It stayed invisible until +the MCP tools became reachable and the init frame that enumerates them grew to +roughly 9.5 KB — past a pipe buffer, which is not always the 64 KiB default +(8 KiB inside the Docker Desktop VM this suite runs in). So the pipe is created +`O_CLOEXEC` only and `O_NONBLOCK` is set afterwards on `pipes[0]` alone, the +read end this process polls; that one is load bearing, because the post-exit +drain loop has no `poll` and would otherwise park on a write end some surviving +grandchild still holds. + +A blocking child cannot wedge the launcher. `serve()` runs in its own forked +handler per connection, so one worker's backpressure never reaches another +turn; `supervise` drains the pipe on every pass of a 250 ms `poll`, and both +bounds act on a child that is asleep in `write()`: crossing `DBL_MAX_OUTPUT` +stops reading (`p[1].events = 0`) and `kill(-pid, SIGKILL)`s the worker's whole +process group in the same iteration, and a client disconnect does the same — +neither is refusable by a process sleeping on a pipe. `worker_spill_case` is +the cover: the fixture shrinks its own stdout pipe to the kernel minimum, +reports the capacity it actually got, and writes four times that in one +`write`, so it straddles the buffer on any host without assuming 64 KiB while +staying under `DBL_MAX_OUTPUT`. + +**Crossing `DBL_MAX_OUTPUT` is a reported status, not a lost turn.** This is +worth stating because it has been guessed at twice: a trip sets +`output_limited`, stops reading, `SIGKILL`s the worker's process group, reaps +it, and then — `disconnected` is still 0, so the branch at the end of +`supervise` runs — writes the complete 128-byte result frame with +`DBL_STATUS_OUTPUT_FAILED`, `DBL_STAGE_OUTPUT`, `DBL_FAILURE_OUTPUT_LIMIT` and +`output_length = 0`. `closed_result` admits exactly that shape, the client +relays it, and `decodeNativeBrokerResult` raises a named +`NativeBrokerTurnFailure`. So a trip costs the turn its *text* and nothing +else: the broker still seals the turn and still meters the spend the proxy +measured. A lost terminal frame, an unnamed transport failure or an unmetered +turn therefore cannot be explained by this bound, and the only branch that +sends nothing at all is a client that already disconnected. + +**Both readers of that buffer trip the same bound**, through +`output_limit_crossed`. The poll loop always did; the post-exit drain did not, +so a worker that exited with more than `DBL_MAX_OUTPUT` still in the pipe left +`used` at the buffer's last byte with `output_limited` clear, and the turn was +published `DBL_STATUS_OK` with `output_length = DBL_MAX_OUTPUT + 1` — which +`closed_result` refuses, so the client replaced it with a fabricated +`prelaunch_failed`/`protocol` frame carrying no pid and no start ticks. That +frame says the worker never ran, about a turn that ran and whose work may have +succeeded, which is the one class of lie this boundary must never tell. +The window is real but narrow: `poll` is level-triggered, so the loop sees any +buffered byte, and the drain can only inherit data written in the gap between +`poll()` returning and `waitpid()` reaping. It is therefore **not +reproducible on demand in this suite** — the fix is by construction, and the +adversarial cases that do cross the bound (`output_boundary_case`, +`worker_flood_case`, `worker_spill_case`) only prove it did not regress. Do not +add a test that claims to cover it by feeding the bound through the poll loop: +that routes around the defect. + +The bound is the whole turn's stdout, not one frame, and it is now 256 KiB. +**This supersedes the "known limit, deliberately not raised yet" this file +carried while the pipe's blocking mode was the variable under test.** The +measurement that decided it: a live brokered turn emitted 26,482 bytes for +four tool calls, 23,320 of them one tool-result frame carrying all four +(`.runtime/grok-p1b/worker-a2-output.jsonl`); JSON framing and escaping +inflated those payloads by 1.007x, so a turn's stdout is close to the sum of +its tool results. The nine-tool-call turn this was raised for is about 210 KB +of the same shape, against a 64 KiB bound — so 64 KiB was reachable by an +ordinary working turn, and crossing it costs that turn its whole text. The new +number is not headroom-by-guess: it is the control protocol's own `text` bound +(`engineBrokerProtocol.ts`, 262144), the next boundary this output has to +cross, so a larger launcher bound would only move the refusal one layer up. +`worker_turn_case` writes exactly that measured shape — a 9,728-byte init +frame and nine 23,320-byte frames, 219,608 bytes — and asserts it is published +whole; restoring 65536 turns it red. + +**A bound that hangs would be worse than no bound, and this one does not.** +The hypothesis that a worker parks forever in `write()` once the bound is +crossed — plausible after the pipe became blocking, because a write that +cannot complete now blocks instead of erroring — was tested, not reasoned +about, and it is false. `worker_stream_case` writes eight times the bound in +frame-sized writes and then sleeps far longer than this suite, so it is asleep +inside `write()` with its pipe full when the trip fires and nothing but the +launcher can end it; the launcher answers `output_limit` with `term_signal` +SIGKILL in seconds. Its socket carries a 30-second deadline so a park fails +red instead of parking the runner. Deleting the `output_limited` half of +`if (disconnected || output_limited) kill(-pid, SIGKILL)` is the mutation that +proves it: the case then times out on that deadline, and +`output_boundary_case` does not notice, because its worker has already exited +by the time the trip fires. That is the boundary the two cases straddle — +a worker gone at the trip against a worker alive and blocked at it. + +The result frame's last word is `diagnostic_length`, not padding: on +`DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` +bytes of the worker's merged stdout/stderr and sends them after the fixed +frame, while `output_length` stays 0 as before. Every other failure sends none, +and `closed_result` refuses a frame that mixes the two. The bytes are the +worker's own, so the broker redacts them before they cross any boundary. + +**The window keeps both ends.** A worker that dies early prints its error +first and then echoes its own input, so a pure tail kept the echo: the one live +capture this had ever produced was 512 bytes of the agent's own prompt read +back, with the error already off the front and erased here. `diagnostic_window` +keeps the first `DBL_MAX_DIAGNOSTIC / 2`, then `DBL_DIAGNOSTIC_ELISION` naming +the bytes dropped, then the last `DBL_MAX_DIAGNOSTIC / 2`, all inside the same +bound — the marker is sized against `used`, the largest count it can carry, so +the budget holds for every input, and a `snprintf` that will not fit falls back +to the tail. Output that already fits is left in place, byte-identical, with no +marker. The marker text is byte-identical to the TypeScript window's +(`boundedDiagnosticWindow`), so one grep finds an elision on either side of the +boundary. + +That elision is a *cut*, and a cut can split a turn capability in half, leaving +a fragment exact redaction can never match. The answer used where Daimon owns +both ends — retain one whole secret more than is reported — cannot work here, +because what this window keeps is exactly what it sends: a margin reserved here +would be sent too. So the fragment is scrubbed where the capabilities are +known, in `engineBrokerNativeClient.ts` (`scrubCutFragments`), on both sides of +every marker and at the window's outer ends. + +Changing any of the six pinned launcher sources means rebuilding: `node +--import tsx src/runtime/native/build.ts`, then re-pin +`artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256` in the contract manifest and +re-emit it. `artifactsManifest.test.ts` fails by design until that is done. The +adversarial suite is `docker build -f Dockerfile.integration -t .` in this +folder and `docker run --rm --privileged `; `worker_flood_case` is the +head-and-tail cover and fails first if the window regresses to a tail. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index daeaf6e..7d34af4 100755 Binary files a/src/runtime/native/artifacts/daimon-engine-broker-arm64 and b/src/runtime/native/artifacts/daimon-engine-broker-arm64 differ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json index 6eb9ffd..a694add 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","binary_sha256":"sha256:c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64 b/src/runtime/native/artifacts/daimon-engine-broker-x64 index fce563c..f7c022d 100755 Binary files a/src/runtime/native/artifacts/daimon-engine-broker-x64 and b/src/runtime/native/artifacts/daimon-engine-broker-x64 differ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index f2bcf17..4dba2b9 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","binary_sha256":"sha256:36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/copyArtifact.ts b/src/runtime/native/copyArtifact.ts index 1166aa6..be3c30d 100644 --- a/src/runtime/native/copyArtifact.ts +++ b/src/runtime/native/copyArtifact.ts @@ -4,8 +4,14 @@ import { fileURLToPath } from 'node:url'; await import('./verifyArtifacts.ts'); const root = path.dirname(fileURLToPath(import.meta.url)); -const architecture = process.arch; -if (!['x64', 'arm64'].includes(architecture) || process.platform !== 'linux') { +// The broker artifacts are prebuilt, provenance-verified Linux executables checked into +// this repository, so staging one is a packaging step and not a host capability: the +// published tarball must carry `dist/runtime/native/daimon-engine-broker` on every packing +// host, because the runtime image installs that tarball with no lifecycle script that could +// stage it later. `DAIMON_ENGINE_BROKER_ARCH` selects the packaged Linux target when it is +// not the host's own architecture. +const architecture = process.env.DAIMON_ENGINE_BROKER_ARCH?.trim() || process.arch; +if (!['x64', 'arm64'].includes(architecture)) { if (process.env.DAIMON_REQUIRE_ENGINE_BROKER === '1') throw new Error('native engine broker is Linux x64/arm64 only'); process.exit(0); } diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index d3a9512..7542b51 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -8,7 +8,28 @@ #define DBL_MAX_PROMPT 65536u #define DBL_MAX_TOKEN 4096u #define DBL_MAX_CAPABILITY_BUNDLE (DBL_MAX_TOKEN * 2u + 4u) -#define DBL_MAX_OUTPUT 65536u +/* The WHOLE turn's stdout, not one frame, and sized against real turns rather + than headroom-by-guess. A live four-tool-call brokered turn emitted 26,482 + bytes, 23,320 of them one tool-result frame carrying four results + (`.runtime/grok-p1b/worker-a2-output.jsonl`), so 64 KiB was reachable by an + ordinary working turn: the nine-tool-call turn this was raised for lands + around 210 KB of the same shape, and a trip costs the turn its whole text. + The number is the control protocol's own `text` bound + (`engineBrokerProtocol.ts`, 262144), because that is the next boundary the + output must cross: a larger launcher bound would only move the refusal one + layer up. A runaway worker is still stopped here — crossing it stops + reading, SIGKILLs the worker's process group and reports `output_limit`. */ +#define DBL_MAX_OUTPUT 262144u +/* Bounded tail of the worker's own merged stdout/stderr, kept only for a + worker that exited on its own account (`DBL_STATUS_WORKER_FAILED`), so the + reason reaches the host instead of `exit=1`. It is a diagnostic, never the + turn's output: `output_length` stays 0 on every failure. */ +#define DBL_MAX_DIAGNOSTIC 512u +/* The marker that joins the two ends of an elided diagnostic, byte-identical + to the TypeScript window's (`boundedDiagnosticWindow` in + `src/pi/cliChildOutput.ts`), so one grep finds every elision on either side + of the boundary. Its own bytes are paid for out of DBL_MAX_DIAGNOSTIC. */ +#define DBL_DIAGNOSTIC_ELISION "[\xe2\x80\xa6 %llu bytes elided \xe2\x80\xa6]" #ifndef DBL_REGISTRY #define DBL_REGISTRY "/etc/daimon-engine-broker/registrations.bin" #endif @@ -73,13 +94,13 @@ struct dbl_result { int32_t worker_pid, exit_code, term_signal; uint64_t start_ticks; char turn_id[65]; - uint32_t stage, failure_class, profile_applied, reserved; + uint32_t stage, failure_class, profile_applied, diagnostic_length; }; #define DBL_RESULT_SIZE 128u #define DBL_RESULT_STAGE_OFFSET 108u #define DBL_RESULT_FAILURE_CLASS_OFFSET 112u #define DBL_RESULT_PROFILE_APPLIED_OFFSET 116u -#define DBL_RESULT_RESERVED_OFFSET 120u +#define DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET 120u _Static_assert(sizeof(struct dbl_result) == DBL_RESULT_SIZE, "dbl_result ABI size"); _Static_assert(__builtin_offsetof(struct dbl_result, stage) == @@ -91,8 +112,8 @@ _Static_assert(__builtin_offsetof(struct dbl_result, failure_class) == _Static_assert(__builtin_offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET, "dbl_result profile offset"); -_Static_assert(__builtin_offsetof(struct dbl_result, reserved) == - DBL_RESULT_RESERVED_OFFSET, - "dbl_result reserved offset"); +_Static_assert(__builtin_offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, + "dbl_result diagnostic length offset"); #endif diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index cb2b9ec..332fb0a 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -25,7 +25,6 @@ #include static __attribute__((noreturn)) void die(void) { _exit(111); } -static int bounded(const char *s, size_t n) { return memchr(s, 0, n) != NULL; } static int safe_component(const char *s, size_t n) { size_t i, l = strnlen(s, n); if (!l || l == n) @@ -157,12 +156,27 @@ static int load_registration(uint32_t slot, struct dbl_registration *out) { close(fd); return -1; } +/* Absolute, NUL-terminated within n, no empty, "." or ".." component and no + trailing slash: the launcher derives HOME, GROK_HOME and TMPDIR from it. */ +static int canonical_path(const char *s, size_t n) { + size_t l = strnlen(s, n), i = 0; + if (l < 2 || l == n || s[0] != '/' || s[l - 1] == '/') + return 0; + while (i < l) { + size_t start = ++i; + while (i < l && s[i] != '/') + i++; + if (i == start || (i - start == 1 && s[start] == '.') || + (i - start == 2 && s[start] == '.' && s[start + 1] == '.')) + return 0; + } + return 1; +} static int valid_registration(const struct dbl_registration *r, const struct dbl_request *q) { return r->uid >= 2200 && r->gid >= 2200 && - bounded(r->workspace, sizeof(r->workspace)) && - bounded(r->home, sizeof(r->home)) && r->workspace[0] == '/' && - r->home[0] == '/' && + canonical_path(r->workspace, sizeof(r->workspace)) && + canonical_path(r->home, sizeof(r->home)) && safe_component(r->agent_id, sizeof(r->agent_id)) && strcmp(r->agent_id, q->agent_id) == 0; } @@ -248,143 +262,3 @@ static uint64_t start_ticks(pid_t pid) { } return 0; } - -static __attribute__((noreturn)) void launch_fail(int status_fd, - uint32_t code) { - (void)full_write(status_fd, &code, sizeof(code)); - _exit(111); -} -static pid_t launch(const struct dbl_registration *r, int executable, - int prompt, int capability, int output, uint32_t *failure, - uint64_t *observed_start_ticks) { - unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; - int status_pipe[2]; - if (capability_bundle(capability, provider, mcp) || - pipe2(status_pipe, O_CLOEXEC)) { - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - pid_t p = fork(); - if (p < 0) { - close(status_pipe[0]); - close(status_pipe[1]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - if (p != 0) { - uint32_t code = 0; - close(status_pipe[1]); - if (full_read(status_pipe[0], observed_start_ticks, - sizeof(*observed_start_ticks)) || - !*observed_start_ticks) { - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - ssize_t got = read(status_pipe[0], &code, sizeof(code)); - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - if (got == 0) - return p; - if (got == (ssize_t)sizeof(code)) - *failure = code; - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - close(status_pipe[0]); - if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) - launch_fail(status_pipe[1], 1); - uint64_t identity = start_ticks(getpid()); - if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) - launch_fail(status_pipe[1], 1); - struct rlimit z = {0, 0}; - if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || - prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || - setresuid(r->uid, r->uid, r->uid)) - launch_fail(status_pipe[1], 2); - zero_caps(); - if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || - prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) - launch_fail(status_pipe[1], 3); - /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto - itself keeps close-on-exec (the prompt vanished at exec) and an - overlapping order clobbers a source. Lift all of them above the targets - first so every dup2 below changes the fd number and clears CLOEXEC. */ - int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); - if (status_fd < 0) - launch_fail(status_pipe[1], 4); - int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); - int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), - high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), - high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), - high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); - if (null_input < 0 || high_prompt < 0 || high_capability < 0 || - high_output < 0 || high_executable < 0 || - dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || - dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || - dup2(high_output, STDERR_FILENO) < 0) - launch_fail(status_fd, 4); - /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok - image descriptor out of the worker and every tool child (execveat with - AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ - if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) - launch_fail(status_fd, 5); - close_other_fds(status_fd); - char *const argv[] = {"grok", - "--sandbox", - "daimon-strict", - "--always-approve", - "--no-subagents", - "--prompt-file", - "/proc/self/fd/3", - "--no-memory", - "--disable-web-search", - "--no-plan", - "--verbatim", - "--system-prompt-override", - DBL_GROK_SYSTEM_PROMPT, - "--tools", - DBL_GROK_TOOLS, - "--max-turns", - DBL_GROK_MAX_TURNS, - "--cwd", - (char *)r->workspace, - "--output-format", - "streaming-messages-json", - "--model", - "daimon-broker-grok", - NULL}; - char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24], - provider_env[DBL_MAX_TOKEN + 32]; - snprintf(home, sizeof(home), "HOME=%s", r->home); - snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); - snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); - /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so - the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped - proxy capability from here. It is as exposed as the MCP capability. */ - snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", - provider); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - char *const envp[] = {home, - grok, - mcp_env, - provider_env, - "DAIMON_CAPABILITY_FD=4", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LANG=C.UTF-8", - "TZ=UTC", - NULL}; - syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); - erase(provider_env, sizeof(provider_env)); - erase(mcp_env, sizeof(mcp_env)); - launch_fail(status_fd, 6); -} - diff --git a/src/runtime/native/engineBrokerLauncherIntegrationCore.inc b/src/runtime/native/engineBrokerLauncherIntegrationCore.inc index 25ff703..0194abf 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationCore.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationCore.inc @@ -215,7 +215,7 @@ static void client_case(void) { result_bytes == sizeof(result) && result_padding_zero(&result) && result.status == DBL_STATUS_OK && result.stage == DBL_STAGE_OUTPUT && result.failure_class == DBL_FAILURE_NONE && !result.profile_applied && - !result.reserved && result.output_length < DBL_MAX_OUTPUT, + !result.diagnostic_length && result.output_length < DBL_MAX_OUTPUT, "client result"); char *out = calloc(1, result.output_length + 1); check(read(output[0], out, result.output_length) == @@ -369,7 +369,7 @@ static void malformed_response_case(int truncated) { if (truncated) write(peer, &result, sizeof(result) / 2); else { - result.reserved = 1; + result.diagnostic_length = DBL_MAX_DIAGNOSTIC + 1; write(peer, &result, sizeof(result)); } close(peer); @@ -380,11 +380,11 @@ static void malformed_response_case(int truncated) { observed.status == DBL_STATUS_PRELAUNCH_FAILED && observed.stage == DBL_STAGE_REQUEST && observed.failure_class == DBL_FAILURE_PROTOCOL && - !observed.reserved, + !observed.diagnostic_length, "fixed protocol diagnostic"); close(output[0]); int status; waitpid(child, &status, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, - truncated ? "truncated result" : "reserved result"); + truncated ? "truncated result" : "over-bound diagnostic result"); } diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index ee189d9..7b4bdc8 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -83,6 +83,198 @@ static void output_boundary_case(const char *provider, int overflow) { close(p); close(c); } +/* A worker that exits nonzero publishes no output, but its own last words — + stderr and stdout share one pipe — must survive as the bounded diagnostic + tail, because the host otherwise sees nothing but `exit=1`. */ +static void worker_failure_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-failure", "mcp.Failure-2"); + struct dbl_request q = request(); + struct dbl_result r; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_WORKER_FAILED && r.stage == DBL_STAGE_WAIT && + r.failure_class == DBL_FAILURE_EXEC && r.exit_code == 1 && + r.term_signal == 0 && r.worker_pid > 0 && r.worker_uid == 2200 && + r.start_ticks > 0 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker failure diagnostic length"); + char *reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length) && + strstr(reason, "grok: session store unwritable"), + "worker failure reason"); + char extra; + check(read(s, &extra, 1) == 0, "worker failure EOF"); + free(reason); + close(s); + close(p); + close(c); +} +/* A worker that dies after printing far more than the window: the error is at + the START of what it printed and the echo at the end, which is the shape a + pure tail got wrong. The distinctive head run STRADDLES the head's own cut — + its first bytes are inside the retained head and its later bytes are elided — + so a tail-only window loses it entirely and this case fails. */ +static void worker_flood_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-flood", "mcp.Flood-2"); + struct dbl_request q = request(); + struct dbl_result r; + size_t printed = strlen("HEAD-OF-ERROR grok: profile refused ") + 4096u + + strlen(" TAIL-OF-ECHO"); + unsigned long long elided = 0; + char *reason, *marker, *shown; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && r.status == DBL_STATUS_WORKER_FAILED && + r.exit_code == 1 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker flood diagnostic length"); + reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length), "worker flood diagnostic"); + check(!strncmp(reason, "HEAD-OF-ERROR grok: profile refused", + strlen("HEAD-OF-ERROR grok: profile refused")), + "worker flood head"); + shown = strstr(reason, " TAIL-OF-ECHO"); + check(shown && shown[strlen(" TAIL-OF-ECHO")] == 0, "worker flood tail"); + marker = strstr(reason, "[\xe2\x80\xa6 "); + check(marker && sscanf(marker, "[\xe2\x80\xa6 %llu bytes elided", &elided) == 1, + "worker flood marker"); + /* The count is exactly what was dropped: everything printed, less the two + ends that survived (the whole window less the marker itself). */ + size_t marker_length = (size_t)(strchr(marker, ']') - marker) + 1u; + check((size_t)elided + (size_t)r.diagnostic_length - marker_length == printed, + "worker flood elided count"); + char extra; + check(read(s, &extra, 1) == 0, "worker flood EOF"); + free(reason); + close(s); + close(p); + close(c); +} +/* A worker's single write larger than the pipe buffer must complete, not kill + it. The launcher created its stdout/stderr pipe O_NONBLOCK as a whole, and + O_NONBLOCK is a property of the open file description, so the worker + inherited a non-blocking stdout: Grok 1.0.34 turns the first EAGAIN from a + headless stdout write into `stdout write failed: Resource temporarily + unavailable` and exits 1 before it ever reaches the model. Once its init + frame outgrew a pipe buffer that was every turn, at $0 apiece. + The fixture shrinks its own stdout pipe to the kernel minimum, reports the + capacity it actually got, and writes four times that in ONE write, so the + case straddles the buffer on any host instead of assuming 64 KiB — and + stays under DBL_MAX_OUTPUT, so it is the write that is bounded here, never + the turn's output. */ +static void worker_spill_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("spill-output", "mcp.Spill-2"); + struct dbl_request q = request(); + struct dbl_result r; + unsigned capacity = 0, count = 0; + size_t head, filler = 0; + char *out, *line, extra; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && + r.output_length > 0 && r.output_length <= DBL_MAX_OUTPUT, + "worker spill survived its own oversized write"); + out = calloc(1, (size_t)r.output_length + 1u); + check(out && read_all(s, out, r.output_length), "worker spill output"); + line = strchr(out, '\n'); + check(line && sscanf(out, "SPILL cap=%u count=%u", &capacity, &count) == 2 && + capacity > 0 && count == r.output_length && + count >= capacity * 4u, + "worker spill straddles the pipe buffer"); + head = (size_t)(line - out) + 1u; + while (head + filler + 10u < (size_t)r.output_length && + out[head + filler] == 's') + filler++; + check(head + filler + 10u == (size_t)r.output_length && + !memcmp(out + r.output_length - 10, "SPILL-TAIL", 10), + "worker spill bytes arrived intact"); + check(read(s, &extra, 1) == 0, "worker spill EOF"); + free(out); + close(s); + close(p); + close(c); +} +/* An ordinary working turn must not be able to hit the bound, and its output + must arrive whole. The fixture writes the shape one live turn measured — a + 9,728-byte init frame and nine 23,320-byte tool-result frames, 219,608 + bytes in all — in frame-sized writes. That is more than three times the + 64 KiB this bound used to be, so at the old value this same turn was + published as `output_limit` with its whole text discarded; it is the case + that straddles the raise, and restoring 65536 turns it red. */ +static void worker_turn_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("turn-output", "mcp.Turn-2"); + struct dbl_request q = request(); + struct dbl_result r; + const uint32_t expected = 9728u + 23320u * 9u; + char *out, extra; + size_t index; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + expected > 65536u && + r.output_length == expected, + "ordinary turn published whole"); + out = calloc(1, (size_t)expected + 1u); + check(out && read_all(s, out, expected), "ordinary turn output"); + check(!memcmp(out, "TURN-HEAD", 9) && + !memcmp(out + expected - 9, "TURN-TAIL", 9), + "ordinary turn ends intact"); + for (index = 9; index < (size_t)expected - 9u; index++) + if (out[index] != 'u') + break; + check(index == (size_t)expected - 9u, "ordinary turn bytes intact"); + check(read(s, &extra, 1) == 0, "ordinary turn EOF"); + free(out); + close(s); + close(p); + close(c); +} +/* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is + still working and still writing. `output_boundary_case` writes one byte past + the bound and then exits on its own account, so it proves the arithmetic and + not the termination: the worker was already gone when the trip fired. This + one straddles that boundary from the other side. The fixture writes eight + times the bound in ordinary frame-sized writes and then sleeps far longer + than this suite, so it fills its own pipe repeatedly and is asleep inside + `write()` when the trip happens, and NOTHING but the launcher can end it. + A launcher that stopped reading without killing — or killed without + answering — parks the worker and this client forever, which is why the + socket carries a deadline: a park has to fail red here, not hang the runner. + The deadline also bounds the answer: it must arrive while the fixture is + still sleeping, so a pass cannot be the fixture exiting by itself. */ +static void worker_stream_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stream-output", "mcp.Stream-2"); + struct dbl_request q = request(); + struct dbl_result r; + struct timeval deadline = {.tv_sec = 30, .tv_usec = 0}; + char extra; + check(!setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &deadline, sizeof(deadline)), + "worker stream deadline"); + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r), + "worker stream answered before the deadline"); + check(r.status == DBL_STATUS_OUTPUT_FAILED && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_OUTPUT_LIMIT && + r.output_length == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && r.start_ticks > 0 && + r.term_signal == SIGKILL, + "worker stream stopped at the bound"); + check(read(s, &extra, 1) == 0, "worker stream EOF"); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -90,6 +282,11 @@ static void org_cases(void) { client_input_reject(5, 1); output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); + worker_failure_case(); + worker_flood_case(); + worker_spill_case(); + worker_turn_case(); + worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); @@ -110,7 +307,7 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && - strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null\n") && + strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null tmpdir=/tmp/worker-home/tmp\n") && !strstr(out, "EVIL"), "fixed worker boundary"); check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index 1191e2d..ef59692 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -1,3 +1,61 @@ +/* Non-canonical or unterminated homes/workspaces are refused at registration: + HOME, GROK_HOME and TMPDIR are derived from them. */ +static void noncanonical_registration_cases(struct dbl_registration r) { + const char *homes[] = {"/tmp/worker-home/../other", "/tmp//worker-home", + "/tmp/worker-home/", "/tmp/./worker-home", + "/tmp/worker-home/..", "relative/home", NULL}; + struct dbl_registration bad[8]; + size_t count = 0; + for (; homes[count]; count++) { + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 0, sizeof(bad[count].home)); + strcpy(bad[count].home, homes[count]); + } + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 'a', sizeof(bad[count].home)); /* over-long: no NUL */ + bad[count].home[0] = '/'; + count++; + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + strcpy(bad[count].workspace, "/tmp/workspace/../etc"); + count++; + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r) && + write(f, bad, sizeof(bad[0]) * count) == + (ssize_t)(sizeof(bad[0]) * count), + "noncanonical registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + for (size_t i = 0; i < count; i++) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Path-1", "mcp.Path-2"); + struct dbl_request q = request(); + struct dbl_result result; + q.slot = 100 + (uint32_t)i; + send_request(s, &q, p, c); + if (!read_all(s, &result, sizeof(result)) || + result.status != DBL_STATUS_PRELAUNCH_FAILED || + result.stage != DBL_STAGE_REGISTRATION || result.worker_pid != 0) + _exit(10 + (int)i); + close(s); + close(p); + close(c); + } + _exit(0); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "noncanonical registration refused"); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} /* A worker that fails before or at exec must not run anything: the launcher child exits instead. The registered "executable" is a #! script; fd 5 is close-on-exec, so execveat(AT_EMPTY_PATH) cannot run it (a script needs @@ -56,7 +114,8 @@ int main(void) { DBL_RESULT_FAILURE_CLASS_OFFSET && offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET && - offsetof(struct dbl_result, reserved) == DBL_RESULT_RESERVED_OFFSET, + offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, "result ABI"); mkdir("/run/daimon-engine-broker", 0755); mkdir("/etc/daimon-engine-broker", 0755); @@ -113,6 +172,8 @@ int main(void) { check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); exec_failure_case(r); puts("native-stage exec failure complete"); + noncanonical_registration_cases(r); + puts("native-stage noncanonical registration complete"); kill(broker, SIGKILL); waitpid(broker, 0, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); diff --git a/src/runtime/native/engineBrokerLauncherModes.inc b/src/runtime/native/engineBrokerLauncherModes.inc index ac78544..012da6b 100644 --- a/src/runtime/native/engineBrokerLauncherModes.inc +++ b/src/runtime/native/engineBrokerLauncherModes.inc @@ -2,7 +2,7 @@ static int client_mode(void) { struct dbl_request request; uint32_t prompt_length = 0, capability_length = 0; unsigned char *prompt = NULL, capability[DBL_MAX_CAPABILITY_BUNDLE] = {0}, - output[DBL_MAX_OUTPUT] = {0}, extra; + output[DBL_MAX_OUTPUT + DBL_MAX_DIAGNOSTIC] = {0}, extra; struct dbl_result result; struct sockaddr_un a = {.sun_family = AF_UNIX}; int s = -1, p = -1, c = -1, ok = -1; @@ -37,7 +37,8 @@ static int client_mode(void) { 1) || client_send(s, &request, p, c) || full_read(s, &result, sizeof(result)) || !closed_result(&result, request.turn_id) || - full_read(s, output, result.output_length) || read(s, &extra, 1) != 0) { + full_read(s, output, result.output_length + result.diagnostic_length) || + read(s, &extra, 1) != 0) { struct dbl_result failure; memset(&failure, 0, sizeof(failure)); failure.version = DBL_VERSION; @@ -50,8 +51,11 @@ static int client_mode(void) { ok = 0; goto done; } + /* The trailer is the turn's output on success and the worker's bounded + diagnostic tail on failure; `closed_result` keeps the two exclusive. */ if (full_write(STDOUT_FILENO, &result, sizeof(result)) || - full_write(STDOUT_FILENO, output, result.output_length)) + full_write(STDOUT_FILENO, output, + result.output_length + result.diagnostic_length)) goto done; ok = 0; done: diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index a6cb347..bd3dca2 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -1,3 +1,210 @@ +static __attribute__((noreturn)) void launch_fail(int status_fd, + uint32_t code) { + (void)full_write(status_fd, &code, sizeof(code)); + _exit(111); +} +static pid_t launch(const struct dbl_registration *r, int executable, + int prompt, int capability, int output, uint32_t *failure, + uint64_t *observed_start_ticks) { + unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; + int status_pipe[2]; + if (capability_bundle(capability, provider, mcp) || + pipe2(status_pipe, O_CLOEXEC)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + pid_t p = fork(); + if (p < 0) { + close(status_pipe[0]); + close(status_pipe[1]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + if (p != 0) { + uint32_t code = 0; + close(status_pipe[1]); + if (full_read(status_pipe[0], observed_start_ticks, + sizeof(*observed_start_ticks)) || + !*observed_start_ticks) { + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + ssize_t got = read(status_pipe[0], &code, sizeof(code)); + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + if (got == 0) + return p; + if (got == (ssize_t)sizeof(code)) + *failure = code; + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + close(status_pipe[0]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) + launch_fail(status_pipe[1], 1); + uint64_t identity = start_ticks(getpid()); + if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) + launch_fail(status_pipe[1], 1); + struct rlimit z = {0, 0}; + if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || + prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || + setresuid(r->uid, r->uid, r->uid)) + launch_fail(status_pipe[1], 2); + zero_caps(); + if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + launch_fail(status_pipe[1], 3); + /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto + itself keeps close-on-exec (the prompt vanished at exec) and an + overlapping order clobbers a source. Lift all of them above the targets + first so every dup2 below changes the fd number and clears CLOEXEC. */ + int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); + if (status_fd < 0) + launch_fail(status_pipe[1], 4); + int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); + int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), + high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), + high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), + high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); + if (null_input < 0 || high_prompt < 0 || high_capability < 0 || + high_output < 0 || high_executable < 0 || + dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || + dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || + dup2(high_output, STDERR_FILENO) < 0) + launch_fail(status_fd, 4); + /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok + image descriptor out of the worker and every tool child (execveat with + AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ + if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) + launch_fail(status_fd, 5); + close_other_fds(status_fd); + char *const argv[] = {"grok", + "--sandbox", + "daimon-strict", + "--always-approve", + "--no-subagents", + "--prompt-file", + "/proc/self/fd/3", + "--no-memory", + "--disable-web-search", + "--no-plan", + "--verbatim", + "--system-prompt-override", + DBL_GROK_SYSTEM_PROMPT, + "--tools", + DBL_GROK_TOOLS, + "--max-turns", + DBL_GROK_MAX_TURNS, + "--cwd", + (char *)r->workspace, + "--output-format", + "streaming-messages-json", + "--model", + "daimon-broker-grok", + NULL}; + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], + provider_env[DBL_MAX_TOKEN + 32]; + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). + A truncated path must never name a different directory: fail instead. */ + if ((size_t)snprintf(home, sizeof(home), "HOME=%s", r->home) >= sizeof(home) || + (size_t)snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home) >= + sizeof(grok) || + (size_t)snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home) >= + sizeof(tmp)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + launch_fail(status_fd, 6); + } + snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); + /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so + the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped + proxy capability from here. It is as exposed as the MCP capability. */ + snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", + provider); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + char *const envp[] = {home, + grok, + tmp, + mcp_env, + provider_env, + "DAIMON_CAPABILITY_FD=4", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LANG=C.UTF-8", + "TZ=UTC", + NULL}; + syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); + erase(provider_env, sizeof(provider_env)); + erase(mcp_env, sizeof(mcp_env)); + launch_fail(status_fd, 6); +} + +/* Both ends of a failed worker's own output, inside DBL_MAX_DIAGNOSTIC. + A pure tail was the wrong end for the process this exists for: a worker that + dies early prints its error first and then echoes its input, so the tail is + the echo. One live turn reported 512 bytes of the agent's own prompt read + back, with the error already off the front and erased here — the head is + only recoverable where it still exists, which is here. + The marker is paid for out of the same budget and sized against `used`, the + largest count it can ever carry, so the result never exceeds the bound for + any input; output that already fits is left exactly as it is, in place and + byte-identical, with no marker at all. A snprintf that will not fit falls + back to the tail this replaced. */ +static size_t diagnostic_window(unsigned char *bytes, size_t used) { + char marker[64]; + int width, final; + size_t budget, head, tail; + if (used <= DBL_MAX_DIAGNOSTIC) + return used; + width = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)used); + budget = (width > 0 && (size_t)width + 2u <= DBL_MAX_DIAGNOSTIC) + ? DBL_MAX_DIAGNOSTIC - (size_t)width + : 0u; + if (!budget) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + head = budget / 2u; + tail = budget - head; + final = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)(used - head - tail)); + if (final <= 0 || (size_t)final > (size_t)width) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + /* The head stays where it is; the tail moves up behind the marker. */ + memmove(bytes + head + (size_t)final, bytes + (used - tail), tail); + memcpy(bytes + head, marker, (size_t)final); + return head + (size_t)final + tail; +} +/* The one place the total-output bound is decided, because both readers fill + the same buffer and both must trip it. The post-exit drain had no check: a + worker that exited with more than DBL_MAX_OUTPUT still in the pipe left + `used` at the buffer's last byte with `output_limited` clear, so the turn + was published OK with `output_length = DBL_MAX_OUTPUT + 1` — which + `closed_result` refuses, so the client replaced it with a fabricated + `prelaunch_failed`/`protocol` frame. That frame says the worker never ran, + about a turn that ran and whose work may well have succeeded, and it carries + neither the worker's pid nor its start ticks. An honest `output_failed` is + the whole of what changes here. */ +static int output_limit_crossed(size_t used, struct dbl_result *out) { + if (used <= DBL_MAX_OUTPUT) + return 0; + out->status = DBL_STATUS_OUTPUT_FAILED; + out->stage = DBL_STAGE_OUTPUT; + out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; + return 1; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -23,12 +230,9 @@ static void supervise(int client, pid_t pid, int output, ssize_t got = read(output, bytes + used, sizeof(bytes) - used); if (got > 0) used += (size_t)got; - if (used > DBL_MAX_OUTPUT) { + if (output_limit_crossed(used, out)) { output_limited = 1; p[1].events = 0; - out->status = DBL_STATUS_OUTPUT_FAILED; - out->stage = DBL_STAGE_OUTPUT; - out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; } } if (disconnected || output_limited) @@ -47,6 +251,8 @@ static void supervise(int client, pid_t pid, int output, if (got <= 0) break; used += (size_t)got; + if (output_limit_crossed(used, out)) + output_limited = 1; } if (WIFEXITED(status)) out->exit_code = WEXITSTATUS(status); @@ -75,12 +281,25 @@ static void supervise(int client, pid_t pid, int output, out->output_length = (uint32_t)used; } if (out->status != DBL_STATUS_OK) { + /* A failed turn publishes no output, but a worker that exited on its own + account said why on the pipe it shares with stdout, and that window is + the only reason the host can ever see: without it a failure reads + `exit=1`. Keep a bounded window of it and erase the rest here; the + broker redacts it before it crosses any boundary. The other failures get + none: an output-limit window is the very payload the bound refused to + publish, a cancelled turn has no reader left, and a prelaunch failure ran + nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED + ? 0 + : diagnostic_window(bytes, used); + erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; - erase(bytes, sizeof(bytes)); + out->diagnostic_length = (uint32_t)keep; + used = keep; } if (!disconnected) { full_write(client, out, sizeof(*out)); - if (out->status == DBL_STATUS_OK) + if (used) full_write(client, bytes, used); } erase(bytes, sizeof(bytes)); @@ -126,7 +345,14 @@ static void serve(int client) { goto done; out.stage = DBL_STAGE_EXEC; out.failure_class = DBL_FAILURE_EXEC; - if (pipe2(pipes, O_CLOEXEC | O_NONBLOCK)) + /* O_NONBLOCK lives on the open file description, so setting it on the pipe + as a whole handed it to the worker with `pipes[1]`: Grok 1.0.34 makes one + EAGAIN from a headless stdout write fatal and exits 1 before any model + request. Only the read end this process polls is non-blocking, so the + post-exit drain cannot park; a blocking child cannot wedge the launcher, + because `supervise` drains every pass and SIGKILLs the worker's process + group the moment the bound is crossed (`native/AGENTS.md`). */ + if (pipe2(pipes, O_CLOEXEC) || fcntl(pipes[0], F_SETFL, O_NONBLOCK)) goto done; pid = launch(&r, exe, fds[0], fds[1], pipes[1], &launch_failure, &launch_start_ticks); @@ -184,7 +410,8 @@ static int client_send(int socket_fd, const struct dbl_request *r, int prompt, return sendmsg(socket_fd, &m, MSG_NOSIGNAL) == (ssize_t)sizeof(*r) ? 0 : -1; } static int closed_result(const struct dbl_result *r, const char turn_id[65]) { - if (r->version != DBL_VERSION || r->reserved || r->profile_applied > 1 || + if (r->version != DBL_VERSION || r->diagnostic_length > DBL_MAX_DIAGNOSTIC || + r->profile_applied > 1 || r->stage > DBL_STAGE_ATTESTATION || r->failure_class > DBL_FAILURE_ATTESTATION_PROFILE_INVALID || r->output_length > DBL_MAX_OUTPUT || memcmp(r->turn_id, turn_id, 65)) @@ -193,12 +420,14 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_NONE && r->profile_applied == 0 && r->worker_pid > 0 && r->worker_uid >= 2200 && r->start_ticks && - r->exit_code == 0 && r->term_signal == 0; + r->exit_code == 0 && r->term_signal == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_PRELAUNCH_FAILED) return r->stage >= DBL_STAGE_PEER && r->stage <= DBL_STAGE_EXEC && r->failure_class >= DBL_FAILURE_PEER && r->failure_class <= DBL_FAILURE_EXEC && r->worker_pid == 0 && - r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0; + r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_WORKER_FAILED) return r->stage == DBL_STAGE_WAIT && (r->failure_class == DBL_FAILURE_EXEC || @@ -208,10 +437,12 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { if (r->status == DBL_STATUS_OUTPUT_FAILED) return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_OUTPUT_LIMIT && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_CANCELLED) return r->stage == DBL_STAGE_WAIT && r->failure_class == DBL_FAILURE_CANCELLED && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; return 0; } diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index b26a4ae..14fd55a 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -4,6 +4,20 @@ #include #include #include +#include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target);for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");int turn=!strcmp(provider,"turn-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(turn){char*blob=malloc(TURN_BYTES);if(!blob)return 39;memset(blob,'u',TURN_BYTES);memcpy(blob,"TURN-HEAD",9);memcpy(blob+TURN_BYTES-9,"TURN-TAIL",9);size_t sent=0;while(sentd_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i => { /** The compiled worker argv, token by token, exactly as `launch()` passes it to `execveat`. */ const compiledArgv = (): readonly string[] => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; const block = source.match(/char \*const argv\[\] = \{([\s\S]*?)NULL\};/u); assert.ok(block, "launcher argv array not found"); const values = defines(); @@ -55,8 +55,16 @@ test("the compiled system prompt is byte-identical to the contract prompt pinned }); test("the launcher exports the turn provider capability under the env_key the worker config reads", () => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); - assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*mcp_env,\s*provider_env,/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); }); + +test("the launcher gives every worker a private TMPDIR under its registered home", () => { + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\) >=\s*sizeof\(tmp\)/u); + assert.match(source, /canonical_path\(r->home, sizeof\(r->home\)\)/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); + assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); +}); diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts new file mode 100644 index 0000000..67b1a48 --- /dev/null +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -0,0 +1,187 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeHost, type OrganizationRuntimeWakeRequest } from "./organizationRuntime.js"; +import { createOrganizationRuntimeControlHostWithCoreForTest } from "./organizationRuntimeControl.js"; +import { ACTIVITY_V2_VERSION } from "./wakeAcceptanceTypes.js"; + +const token = "control-secret"; +const config = { + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_CLOSURE_TOKEN" }, + agents: [{ id: "alpha", name: "Alpha", instructions: "Act.", workspacePath: "/runtime/workspace", runtimeHomePath: "/runtime/home", engine: { kind: "codex" as const } }] +}; +const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; +const delivery = (deliveryId = "delivery-1") => ({ token, agent_id: "alpha", delivery_id: deliveryId, event: { version: "noopolis.daimon.wake.v2", kind: "manual" as const, text: "hello", occurred_at: "2026-09-18T00:00:00.000Z" } }); + +const core = { + async start(): Promise {}, + async wake(request: OrganizationRuntimeWakeRequest) { return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; }, + async health() { return { version: "noopolis.daimon.organization-runtime-health.v1" as const, state: "running" as const, agents: [{ agentId: "alpha", engine: "codex" as const, state: "idle" as const }] }; }, + async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; }, + async stop() { return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } +} as unknown as OrganizationRuntimeHost; + +/** + * A caller proving that a native execution closed has exactly one authority to + * read — the v2 activity projection — and the runtime that owns it is stopped by + * the time the proof is taken: the worker stops its own host as soon as a delivery + * closes its execution and stays deferred. Before this, `activityV2` answered + * `undefined` there (HTTP 503 `native_host_unavailable` through the caller's + * worker route), so a trial whose subject really ran, spent its budget and simply + * did not do the work reported as an unscorable infrastructure failure. + */ +test("a stopped control host still answers the closure query it alone can settle", async () => { + const root = await privateRoot(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + // Nothing has started: there is no projection to seal and none is invented. + assert.equal(await control.activityV2(token), undefined); + await control.start(); + const accepted = await control.accept(delivery()); + assert.equal(accepted.state, "accepted"); + const live = await control.activityV2(token); + assert.equal(live?.state, "running"); + assert.equal(live?.items.length, 1); + + assert.equal((await control.stop()).state, "stopped"); + const sealed = await control.activityV2(token); + assert.equal(sealed?.version, ACTIVITY_V2_VERSION); + // The same projection, said to be final: nothing can be admitted after it, so + // an empty execution list is a stronger quiescence statement than a live poll. + assert.equal(sealed?.state, "stopped"); + assert.deepEqual(sealed?.executions, []); + assert.equal(sealed?.items.length, 1); + assert.equal(sealed?.items[0]?.delivery_id, "delivery-1"); + assert.equal(sealed?.items[0]?.active, false); + // Repeating the query repeats the seal rather than draining it. + assert.deepEqual(await control.activityV2(token), sealed); + // The seal is not a bypass of authentication, and the store-backed routes that + // have no post-stop answer still report absence instead of an empty runtime. + assert.equal(await control.activityV2("wrong-token"), undefined); + assert.equal(await control.availability(token), undefined); + assert.equal(await control.wakeReceipt(token, accepted.state === "accepted" ? accepted.acceptance_id : ""), undefined); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +/** A host that was never started cannot attest anything, and a second stop keeps the seal. */ +test("an unstarted host seals nothing and a repeated stop does not erase the seal", async () => { + const root = await privateRoot(); + const unstarted = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + assert.equal((await unstarted.stop()).state, "stopped"); + assert.equal(await unstarted.activityV2(token), undefined); + + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + await control.start(); + await control.accept(delivery("delivery-2")); + await control.stop(); + const sealed = await control.activityV2(token); + assert.equal(sealed?.state, "stopped"); + await control.stop(); + assert.deepEqual(await control.activityV2(token), sealed); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +async function privateRoot(): Promise { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-closure-")); await chmod(root, 0o700); return root; } + +/** + * A delivery returned to the inbox for restart must say which outcome returned it. + * + * `attentionDispatcher` reclaims an undisposed delivery to `accepted` on two + * conditions — the dispatcher halting, and a wake that came back `stopped` — and it + * recorded neither, so the receipt an evaluator reads was identical for both. A + * live trial closed its execution, spent real money and reported an `accepted` + * delivery with no marker and no reason, and four investigations went into telling + * those two apart from the outside. The wake's own code is exact and is now kept; + * a halt has no code of its own and stays absent, because a plausible name for an + * undetermined cause gets acted on and a missing one does not. + */ +test("a delivery reclaimed for restart records the stopped wake's own code", async () => { + const root = await privateRoot(); + const attention = { version: ORGANIZATION_RUNTIME_VERSION, host: config.host, + agents: [{ ...config.agents[0]!, attention: { maxBatchMessages: 4, maxBatchBytes: 4096, maxExecutions: 8, maxTokens: 100_000 } }] }; + let stopWake = false; + const stopping = { + ...core, + async wake(request: OrganizationRuntimeWakeRequest) { + if (!stopWake) return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; + // Exactly what organizationRuntimeHost settles an in-flight wake with at shutdown. + return { version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: request.agentId, wakeId: request.event.id, code: "active_wake_aborted" } as const; + } + } as unknown as OrganizationRuntimeHost; + const control = createOrganizationRuntimeControlHostWithCoreForTest(attention, stopping, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + await control.start(); + stopWake = true; + const accepted = await control.accept(delivery("restart-delivery")); + assert.equal(accepted.state, "accepted"); + await waitFor(async () => (await control.activityV2(token))?.items.some((item) => item.state === "accepted" && item.code !== undefined) === true); + const item = (await control.activityV2(token))?.items.find((row) => row.delivery_id === "restart-delivery"); + // Returned for restart, undisposed, and no longer silent about which outcome did it. + assert.equal(item?.state, "accepted"); + // Exactly the live shape: the running transition left deferred FALSE and the + // reclaim does not clear it, which is what distinguishes it from a real deferral. + assert.equal(item?.deferred, false); + assert.equal(item?.code, "active_wake_aborted"); + } finally { await control.stop().catch(() => undefined); await rm(root, { recursive: true, force: true }); } +}); + +async function waitFor(predicate: () => Promise, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + do { if (await predicate()) return; await new Promise((resolve) => setTimeout(resolve, 10)); } while (Date.now() < deadline); + throw new Error("timed out waiting for the reclaimed delivery"); +} + +/** + * The other half, and the one the reclaim path kept getting wrong. A wake that + * COMPLETED is a wake outcome; the dispatcher happening to be halting when it + * lands is not. Keying the reclaim on the host's own `stopping` latch discarded + * that outcome and wrote `accepted, deferred: false, execution retained, no code` + * — a record byte-identical to "never ran" and to "ran but forgotten". Production + * survived it because a restart re-delivers and the agent redoes the work; a + * one-shot isolated trial has no restart, so the evidence was simply lost and a + * subject that really ran and made a choice reported as infrastructure failure. + * An agent that read a delivery and declined to dispose of it is DEFERRED, + * whichever way the host is heading, and a restart must not re-deliver it as + * fresh work. + */ +test("a completed wake under a halting dispatcher is deferred, not reclaimed for restart", async () => { + const root = await privateRoot(); + const attention = { version: ORGANIZATION_RUNTIME_VERSION, host: config.host, + agents: [{ ...config.agents[0]!, attention: { maxBatchMessages: 4, maxBatchBytes: 4096, maxExecutions: 8, maxTokens: 100_000 } }] }; + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + let arrived!: () => void; + const waking = new Promise((resolve) => { arrived = resolve; }); + const blocking = { ...core, + async wake(request: OrganizationRuntimeWakeRequest) { + arrived(); + await held; + return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; + } + } as unknown as OrganizationRuntimeHost; + const control = createOrganizationRuntimeControlHostWithCoreForTest(attention, blocking, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + await control.start(); + await control.accept(delivery("halted-delivery")); + await waking; + // The halt lands while the wake is in flight; the wake then completes anyway. + const stopping = control.stop(); + release(); + await stopping; + const item = (await control.activityV2(token))?.items.find((row) => row.delivery_id === "halted-delivery"); + assert.equal(item?.state, "accepted"); + // The wake's own outcome decides the record: read, undisposed, deferred. + assert.equal(item?.deferred, true); + // A completed wake releases its execution, so a restart waits for new input + // instead of replaying the delivery as work nobody has seen. + assert.equal(item?.execution_id, undefined); + // Still silent: a completed wake is no more a named reclaim outcome than a + // halt is, and a plausible name for an undetermined cause gets acted on. + assert.equal(item?.code, undefined); + } finally { await control.stop().catch(() => undefined); await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/organizationRuntimeControl.ts b/src/runtime/organizationRuntimeControl.ts index e4d9c5c..386b6c5 100644 --- a/src/runtime/organizationRuntimeControl.ts +++ b/src/runtime/organizationRuntimeControl.ts @@ -47,6 +47,19 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt let fusePoll: ReturnType | undefined; let started = false; let stopping = false; + let sealedActivity: OrganizationRuntimeActivityV2 | undefined; + + /** + * One projection, read the same way live and at shutdown. `active` is decided by + * the dispatcher's own execution authority rather than the record's flag alone, + * so a stopped host — whose dispatcher has already awaited every in-flight turn + * — reports exactly the executions that were still admitted when it stopped. + */ + const projectActivity = async (current: WakeAcceptanceStore, state: "running" | "stopped"): Promise => { + const executions = dispatcher?.activeExecutions() ?? []; + const items = (await current.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); + return { version: ACTIVITY_V2_VERSION, state, items, executions }; + }; const hardReason = (): BlockReason | undefined => { if (!started || stopping) return stopping ? "host_stopping" : "host_stopped"; @@ -132,10 +145,15 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt return await store.status(acceptanceId); }, async activityV2(token) { - if (!tokensEqual(expectedToken, token) || store === undefined) return undefined; - const executions = dispatcher?.activeExecutions() ?? []; - const items = (await store.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); - return { version: ACTIVITY_V2_VERSION, items, executions }; + if (!tokensEqual(expectedToken, token)) return undefined; + // A stopped host is not an unanswerable one. Its sealed projection is a + // *stronger* statement about quiescence than a live poll, because nothing + // can be admitted after it, and a caller proving that an execution closed + // has no other authority to read. A host that never started, or one whose + // seal could not be taken, still answers nothing: absence stays absence + // rather than becoming a fabricated idle runtime. + if (store === undefined) return sealedActivity; + return await projectActivity(store, "running"); }, async availability(token) { if (!tokensEqual(expectedToken, token) || !store || !fuse) return undefined; @@ -161,6 +179,11 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt await Promise.allSettled(persistence); const result = await host.stop(); await dispatcher?.stop(); + // The last moment the store can be read, and the only one at which the + // dispatcher has finished every admitted turn. Seal the projection here so + // the closure query keeps an accurate answer once the store is closed; a + // fault leaves it absent instead of inventing one. + if (store) { try { sealedActivity = await projectActivity(store, "stopped"); } catch { /* an unreadable final state stays absent */ } } await store?.close(); await fuse?.close(); store = undefined; fuse = undefined; schedules = undefined; started = false; return result; diff --git a/src/runtime/physicalReadiness.test.ts b/src/runtime/physicalReadiness.test.ts index bdbb248..b645cce 100644 --- a/src/runtime/physicalReadiness.test.ts +++ b/src/runtime/physicalReadiness.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; +import { assertRuntimeDirectory, prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; const agent = (workspacePath: string, runtimeHomePath: string): OrganizationRuntimeAgentConfig => ({ @@ -46,3 +46,68 @@ test("preflight requires safe workspace and private runtime roots, and proves ph await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, workspace)]), /overlap/); } finally { await rm(root, { recursive: true, force: true }); } }); + +const withRoots = async (body: (root: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-physical-")); + try { await body(root); } finally { await rm(root, { force: true, recursive: true }); } +}; + +const runtime = { uid: 2000, gid: 2000, firstWorkerUid: 2200 }; +const entry = (mode: number, uid = 2000, gid = 2000, kind: "dir" | "link" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : 0o120000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); + +test("a brokered Grok runtime home is accepted at exactly 2000: 0710 and nothing wider", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2200), "runtimeHomePath", "worker-traversable", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2201), "runtimeHomePath", "worker-traversable", runtime)); + const refusals: Record> = { + "0700 (no worker traversal, pre-P1b layout)": entry(0o700, 2000, 2200), + "0711 (world traverse)": entry(0o711, 2000, 2200), + "0712": entry(0o712, 2000, 2200), + "0714": entry(0o714, 2000, 2200), + "0730 (group write)": entry(0o730, 2000, 2200), + "0750 (group read)": entry(0o750, 2000, 2200), + "0770": entry(0o770, 2000, 2200), + "0777": entry(0o777, 2000, 2200), + "2710 (setgid)": entry(0o2710, 2000, 2200), + "owned by a worker": entry(0o710, 2200, 2200), + "owned by root": entry(0o710, 0, 2200), + "group is the runtime's own": entry(0o710, 2000, 2000), + "group below the worker range": entry(0o710, 2000, 2100), + "a symlink": entry(0o710, 2000, 2200, "link") + }; + for (const [label, candidate] of Object.entries(refusals)) { + assert.throws(() => assertRuntimeDirectory(candidate, "runtimeHomePath", "worker-traversable", runtime), /runtimeHomePath/u, label); + } +}); + +test("every other engine's runtime home stays exactly 0700, and a workspace stays group/other-write free", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "runtimeHomePath", "private", runtime)); + for (const mode of [0o710, 0o701, 0o750, 0o770, 0o711, 0o755, 0o2700]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "runtimeHomePath", "private", runtime), /must have mode 0700/u, mode.toString(8)); + } + // The brokered Grok workspace contract (2000: 0750) passes the workspace shape. + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o750, 2000, 2200), "workspacePath", "safe", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "workspacePath", "safe", runtime)); + for (const mode of [0o770, 0o720, 0o702, 0o777]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "workspacePath", "safe", runtime), /must not grant group or other write/u, mode.toString(8)); + } +}); + +test("the engine kind decides the runtime home shape on a real filesystem", async () => { + await withRoots(async (root) => { + const workspace = path.join(root, "workspace"), home = path.join(root, "home"); + await mkdir(workspace, { mode: 0o700 }); + await mkdir(home, { mode: 0o710 }); + const grok = { ...agent(workspace, home), engine: { kind: "grok" as const, model: "grok-4.6" as const, reasoningEffort: "low" as const } }; + // 0710 reaches the Grok branch: only the worker-group requirement is left to refuse it here. + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /group-owned by the agent's Grok worker group/u); + // The same home refuses a Codex agent for being wider than 0700. + await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, home)]), /must have mode 0700/u); + await chmod(home, 0o700); + const authority = await prepareOrganizationRuntimePaths([agent(workspace, home)]); + await authority.close(); + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /must have mode 0710 for a brokered Grok agent/u); + }); +}); diff --git a/src/runtime/physicalReadiness.ts b/src/runtime/physicalReadiness.ts index 075ea47..50ce64a 100644 --- a/src/runtime/physicalReadiness.ts +++ b/src/runtime/physicalReadiness.ts @@ -2,9 +2,26 @@ import { constants, type Stats } from "node:fs"; import { lstat, open, realpath } from "node:fs/promises"; import path from "node:path"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; type Identity = Readonly<{ dev: number; ino: number; uid: number; mode: number }>; +/** + * `private` is every engine's runtime home: 0700, nothing but the runtime user. + * + * `worker-traversable` is the brokered Grok shape, and only that shape: the + * agent's own sandboxed worker runs as another uid and must be able to *walk + * into* this home to read the setgid `tool-output/` spill directory the + * truncation notice sends it to (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`). + * Traverse-only means `0710`: no group read (the worker cannot list the home or + * see the acceptance store, telemetry, memory or credential names) and no group + * write. Anything wider — `0711`, `0750`, `0770`, any world bit — is refused, + * as is a group that is not a worker group. Daimon cannot tell *which* worker + * gid belongs to this agent; the per-slot mapping is the deployment's + * provisioning contract, re-checked by the slot preflight receipt's worker-uid + * canaries. + */ +type DirectoryShape = "safe" | "private" | "worker-traversable"; type Directory = { readonly configured: string; readonly real: string; readonly fd: Awaited>; readonly identity: Identity; closed: boolean }; /** @@ -20,6 +37,11 @@ export type OrganizationRuntimePathAuthority = Readonly<{ close(): Promise; }>; +/** Only a brokered Grok agent's home is worker-traversable; every other engine keeps 0700. */ +function runtimeHomeShape(agent: OrganizationRuntimeAgentConfig): DirectoryShape { + return agent.engine.kind === "grok" ? "worker-traversable" : "private"; +} + export async function prepareOrganizationRuntimePaths( agents: readonly OrganizationRuntimeAgentConfig[] ): Promise { @@ -28,7 +50,7 @@ export async function prepareOrganizationRuntimePaths( try { for (const agent of agents) { workspaces.set(agent.id, await verifyDirectory(agent.workspacePath, "workspacePath", "safe")); - homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", "private")); + homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", runtimeHomeShape(agent))); } const roots = [...workspaces.values(), ...homes.values()]; for (let left = 0; left < roots.length; left += 1) for (let right = left + 1; right < roots.length; right += 1) { @@ -51,7 +73,7 @@ export async function prepareOrganizationRuntimePaths( if (workspace === undefined || home === undefined) throw new Error(`no runtime path authority for ${agent.id}`); await Promise.all([ verifyIdentity(workspace, "workspacePath", "safe"), - verifyIdentity(home, "runtimeHomePath", "private") + verifyIdentity(home, "runtimeHomePath", runtimeHomeShape(agent)) ]); }; return { @@ -70,10 +92,10 @@ export async function prepareOrganizationRuntimePaths( }; } -async function verifyDirectory(configured: string, label: string, mode: "safe" | "private"): Promise { +async function verifyDirectory(configured: string, label: string, shape: DirectoryShape): Promise { await assertNoSymlinkComponents(configured); const before = await lstat(configured); - assertDirectory(before, label, mode); + assertDirectory(before, label, shape); const fd = await open(configured, constants.O_RDONLY | directoryFlag() | noFollow()); try { const opened = await fd.stat(); @@ -89,7 +111,7 @@ async function verifyDirectory(configured: string, label: string, mode: "safe" | } } -async function verifyIdentity(directory: Directory, label: string, mode: "safe" | "private"): Promise { +async function verifyIdentity(directory: Directory, label: string, shape: DirectoryShape): Promise { if (directory.closed) throw new Error(`${label} authority is closed`); await assertNoSymlinkComponents(directory.configured); const entry = await lstat(directory.configured); @@ -97,7 +119,7 @@ async function verifyIdentity(directory: Directory, label: string, mode: "safe" if (!sameIdentity(identity(entry), directory.identity) || !sameIdentity(identity(opened), directory.identity)) { throw new Error(`${label} changed after readiness validation`); } - assertDirectory(entry, label, mode); + assertDirectory(entry, label, shape); if (await realpath(directory.configured) !== directory.real) throw new Error(`${label} changed after readiness validation`); } @@ -112,12 +134,28 @@ async function assertNoSymlinkComponents(target: string): Promise { } } -function assertDirectory(entry: Stats, label: string, mode: "safe" | "private"): void { +/** Identity of the process Daimon runs as; a seam so every refusal is testable unprivileged. */ +export type RuntimeIdentity = Readonly<{ uid: number; gid: number; firstWorkerUid?: number }>; +type DirectoryEntry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean; isSymbolicLink(): boolean }>; + +/** Pure shape check for a caller-prepared runtime root. */ +export function assertRuntimeDirectory(entry: DirectoryEntry, label: string, shape: DirectoryShape, runtime: RuntimeIdentity): void { if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error(`${label} must be an existing real directory`); - if (entry.uid !== process.getuid?.()) throw new Error(`${label} must be owned by the runtime user`); - const permissions = entry.mode & 0o777; - if (mode === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); - if (mode === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); + if (entry.uid !== runtime.uid) throw new Error(`${label} must be owned by the runtime user`); + const permissions = Number(entry.mode) & 0o7777; + if (shape === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); + if (shape === "worker-traversable") { + const home = GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome; + if (permissions !== home.mode) throw new Error(`${label} must have mode 0710 for a brokered Grok agent`); + if (entry.gid < (runtime.firstWorkerUid ?? GROK_ENGINE_BROKER.identities.firstWorkerUid) || entry.gid === runtime.gid) { + throw new Error(`${label} must be group-owned by the agent's Grok worker group`); + } + } + if (shape === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); +} + +function assertDirectory(entry: Stats, label: string, shape: DirectoryShape): void { + assertRuntimeDirectory(entry, label, shape, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); } function identity(entry: Stats): Identity { return { dev: entry.dev, ino: entry.ino, uid: entry.uid, mode: entry.mode & 0o7777 }; } diff --git a/src/runtime/productionAgentTools.ts b/src/runtime/productionAgentTools.ts index a23f304..1b7c702 100644 --- a/src/runtime/productionAgentTools.ts +++ b/src/runtime/productionAgentTools.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, mkdir, open, readdir, rename, unlink } from "node:fs/promises"; +import { lstat, open, readdir, rename, unlink } from "node:fs/promises"; import path from "node:path"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; @@ -13,6 +13,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/ import type { OrganizationRuntimeAgentConfig, OrganizationRuntimeMcpServer } from "./organizationRuntime.js"; import { moltnetOperationResult, readMoltnetPages } from "./moltnetMachineRead.js"; import { McpToolCallError, MCP_TOOL_RESULT_MAX_BYTES, renderMcpToolResult, replayMcpReceipt, type McpUpstreamResult } from "./mcpToolResult.js"; +import { ensureRuntimeHomeDirectory } from "./runtimeHomeLayout.js"; import { capToolResult, resolveExemptToolNames, resolveToolResultMaxBytes, TOOL_OUTPUT_DIRECTORY_NAME } from "./toolResultSpill.js"; import { cliChildEnvironment } from "../pi/cliEnvironment.js"; import type { PiWakeEnvironmentContextRef } from "../pi/piAgentWakeSupport.js"; @@ -31,7 +32,7 @@ const MAX_RESULT = 65_536; const TIMEOUT = 10_000; const DAIMON_ACTION_ID_PREFIX = "daimon-"; export async function createProductionAgentTools(agent: OrganizationRuntimeAgentConfig, wakeContext: PiWakeEnvironmentContextRef = {}): Promise { - await mkdir(path.join(agent.runtimeHomePath, "tool-state"), { recursive: true, mode: 0o700 }); + await ensureRuntimeHomeDirectory(agent.runtimeHomePath, "tool-state"); // Resolved once, at agent start: a malformed bound is a configuration error // that should refuse the agent, not a surprise thrown from the middle of a // tool call the model is waiting on. diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts new file mode 100644 index 0000000..8b1a867 --- /dev/null +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -0,0 +1,218 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; +import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory, RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; + +/** + * A brokered Grok runtime home is traversable by its worker uid (0710), so + * anything Daimon creates inside it must stay 0700 — a default `mkdir` would + * make telemetry (prompts, replies, trajectories) readable by the model's own + * sandboxed worker. + */ +const withTraversableHome = async (body: (home: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-home-layout-")); + const home = path.join(root, "runtime-home"); + await mkdir(home, { mode: 0o710 }); + try { await body(home); } finally { await rm(root, { force: true, recursive: true }); } +}; +const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; + +const traceRecord = { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } +} as unknown as Parameters[1]; + +test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { + assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); +}); + +test("telemetry directories Daimon creates in a traversable runtime home are private", async () => { + await withTraversableHome(async (home) => { + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + await rm(path.join(home, "telemetry"), { recursive: true }); + + await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +/** + * The half the mode argument never covered. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and `assertRuntimeDirectory` checks the home and not what Daimon creates + * inside it. So a `telemetry/` left at 0755 by a pre-branch Daimon — or + * pre-created by a deployment — stayed 0755 under a Grok agent's deliberately + * traversable 0710 home, where it is the sandboxed worker reading its own + * agent's prompts, replies and causal history. + * + * The boundary these assertions straddle: a fresh install against an existing + * one. Every writer below is reached through its real entry point, because the + * hole was never in the mode constant — it was in what the call sites did with + * it. + * + * Mutation: restore `mkdir(directory, { recursive: true, mode })` in + * `ensureRuntimeHomeDirectory` and every assertion here goes red while the + * fresh-install test above stays green, which is exactly how this shipped. + */ +test("a runtime-home subdirectory that already exists is made private, not left as it was found", async () => { + await withTraversableHome(async (home) => { + // Pre-created by a deployment, or by a Daimon that predates the mode. + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // An existing ancestor is the same hole one level up: `telemetry/turns` can + // be created privately under a `telemetry/` that stays world-readable. + await chmod(path.join(home, "telemetry"), 0o755); + await mkdir(path.join(home, "telemetry", "turns"), { mode: 0o755 }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE, "the ancestor is corrected too"); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // And the home itself is never touched: which mode it should carry is + // `physicalReadiness.ts`'s judgement, and for a Grok agent it is 0710. + assert.equal(await mode(home), 0o710); + }); +}); + +/** + * Refuse rather than widen, and never follow a link to do it. + * + * A directory the runtime does not own cannot be made private by it, and + * writing an agent's telemetry into it anyway is the failure the correction + * exists to prevent. A symlink planted where a directory belongs is the same + * fault with an attacker attached, so the correction goes through an + * `O_DIRECTORY|O_NOFOLLOW` handle and the `fchmod` lands on the directory that + * was stat'd. + * + * Mutation: drop the owner check and the first case silently proceeds; drop + * `O_NOFOLLOW` and the second chmods the link's target instead of refusing. + */ +test("a runtime-home subdirectory owned by another user, or replaced by a symlink, is refused", async () => { + await withTraversableHome(async (home) => { + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + const foreign = (process.getuid?.() ?? 0) + 4_242; + await assert.rejects(ensureRuntimeHomeDirectory(home, "telemetry", foreign), /owned by the runtime user/u); + assert.equal(await mode(path.join(home, "telemetry")), 0o755, "a refusal corrects nothing and widens nothing"); + + const elsewhere = path.join(home, "elsewhere"); + await mkdir(elsewhere, { mode: 0o755 }); + await symlink(elsewhere, path.join(home, "tool-state")); + await assert.rejects(ensureRuntimeHomeDirectory(home, "tool-state")); + assert.equal(await mode(elsewhere), 0o755, "the link's target must not be chmod'ed through it"); + }); +}); + +test("the home itself is created when absent and never re-moded when present", async () => { + await withTraversableHome(async (home) => { + const fresh = path.join(home, "fresh-home"); + assert.equal(await ensureRuntimeHome(fresh), fresh); + assert.equal(await mode(fresh), RUNTIME_HOME_SUBDIRECTORY_MODE); + await chmod(fresh, 0o710); + await ensureRuntimeHome(fresh); + assert.equal(await mode(fresh), 0o710, "a Grok agent's traversable home is not this helper's judgement"); + }); +}); + +/** + * The rule that keeps the correction from being reintroducible. + * + * `mkdir(..., { mode })` reads like a guarantee and is one only for a + * directory that does not exist yet, so the mode constant stays private to + * this module: a call site that wants a private directory under a runtime home + * asks `ensureRuntimeHomeDirectory` for one and gets the assertion with it. + * + * Mutation: import the constant into any writer and pass it to `mkdir` again, + * and this goes red — which is the shape the 0755 hole had. + */ +test("the private mode is used only where it is also asserted", async () => { + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || target === "src/runtime/runtimeHomeLayout.ts") continue; + if ((await readFile(target, "utf8")).includes("RUNTIME_HOME_SUBDIRECTORY_MODE")) offenders.push(target); + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); + +// Creations that are not inside an agent's runtime home. +const OUTSIDE_RUNTIME_HOME = [ + "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" +]; + +/** + * Files that may name a runtime home in a `mkdir` of their own. + * + * `runtimeHomeLayout.ts` is the correction itself. `wakeAcceptanceFs.ts` + * creates the home and its store directory and then *asserts* each one — + * refusing a directory it finds wider rather than correcting it — which closes + * the same hole the other way and is its own documented contract. + */ +const MAY_MKDIR_A_RUNTIME_HOME = ["src/runtime/runtimeHomeLayout.ts", "src/pi/wakeAcceptanceFs.ts"]; + +const mkdirCalls = (source: string): string[] => { + const calls: string[] = []; + for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { + let depth = 0; + for (let cursor = index + "mkdir".length; cursor < source.length; cursor += 1) { + if (source[cursor] === "(") depth += 1; + else if (source[cursor] === ")") { depth -= 1; if (depth === 0) { calls.push(source.slice(index, cursor + 1)); break; } } + } + } + return calls; +}; + +test("no runtime-home directory is created without an explicit private mode, and none is created outside the layout", async () => { + // Source policy: a default `mkdir` under an agent's runtime home would be 0755, + // the home of a brokered Grok agent is traversable by its worker uid, and a + // `mode:` argument only covers the install where the directory is new. + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; + const source = await readFile(target, "utf8"); + for (const call of mkdirCalls(source)) { + // A `mode:` argument is a create-only mode: it decides nothing for a + // directory that already exists, so naming a runtime home in a `mkdir` + // is the defect whether or not a mode is passed. + if (/runtimeHome/iu.test(call) && !MAY_MKDIR_A_RUNTIME_HOME.includes(target)) { + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + continue; + } + // The workspace is a caller-prepared root with its own contract (group-readable for Grok). + if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + } + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts new file mode 100644 index 0000000..d1b5c1f --- /dev/null +++ b/src/runtime/runtimeHomeLayout.ts @@ -0,0 +1,84 @@ +import { constants } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; + +/** + * Mode for every directory Daimon creates inside an agent's runtime home. + * + * A brokered Grok agent's runtime home is `0710` + * (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`) so its sandboxed + * worker can traverse into the setgid `tool-output/` spill directory. Traverse + * is all it may have: anything Daimon creates in that home — telemetry traces + * (prompts, replies, world trajectories), tool state, receipts, the engine's + * XDG directories and the private `.tmp` — stays `0700`, so a default + * `mkdir` (0755 under the usual umask) never turns a traversable home into a + * readable one. + */ +export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; + +/** + * The home itself, created if it is absent and otherwise left exactly as it is. + * + * Create-only is the whole contract here. A brokered Grok agent's home is + * deliberately `0710` and an organization's may be `0700`; which of the two is + * correct is `physicalReadiness.ts`'s judgement, made against the agent's + * declared engine, and a layout helper that "corrected" a traversable home to + * `0700` would break the worker's only route to its own spills. + */ +export const ensureRuntimeHome = async (runtimeHomePath: string): Promise => { + await mkdir(runtimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + return runtimeHomePath; +}; + +/** + * One directory Daimon owns *below* a runtime home, private on every install. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and that is the common case rather than the exotic one: a `telemetry/` left + * at `0755` by a pre-branch Daimon, or pre-created by a deployment, stayed + * `0755` forever. Under a Grok agent's traversable `0710` home that is the + * worker reading its own agent's prompts, replies and causal history — the + * home is traverse-only precisely so that nothing but `tool-output/` is + * readable. `assertRuntimeDirectory` checks the home, and nothing checked what + * Daimon created inside it. + * + * So every level below the home is asserted and corrected on the way down, + * through a handle rather than a path: `O_DIRECTORY|O_NOFOLLOW` refuses a + * symlink planted where a directory belongs, and the `fchmod` that follows + * lands on the directory that was stat'd. A directory owned by anyone but the + * runtime user is **refused**, never widened and never silently accepted — the + * runtime cannot make someone else's directory private, and proceeding would + * write an agent's telemetry into it anyway. + * + * `owner` is a seam so both refusals are testable unprivileged, exactly as + * `physicalReadiness.ts`'s `RuntimeIdentity` is. + */ +export async function ensureRuntimeHomeDirectory(runtimeHomePath: string, relative: string, owner: number = process.getuid?.() ?? -1): Promise { + const segments = relative.split("/").filter((segment) => segment.length > 0); + if (segments.length === 0 || segments.some((segment) => segment === "." || segment === "..")) { + throw new Error(`runtime home subdirectory must name a path below the home: ${relative}`); + } + let current = await ensureRuntimeHome(runtimeHomePath); + for (const segment of segments) { + current = path.join(current, segment); + await mkdir(current, { mode: RUNTIME_HOME_SUBDIRECTORY_MODE }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + await assertPrivateDirectory(current, owner); + } + return current; +} + +const flag = (name: "O_DIRECTORY" | "O_NOFOLLOW"): number => (constants as typeof constants & Partial>)[name] ?? 0; + +async function assertPrivateDirectory(directory: string, owner: number): Promise { + const handle = await open(directory, constants.O_RDONLY | flag("O_DIRECTORY") | flag("O_NOFOLLOW")); + try { + const entry = await handle.stat(); + if (!entry.isDirectory()) throw new Error(`runtime home path is not a directory: ${directory}`); + if (entry.uid !== owner) throw new Error(`runtime home subdirectory must be owned by the runtime user: ${directory}`); + if ((entry.mode & 0o7777) !== RUNTIME_HOME_SUBDIRECTORY_MODE) await handle.chmod(RUNTIME_HOME_SUBDIRECTORY_MODE); + } finally { + await handle.close(); + } +} diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index b83a587..a4b94c6 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { chmod, chown, lstat, mkdir, mkdtemp, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { mock } from "node:test"; import { renderMcpToolResult, MCP_TOOL_RESULT_MAX_BYTES, type McpUpstreamResult } from "./mcpToolResult.js"; import { + assertSpillDirectoryStat, capToolResult, DEFAULT_TOOL_RESULT_MAX_BYTES, MIN_TOOL_RESULT_MAX_BYTES, @@ -150,3 +151,91 @@ test("the bound and the exemption list come from the environment, and a nonsense assert.deepEqual([...resolveExemptToolNames({})], []); assert.deepEqual([...resolveExemptToolNames({ [TOOL_RESULT_EXEMPT_ENV]: " mcp_a_b , mcp_c_d ," })], ["mcp_a_b", "mcp_c_d"]); }); + +test("spilled files are readable by the directory's (worker) group and never by other users", async () => { + await withDirectory(async (directory) => { + // What a deployment provisions for a brokered worker: setgid tool-output in a group that is not the runtime's own. + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { await chown(directory, process.getuid?.() ?? -1, workerGroup); await chmod(directory, 0o2750); } else await chmod(directory, 0o700); + const previous = process.umask(0o077); + let capped; + try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } + assert.ok(capped.spillPath, "the spill was written"); + const file = await stat(capped.spillPath); + assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); + assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); + }); +}); + +const big = { content: [{ type: "text" as const, text: `HEAD${"y".repeat(100_000)}TAIL` }] }; +const spillName = "daimon-abc123.mcp_desk_archive_dump.log"; + +test("a symlinked, world-open, or group-open-without-setgid spill directory is refused and nothing is written", async () => { + await withDirectory(async (root) => { + const target = path.join(root, "target"); await mkdir(target, { mode: 0o700 }); + const linked = path.join(root, "linked"); await symlink(target, linked); + const capped = await cap(big, { spillDirectory: linked }); + assert.equal(capped.spillPath, undefined); + assert.equal(capped.details.full_output_saved, false); + assert.deepEqual(await readdir(target), [], "nothing written through the symlink"); + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { + // A foreign group without setgid: files would not inherit it, so it is refused. + const noSetgid = path.join(root, "foreign-group-no-setgid"); await mkdir(noSetgid); await chown(noSetgid, process.getuid?.() ?? -1, workerGroup); await chmod(noSetgid, 0o750); + assert.equal((await cap(big, { spillDirectory: noSetgid })).spillPath, undefined, "0750 foreign group without setgid"); + } + // 2750 in the runtime's own group is not a worker grant; 0701/0704 exceed 2750. + for (const mode of [0o777, 0o755, 0o750, 0o2770, 0o2757, 0o2750, 0o701, 0o704]) { + const directory = path.join(root, `mode-${mode.toString(8)}`); await mkdir(directory); await chmod(directory, mode); + const refused = await cap(big, { spillDirectory: directory }); + assert.equal(refused.spillPath, undefined, mode.toString(8)); + assert.deepEqual((await readdir(directory)).filter((name) => !name.startsWith(".")), [], mode.toString(8)); + } + }); +}); + +test("a destination symlink or a pre-existing 0666 file is replaced by a 0640 regular file without touching the target", async () => { + await withDirectory(async (root) => { + const directory = path.join(root, "tool-output"); await mkdir(directory, { mode: 0o700 }); + const victim = path.join(root, "victim.txt"); await writeFile(victim, "VICTIM", { mode: 0o644 }); + await symlink(victim, path.join(directory, spillName)); + const first = await cap(big, { spillDirectory: directory }); + assert.equal(first.spillPath, path.join(directory, spillName)); + const replaced = await lstat(first.spillPath!); + assert.ok(replaced.isFile() && !replaced.isSymbolicLink()); + assert.equal(replaced.mode & 0o777, 0o640); + assert.equal(await readFile(victim, "utf8"), "VICTIM", "the symlink target is untouched"); + + await rm(first.spillPath!); await writeFile(first.spillPath!, "stale", { mode: 0o666 }); await chmod(first.spillPath!, 0o666); + const second = await cap(big, { spillDirectory: directory }); + const rewritten = await lstat(second.spillPath!); + assert.equal(rewritten.mode & 0o777, 0o640); + assert.equal(await readFile(second.spillPath!, "utf8"), big.content[0].text); + }); +}); + +test("the spill directory must be owned by the runtime itself", () => { + const runtime = { uid: 2000, gid: 2000 }; + const entry = (uid: number, gid: number, mode: number) => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2000, 0o700), runtime)); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2200, 0o2750), runtime)); + for (const [label, candidate] of [["owned by a worker", entry(2200, 2200, 0o700)], ["owned by root", entry(0, 2200, 0o2750)], ["not a directory", { ...entry(2000, 2000, 0o700), isDirectory: () => false }]] as const) { + assert.throws(() => assertSpillDirectoryStat(candidate, runtime), /spill directory/u, label); + } +}); + +test("a spill directory whose opened inode differs from the named one is refused", async () => { + await withDirectory(async (directory) => { + const probe = await open(directory, "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; let calls = 0; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + calls += 1; + return calls === 1 ? Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }) : real; + }); + try { assert.equal((await cap(big, { spillDirectory: directory })).spillPath, undefined); } finally { swapped.mock.restore(); } + assert.deepEqual(await readdir(directory), []); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index 6e993a2..5955577 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -116,7 +116,8 @@ const measure = (content: readonly unknown[], details: Record): Buffer.byteLength(safeStringify({ content, structuredContent: details }), "utf8"); /** Cut on a code-point boundary, from the front. */ -const headUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const headUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let end = Math.max(0, maxBytes); @@ -125,7 +126,8 @@ const headUtf8 = (value: string, maxBytes: number): string => { }; /** Cut on a code-point boundary, from the back. */ -const tailUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const tailUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let start = Math.max(0, bytes.byteLength - maxBytes); @@ -181,6 +183,20 @@ const notice = (input: Readonly<{ + ` or \`grep -n "" ${input.spillPath}\` — and do NOT repeat this tool call to see it: an identical call returns this same truncation.]`; }; +/** + * Spilled files are group-readable and never other-readable. + * + * A brokered Grok worker runs as its own uid and reads a spill with + * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to + * the very agent the notice sends there. The group grant reaches exactly that + * agent's worker only when the deployment provisions `tool-output` as + * `: 2750` (setgid, so each file inherits the worker + * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon + * creates itself stays 0700, so for every other engine nothing new is exposed. + */ +export const SPILL_FILE_MODE = 0o640; +export const SPILL_DIRECTORY_MAX_MODE = 0o2750; + /** * Write the full payload where the agent can read it, atomically. * @@ -190,12 +206,66 @@ const notice = (input: Readonly<{ */ const writeSpill = async (directory: string, name: string, text: string): Promise => { await mkdir(directory, { recursive: true, mode: 0o700 }); + const pinned = await pinSpillDirectory(directory); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } - return file; + try { + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + let written: Awaited>; + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); written = await handle.stat(); } finally { await handle.close(); } + if (!written.isFile() || written.nlink !== 1 || written.uid !== process.getuid?.()) throw new Error("spill file is not a private regular file"); + // Node has no openat: re-check that the directory entry still names the pinned inode before publishing into it. + await assertSameDirectory(directory, pinned); + // rename replaces a pre-existing destination entry (a symlink included) without following it. + await rename(temporary, file); + const published = await lstat(file); + if (!published.isFile() || published.dev !== written.dev || published.ino !== written.ino) throw new Error("spill file was replaced"); + return file; + } catch (error) { + await unlink(temporary).catch(() => undefined); + throw error; + } finally { + await pinned.handle.close(); + } +}; + +type PinnedDirectory = Readonly<{ handle: Awaited>; dev: number; ino: number }>; + +/** + * The spill directory must be a real directory owned by this runtime and no + * wider than `2750`: either Daimon's own `0700`, or a deployment-provisioned + * setgid directory whose group is not the runtime's own (a worker group). A + * symlinked, foreign-owned, world-accessible, or group-open-without-setgid + * directory is refused and nothing is written. Daimon cannot know *which* + * worker gid belongs to this agent; that mapping is the deployment's + * provisioning contract (`GROK_ENGINE_BROKER.worker.home.spillDirectory`). + */ +const pinSpillDirectory = async (directory: string): Promise => { + const before = await lstat(directory); + if (before.isSymbolicLink() || !before.isDirectory()) throw new Error("spill directory is not a real directory"); + const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("spill directory was replaced"); + assertSpillDirectoryStat(opened, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); + return { handle, dev: Number(opened.dev), ino: Number(opened.ino) }; + } catch (error) { await handle.close(); throw error; } +}; + +/** Pure so a foreign owner is testable without root. */ +export const assertSpillDirectoryStat = (entry: Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>, runtime: Readonly<{ uid: number; gid: number }>): void => { + const mode = Number(entry.mode) & 0o7777; + const groupOpen = (mode & 0o070) !== 0; + if (!entry.isDirectory() || entry.uid !== runtime.uid || (mode & ~SPILL_DIRECTORY_MAX_MODE) !== 0 + || (groupOpen && ((mode & 0o2000) === 0 || entry.gid === runtime.gid))) { + throw new Error("spill directory is not a private or provisioned worker-group directory"); + } +}; + +const assertSameDirectory = async (directory: string, pinned: PinnedDirectory): Promise => { + const [now, held] = await Promise.all([lstat(directory), pinned.handle.stat()]); + if (now.isSymbolicLink() || Number(now.dev) !== pinned.dev || Number(now.ino) !== pinned.ino || Number(held.ino) !== pinned.ino) throw new Error("spill directory was replaced"); }; /** Newest-first retention, so a busy agent cannot fill its own runtime home. */ diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index bb33992..d0d2153 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -110,7 +110,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st * (the provider response the proxy saw), or `estimated` (no valid usage; the * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). */ -export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated" }>; +/** + * `toolCalls` are the tool-call names that request's response carried, as the + * proxy read them (`grokBrokerTurnMeter.ts`): names only, bounded, `[]` for a + * response that called nothing, and absent when no response could be decoded. + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated"; toolCalls?: readonly string[] }>; /** * `requestCount` is the turn's admitted request count when it exceeds the rows: * a killed turn's in-flight request was sent upstream but never reported usage, @@ -125,6 +130,22 @@ export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the * broker idempotency key and `thread` the Grok session id when the stream * named one. + * + * `tool_calls` is what a turn's rows could not say before: whether the model + * ever *tried* to call anything. Two live turns ended with correctly mounted + * tools and no visible attempt, and the rows recorded timings and tokens only, + * so the question could not be answered after the fact. It is names only — + * never arguments, never message content, never a bearer — bounded, `[]` for a + * response that called nothing, and absent for a response that could not be + * decoded, because a fabricated empty list is byte-identical to a measured one. + * + * It is an additive field inside the unchanged + * `noopolis.daimon.turn-requests.v1` row, deliberately without a version bump: + * Spawnfile's usage reader (`spawnfile/src/runtime/usageLedger.ts`) drops every + * line whose `v` it does not recognise while ignoring fields it does not know, + * and Paideia only relocates this stream's path + * (`DAIMON_TURN_REQUESTS_LEDGER_PATH`). A bump is what would blind them; a new + * field is not. */ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { const at = entry.at ?? new Date().toISOString(); @@ -146,6 +167,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string output: request.output, total: request.total, ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), + ...(request.toolCalls === undefined ? {} : { tool_calls: request.toolCalls }), ...requestClockFields(request) })}\n`).join(""); }; diff --git a/src/runtime/wakeAcceptanceReconciliation.test.ts b/src/runtime/wakeAcceptanceReconciliation.test.ts index fb2bd4e..f7ab14c 100644 --- a/src/runtime/wakeAcceptanceReconciliation.test.ts +++ b/src/runtime/wakeAcceptanceReconciliation.test.ts @@ -48,9 +48,16 @@ test("offline reconciliation blocks untrusted proof, identity mismatch, and conc const mismatch = await reconcileOfflineWakeTransition({ ...request, lock: { ...request.lock, ino: request.lock.ino + 1 } }, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => ({ request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }) }); assert.equal(mismatch.state, "blocked"); let release!: () => void; + let leaseCreated!: () => void; const paused = new Promise((resolve) => { release = resolve; }); - const first = reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => { await paused; return { request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }; } }); - await new Promise((resolve) => setTimeout(resolve, 5)); + // `verifyDeploymentAttestation` runs only after `acquireLease` has published the + // lease, so signalling from inside it is a real happens-after of that publication. + // A sleep is not: publishing the lease is several fsynced filesystem operations and + // takes ~5 ms even on an idle machine, so a 5 ms timer raced it and the store then + // opened against a store no one had reserved yet. + const leased = new Promise((resolve) => { leaseCreated = resolve; }); + const first = reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => { leaseCreated(); await paused; return { request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }; } }); + await leased; await assert.rejects(WakeAcceptanceStore.open(root, testStoreOptions), /reserved for offline reconciliation/); const concurrent = await reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => ({ request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }) }); assert.equal(concurrent.state, "blocked"); diff --git a/src/runtime/wakeAcceptanceRecord.ts b/src/runtime/wakeAcceptanceRecord.ts index ddf544b..7e5ae32 100644 --- a/src/runtime/wakeAcceptanceRecord.ts +++ b/src/runtime/wakeAcceptanceRecord.ts @@ -40,8 +40,12 @@ export function parseStoredWakeAcceptance(value: unknown): StoredWakeAcceptanceR if (executionError === "" || record.execution_error !== undefined && Buffer.byteLength(string(record.execution_error)) > MAX_EXECUTION_ERROR_BYTES) throw new Error("wake acceptance execution error is invalid"); const completionText = record.text === undefined ? undefined : sanitizeWakeCompletionText(string(record.text)); if (claimGeneration !== undefined && !uuid(claimGeneration)) throw new Error("wake acceptance record is invalid"); - if (code !== undefined && !(["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] as const).includes(code)) throw new Error("wake acceptance record is invalid"); - if ((state === "accepted" || state === "running" || state === "completed") && code !== undefined) throw new Error("wake acceptance record is invalid"); + if (code !== undefined && !(["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] as const).includes(code)) throw new Error("wake acceptance record is invalid"); + // `accepted` is the one non-terminal state a record can be arrived at FROM an ended + // execution: the dispatcher returns an undisposed delivery there for restart, and the + // outcome that returned it is the only account of why. `running` and `completed` still + // refuse a code, where one would be nonsense rather than evidence. + if ((state === "running" || state === "completed") && code !== undefined) throw new Error("wake acceptance record is invalid"); if ((state === "failed" || state === "stopped") && code === undefined) throw new Error("wake acceptance record is invalid"); if ((state !== "completed" && state !== "failed" && completionText !== undefined) || completionText !== record.text) throw new Error("wake acceptance record is invalid"); if (string(record.request_digest) !== wakeAcceptanceDigest(parsed) || !uuid(string(record.acceptance_id))) throw new Error("wake acceptance record is invalid"); diff --git a/src/runtime/wakeAcceptanceStore.ts b/src/runtime/wakeAcceptanceStore.ts index bad2dd6..7c456ed 100644 --- a/src/runtime/wakeAcceptanceStore.ts +++ b/src/runtime/wakeAcceptanceStore.ts @@ -198,7 +198,13 @@ export class WakeAcceptanceStore { await this.afterFinalLockAssertion?.(); await this.assertTransitionLock(record, lock); const target = this.fileFor(record.agent_id, record.delivery_id); - const next: Stored = { ...record, state, updated_at: new Date().toISOString(), claim_generation: claim.generation, ...(code === undefined ? {} : { code }), ...(completedText === undefined ? {} : { text: sanitizeWakeCompletionText(completedText) }), ...(attention === undefined ? {} : { execution_id: attention.clear_execution ? undefined : attention.execution_id ?? record.execution_id, deferred: attention.deferred ?? record.deferred, execution_error: attention.execution_error === null ? undefined : attention.execution_error === undefined ? record.execution_error : sanitizeExecutionError(attention.execution_error) || undefined }) }; + // The code explains the transition that produced the CURRENT state, so a + // transition that names none clears it. While codes existed only on terminal + // records this could not matter — a terminal record returns above and is never + // rewritten — but a delivery reclaimed to `accepted` with its wake's outcome is + // claimed again later, and a carried-over code would describe the wrong state. + const { code: _replaced, ...carried } = record; + const next: Stored = { ...carried, state, updated_at: new Date().toISOString(), claim_generation: claim.generation, ...(code === undefined ? {} : { code }), ...(completedText === undefined ? {} : { text: sanitizeWakeCompletionText(completedText) }), ...(attention === undefined ? {} : { execution_id: attention.clear_execution ? undefined : attention.execution_id ?? record.execution_id, deferred: attention.deferred ?? record.deferred, execution_error: attention.execution_error === null ? undefined : attention.execution_error === undefined ? record.execution_error : sanitizeExecutionError(attention.execution_error) || undefined }) }; await this.replace(target, next); if (claim.acceptance_ids === undefined && (isTerminal(state) || state === "accepted")) { const currentClaim = await this.readClaimOptional(this.claimFor(record)); diff --git a/src/runtime/wakeAcceptanceTypes.ts b/src/runtime/wakeAcceptanceTypes.ts index 1bc8e40..8c6ad3a 100644 --- a/src/runtime/wakeAcceptanceTypes.ts +++ b/src/runtime/wakeAcceptanceTypes.ts @@ -25,12 +25,21 @@ export const WAKE_ACCEPTANCE_REQUEST_SCHEMA = { export const WAKE_RECEIPT_STATUS_SCHEMA = { $schema: "https://json-schema.org/draft/2020-12/schema", $id: WAKE_RECEIPT_STATUS_VERSION, type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at"], properties: { - version: { const: WAKE_RECEIPT_STATUS_VERSION }, acceptance_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, agent_id: { type: "string" }, delivery_id: { type: "string" }, request_digest: { type: "string", pattern: "^[a-f0-9]{64}$" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: { type: "string" }, updated_at: { type: "string" }, execution_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, deferred: { type: "boolean" }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] }, text: { type: "string", maxLength: MAX_WAKE_COMPLETION_TEXT_BYTES } + version: { const: WAKE_RECEIPT_STATUS_VERSION }, acceptance_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, agent_id: { type: "string" }, delivery_id: { type: "string" }, request_digest: { type: "string", pattern: "^[a-f0-9]{64}$" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: { type: "string" }, updated_at: { type: "string" }, execution_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, deferred: { type: "boolean" }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] }, text: { type: "string", maxLength: MAX_WAKE_COMPLETION_TEXT_BYTES } } } as const; export type WakeReceiptState = "accepted" | "running" | "completed" | "failed" | "stopped"; -export type WakeReceiptCode = "engine_failed" | "host_stopped" | "host_stopping" | "queue_full" | "unknown_agent"; +/** + * Why a receipt reached its state, and every member is a state the runtime really + * produces: `queued_wake_stopped` and `active_wake_aborted` are the two shapes a + * shutdown gives a wake (`organizationRuntimeHost.ts` settles a queued job with the + * first and the in-flight one with the second), and a delivery reclaimed for restart + * used to record neither, because the only caller that could name them passed no + * code at all. A code that cannot be determined stays ABSENT: a plausible name for + * an undetermined cause is worse than no name, because it is acted on. + */ +export type WakeReceiptCode = "engine_failed" | "host_stopped" | "host_stopping" | "queued_wake_stopped" | "active_wake_aborted" | "queue_full" | "unknown_agent"; export type OrganizationRuntimeWakeAcceptanceRequest = Readonly<{ token: string | undefined; agent_id: string; @@ -71,6 +80,12 @@ export type OrganizationRuntimeActivityV2Item = OrganizationRuntimeWakeReceiptSt }>; export type OrganizationRuntimeActivityV2 = Readonly<{ version: typeof ACTIVITY_V2_VERSION; + /** + * Whether this projection was read from a live runtime or sealed as the host + * stopped. Optional on the wire because a projection published before the seal + * existed must still parse; its absence means "not stated", never "running". + */ + state?: "running" | "stopped"; items: readonly OrganizationRuntimeActivityV2Item[]; executions?: readonly Readonly<{ agent_id: string; execution_id: string; state: "running"; delivery_ids: readonly string[] }>[]; }>;