diff --git a/README.md b/README.md index 60cf74b..ce35a4e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,22 @@ import { PiHarnessAdapter } from "@noopolis/daimon/pi"; ## Organization-runtime contract +For a strict Codex agent in an already prepared runtime, the public +`resolveOrganizationCodexSandboxProjection(config, agentId, { acceptanceStorePath })` +API returns `noopolis.daimon.codex-sandbox-projection.v1`: canonical workspace/home +paths, the resolved executable, and `sandboxArgs` containing the exact permission +profile rendered for production. Append only `--` and a mechanical command when +checking whether that policy can execute in the caller's container. + +The resolver opens and verifies caller-owned path identities and executes the +Codex version probe. It does not import/read authentication, create an agent, +accept a wake or invoke a model. The acceptance-store path must be the same one +given to the control host. This API projects command permissions, not the complete +cognition invocation: Codex's `sandbox` subcommand lacks the strict configuration +flags accepted by `exec`. A caller must therefore use a fresh empty `HOME` and +`CODEX_HOME` and a restricted environment for its mechanical probe. Do not infer +successful tool use or completed agent work from this projection alone. + `@noopolis/daimon/runtime` exports a standard JSON Schema for structural validation plus the strict, side-effect-free semantic `validateOrganizationRuntimeConfig` / `parseOrganizationRuntimeConfig` API and diff --git a/docs/engines.md b/docs/engines.md index 7512b3a..6342210 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -81,6 +81,38 @@ the broker owns refresh and stale-credential recovery. The runtime checks broker readiness before admitting Grok agents and verifies their sandbox policy before turns. The older credential-lease helper is not the production host path. +The broker worker is pinned to Grok CLI 1.0.34 and runs lean: a fixed Daimon +system prompt, six tools (`run_terminal_command`, `read_file`, `grep`, +`list_dir`, and the MCP meta-tools `search_tool`/`use_tool`), no bundled +skills, workflows, plan mode, subagents, memory or web search, and a declared +model and reasoning effort from a closed list (default `grok-4.6` at `low`). +The broker proxy refuses any request outside that shape before it spends. + +Each broker registration (`service.json` v2) declares its model and effort, +its usage ledger, and turn limits `{maxRequests, maxTokens, timeoutMs}`. A wake +may only lower them (`DAIMON_ENGINE_WAKE_TIMEOUT_MS`, +`DAIMON_ENGINE_WAKE_TOKEN_CEILING`; the `DAIMON_CODEX_WAKE_*` names are +aliases). The proxy refuses request `maxRequests + 1` and any request after the +deadline with HTTP 429 before upstream, and stops admitting requests once the +upstream-reported running total (cached input included) reaches `maxTokens`, so +a turn overshoots its token ceiling by at most one request. A tripped limit +kills the worker. The broker seals every terminal turn with its usage, request +count, declared model and limit reason, and writes one usage row (keyed by +`turn`) plus per-request rows for completed and failed turns alike; a replayed +turn is never metered twice. `resolveOrganizationGrokBrokerProjection` exposes +a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v2` +receipts bind a slot's denied-path canaries to that projection's digest and to +one recycle (the caller's nonce and the slot's increasing generation). + +Evaluators (Paideia judges and the optimizer, organization uid only) borrow the +same credential through inference grants: `request_inference_grant` over the +control socket returns a ten-minute token for one declared model and effort, +which the evaluator's Grok CLI presents to the provider proxy through +`env_key` in a config rendered by `renderGrokInferenceClientConfig`. Grant +requests must carry no tools, are metered like a turn, and are written only to +the broker's separate `inferenceLedgerPath` (`kind: "inference"` rows), never +to a subject usage ledger or the wake fuse. + AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: diff --git a/package-lock.json b/package-lock.json index d046ccf..ffeb234 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "bin": { "daimon-runtime": "dist/runtime/cli.js" diff --git a/package.json b/package.json index dc42793..1f22e90 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^24.12.4", diff --git a/scripts/liveGrokBrokerSession.ts b/scripts/liveGrokBrokerSession.ts index 092bb9b..83c8772 100644 --- a/scripts/liveGrokBrokerSession.ts +++ b/scripts/liveGrokBrokerSession.ts @@ -8,8 +8,11 @@ import { readChild } from "../src/pi/cliChildOutput.ts"; import { terminateChild, trackCliChild } from "../src/pi/cliProcess.ts"; import { decodeGrokHeadlessTurn } from "../src/pi/grokHeadlessResult.ts"; import { readGrokBrokerCredential } from "../src/runtime/grokBrokerCredentialReader.ts"; +import { DEFAULT_GROK_BROKER_TURN_LIMITS } from "../src/runtime/engineBrokerTurnAccounting.ts"; import { startGrokBrokerProxy } from "../src/runtime/grokBrokerProxy.ts"; -import { renderGrokBrokerWorkerConfig } from "../src/runtime/grokBrokerWorkerConfig.ts"; +import { GrokBrokerTurnMeter } from "../src/runtime/grokBrokerTurnMeter.ts"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "../src/runtime/grokBrokerModelPolicy.ts"; +import { GROK_BROKER_PROVIDER_CAPABILITY_ENV, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; // Explicit live auth/transport check, not the Linux native worker/isolation E2E. // Read the operator credential only in this process; never stage or rotate it. @@ -37,20 +40,21 @@ try { const capability = proxy.capabilities.issue("local-auth-probe", turnId); // This local transport probe deliberately does not attest a native worker. proxy.registerIsolationGuard(turnId, async () => undefined); - const helper = path.join(home, "auth-helper"); - await writeFile(helper, `#!/bin/sh\nprintf '{"access_token":"${capability}","expires_in":600}\\n'\n`, { mode: 0o700 }); + // The proxy forwards nothing unmetered; the probe runs under the default v1 limits. + proxy.registerTurn(turnId, { policy: DEFAULT_GROK_BROKER_MODEL_POLICY, meter: new GrokBrokerTurnMeter(DEFAULT_GROK_BROKER_TURN_LIMITS) }); // No MCP tools are needed for this exact-reply authentication probe. - await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfig(helper, proxy.port).split("[mcp_servers.daimon]")[0]); + await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); const prompt = path.join(home, "prompt.txt"); await writeFile(prompt, `Reply exactly ${sentinel}. Do not use tools.`); stage = `model turn ${round}`; - const child = trackCliChild(spawn("grok", [ - "--sandbox", "strict", "--prompt-file", prompt, "--no-memory", "--no-subagents", - "--disable-web-search", "--max-turns", "1", "--permission-mode", "dontAsk", - "--model", "daimon-broker-grok", "--output-format", "streaming-messages-json", - ], { + // The proxy only forwards the lean worker request shape (pinned client + // version, exact tool set, declared effort), so the probe uses the same + // argv as the native launcher with the built-in strict profile. + const args = [...renderGrokBrokerWorkerArgs(prompt, home)].map((value) => value === "daimon-strict" ? "strict" : value); + args[args.indexOf("--max-turns") + 1] = "1"; + const child = trackCliChild(spawn("grok", args, { cwd: home, detached: process.platform !== "win32", - env: { PATH: process.env.PATH, HOME: home, GROK_HOME: home, LANG: "C", LC_ALL: "C", TZ: "UTC" }, + env: { PATH: process.env.PATH, HOME: home, GROK_HOME: home, LANG: "C", LC_ALL: "C", TZ: "UTC", [GROK_BROKER_PROVIDER_CAPABILITY_ENV]: capability }, stdio: ["ignore", "pipe", "pipe"], })); let output: string; diff --git a/scripts/scriptSourcePolicy.test.ts b/scripts/scriptSourcePolicy.test.ts index 1a73f68..bea5b23 100644 --- a/scripts/scriptSourcePolicy.test.ts +++ b/scripts/scriptSourcePolicy.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -37,3 +37,35 @@ test("script source policy detects a maintained JavaScript regression", async () await rm(root, { force: true, recursive: true }); } }); + +// Raw control bytes make review tooling classify a source file as binary and skip it. +// Escape them (`\u0000`) instead; tab, newline and carriage return are the only exceptions. +const RAW_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u; +const textSourcesWithRawControlBytes = async (roots: string[]): Promise => { + const results: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "artifacts" && entry.name !== "node_modules") await walk(entryPath); continue; } + if (!/\.(?:ts|mts|mjs|js|json|jsonl|md|c|h|inc|toml|sh|yml|yaml)$/u.test(entry.name)) continue; + if (RAW_CONTROL.test(await readFile(entryPath, "latin1"))) results.push(entryPath); + } + }; + await Promise.all(roots.map(walk)); + return results.sort(); +}; + +test("maintained text sources contain no raw control bytes", async () => { + assert.deepEqual(await textSourcesWithRawControlBytes(["src", "scripts", "docs"]), []); +}); + +test("raw control byte policy detects a regression", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-control-policy-")); + try { + await writeFile(path.join(root, "regex.ts"), `export const r = /[${String.fromCharCode(0)}-${String.fromCharCode(0x1f)}]/u;\n`); + await writeFile(path.join(root, "clean.ts"), "export const r = /[\\u0000-\\u001f]/u;\n"); + assert.deepEqual(await textSourcesWithRawControlBytes([root]), [path.join(root, "regex.ts")]); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts new file mode 100644 index 0000000..2c49b12 --- /dev/null +++ b/src/contracts/grokWorkerContract.ts @@ -0,0 +1,95 @@ +/** + * Fixed operating contract every broker-launched Grok worker receives through + * `--system-prompt-override`. + * + * The override replaces Grok's ~12k-token coding-agent system prompt and stops + * cwd `AGENTS.md` injection; the agent's identity and instructions still arrive + * in the prompt file. It is compiled into the native launcher byte-for-byte + * (`DBL_GROK_SYSTEM_PROMPT`), pinned by sha256 in the runtime contract + * manifest, and must stay ASCII without quotes or backslashes so the C literal + * needs no escaping. + * + * Daimon's per-wake tools reach Grok as deferred MCP tools named + * `daimon__` (server `[mcp_servers.daimon]`). Naming them lets `use_tool` + * run directly and saves one `search_tool` round trip per tool (P0: 3 → 2 + * requests). + */ + +/** + * The atoms of that route, and its single definition. + * + * Both texts a Grok worker receives are rendered from them: the pinned system + * prompt below, and the caller's identity envelope + * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). + * They were worded independently once, and the envelope told the model to call + * the tools by their bare names - which Grok 1.0.34 refuses outright, before + * any HTTP: `'moltnet_read' is not a valid MCP tool name. Tool names must be + * qualified as \`server__tool\`` (local rig, real CLI, real rendered config). + * There is exactly one valid spelling, so two independently worded naming rules + * are one rule too many; this is the single definition both render from. + */ +export const DAIMON_GROK_MCP_SERVER = "daimon" as const; +/** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ +export const DAIMON_GROK_TOOL_PREFIX = `${DAIMON_GROK_MCP_SERVER}__` as const; +export const grokDaimonToolName = (tool: string): string => `${DAIMON_GROK_TOOL_PREFIX}${tool}`; +/** Grok's two MCP meta-tools, and the argument that names a tool for the first. */ +export const GROK_MCP_INVOKE_TOOL = "use_tool" as const; +export const GROK_MCP_SEARCH_TOOL = "search_tool" as const; +export const GROK_MCP_TOOL_NAME_ARGUMENT = "tool_name" as const; +/** Illustrative Daimon tools for the system prompt, which cannot know a wake's real mount. */ +const DAIMON_GROK_EXAMPLE_TOOLS = Object.freeze(["moltnet_read", "moltnet_send", "memory_search", "memory_register"] as const); + +export const DAIMON_GROK_SYSTEM_PROMPT = [ + "You are a headless Daimon agent; no human is present.", + "Your identity, instructions and wake event are in the user prompt.", + `Daimon tools are MCP tools on server ${DAIMON_GROK_MCP_SERVER}: call a known one directly with ${GROK_MCP_INVOKE_TOOL} (${GROK_MCP_TOOL_NAME_ARGUMENT} ${DAIMON_GROK_EXAMPLE_TOOLS.map(grokDaimonToolName).join(", ")}, or another ${DAIMON_GROK_TOOL_PREFIX} name you were given); use ${GROK_MCP_SEARCH_TOOL} only for a name you do not know.`, + "If a tool result says output was saved to a file, read that path with read_file.", + "If a tool fails, do not retry it in a loop: stop and report the failure.", + "Your final answer is a private note to the runtime: one line, or empty." +].join(" "); + +/** + * The same route, stated once for the caller's identity envelope, where a + * wake's real mounted tools are known. + * + * It contributes exactly what the pinned prompt cannot know - the wake's real + * mounted names - and the one rule that makes them callable. It asserts rather + * than corrects: one bare catalogue, one prefix rule, one example, and the + * prefixed form named as the *only* valid form, because that is the CLI's own + * verdict on a bare name rather than a preference. + * + * What it deliberately leaves out is as load bearing. It never claims the + * agent's own instructions spell a tool wrongly, never offers a shell or CLI + * route, never repeats the catalogue in prefixed form - and never restates the + * `search_tool` rule. A Grok worker already reads two authoritative sentences + * about `search_tool`: the pinned prompt's ("only for a name you do not know") + * and Grok's own injected notice, which says the model MUST call it before any + * MCP tool. Observed on the rig: that contradiction is not enforced, and + * `use_tool` works with no prior `search_tool`. A third wording would only add + * a voice, so this sentence stays out of that argument entirely. + */ +export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { + const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); + return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}, and its only valid tool name is ` + + `${DAIMON_GROK_TOOL_PREFIX}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${example}. ` + + "A bare name is not a valid MCP tool name and reaches nothing."; +}; + +/** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ +export const GROK_BROKER_MODELS = Object.freeze(["grok-4.6", "grok-4.5", "grok-build"] as const); +export const GROK_BROKER_REASONING_EFFORTS = Object.freeze(["low", "medium", "high"] as const); + +/** `--tools` input ids. These are NOT the model-visible names (see below). */ +export const GROK_WORKER_TOOL_IDS = Object.freeze(["run_terminal_cmd", "read_file", "grep", "list_dir", "search_tool", "use_tool"] as const); + +/** + * The exact tool names a lean worker request body must carry. + * + * Grok 1.0.34 fails open on an unmappable `--tools` entry and ships all 19 + * tools, so the proxy compares every upstream body against this set. + */ +export const GROK_WORKER_VISIBLE_TOOLS = Object.freeze(["grep", "list_dir", "read_file", "run_terminal_command", "search_tool", "use_tool"] as const); + +/** `--max-turns` backstop compiled into the launcher; per-wake ceilings belong to the broker. */ +export const GROK_WORKER_MAX_TURNS = 48 as const; diff --git a/src/contracts/organizationRuntimeContract.ts b/src/contracts/organizationRuntimeContract.ts index 8f2b9ca..ce87b07 100644 --- a/src/contracts/organizationRuntimeContract.ts +++ b/src/contracts/organizationRuntimeContract.ts @@ -1,3 +1,5 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokWorkerContract.js"; + /** The data-only organization-runtime constants shared by product code and artifacts. */ export const ORGANIZATION_RUNTIME_VERSION = "noopolis.daimon.organization-runtime.v1" as const; export const ORGANIZATION_RUNTIME_V2_VERSION = "noopolis.daimon.organization-runtime.v2" as const; @@ -74,7 +76,11 @@ export const ORGANIZATION_RUNTIME_CONFIG_SCHEMA = { codexSandbox: { type: "object", additionalProperties: false, required: ["mode", "networkAccess", "webSearch"], properties: { mode: { const: "workspace-write" }, networkAccess: { const: false }, webSearch: { const: "disabled" } } } - } }, + }, allOf: [ + // grok: a declared model is the closed broker pair, both or neither; never a Codex sandbox. + { if: { properties: { kind: { const: "grok" } } }, then: { properties: { model: { enum: GROK_BROKER_MODELS }, reasoningEffort: { enum: GROK_BROKER_REASONING_EFFORTS }, codexSandbox: false }, dependentRequired: { model: ["reasoningEffort"], reasoningEffort: ["model"] } } }, + { if: { properties: { kind: { const: "agy" } } }, then: { properties: { model: false, reasoningEffort: false, codexSandbox: false } } } + ] }, ...PRODUCTION_TOOL_PROPERTIES } } } diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index b462d54..687b104 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -1,4 +1,5 @@ import { WORK_AVAILABILITY_SCHEMA, WORK_BLOCKED_SCHEMA } from "./attentionContract.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS, GROK_WORKER_VISIBLE_TOOLS } from "./grokWorkerContract.js"; import { ORGANIZATION_RUNTIME_CONFIG_SCHEMA, ORGANIZATION_RUNTIME_CONFIG_V2_SCHEMA, @@ -34,11 +35,107 @@ export const GROK_ENGINE_BROKER = { providerProxy: { host: "127.0.0.1", port: 43_123 }, mcpFacade: { host: "127.0.0.1", port: 43_124, path: "/mcp" }, identities: { organizationUid: 2_000, brokerUid: 2_100, firstWorkerUid: 2_200 }, - bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + grokCliVersion: "1.0.34", + grokCliBuild: "3736acbc8658", + grokCliArtifacts: { + arm64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64", sha256: "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", bytes: 136_090_504 }, + x64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64", sha256: "be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d", bytes: 163_035_648 } + }, + worker: { + modelId: "daimon-broker-grok", + models: GROK_BROKER_MODELS, + reasoningEfforts: GROK_BROKER_REASONING_EFFORTS, + defaultModel: "grok-4.6", + defaultReasoningEffort: "low", + toolIds: GROK_WORKER_TOOL_IDS, + visibleTools: GROK_WORKER_VISIBLE_TOOLS, + maxTurns: GROK_WORKER_MAX_TURNS, + systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", + // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. + configSha256: { + "grok-4.6": { low: "ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e", medium: "df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f", high: "65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3" }, + "grok-4.5": { low: "8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6", medium: "59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155", high: "c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f" }, + "grok-build": { low: "fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5", medium: "8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656", high: "a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3" } + }, + // Worker `GROK_HOME` layout the broker attests before every turn. The home and + // its `sessions/` directory are root-owned, worker-group writable and sticky so + // Grok can create its own state but never replace a root-owned file. + home: { + directory: { uid: 0, group: "worker", mode: 0o1771 }, + sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, + readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, + // The launcher exports TMPDIR=/tmp; Grok's strict profile grants TMPDIR read-write. + privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, + // Strict also grants shared /tmp and /var/tmp read-write and refuses to start if either is + // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker + // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. + sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // The organization runtime home of a brokered Grok agent: traverse-only for the + // worker group so the worker can reach `tool-output/` and nothing else (no group + // read, no group write, no world bits; `physicalReadiness.ts` refuses anything else). + organizationRuntimeHome: { owner: "organization", group: "worker", mode: 0o710 }, + // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, + // files written 0640 by the runtime, never other-readable. + spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } + } + }, + bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 262_144 }, + // Accounting and limits (P2). The broker is the single sealed usage writer. + controlProtocolVersion: "noopolis.daimon.engine-broker.v2", + turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], + serviceConfigVersions: ["noopolis.daimon.engine-broker-service.v1", "noopolis.daimon.engine-broker-service.v2"], + turnLimits: { + keys: ["maxRequests", "maxTokens", "timeoutMs"], + v1Defaults: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + bounds: { maxRequests: [1, GROK_WORKER_MAX_TURNS], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, + limitReasons: ["tokens", "requests", "timeout", "none"], + wakeMayOnlyLower: true, + tokenCeilingOvershoot: "at-most-one-request", + maxInFlightRequests: 1, + // A per-request usage block above this is implausible (beyond the model + // context window) and treated as invalid rather than added to any total. + requestUsageMaxTokens: 500_000, + // A request whose response carries no valid usage is charged this estimate. + missingUsageEstimate: { inputBytesPerToken: 2, outputTokens: 4_096 } + }, + wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, + // Evaluator inference grants (P2c). Judges and the optimizer (organization uid + // only, over the control socket) borrow the broker's Grok credential through + // the provider proxy; they never hold it, and their spend never reaches the + // subject usage ledger or wake fuse. + inferenceGrants: { + requestKinds: ["request_inference_grant", "release_inference_grant"], + purposes: ["judge", "optimizer"], + tokenPrefix: "inference_", + ttlMs: 600_000, + limits: { maxRequests: 64, maxTokens: 2_000_000 }, + maxLiveGrants: 8, + maxInFlightRequestsPerGrant: 1, + // Top-level request members Grok 1.0.34 sends for a Paideia judge/optimizer call + // (live stub capture); `tools` and `tool_choice` are refused outright. + bodyMembers: ["messages", "model", "reasoning_effort", "response_format", "stream", "stream_options"], + messageRoles: ["system", "user", "assistant"], + failureCodes: ["auth_stale", "grant_limit", "invalid_request", "unavailable"], + ledgerVersion: "noopolis.daimon.inference-usage.v1", + ledgerDedupeKey: ["grant", "request"], + client: { + modelId: "daimon-inference-grok", + envKey: "DAIMON_INFERENCE_GRANT", + // sha256 of `renderGrokInferenceClientConfig` for the production proxy base URL and this env key. + configSha256: { + "grok-4.6": { low: "79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f", medium: "6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8", high: "5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac" }, + "grok-4.5": { low: "a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74", medium: "1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9", high: "ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2" }, + "grok-build": { low: "ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1", medium: "01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d", high: "98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c" } + } + } + }, + projectionVersion: "noopolis.daimon.grok-broker-projection.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58", - x64Sha256: "e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd", - arm64Sha256: "ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d" + sourceSha256: "dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24", + x64Sha256: "67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7", + arm64Sha256: "c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98" } } as const; export const AGY_SUBSCRIPTION_REALM = { @@ -88,5 +185,5 @@ export const RUNTIME_CONTRACT_MANIFEST = { ] }, healthResponseSchema: { type: "object", additionalProperties: false, required: ["version", "state", "agents"], properties: { version: { const: "noopolis.daimon.organization-runtime-health.v1" }, state: { enum: ["starting", "running", "stopping", "stopped"] }, agents: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agentId", "state"], properties: { agentId: text, state: { enum: ["starting", "running", "stopping", "stopped", "idle", "failed"] } } } } } }, activityResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: "noopolis.daimon.organization-runtime-activity.v1" }, items: { type: "array", maxItems: 100, items: activityItem }, nextCursor: { type: "string", minLength: 1, maxLength: 16, pattern: "^(0|[1-9][0-9]{0,15})$" } } }, - activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } + activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } } as const; diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 044b4ca..b594157 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -1,7 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; -import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; +import { appendFile, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -106,8 +107,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => }; const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { - const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true }); + const directory = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -182,7 +182,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true }); + await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry"); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -207,7 +207,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true }); + await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index c6dbd7f..bc55516 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -1,7 +1,8 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -206,8 +207,7 @@ export class OrgObserver { } async write(runtimeRoot: string): Promise { - const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true }); + const telemetryDir = await ensureRuntimeHomeDirectory(runtimeRoot, "telemetry"); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/AGENTS.md b/src/pi/AGENTS.md new file mode 100644 index 0000000..4685c74 --- /dev/null +++ b/src/pi/AGENTS.md @@ -0,0 +1,21 @@ +# Pi runtime implementation + +This directory owns the Pi harness and native CLI adaptation. Keep public +runtime contracts independent of Pi types. Files stay below 400 lines and +tests live beside their implementations. + +Codex permission rendering must preserve the effective path permissions while +removing redundant nested denies that cannot be mounted by its Linux sandbox. +An intervening readable path or workspace root makes a deeper deny necessary. +Verify changes with generated production arguments and real local sandbox +commands; a model call is neither required nor permitted for this check. + +The per-wake MCP mount is torn down on the wake's own completion path, so that +teardown must be bounded. `Server.close()` waits for every open connection, and +a connection the MCP transport has no record of — a socket opened before +`initialize`, or an idle keep-alive socket a client's pool still holds, which is +what relaying a turn through the broker MCP facade leaves behind — is not the +transport's to end. Close the transport first, then end the remaining +connections; never wait for the client to release them. A finished turn that +parks here publishes nothing and dies to an outer deadline, which loses exactly +the terminal evidence the turn existed to produce. diff --git a/src/pi/CLAUDE.md b/src/pi/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/src/pi/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/pi/cliChildOutput.ts b/src/pi/cliChildOutput.ts index 19b5ca2..78a87d4 100644 --- a/src/pi/cliChildOutput.ts +++ b/src/pi/cliChildOutput.ts @@ -1,13 +1,14 @@ import type { ChildProcess } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { headUtf8, tailUtf8 } from "../runtime/toolResultSpill.js"; import type { TurnUsageFailureReason } from "../runtime/turnUsageLedger.js"; import { decodeCodexTurnUsage, type CodexTurnUsage } from "./codexHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; /** Maximum assistant reply bytes retained from stdout. */ export const CLI_ENGINE_MAX_OUTPUT_BYTES = 64 * 1024; -/** Tail bytes retained from stderr only for a failed-child diagnostic. */ +/** Diagnostic bytes retained from stderr for a failed child: its head and its tail together. */ export const CLI_ENGINE_MAX_DIAGNOSTIC_BYTES = 768; const CLI_ENGINE_FAILURE_SCAN_CHARS = 256; @@ -39,18 +40,36 @@ const redactChildOutput = (value: string, secretValues: readonly string[]): stri return redactCredentialText(value, secretValues, CLI_ENGINE_MAX_OUTPUT_BYTES); }; -const utf8Tail = (value: string, maxBytes: number): string => { - const bytes = Buffer.from(value, "utf8"); - if (bytes.length <= maxBytes) return value; - let result = bytes.subarray(bytes.length - maxBytes).toString("utf8"); - while (result.startsWith("\uFFFD")) result = result.slice(1); - return result; +/** + * One bounded window over a failed child's own output: its head AND its tail, + * with an explicit marker naming the bytes elided between them. + * + * A pure tail is the wrong end for the process this exists for. A worker that + * dies early prints its error first and then echoes its own input, so the tail + * is the echo: one live brokered turn reported 512 bytes of its own prompt + * read back, with the actual error already off the front and discarded. Both + * ends cost the same window, and the marker is the one oversized tool results + * already use (`toolResultSpill.ts`), so a reader meets one shape everywhere. + * + * The marker is paid for out of the same budget — it is sized against the + * largest count it could carry — so the result never exceeds `maxBytes`, and + * output that fits is returned byte-identical with no marker at all. + */ +export const boundedDiagnosticWindow = (value: string, maxBytes: number): string => { + const total = Buffer.byteLength(value, "utf8"); + if (total <= maxBytes) return value; + const budget = Math.max(0, maxBytes - Buffer.byteLength(elisionMarker(total), "utf8")); + const head = headUtf8(value, Math.floor(budget / 2)); + const tail = tailUtf8(value, budget - Buffer.byteLength(head, "utf8")); + const elided = total - Buffer.byteLength(head, "utf8") - Buffer.byteLength(tail, "utf8"); + return `${head}${elisionMarker(elided)}${tail}`; }; +const elisionMarker = (elided: number): string => `[… ${elided} bytes elided …]`; const childDiagnostic = (stdout: string, stderr: string, secretValues: readonly string[]): string => { const output = stderr.trim().length > 0 ? stderr : stdout; const redacted = redactCredentialText(output, secretValues, Number.MAX_SAFE_INTEGER).trim(); - const bounded = utf8Tail(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + const bounded = boundedDiagnosticWindow(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return bounded.length > 0 ? `: ${bounded}` : ""; }; @@ -104,7 +123,13 @@ export const readChild = ( const stdout: Buffer[] = []; let stdoutTail = Buffer.alloc(0); let droppingStdoutLine = false; + // Both ends of stderr, retained as it streams: the head frozen once it is + // full, the tail sliding. A single sliding tail dropped the head at capture + // time, which is where the cause of an early death lives — no later window + // can recover what was never kept. + let stderrHead = Buffer.alloc(0); let stderrTail = Buffer.alloc(0); + let stderrBytes = 0; let stdoutBytes = 0; let stdoutRemainder = Buffer.alloc(0); let droppingNdjsonLine = false; @@ -113,8 +138,18 @@ export const readChild = ( let cleanupStarted = false; let classifiedFailure: Error | undefined; let failureScanTail = ""; - const stderrRetentionBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - + Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + /** + * Each retained end carries its reported share PLUS one whole secret, which + * is the invariant that keeps exact redaction exact: a secret that reaches + * the reported window can extend at most its own length past that window's + * cut, so retaining that much more on each side means the redactor always + * sees the secret whole. Halving one shared budget instead broke it — a + * 2000-byte secret was cut in the middle and its tail fragment + * (`…qqq-secret-end`) reached the diagnostic verbatim. + */ + const stderrSecretAllowance = Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + const stderrHeadBytes = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; + const stderrTailBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; const settle = (action: () => void): void => { if (settled) return; settled = true; @@ -221,18 +256,33 @@ export const readChild = ( } stdout.push(value); }; - const retainStderrTail = (chunk: Buffer): void => { + const retainStderrWindow = (chunk: Buffer): void => { const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); classifyFailure(value); - if (value.length >= stderrRetentionBytes) { - stderrTail = Buffer.from(value.subarray(value.length - stderrRetentionBytes)); + stderrBytes += value.length; + if (stderrHead.length < stderrHeadBytes) stderrHead = Buffer.concat([stderrHead, value.subarray(0, stderrHeadBytes - stderrHead.length)]); + if (value.length >= stderrTailBytes) { + stderrTail = Buffer.from(value.subarray(value.length - stderrTailBytes)); return; } - const overflow = stderrTail.length + value.length - stderrRetentionBytes; + const overflow = stderrTail.length + value.length - stderrTailBytes; stderrTail = Buffer.concat([overflow > 0 ? stderrTail.subarray(overflow) : stderrTail, value]); }; + /** + * The retained stderr, reassembled exactly. + * + * Nothing was elided while the whole output fitted the budget, and then the + * two ends overlap: they tile the stream, so dropping the overlap from the + * tail rebuilds it byte-identically. Above the budget the ends are joined by + * the marker, which names how many bytes never reached this process at all. + */ + const retainedStderr = (): string => { + const elided = stderrBytes - stderrHead.length - stderrTail.length; + if (elided <= 0) return Buffer.concat([stderrHead, stderrTail.subarray(stderrHead.length + stderrTail.length - stderrBytes)]).toString("utf8"); + return `${stderrHead.toString("utf8")}${elisionMarker(elided)}${stderrTail.toString("utf8")}`; + }; child.stdout?.on("data", retainStdout); - child.stderr?.on("data", retainStderrTail); + child.stderr?.on("data", retainStderrWindow); const timer = timeoutMs === undefined ? undefined : setTimeout(() => abort(tagCliChildFailure(new Error(options.timeoutErrorMessage ?? "CLI engine timed out"), "wake_timeout")), timeoutMs); child.once("error", abort); child.once("close", (code, signal) => { @@ -244,7 +294,7 @@ export const readChild = ( return; } settle(() => reject(tagCliChildFailure(classifiedFailure ?? new Error(`CLI engine exited ${code ?? signal}${childDiagnostic( - retainedStdout.toString("utf8"), stderrTail.toString("utf8"), secretValues + retainedStdout.toString("utf8"), retainedStderr(), secretValues )}`), "engine_exit"))); }); }); diff --git a/src/pi/cliEngineSpawn.ts b/src/pi/cliEngineSpawn.ts index 861cc58..3b4ef99 100644 --- a/src/pi/cliEngineSpawn.ts +++ b/src/pi/cliEngineSpawn.ts @@ -2,6 +2,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { trackCliChild } from "./cliProcess.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; +import { codexFilesystemRules } from "./codexFilesystemRules.js"; import type { CliEngineOptions, CliSessionInput } from "./cliSession.js"; export const GROK_STRICT_SANDBOX_PROFILE = "strict"; @@ -49,7 +50,7 @@ export const renderCodexArgs = ( "--strict-config", "-c", "web_search=\"disabled\"", "-c", `default_permissions=${JSON.stringify(profileName)}`, - "-c", renderCodexPermissionProfile(profileName, options.codexSandboxProtectedPaths ?? [], options.codexSandboxReadablePaths ?? []), + "-c", renderCodexPermissionProfile(profileName, options.codexSandboxProtectedPaths ?? [], options.codexSandboxReadablePaths ?? [], cwd), "-c", "approval_policy=\"never\"", "-c", `mcp_servers={daimon={url=\"${endpoint}\",enabled=true,default_tools_approval_mode=\"approve\"}}` ]; @@ -66,13 +67,13 @@ export const renderCodexArgs = ( export const renderCodexPermissionProfile = ( profileName: string, protectedPaths: readonly string[], - readablePaths: readonly string[] = [] + readablePaths: readonly string[] = [], + workspacePath?: string ): string => { const filesystem: Record> = { - ":workspace_roots": { ".": "write" } + ":workspace_roots": { ".": "write" }, + ...codexFilesystemRules(protectedPaths, readablePaths, workspacePath) }; - for (const readablePath of readablePaths) filesystem[readablePath] = "read"; - for (const protectedPath of protectedPaths) filesystem[protectedPath] = "deny"; return `permissions=${tomlInline({ [profileName]: { extends: ":workspace", filesystem, diff --git a/src/pi/cliMcpRegistration.test.ts b/src/pi/cliMcpRegistration.test.ts index f725b3d..efa1776 100644 --- a/src/pi/cliMcpRegistration.test.ts +++ b/src/pi/cliMcpRegistration.test.ts @@ -5,18 +5,9 @@ import { renderAgyArgs } from "./cliEngineSpawn.js"; import { DAIMON_MCP_SERVER_NAME, renderAgyMcpAddArgs, - renderAgyMcpRemoveArgs, - renderGrokMcpAddArgs, - renderGrokMcpRemoveArgs + renderAgyMcpRemoveArgs } from "./cliMcpRegistration.js"; -test("Grok's registration arguments are unchanged by the AGY generalization", () => { - assert.deepEqual(renderGrokMcpAddArgs([], "strict", "http://127.0.0.1:1/mcp"), - ["--sandbox", "strict", "mcp", "add", "--transport", "http", "--scope", "project", "daimon", "http://127.0.0.1:1/mcp"]); - assert.deepEqual(renderGrokMcpRemoveArgs([], "strict"), - ["--sandbox", "strict", "mcp", "remove", "--scope", "project", "daimon"]); -}); - test("AGY registers the per-wake endpoint as an http server, flags before the name", () => { const args = renderAgyMcpAddArgs([], "http://127.0.0.1:54321/mcp"); assert.deepEqual(args, ["mcp", "add", "--type", "http", DAIMON_MCP_SERVER_NAME, "http://127.0.0.1:54321/mcp"]); diff --git a/src/pi/cliMcpRegistration.ts b/src/pi/cliMcpRegistration.ts index 83d5993..0d5118f 100644 --- a/src/pi/cliMcpRegistration.ts +++ b/src/pi/cliMcpRegistration.ts @@ -2,16 +2,17 @@ import { spawn, type ChildProcess } from "node:child_process"; import { readChild } from "./cliChildOutput.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; -import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; /** * Per-wake MCP endpoint registration for the CLI engines that cannot take the * endpoint on their own command line. * * Codex takes `-c mcp_servers.daimon.url=` per invocation and needs - * nothing here. Grok and AGY are both config-file driven, so Daimon registers - * the ephemeral endpoint before the turn and removes it afterwards, through - * each CLI's own `mcp add`/`mcp remove` subcommands. + * nothing here. AGY is config-file driven, so Daimon registers the ephemeral + * endpoint before the turn and removes it afterwards through its own `mcp + * add`/`mcp remove` subcommands. Grok no longer uses this path: its project + * scope is ignored in untrusted workspaces on 1.0.34, so its endpoint is written + * into the agent's Daimon-owned GROK_HOME (`grokHomeMcpRegistration.ts`). * * The registration is deliberately performed by the engine CLI rather than by * writing its config file directly: the file format belongs to the engine, and @@ -90,15 +91,9 @@ export const registerCliMcpServer = async ( }; }; -/** The MCP server name both engines register Daimon's per-wake endpoint under. */ +/** The MCP server name every CLI engine registers Daimon's per-wake endpoint under. */ export const DAIMON_MCP_SERVER_NAME = "daimon" as const; -export const renderGrokMcpAddArgs = (commandArgs: readonly string[] | undefined, profile: string, endpoint: string): string[] => - [...renderGrokSandboxArgs(commandArgs, profile), "mcp", "add", "--transport", "http", "--scope", "project", DAIMON_MCP_SERVER_NAME, endpoint]; - -export const renderGrokMcpRemoveArgs = (commandArgs: readonly string[] | undefined, profile: string): string[] => - [...renderGrokSandboxArgs(commandArgs, profile), "mcp", "remove", "--scope", "project", DAIMON_MCP_SERVER_NAME]; - /** * `agy mcp add --type http `. * diff --git a/src/pi/cliSession.test.ts b/src/pi/cliSession.test.ts index 3e76d9d..4c9a489 100644 --- a/src/pi/cliSession.test.ts +++ b/src/pi/cliSession.test.ts @@ -471,16 +471,16 @@ test("disposing from a Server.prototype.listen interleaving never leaves an MCP } }); -test("disposing during Grok registration terminates setup before the engine starts", async (context) => { +test("disposing during AGY MCP registration terminates setup before the engine starts", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-grok-cancel-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-agy-cancel-")); const ready = path.join(root, "add-ready"); const marker = path.join(root, "engine-started"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { writeFileSync } from "node:fs"; const args = process.argv.slice(2); if (args.includes("add")) { writeFileSync(${JSON.stringify(ready)}, "ready"); process.on("SIGTERM", () => undefined); setInterval(() => undefined, 1000); } else if (args.includes("remove")) process.exit(0); else writeFileSync(${JSON.stringify(marker)}, "started");`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); const pending = session.prompt("cancel"); void pending.catch(() => undefined); diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index ac5982a..b045f4b 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; import { createServer, type Server } from "node:http"; import { spawn, type ChildProcess } from "node:child_process"; -import { mkdir } from "node:fs/promises"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; @@ -18,7 +17,6 @@ import type { TurnUsageOutcome } from "../runtime/turnUsageLedger.js"; import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { - GROK_STRICT_SANDBOX_PROFILE, renderCodexArgs, spawnEngine } from "./cliEngineSpawn.js"; @@ -26,10 +24,9 @@ import { registerCliMcpServer, renderAgyMcpAddArgs, renderAgyMcpRemoveArgs, - renderGrokMcpAddArgs, - renderGrokMcpRemoveArgs, type CliMcpRegistration } from "./cliMcpRegistration.js"; +import { registerGrokHomeMcpServer } from "./grokHomeMcpRegistration.js"; import { decodeAgyHeadlessTurn, type AgyTurnUsage } from "./agyHeadlessResult.js"; import { type CodexTurnUsage } from "./codexHeadlessResult.js"; import { createCliTurnMeter, decodeCodexTurn, failedTurnOutcome, publishTurnRequests, publishTurnUsage } from "./cliTurnMetering.js"; @@ -37,6 +34,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -61,36 +59,10 @@ export type CliEngineKind = "agy" | "codex" | "grok"; */ export const AGY_MAX_TOOL_TURNS = 16; -/** - * Codex's per-wake bounds, and the one place they are decided. - * - * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell - * (`exec_command`) is never routed through that gate, so a single Codex turn - * previously had no ceiling at all — one production wake ran 23:32→23:42 - * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream - * reports token usage exactly once, on `turn.completed` — there is no - * incremental total to watch mid-turn (verified against a live multi-tool-call - * turn: `item.completed` fires once per tool call, but usage is reported only - * on the single terminal `turn.completed`) — so the token ceiling is the best - * bound obtainable from that wire shape: it converts an over-budget turn into - * an explicit, killed, named failure instead of a silent success, and the - * wall-clock timeout is what actually interrupts a runaway turn in progress. - */ -export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; -export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; -export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; -export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; - -const positiveInteger = (value: string | undefined, fallback: number, name: string): number => { - if (value === undefined) return fallback; - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); - return parsed; -}; -export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV], DEFAULT_CODEX_WAKE_TIMEOUT_MS, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); -export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV], DEFAULT_CODEX_WAKE_TOKEN_CEILING, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); +export { + DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, + DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides +} from "./engineWakeLimits.js"; export type CliEngineOptions = { readonly commandArgs?: readonly string[]; @@ -166,14 +138,9 @@ type CliTurnEnd = Extract; export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined): Promise => { if (runtimeHomePath === undefined) return; - await Promise.all([ - runtimeHomePath, - `${runtimeHomePath}/.config`, - `${runtimeHomePath}/.local/share`, - `${runtimeHomePath}/.local/state`, - `${runtimeHomePath}/.cache`, - `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true }))); + await ensureRuntimeHome(runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp"] + .map((relative) => ensureRuntimeHomeDirectory(runtimeHomePath, relative))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => @@ -216,7 +183,17 @@ const startMcp = async ( await startupSettled; await transport.close().catch(() => undefined); await mcpServer.close().catch(() => undefined); - if (httpServer.listening) await new Promise((resolve) => httpServer.close(() => resolve())); + // `close` only stops accepting and then waits for every open connection, + // including the ones the transport has no record of and so cannot end (a + // socket opened before `initialize`, or a client pool's idle keep-alive + // socket, which relaying a turn through the broker MCP facade leaves + // behind). That wait is unbounded and sits on the wake's own completion + // path: measured, one such connection parked a finished broker turn with + // its result in hand and published nothing. By here this one wake's engine + // has returned, failed or been cancelled, so anything still connected is a + // leftover — see `AGENTS.md`, and the facade, which bounds itself the same + // way. + if (httpServer.listening) await new Promise((resolve) => { httpServer.close(() => resolve()); httpServer.closeAllConnections(); }); lifecycle = "closed"; })(); const mount = { get endpoint(): string { return endpoint; }, close }; @@ -321,35 +298,33 @@ class CliSession implements PiSessionLike { const controller=new AbortController();this.activeBrokerTurn=controller; try{output=await this.options.grokBrokerTurn(`${this.options.identityPrompt ?? ""}${text}`,mount.endpoint,controller.signal);}finally{if(this.activeBrokerTurn===controller)this.activeBrokerTurn=undefined;} } else { - if ((this.options.engine === "grok" || this.options.engine === "agy") && mount !== undefined) { + if (this.options.engine === "grok" && mount !== undefined) { + await this.options.verifyExecutable?.(); + registration = await registerGrokHomeMcpServer({ + engineHomePath: this.options.engineHomePath, + endpoint: mount.endpoint, + ...(this.options.verifyGrokSandbox !== undefined ? { verify: this.options.verifyGrokSandbox } : {}) + }); + this.mcpRegistration = registration; + } else if (this.options.engine === "agy" && mount !== undefined) { await this.options.verifyExecutable?.(); - const profile = this.options.grokSandboxProfile ?? GROK_STRICT_SANDBOX_PROFILE; - const grok = this.options.engine === "grok"; registration = await registerCliMcpServer({ - addArgs: grok - ? renderGrokMcpAddArgs(this.options.commandArgs, profile, mount.endpoint) - : renderAgyMcpAddArgs(this.options.commandArgs, mount.endpoint), - removeArgs: grok - ? renderGrokMcpRemoveArgs(this.options.commandArgs, profile) - : renderAgyMcpRemoveArgs(this.options.commandArgs), + addArgs: renderAgyMcpAddArgs(this.options.commandArgs, mount.endpoint), + removeArgs: renderAgyMcpRemoveArgs(this.options.commandArgs), command: this.options.command ?? this.options.engine, cwd: this.input.cwd, env: cliChildEnvironment([ ...(this.options.redactedEnvironmentNames ?? []), ...(this.input.daimonSecretEnvironmentNames ?? []) ], this.input.runtimeHomePath, { - ...(this.options.engine === "agy" && this.options.dbusSessionBusAddress !== undefined - ? { dbusSessionBusAddress: this.options.dbusSessionBusAddress } - : {}), + ...(this.options.dbusSessionBusAddress !== undefined ? { dbusSessionBusAddress: this.options.dbusSessionBusAddress } : {}), engine: this.options.engine, executablePath: this.options.command, engineHomePath: this.options.engineHomePath }), - ...(grok ? { failureClassifier: classifyGrokAuthenticationDiagnostic } : {}), onChild: (setupChild) => { this.setupChildren.add(setupChild); }, onChildSettled: (setupChild) => this.setupChildren.delete(setupChild), - secretValues, - ...(grok && this.options.verifyGrokSandbox !== undefined ? { verify: this.options.verifyGrokSandbox } : {}) + secretValues }); this.mcpRegistration = registration; } diff --git a/src/pi/cliSessionMcpMountClose.test.ts b/src/pi/cliSessionMcpMountClose.test.ts new file mode 100644 index 0000000..69b74ed --- /dev/null +++ b/src/pi/cliSessionMcpMountClose.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { connect, type Socket } from "node:net"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; + +import { createCliSessionFactory } from "./cliSession.js"; + +type PublishedTurn = { readonly message: { readonly content: readonly unknown[] } }; + +/** + * A finished turn must not wait on a connection to its own per-wake MCP mount. + * + * The mount's HTTP server is torn down on the wake's completion path, and + * `Server.close()` only stops accepting: it waits for every open connection. + * The MCP transport ends the sessions it knows about, but a connection it has + * no record of — a socket opened before `initialize`, or an idle HTTP + * keep-alive socket a client's connection pool is still holding — is not its + * to end. The broker MCP facade is exactly such a client: relaying a turn's + * session leaves pooled connections to the mount behind it. + * + * Measured before this was bounded: the broker turn returned its result and + * the wake then never completed and never published — the terminal evidence + * that is the whole point of letting a finished turn finish. + */ +const TURN_COMPLETION_BOUND_MS = 5_000; + +test("a finished Grok broker turn publishes without waiting on an open connection to its own MCP mount", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-mount-close-")); + const peers: Socket[] = []; + const hangUp = (): void => { for (const peer of peers) peer.destroy(); }; + try { + const { session } = await createCliSessionFactory({ + engine: "grok", + command: "/nonexistent-engine", + grokBrokerTurn: async (_prompt, mcpEndpoint) => { + // A peer holding a connection the transport never saw an `initialize` + // on: none of this connection is the mount's own session state. + const peer = connect({ host: "127.0.0.1", port: Number(new URL(mcpEndpoint).port) }); + peers.push(peer); + peer.on("error", () => undefined); + await new Promise((resolve, reject) => { peer.once("connect", resolve); peer.once("error", reject); }); + return "Filed and delivered."; + } + })({ cwd: root }); + const published: PublishedTurn[] = []; + session.subscribe((event) => { published.push(event as unknown as PublishedTurn); }); + try { + const settled = session.prompt("wake").then(() => "completed" as const); + const outcome = await Promise.race([settled, delay(TURN_COMPLETION_BOUND_MS).then(() => "parked" as const)]); + // Hang the peer up before asserting, so a regression reports the parked + // wake instead of parking the suite's own teardown behind it. + hangUp(); + assert.equal(outcome, "completed", `the finished wake did not complete within ${TURN_COMPLETION_BOUND_MS}ms`); + assert.equal(published.length, 1); + assert.deepEqual(published[0]?.message.content, [{ type: "text", text: "Filed and delivered." }]); + } finally { hangUp(); await session.disposeAsync?.(); } + } finally { + hangUp(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/src/pi/cliSessionOutput.test.ts b/src/pi/cliSessionOutput.test.ts index 2dc92ac..65d0b2c 100644 --- a/src/pi/cliSessionOutput.test.ts +++ b/src/pi/cliSessionOutput.test.ts @@ -34,11 +34,15 @@ test("verbose progress stderr is drained without invalidating a bounded successf } }); -test("failed verbose stderr retains only a redacted bounded diagnostic tail", async () => { +test("failed verbose stderr retains a redacted bounded window of its head AND its tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-stderr-diagnostic-")); const engine = path.join(root, "verbose-failure.mjs"); const secret = "bounded-diagnostic-secret-value"; await writeFile(engine, [ + // The real shape of an early death: the cause first, then a flood, then + // the last thing the child happened to print. A pure tail kept only the + // last of the three. + `process.stderr.write(${JSON.stringify("first-cause: profile refused\n")});`, `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, `process.stderr.write(${JSON.stringify(` final-error ${secret}`)});`, "process.exitCode = 7;" @@ -54,7 +58,9 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { assert.ok(error instanceof Error); assert.match(error.message, /CLI engine exited 7/); + assert.match(error.message, /first-cause: profile refused/, "the head of the output is the error, and must survive the bound"); assert.match(error.message, /final-error \[REDACTED\]/); + assert.match(error.message, /\[… \d+ bytes elided …\]/, "and what was dropped between the two ends is named"); assert.equal(error.message.includes(secret), false); assert.ok(Buffer.byteLength(error.message) <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES + 80); return true; @@ -64,6 +70,45 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as } }); +/** + * The mirror of the long-secret case, at the other cut. Retaining both ends + * only stays safe while each end carries a whole secret's worth beyond its + * reported share: sizing the two ends by halving one shared budget let a + * 2000-byte secret straddle the cut and leak its own tail verbatim. + */ +test("a secret straddling the head's own retention edge is still redacted whole", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-head-edge-secret-")); + const engine = path.join(root, "head-edge-secret.mjs"); + // The first ten bytes are a distinctive token, so any surviving prefix of + // this credential is visible to the assertion rather than merely shorter. + const secret = `CREDENTIAL-${"z".repeat(200)}-END`; + // Land it across the edge of the head's REPORTED share + // (`CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2`): 80 bytes inside the window, the + // rest past it. Only the extra secret-length the head retains beyond that + // share lets the redactor match it whole; without it those 80 bytes are a + // verbatim credential prefix in the diagnostic. + const pad = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) - 80; + await writeFile(engine, [ + `process.stderr.write("h".repeat(${pad}));`, + `process.stderr.write(${JSON.stringify(secret)});`, + `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, + `process.stderr.write(${JSON.stringify(" terminal-detail")});`, + "process.exitCode = 9;" + ].join("\n")); + try { + const child = spawnEngine({ + command: process.execPath, commandArgs: [engine], engine: "agy", + maxToolTurns: 1, timeoutMs: 10_000 + }, "verbose", { cwd: root }, undefined); + await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /terminal-detail/u, "the tail still reports the last words"); + assert.doesNotMatch(error.message, /CREDENTIAL|z{32}/u, "no fragment of the secret may survive the cut"); + return true; + }); + } finally { await rm(root, { recursive: true, force: true }); } +}); + test("redacts a 2000-byte exact secret before retaining a failed stderr tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-long-secret-")); const engine = path.join(root, "long-secret-failure.mjs"); @@ -98,7 +143,7 @@ test("Grok replies redact both the staged credential and a credential rotated du let reads = 0; try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => ++reads === 1 ? [oldSecret] : [rotatedSecret] })({ cwd: root }); let emitted = ""; @@ -123,7 +168,7 @@ test("Grok auth rejection is typed and never retains raw credential diagnostics" await writeFile(engine, `const a=process.argv.slice(2);if(a.includes("mcp"))process.stdout.write("ok");else{process.stderr.write("Authentication rejected by server ${secret}");process.exitCode=7;}`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => [secret] })({ cwd: root }); await assert.rejects(session.prompt("work"), (error: unknown) => { @@ -146,7 +191,7 @@ test("Grok auth rejection is classified before a long secret and verbose tail ar ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => [secret] })({ cwd: root }); await assert.rejects(session.prompt("work"), (error: unknown) => { diff --git a/src/pi/cliSessionProcess.test.ts b/src/pi/cliSessionProcess.test.ts index 937b347..7b150c9 100644 --- a/src/pi/cliSessionProcess.test.ts +++ b/src/pi/cliSessionProcess.test.ts @@ -6,10 +6,9 @@ import test from "node:test"; import { createCliSessionFactory, readChild, spawnEngine, terminateChild } from "./cliSession.js"; -const grokStream = (text: string): string => [ - { type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text }] } }, - { type: "result", subtype: "success", is_error: false, result: text, stop_reason: "end_turn", session_id: "fake" } -].map((event) => JSON.stringify(event)).join("\n"); +// Only AGY still registers its per-wake MCP endpoint through setup/removal +// children; Grok writes it into its own GROK_HOME (`grokHomeMcpRegistration.ts`). +const agyStream = (text: string): string => JSON.stringify({ event: "result", result: { conversation_id: "fake", status: "SUCCESS", response: text, num_turns: 1, usage: { input_tokens: 1, output_tokens: 1, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 2 } } }); test("terminates a process group after its leader has exited", async (context) => { if (!requirePosixProcessGroups(context)) return; @@ -29,9 +28,9 @@ test("terminates a process group after its leader has exited", async (context) = } }); -test("Grok setup reaps a stubborn descendant after its successful leader exits", async (context) => { +test("AGY MCP setup reaps a stubborn descendant after its successful leader exits", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-setup-group-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-setup-group-")); const descendant = path.join(root, "setup-descendant-pid"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; @@ -41,10 +40,10 @@ test("Grok setup reaps a stubborn descendant after its successful leader exits", child.stdout.once("data", () => { writeFileSync(${JSON.stringify(descendant)}, String(child.pid)); process.exit(0); }); } if (args.includes("remove")) process.exit(0); - process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); await session.prompt("research"); const pid = Number(await readFile(descendant, "utf8")); @@ -56,9 +55,9 @@ test("Grok setup reaps a stubborn descendant after its successful leader exits", } }); -test("Grok removal reaps a stubborn descendant after its successful leader exits", async (context) => { +test("AGY MCP removal reaps a stubborn descendant after its successful leader exits", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-group-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-remove-group-")); const descendant = path.join(root, "remove-descendant-pid"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; @@ -66,10 +65,10 @@ test("Grok removal reaps a stubborn descendant after its successful leader exits if (args.includes("remove")) { const child = spawn(process.execPath, ["-e", "process.on('SIGTERM', () => undefined); process.stdout.write('ready'); setInterval(() => undefined, 1000)"], { stdio: ["ignore", "pipe", "ignore"] }); child.stdout.once("data", () => { writeFileSync(${JSON.stringify(descendant)}, String(child.pid)); process.exit(0); }); - } else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + } else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); await session.prompt("research"); const pid = Number(await readFile(descendant, "utf8")); diff --git a/src/pi/cliSessionRemoval.test.ts b/src/pi/cliSessionRemoval.test.ts index aa880ac..f8320f1 100644 --- a/src/pi/cliSessionRemoval.test.ts +++ b/src/pi/cliSessionRemoval.test.ts @@ -1,23 +1,21 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { createCliSessionFactory } from "./cliSession.js"; -import { GrokSubscriptionAuthenticationRejectedError } from "../runtime/grokAuthenticationError.js"; -const grokStream = (text: string): string => [ - { type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text }] } }, - { type: "result", subtype: "success", is_error: false, result: text, stop_reason: "end_turn", session_id: "fake" } -].map((event) => JSON.stringify(event)).join("\n"); +// Grok has no removal child any more (its endpoint lives in GROK_HOME config), +// so removal-failure semantics are exercised through AGY's `mcp remove`. +const agyStream = (text: string): string => JSON.stringify({ event: "result", result: { conversation_id: "fake", status: "SUCCESS", response: text, num_turns: 1, usage: { input_tokens: 1, output_tokens: 1, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 2 } } }); -test("Grok removal failure rejects without emitting a successful turn", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-failure-")); +test("MCP removal failure rejects without emitting a successful turn", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-remove-failure-")); const grok = path.join(root, "grok.mjs"); - await writeFile(grok, `const args = process.argv.slice(2); if (args.includes("remove")) process.exit(23); else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + await writeFile(grok, `const args = process.argv.slice(2); if (args.includes("remove")) process.exit(23); else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { - const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); + const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); let turns = 0; session.subscribe((event) => { if (event.type === "turn_end") turns += 1; }); await assert.rejects(session.prompt("research"), /CLI engine exited 23/); @@ -29,43 +27,27 @@ test("Grok removal failure rejects without emitting a successful turn", async () } }); -test("Grok authentication rejection keeps precedence over bounded removal failure", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-auth-remove-failure-")); +test("Grok direct sessions register the per-wake MCP endpoint in GROK_HOME, never through a project-scoped child", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-mcp-")); const grok = path.join(root, "grok.mjs"); - const authSecret = "auth-rejection-secret-canary"; - const cleanupSecret = "cleanup-secret-canary"; - await writeFile(grok, `const args=process.argv.slice(2);if(args.includes("remove")){process.stderr.write("cleanup ${cleanupSecret}");process.exit(23)}else if(args.includes("add"))process.exit(0);else{process.stderr.write("RefreshTokenRejected ${authSecret}");process.exit(7)}`); + const engineHomePath = path.join(root, "home", ".grok"); + const seen = path.join(root, "seen"); + await writeFile(grok, `import { appendFileSync, readFileSync } from "node:fs"; const args = process.argv.slice(2); appendFileSync(${JSON.stringify(seen)}, args.includes("mcp") ? "MCP-CHILD\\n" : readFileSync(${JSON.stringify(path.join(engineHomePath, "config.toml"))}, "utf8")); process.stdout.write(${JSON.stringify([{ type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text: "done" }] } }, { type: "result", subtype: "success", is_error: false, result: "done", stop_reason: "end_turn", session_id: "fake" }].map((event) => JSON.stringify(event)).join("\n"))});`); try { - const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, - timeoutMs: 10_000, credentialSecretValues: async () => [authSecret, cleanupSecret] - })({ cwd: root }); - await assert.rejects(session.prompt("research"), (error: unknown) => { - assert.ok(error instanceof GrokSubscriptionAuthenticationRejectedError); - assert.doesNotMatch(error.message, /canary|cleanup|RefreshTokenRejected/u); - assert.ok(error.cause instanceof Error); - assert.match(error.cause.message, /CLI engine exited 23/u); - assert.doesNotMatch(error.cause.message, /canary|cleanup-secret/u); - assert.ok(Buffer.byteLength(error.cause.message) < 1_024); - return true; - }); - } finally { await rm(root, { recursive: true, force: true }); } -}); + const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000, engineHomePath })({ cwd: root }); + await session.prompt("research"); + const during = await readFile(seen, "utf8"); + assert.doesNotMatch(during, /MCP-CHILD/u); + assert.match(during, /\[mcp_servers\.daimon\]\nurl = "http:\/\/127\.0\.0\.1:\d+\/mcp"/u); + assert.match(during, /\[skills\]\ndisabled = \[/u); + const after = await readFile(path.join(engineHomePath, "config.toml"), "utf8"); + assert.doesNotMatch(after, /mcp_servers/u); + await session.disposeAsync?.(); -test("Grok removal auth rejection is typed before verbose diagnostics are truncated", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-auth-rejection-")); - const grok = path.join(root, "grok.mjs"); - const secret = `refresh-start-${"r".repeat(1970)}-refresh-end`; - await writeFile(grok, `const a=process.argv.slice(2);if(a.includes("remove")){process.stderr.write("RefreshTokenRejected "+${JSON.stringify(secret)}+" "+"tail".repeat(250));process.exit(23)}else if(a.includes("add"))process.exit(0);else process.stdout.write(${JSON.stringify(grokStream("complete"))});`); - try { - const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", - credentialSecretValues: async () => [secret] - })({ cwd: root }); - await assert.rejects(session.prompt("research"), (error: unknown) => { - assert.ok(error instanceof GrokSubscriptionAuthenticationRejectedError); - assert.doesNotMatch(error.message, /RefreshToken|refresh-start|refresh-end|r{32}|tail/u); - return true; - }); - } finally { await rm(root, { recursive: true, force: true }); } + const unowned = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); + await assert.rejects(unowned.session.prompt("research"), /Daimon-owned GROK_HOME/u); + await unowned.session.disposeAsync?.(); + } finally { + await rm(root, { recursive: true, force: true }); + } }); diff --git a/src/pi/codexFilesystemRules.test.ts b/src/pi/codexFilesystemRules.test.ts new file mode 100644 index 0000000..dd3dca0 --- /dev/null +++ b/src/pi/codexFilesystemRules.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { renderCodexArgs } from "./cliEngineSpawn.js"; +import { codexFilesystemRules } from "./codexFilesystemRules.js"; + +test("prunes redundant nested denies independently of declaration order", () => { + for (const denies of [["/run", "/run/control", "/run/control/acceptance"], ["/run/control/acceptance", "/run/control", "/run"]]) { + assert.deepEqual(codexFilesystemRules(denies, []), { "/run": "deny" }); + } + assert.deepEqual(codexFilesystemRules(["/run", "/runner", "/other/control"], []), { + "/run": "deny", "/runner": "deny", "/other/control": "deny" + }); +}); + +test("preserves a deny below an intervening readable exception", () => { + assert.deepEqual(codexFilesystemRules([ + "/vault", "/vault/shared/auth", "/vault/shared/auth/deeper", "/vault/secret" + ], ["/vault/shared"]), { + "/vault/shared": "read", "/vault": "deny", "/vault/shared/auth": "deny" + }); +}); + +test("normalizes aliases before testing ancestry, including a shared path that escapes /run", () => { + assert.deepEqual(codexFilesystemRules(["/run/", "/run//control/", "/run/../var/lib/private/control/", "/var/lib/private/control"], []), { + "/run": "deny", "/var/lib/private/control": "deny" + }); + assert.deepEqual(codexFilesystemRules(["/vault/", "/vault/open//secret/"], ["/vault/unused/../open/"]), { + "/vault/open": "read", "/vault": "deny", "/vault/open/secret": "deny" + }); + assert.deepEqual(codexFilesystemRules(["/vault", "/vault/work/secret"], [], "/vault//work/"), { + "/vault": "deny", "/vault/work/secret": "deny" + }); +}); + +test("same-path deny wins over a read and does not open a descendant", () => { + assert.deepEqual(codexFilesystemRules(["/vault", "/vault/auth"], ["/vault"]), { "/vault": "deny" }); +}); + +test("implicit workspace write is an intervening exception, including in production argv", () => { + const denies = ["/vault", "/vault/workspace/secret", "/vault/workspace/secret/deeper"]; + assert.deepEqual(codexFilesystemRules(denies, [], "/vault/workspace"), { + "/vault": "deny", "/vault/workspace/secret": "deny" + }); + const args = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denies }, "/vault/workspace", "http://127.0.0.1:1/mcp"); + const config = args.find(arg => arg.startsWith("permissions="))!; + assert.ok(config.includes('"/vault/workspace/secret"="deny"')); + assert.ok(!config.includes('"/vault/workspace/secret/deeper"')); +}); + +test("root deny subsumes descendants and a same-path workspace deny remains authoritative", () => { + assert.deepEqual(codexFilesystemRules(["/", "/run"], []), { "/": "deny" }); + assert.deepEqual(codexFilesystemRules(["/workspace", "/workspace/secret"], [], "/workspace"), { "/workspace": "deny" }); + assert.deepEqual(codexFilesystemRules([], [], "/workspace"), {}); +}); + +test("pruning preserves the most specific permission across alternating rules", () => { + const deny = ["/v", "/v/a", "/v/a/read/secret", "/v/a/read/secret/deeper", "/v/work/secret"]; + const read = ["/v/a/read", "/v/a/read/secret/open"]; + const before = { "/v/work": "write", ...Object.fromEntries(read.map(p => [p, "read"])), ...Object.fromEntries(deny.map(p => [p, "deny"])) }; + const after = { "/v/work": "write", ...codexFilesystemRules(deny, read, "/v/work") }; + const permission = (rules: Record, target: string): string | undefined => Object.entries(rules) + .filter(([root]) => target === root || target.startsWith(`${root}/`)).sort(([a], [b]) => b.length - a.length)[0]?.[1]; + for (const entry of [...deny, ...read, "/v/work", "/v/work/secret", "/v/ab", "/other"]) { + for (const target of [entry, `${entry}/file`]) assert.equal(permission(after, target), permission(before, target), target); + } +}); diff --git a/src/pi/codexFilesystemRules.ts b/src/pi/codexFilesystemRules.ts new file mode 100644 index 0000000..510ff78 --- /dev/null +++ b/src/pi/codexFilesystemRules.ts @@ -0,0 +1,29 @@ +import path from "node:path"; + +type Permission = "read" | "write" | "deny"; + +/** Codex mounts deny roots read-only; a second deny below one cannot create parents. */ +export function codexFilesystemRules( + protectedPaths: readonly string[], readablePaths: readonly string[], workspacePath?: string +): Record { + const explicit = new Map(); + // Agent roots are canonical, but shared acceptance paths may contain `..` + // or trailing separators. Compare the same absolute locations Codex uses. + for (const readablePath of readablePaths) explicit.set(path.resolve(readablePath), "read"); + for (const protectedPath of protectedPaths) explicit.set(path.resolve(protectedPath), "deny"); + const effective = new Map([ + ...(workspacePath === undefined ? [] : [[path.resolve(workspacePath), "write"] as const]), + ...explicit + ]); + return Object.fromEntries([...explicit].filter(([target, permission]) => { + if (permission !== "deny") return true; + let ancestor = path.dirname(target); + while (ancestor !== target) { + const inherited = effective.get(ancestor); + if (inherited !== undefined) return inherited !== "deny"; + target = ancestor; + ancestor = path.dirname(target); + } + return true; + })); +} diff --git a/src/pi/codexRolloutUsage.test.ts b/src/pi/codexRolloutUsage.test.ts index e7fc317..ea41272 100644 --- a/src/pi/codexRolloutUsage.test.ts +++ b/src/pi/codexRolloutUsage.test.ts @@ -46,6 +46,14 @@ test("a real multi-request rollout yields one row per model request, cached and [288, 228, 50_292] ]); assert.deepEqual(requests.map((request) => request.cacheWrite), [0, 0, 0, 0]); + // End is the usage frame; start is the first non-usage frame after the previous + // request's usage frame, else the previous request's end. + assert.deepEqual(requests.map((request) => [request.startedAt, request.endedAt]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); test("reasoning tokens, which the per-wake ledger drops entirely, survive per request", async () => { diff --git a/src/pi/codexRolloutUsage.ts b/src/pi/codexRolloutUsage.ts index b803f5e..0c872bf 100644 --- a/src/pi/codexRolloutUsage.ts +++ b/src/pi/codexRolloutUsage.ts @@ -44,6 +44,17 @@ export type CodexRequestUsage = Readonly<{ output: number; reasoning: number; total: number; + /** + * When the request began and ended, from the rollout's own frame + * `timestamp`s. `endedAt` is the usage frame's timestamp (Codex appends it at + * `response.completed`); `startedAt` is the first non-usage frame after the + * previous request's usage frame — the tool output or turn context that + * triggers the next request — falling back to the previous request's end. + * Either is absent when the frames carry no valid timestamp: a wake-end stamp + * substituted here would be indistinguishable from a measured one. + */ + startedAt?: string; + endedAt?: string; }>; /** @@ -177,17 +188,20 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado const requests: CodexRequestUsage[] = []; const fallback: CodexRequestUsage[] = []; let previousFallback = ""; + const clocks = { record: requestClock(), fallback: requestClock() }; for (const line of text.split("\n")) { if (line.trim().length === 0) continue; let frame: unknown; try { frame = JSON.parse(line); } catch { continue; } if (!isRecord(frame)) continue; const block = usageBlock(frame, threadId); + const usageFrame = frame.type === "token_usage_record" || (frame.type === "event_msg" && isRecord(frame.payload) && frame.payload.type === "token_count"); + if (!usageFrame) { if (frame.type !== "session_meta") { clocks.record.observe(frame.timestamp); clocks.fallback.observe(frame.timestamp); } continue; } if (block === undefined) continue; if (frame.type === "token_usage_record") { const decoded = decodeRequestUsage(block.usage, requests.length); if (decoded === undefined) return []; - requests.push(decoded); + requests.push({ ...decoded, ...clocks.record.close(frame.timestamp) }); continue; } // `token_count` is NOT one frame per request: the captured fixture carries @@ -201,7 +215,7 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado previousFallback = serialized; const decoded = decodeRequestUsage(block.usage, fallback.length); if (decoded === undefined) return []; - fallback.push(decoded); + fallback.push({ ...decoded, ...clocks.fallback.close(frame.timestamp) }); } // A Codex version that emits both shapes emits `token_usage_record` once per // request, so the richer one wins outright rather than being merged into a @@ -209,6 +223,22 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado return requests.length > 0 ? requests : fallback; }; +const timestampOf = (value: unknown): string | undefined => + typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u.test(value) && !Number.isNaN(Date.parse(value)) ? value : undefined; + +/** Tracks one request stream's start/end stamps; see {@link CodexRequestUsage.startedAt}. */ +const requestClock = () => { + let start: string | undefined, previousEnd: string | undefined; + return { + observe(value: unknown): void { start ??= timestampOf(value); }, + close(value: unknown): { startedAt?: string; endedAt?: string } { + const endedAt = timestampOf(value), startedAt = start ?? previousEnd; + start = undefined; previousEnd = endedAt; + return { ...(startedAt === undefined ? {} : { startedAt }), ...(endedAt === undefined ? {} : { endedAt }) }; + } + }; +}; + /** * Read one turn's per-request usage. Never throws. * diff --git a/src/pi/engineWakeLimits.test.ts b/src/pi/engineWakeLimits.test.ts new file mode 100644 index 0000000..e514af1 --- /dev/null +++ b/src/pi/engineWakeLimits.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "./engineWakeLimits.js"; + +test("engine-neutral wake bounds drive Codex, with the Codex names kept as aliases", () => { + assert.equal(resolveCodexWakeTimeoutMs({}), 240_000); + assert.equal(resolveCodexWakeTokenCeiling({}), 300_000); + assert.equal(resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "5000" }), 5_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.throws(() => resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "8000" }), /disagree/u); + assert.throws(() => resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "0" }), /positive integer/u); +}); + +test("the broker receives only the bounds an operator actually set, as lowering limits", () => { + assert.equal(resolveEngineWakeLimitOverrides({}), undefined); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "400000" }), { maxTokens: 400_000 }); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_CODEX_WAKE_TIMEOUT_MS: "480000", DAIMON_ENGINE_WAKE_TOKEN_CEILING: "1" }), { timeoutMs: 480_000, maxTokens: 1 }); +}); diff --git a/src/pi/engineWakeLimits.ts b/src/pi/engineWakeLimits.ts new file mode 100644 index 0000000..a4a0299 --- /dev/null +++ b/src/pi/engineWakeLimits.ts @@ -0,0 +1,58 @@ +/** + * Per-wake engine bounds, and the one place they are decided. + * + * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell + * (`exec_command`) is never routed through that gate, so a single Codex turn + * previously had no ceiling at all — one production wake ran 23:32→23:42 + * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream + * reports token usage exactly once, on `turn.completed` — there is no + * incremental total to watch mid-turn (verified against a live multi-tool-call + * turn: `item.completed` fires once per tool call, but usage is reported only + * on the single terminal `turn.completed`) — so the token ceiling is the best + * bound obtainable from that wire shape: it converts an over-budget turn into + * an explicit, killed, named failure instead of a silent success, and the + * wall-clock timeout is what actually interrupts a runaway turn in progress. + * + * The names are engine-neutral: `DAIMON_ENGINE_WAKE_TIMEOUT_MS` and + * `DAIMON_ENGINE_WAKE_TOKEN_CEILING` bound Codex locally and are passed to the + * Grok broker as the wake's *lowering* limits (the broker refuses a value above + * its registration). The `DAIMON_CODEX_*` names remain aliases; setting both + * names of one bound to different values is refused rather than guessed. + */ +export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; +export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; +export const DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV = "DAIMON_ENGINE_WAKE_TIMEOUT_MS"; +export const DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV = "DAIMON_ENGINE_WAKE_TOKEN_CEILING"; +export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; +export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; + +const positiveInteger = (value: string, name: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +}; + +const declared = (environment: NodeJS.ProcessEnv, neutral: string, alias: string): number | undefined => { + const primary = environment[neutral], legacy = environment[alias]; + const value = primary === undefined ? undefined : positiveInteger(primary, neutral); + const aliased = legacy === undefined ? undefined : positiveInteger(legacy, alias); + if (value !== undefined && aliased !== undefined && value !== aliased) throw new Error(`${neutral} and ${alias} disagree; set one`); + return value ?? aliased; +}; + +export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV) ?? DEFAULT_CODEX_WAKE_TIMEOUT_MS; +export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV) ?? DEFAULT_CODEX_WAKE_TOKEN_CEILING; + +/** + * The limits a wake asks a broker to lower to: only the bounds the operator + * actually set, never the Codex defaults (a broker registration's declared + * limits already are the defaults there). + */ +export const resolveEngineWakeLimitOverrides = (environment: NodeJS.ProcessEnv = process.env): Readonly<{ timeoutMs?: number; maxTokens?: number }> | undefined => { + const timeoutMs = declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); + const maxTokens = declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); + if (timeoutMs === undefined && maxTokens === undefined) return undefined; + return { ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(maxTokens === undefined ? {} : { maxTokens }) }; +}; diff --git a/src/pi/fixtures/README.md b/src/pi/fixtures/README.md index f158a6d..0cf75ed 100644 --- a/src/pi/fixtures/README.md +++ b/src/pi/fixtures/README.md @@ -117,3 +117,23 @@ the previous block and why `token_usage_record` wins outright when both exist. The four requests also show exactly the shape the study predicted, in one wake: fresh input 15,742 → 248 → 4,276 → 10,068 against a context that only grows from 34,686 to 50,004 — most of every request after the first is cache-read replay. + + +# Grok 1.0.34 per-request stream fixture + +`grok-1.0.34-streaming-two-requests.jsonl` is a real two-request turn captured on +2026-09-17 from `grok 1.0.34` (macOS arm64) with the lean worker flags and +`--output-format streaming-messages-json` (P0 host matrix cell c14: one MCP +`use_tool` call, then the answer). Sanitization before commit: the capturing +scratchpad `cwd` was replaced with `/workspace`; every frame is otherwise +verbatim. + +It pins what `grokStreamUsage.ts` reads and the broker meters per request: + + assistant.message.id one request per distinct id + assistant.message.usage that request's own four buckets + result.modelUsage keys "grok-4.6-build" for grok-4.6 + +The two per-request totals (2,775 + 2,810) sum exactly to the terminal +`result.usage` (5,585), which is why a failed turn's frames are trusted as its +partial usage. diff --git a/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl new file mode 100644 index 0000000..3eaabd0 --- /dev/null +++ b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl @@ -0,0 +1,5 @@ +{"type":"system","subtype":"init","session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","apiKeySource":"oauth","model":"grok-4.6","cwd":"/workspace","permissionMode":"bypassPermissions","tools":["run_terminal_command","read_file","list_dir","grep","search_tool","use_tool"],"slash_commands":["compact","always-approve","context","session-info","feedback"],"mcp_servers":[{"name":"probe","status":"connected"}],"skills":[],"uuid":"880b6dc2-0488-4678-9ec8-04b2c2fe9efd"} +{"type":"assistant","message":{"id":"msg_0","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"The user wants me to use moltnet_read on room \"conference\" to get a code word. The instructions say I can call probe__moltnet_read directly with use_tool.","signature":"tl43Ieb3gy7AEz+YdNJWSx3rN7HTKi9cJKhytQwzBfudDDWXQ6UEWS4JxMYQZJRrKQeBXNtXluNa1UHRr5C3OMGlpHodcziT9fqhMn5tgr0xjUjCRq1/KL0fdIGj+tU+8sHeAKtHBCMC2bFFkbtdm4BrWR0kLcHzQKbla9tof1RpzzgFtwyIx3fsVCmp/AYHHNrm0BNZvQpfbb9M1yyN4AD0d7E0b1VZru3uMx9N47RtFoH7uWQ"},{"type":"text","text":"I'll read the conference room now."},{"type":"tool_use","id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","name":"use_tool","input":{"tool_name":"probe__moltnet_read","tool_input":{"room":"conference"}}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2568,"output_tokens":79,"cache_read_input_tokens":128,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"6c8748c9-2ce6-437b-a0d2-24a01ca23b3b"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","content":"{\"type\":\"MCP\",\"tool_name\":\"moltnet_read\",\"server_name\":\"probe\",\"output\":{\"OkayOutput\":\"Brass: the code word is TANGERINE-7.\"}}","is_error":false}]},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"fcd581d3-b43f-4aa0-a9d7-9c25862b4bfc"} +{"type":"assistant","message":{"id":"msg_1","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"Got the code word. Answer briefly.","signature":"KcaYjJIHN4fzMKF4FdR0695QoQUKP7Byln+CLEgdhkDCFs3y0wBDWWYCWDkmvXSZnjMYqtfHn79Tgkif73U"},{"type":"text","text":"TANGERINE-7"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":109,"output_tokens":13,"cache_read_input_tokens":2688,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"b1ddbf87-b115-4ee9-b773-0d81f65a8ef7"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":3300,"duration_api_ms":3118,"num_turns":2,"result":"TANGERINE-7","stop_reason":"end_turn","total_cost_usd":0.00248676,"usage":{"input_tokens":2677,"output_tokens":92,"cache_read_input_tokens":2816,"cache_creation_input_tokens":0,"server_tool_use":{"web_search_requests":0}},"modelUsage":{"grok-4.6-build":{"inputTokens":2677,"outputTokens":92,"cacheReadInputTokens":2816,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.00248676}},"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"0dbd02b1-4721-4c52-8416-25bf2455bb8a"} diff --git a/src/pi/grokHeadlessResult.test.ts b/src/pi/grokHeadlessResult.test.ts index 01d309b..376162e 100644 --- a/src/pi/grokHeadlessResult.test.ts +++ b/src/pi/grokHeadlessResult.test.ts @@ -77,7 +77,7 @@ test("exit-zero cancelled Grok sessions reject without emitting a turn", async ( ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "grok", engineHomePath: path.join(root, ".grok"), timeoutMs: 10_000 })({ cwd: root }); let turns = 0; session.subscribe((event) => { if (event.type === "turn_end") turns += 1; }); @@ -108,7 +108,7 @@ test("successful Grok sessions emit only decoded terminal text", async () => { ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "grok", engineHomePath: path.join(root, ".grok"), timeoutMs: 10_000 })({ cwd: root }); let reply = ""; session.subscribe((event) => { diff --git a/src/pi/grokHomeMcpRegistration.ts b/src/pi/grokHomeMcpRegistration.ts new file mode 100644 index 0000000..2c1fcef --- /dev/null +++ b/src/pi/grokHomeMcpRegistration.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, unlink } from "node:fs/promises"; +import path from "node:path"; + +import { renderGrokLeanBaseConfig } from "../runtime/grokBrokerWorkerConfig.js"; +import type { CliMcpRegistration } from "./cliMcpRegistration.js"; + +/** + * Per-wake MCP registration for the direct (non-broker) Grok CLI path. + * + * `grok mcp add --scope project` writes `/.grok/config.toml`, which Grok + * 1.0.34 skips entirely in an untrusted workspace — and Daimon keeps every + * workspace untrusted so cwd `AGENTS.md` and project skills never load. The + * endpoint therefore goes into the agent's own Daimon-owned `GROK_HOME` + * config, rendered from the same lean base the broker worker uses + * (`renderGrokLeanBaseConfig`), and is removed again after the turn by + * rewriting the base alone. + * + * There is no fallback to the operator's `~/.grok`: without an explicit + * `engineHomePath` the session refuses, rather than editing a human's config. + */ +export async function registerGrokHomeMcpServer(input: Readonly<{ engineHomePath: string | undefined; endpoint: string; verify?: () => Promise }>): Promise { + const home = input.engineHomePath; + if (home === undefined || !path.isAbsolute(home)) { + throw new Error("Grok direct CLI sessions require a Daimon-owned GROK_HOME (engineHomePath): Grok 1.0.34 ignores project-scoped MCP servers in untrusted workspaces"); + } + if (!/^http:\/\/127\.0\.0\.1:\d{1,5}\/[A-Za-z0-9/_-]*$/u.test(input.endpoint)) throw new Error("Grok MCP endpoint must be a loopback http URL"); + await input.verify?.(); + await writeConfig(home, `${renderGrokLeanBaseConfig()}[mcp_servers.daimon]\nurl = ${JSON.stringify(input.endpoint)}\n`); + let closePromise: Promise | undefined; + return { + close: (): Promise => closePromise ??= (async () => { + await input.verify?.(); + await writeConfig(home, renderGrokLeanBaseConfig()); + })() + }; +} + +async function writeConfig(home: string, text: string): Promise { + await mkdir(home, { recursive: true, mode: 0o700 }); + const directory = await lstat(home); + if (!directory.isDirectory() || directory.isSymbolicLink()) throw new Error("Grok engine home is not a private directory"); + const target = path.join(home, "config.toml"); + const temporary = path.join(home, `.config.toml.${randomUUID()}.tmp`); + let handle: Awaited> | undefined; + try { + handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600); + await handle.writeFile(text); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(temporary, target); + } finally { + await handle?.close().catch(() => undefined); + await unlink(temporary).catch(() => undefined); + } +} diff --git a/src/pi/grokSandbox.test.ts b/src/pi/grokSandbox.test.ts index d8bd5d0..b34f194 100644 --- a/src/pi/grokSandbox.test.ts +++ b/src/pi/grokSandbox.test.ts @@ -45,7 +45,8 @@ test("rejects a protected root overlapping the selected agent workspace", async test("rotates its private enforcement receipt before the bounded log is exhausted", async () => { const fixture = await createFixture(); try { - const events = path.join(fixture.engineHomePath, "sandbox-events.jsonl"); + const events = path.join(fixture.engineHomePath, "sessions", "sandbox-events.jsonl"); + await mkdir(path.dirname(events), { mode: 0o700 }); await writeFile(events, "x".repeat(8 * 1024 * 1024), { mode: 0o600 }); await prepareAndVerifyGrokSandbox(fixture.authority); assert.ok((await readFile(events)).byteLength < 64 * 1024); @@ -75,8 +76,8 @@ const profile=fs.readFileSync(path.join(home,"sandbox.toml"),"utf8"); const deny=JSON.parse(profile.split("\\n").find((line)=>line.startsWith("deny = ")).slice(7)); const observed=${JSON.stringify(mode)}==="drop-deny"?deny.slice(1):deny; const event={event_type:"ProfileApplied",profile:"${GROK_DAIMON_SANDBOX_PROFILE}",workspace:fs.realpathSync(args[args.indexOf("--cwd")+1]),platform:"linux/landlock",enforced:${JSON.stringify(mode)}!=="unenforced",restrict_network:true,deny_paths:observed}; -fs.appendFileSync(path.join(home,"sandbox-events.jsonl"),JSON.stringify(event)+"\\n",{mode:0o600}); -fs.chmodSync(path.join(home,"sandbox-events.jsonl"),0o600); +fs.appendFileSync(path.join(home,"sessions","sandbox-events.jsonl"),JSON.stringify(event)+"\\n",{mode:0o600}); +fs.chmodSync(path.join(home,"sessions","sandbox-events.jsonl"),0o600); `); await chmod(command, 0o700); return { diff --git a/src/pi/grokSandbox.ts b/src/pi/grokSandbox.ts index 956095b..1c448ed 100644 --- a/src/pi/grokSandbox.ts +++ b/src/pi/grokSandbox.ts @@ -1,17 +1,20 @@ import { randomUUID } from "node:crypto"; import { spawn } from "node:child_process"; import { constants } from "node:fs"; -import { lstat, open, realpath, rename, unlink } from "node:fs/promises"; +import { lstat, mkdir, open, realpath, rename, unlink } from "node:fs/promises"; import path from "node:path"; import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; +import { assertGrokWorkerDenyPathsPlaceable } from "../runtime/grokWorkerDenyPlacement.js"; +import { GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH, GROK_WORKER_SANDBOX_PROFILE, renderGrokWorkerSandboxProfile } from "../runtime/grokWorkerSandboxProfile.js"; -export const GROK_DAIMON_SANDBOX_PROFILE = "daimon-strict"; +export const GROK_DAIMON_SANDBOX_PROFILE = GROK_WORKER_SANDBOX_PROFILE; const SANDBOX_CONFIG = "sandbox.toml"; -const SANDBOX_EVENTS = "sandbox-events.jsonl"; +/** Grok 1.0.34 logs sandbox events under `sessions/`; the root file stays empty. */ +const SANDBOX_EVENTS = GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH; const MAX_EVENTS_BYTES = 16 * 1024 * 1024; const ROTATE_EVENTS_BYTES = 8 * 1024 * 1024; @@ -34,6 +37,11 @@ export async function prepareAndVerifyGrokSandbox( if (denied.some((entry) => overlaps(entry, cwd) || overlaps(entry, engineHome))) { throw unavailable(); } + // Grok 1.0.34 materializes every deny target inside bubblewrap as the uid it + // runs Grok under — here, this process's own — and refuses the whole profile + // if one cannot be resolved. Refusing now names the path; letting it through + // would kill every turn with a bare `bwrap: Can't create file at …`. + await assertGrokWorkerDenyPathsPlaceable(denied, { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }); await writeProfile(engineHome, denied); const beforeBytes = await eventFileSize(path.join(engineHome, SANDBOX_EVENTS)); const child = trackCliChild(spawn(authority.command, [ @@ -61,13 +69,7 @@ export async function prepareAndVerifyGrokSandbox( await verifyProfile(engineHome, denied); } -const profileText = (denied: readonly string[]): string => [ - `[profiles.${GROK_DAIMON_SANDBOX_PROFILE}]`, - 'extends = "strict"', - "restrict_network = true", - `deny = [${denied.map((entry) => JSON.stringify(entry)).join(", ")}]`, - "" -].join("\n"); +const profileText = (denied: readonly string[]): string => renderGrokWorkerSandboxProfile(denied); async function writeProfile(engineHome: string, denied: readonly string[]): Promise { const target = path.join(engineHome, SANDBOX_CONFIG); @@ -155,6 +157,7 @@ async function eventFileSize(file: string): Promise { return Number(entry.size); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { + await mkdir(path.dirname(file), { mode: 0o700, recursive: true }); const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | noFollow(), 0o600); try { await handle.sync(); } finally { await handle.close(); } await syncDirectory(path.dirname(file)); diff --git a/src/pi/grokStreamUsage.test.ts b/src/pi/grokStreamUsage.test.ts new file mode 100644 index 0000000..8bf0f32 --- /dev/null +++ b/src/pi/grokStreamUsage.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { decodeGrokHeadlessTurn } from "./grokHeadlessResult.js"; +import { decodeGrokStreamUsage } from "./grokStreamUsage.js"; + +const fixture = (): Promise => readFile(fileURLToPath(new URL("./fixtures/grok-1.0.34-streaming-two-requests.jsonl", import.meta.url)), "utf8"); + +test("a real 1.0.34 two-request stream yields one row per request, summing exactly to the terminal result", async () => { + const output = await fixture(); + const stream = decodeGrokStreamUsage(output); + assert.deepEqual(stream.requests, [ + { index: 0, input: 2_568, cacheRead: 128, cacheWrite: 0, output: 79, total: 2_775 }, + { index: 1, input: 109, cacheRead: 2_688, cacheWrite: 0, output: 13, total: 2_810 } + ]); + assert.equal(stream.sessionId, "01a0ad21-a90f-7f71-8054-93fdb4334d6a"); + assert.deepEqual(stream.reportedModels, ["grok-4.6-build"]); + const terminal = decodeGrokHeadlessTurn(output).usage!; + assert.equal(stream.requests.reduce((sum, request) => sum + request.total, 0), terminal.total); +}); + +test("a malformed per-request usage block discards every request instead of reporting part of the turn", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":"109"'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); + +test("frames repeating one message id are one request, and a torn line is skipped", async () => { + const lines = (await fixture()).split("\n").filter((line) => line.length > 0); + const repeated = lines.find((line) => line.includes('"msg_0"'))!; + const stream = decodeGrokStreamUsage([...lines.slice(0, 2), repeated, ...lines.slice(2), '{"type":"assist'].join("\n")); + assert.equal(stream.requests.length, 2); +}); + +test("the captured fixture carries no capturing machine's environment", async () => { + assert.doesNotMatch(await fixture(), /\/Users\/|\/private\/|scratchpad|\/home\//u); +}); + +test("a per-request stream block beyond the context-window bound is invalid, not counted", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":900000000'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); diff --git a/src/pi/grokStreamUsage.ts b/src/pi/grokStreamUsage.ts new file mode 100644 index 0000000..bcb39e1 --- /dev/null +++ b/src/pi/grokStreamUsage.ts @@ -0,0 +1,57 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +/** + * Per-request token accounting read off a Grok `streaming-messages-json` + * stream. + * + * Grok 1.0.34 puts the usage of each model request on that request's + * top-level `assistant` frame (`message.usage`, the same four disjoint + * Messages API buckets as the terminal `result.usage`), before any tool result + * of that request, and the per-request frames sum exactly to the terminal + * result (P0 host matrix). That makes the stream usable in two places the + * terminal frame is not: a turn that failed before its `result` frame, and a + * per-request ledger row. + * + * Never throws, and never invents a number: a usage block that is present but + * does not decode discards *all* requests, because one fabricated zero is + * byte-identical to a measured one. + */ +export type GrokRequestUsage = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>; +export type GrokStreamUsage = Readonly<{ requests: readonly GrokRequestUsage[]; sessionId?: string; reportedModels: readonly string[] }>; + +type JsonRecord = Readonly>; +const isRecord = (value: unknown): value is JsonRecord => typeof value === "object" && value !== null && !Array.isArray(value); +const tokenCount = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +const SESSION_ID = /^[A-Za-z0-9][A-Za-z0-9-]{0,63}$/u; +const MODEL_KEY = /^[a-z0-9][a-z0-9.-]{0,63}$/u; + +const decodeUsage = (usage: unknown): Omit | undefined => { + if (!isRecord(usage)) return undefined; + const input = tokenCount(usage.input_tokens), output = tokenCount(usage.output_tokens), cacheRead = tokenCount(usage.cache_read_input_tokens), cacheWrite = tokenCount(usage.cache_creation_input_tokens); + if (input === undefined || output === undefined || cacheRead === undefined || cacheWrite === undefined) return undefined; + const total = input + cacheRead + cacheWrite + output; + // Beyond the model context window one request cannot have spent it: invalid, like a malformed block. + if (total > GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens) return undefined; + return { input, cacheRead, cacheWrite, output, total }; +}; + +export const decodeGrokStreamUsage = (output: string): GrokStreamUsage => { + const byMessage = new Map>(); + const reportedModels = new Set(); + let sessionId: string | undefined, corrupt = false; + for (const line of output.split(/\r?\n/u)) { + if (line.trim().length === 0) continue; + let event: unknown; + try { event = JSON.parse(line); } catch { continue; } + if (!isRecord(event)) continue; + if (typeof event.session_id === "string" && SESSION_ID.test(event.session_id)) sessionId ??= event.session_id; + if (event.type === "result" && isRecord(event.modelUsage)) for (const key of Object.keys(event.modelUsage)) reportedModels.add(MODEL_KEY.test(key) ? key : "invalid"); + if (event.type !== "assistant" || event.parent_tool_use_id !== null || !isRecord(event.message) || event.message.usage === undefined) continue; + const decoded = decodeUsage(event.message.usage); + if (decoded === undefined || typeof event.message.id !== "string") { corrupt = true; continue; } + // One request can surface as more than one frame of the same message; it is one request. + byMessage.set(event.message.id, decoded); + } + const requests = corrupt ? [] : [...byMessage.values()].map((usage, index) => Object.freeze({ index, ...usage })); + return { requests, ...(sessionId === undefined ? {} : { sessionId }), reportedModels: [...reportedModels].sort() }; +}; diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 19a1ace..cc4e235 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,6 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -97,18 +98,12 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { async startAgent(input: AgentStartInput): Promise { validatePiRawTrainingCaptureOptions(this.options.rawTrainingCapture); - await Promise.all([ - input.runtimeHomePath, - `${input.runtimeHomePath}/.config`, - `${input.runtimeHomePath}/.local/share`, - `${input.runtimeHomePath}/.local/state`, - `${input.runtimeHomePath}/.cache`, - `${input.runtimeHomePath}/.tmp`, - `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true }))); + await ensureRuntimeHome(input.runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp", "tool-state"] + .map((relative) => ensureRuntimeHomeDirectory(input.runtimeHomePath, relative))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true }); + await ensureRuntimeHome(memoryRuntimeHomePath); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 0f0f45c..1833ed0 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -1,11 +1,12 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -268,8 +269,7 @@ export const writeTurnTraceRecord = async ( record: PiTurnTraceRecord ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); - const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true }); + const turnsPath = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry/turns"); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 93d59df..9ee6f8b 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -1,10 +1,11 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -188,8 +189,7 @@ export const persistPiWorldTrajectory = async ( } }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); - const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true }); + const trajectoriesPath = await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry/world-trajectories"); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 93723c1..f53a605 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -13,11 +13,497 @@ coordinate wakes. Every source file stays below 400 lines. Keep tests beside the contract they cover. -`grokBrokerProxyRequest.ts` preserves the worker CLI's bounded -`x-grok-client-version` and supplies its `grok-shell` client identity when -rebuilding provider headers. Dropping the version makes the subscription -provider reject an otherwise valid login with HTTP 426; never replace it with -a fabricated version or pass arbitrary worker headers through. +`grokBrokerProxyRequest.ts` preserves the worker CLI's `x-grok-client-version` +and supplies its `grok-shell` client identity when rebuilding provider headers. +Dropping the version makes the subscription provider reject an otherwise valid +login with HTTP 426; never replace it with a fabricated version or pass +arbitrary worker headers through. The version must equal the pinned +`GROK_ENGINE_BROKER.grokCliVersion` (1.0.34) exactly. + +The proxy is also the spend gate for the lean Grok worker. Before a bearer is +attached it refuses any body whose tool names are not exactly +`GROK_WORKER_VISIBLE_TOOLS` (Grok 1.0.34 turns an unmappable `--tools` entry +into its full 19-tool set, and its `session_title` request carries one forced +tool), and any body whose `model`/`reasoning_effort` differ from the declared +`grokBrokerModelPolicy.ts` policy (closed lists; default `grok-4.6`/`low`). The +model override header follows that declaration. + +The proxy is the per-turn limit gate too. Every broker turn registers a +`grokBrokerTurnMeter.ts` meter with its registration's model policy, and the +proxy forwards nothing for a turn without one. After a body is proven a lean +worker request and before any upstream call, the meter refuses request +`maxRequests + 1`, any request past `timeoutMs`, and any request once the +upstream-reported running total (prompt tokens *including* cached, plus +completion) has reached `maxTokens` — HTTP 429, and the tripped limit aborts +the worker through the ordinary cancel/kill path. The token ceiling is checked +between requests, so a turn overshoots it by at most the last admitted +request. That bound holds only because a turn has at most one upstream request +in flight: an overlapping request is refused (429, uncounted), and Grok's loop +is sequential in every live capture. A per-request usage block above +`turnLimits.requestUsageMaxTokens` (500k) is invalid, and a response without +valid usage is charged `ceil(bodyBytes/2) + 4096` tokens (rows say +`usage_source: "estimated"`, usage rows `estimated_requests`), so a missing +`usage` never disables the ceiling. A broker timer also trips `timeout` for a +worker that is mid-request, and any trip aborts the in-flight upstream call. Limits come from `service.json` v2 +(`engineBrokerServiceConfig.ts`; v1 gets `GROK_ENGINE_BROKER.turnLimits.v1Defaults`) +and a wake may only lower them: a raise is refused as `invalid_request`, never +clamped. + +The broker stays the single sealed usage writer. `grokEngineBrokerTurn.ts` +seals every terminal turn — completed, failed, limit, cancelled — through +`finishBrokerTurnWithUsage` (`grokEngineBrokerMetering.ts`): the turn registry +record v2 stores the control-protocol v2 terminal response *with* its +numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, +closed `limitReason`) *and the exact ledger bytes it owes*, and only then are +those bytes appended. A replay returns the sealed accounting and never meters +again; it only appends the sealed bytes when the ledger has no row for that +`turn` (a crash between seal and append). Two replays of one sealed turn in +the same broker may both append those identical bytes (a second broker cannot +exist: the realm lease is an exclusive lock), so **every ledger consumer — +`wakeFuse.ts`, Spawnfile's reader (P3), Paideia's evidence reader (P4) — MUST +dedupe usage rows by `turn`** (`dedupeTurnUsageRows`). The turn record's rename +is its publish point: a directory-sync failure after it is reported, never +raised, so a published completed turn is never re-sealed as failed. The window not closed: a crash +before the record's rename seals the turn `failed` with `usage: null` on the +next boot. Once a completed record is sealed, nothing after it can re-seal the +turn as failed. v1 records still replay (upgraded with `usage: null`). Completed usage is the terminal `result.usage`; +a failed turn's partial usage is its per-request stream frames +(`../pi/grokStreamUsage.ts`) when output arrived, else the upstream usage the +proxy saw. Usage rows carry `turn` (the idempotency key readers dedupe on — +`wakeFuse.ts` does), `limit_reason` and `model`; per-request rows go to +`requests.jsonl` beside the registration's `usageLedgerPath` with proxy-measured +`started_at`/`ended_at`. A provider-reported model key must map to the declared +model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected and +is still metered. Control protocol v2 is refused-v1 on the wire because both +ends ship in this package. + +A failed brokered turn also carries the worker's own last words. The launcher +gives the worker one pipe for stdout and stderr and publishes no output for a +failure, so a `worker_failed` turn used to reach the host as nothing but +`exit=1` — the reason the worker printed died with the container's tmpfs. +`DBL_MAX_DIAGNOSTIC` (512 bytes) is now the launcher's bounded tail of that +pipe, sent beside the fixed result frame in `diagnostic_length` and kept only +for a worker that exited on its own account: an output-limit tail would be the +very payload the bound refused, a cancelled turn has no reader left, and a +prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail +exactly as the CLI child path redacts a failed engine child +(`redactCredentialText` with the turn's own provider/MCP capabilities as exact +secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to +one line as `diagnostic.reason`. + +Two rules that live capture taught, both cheap and both load bearing. The +bytes are **decoded**, never stringified: `Uint8Array.prototype.toString("utf8")` +ignores its argument and renders bytes as comma-separated decimals, and a +worker's last words reached an operator as +`reason=108,111,110,101,46,32,87,104,101,110,...` — a string, control-character +free, inside the bound, and passing every check on the way out. So the frame is +normalized to a `Buffer` once on entry and the diagnostic goes through an +explicit `TextDecoder`, which also replaces rather than throws on the +multi-byte sequence a byte-counted window can cut in half. And the window +keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early +prints its error before it echoes its input, so a pure tail is the echo. The +marker is paid out of the same budget, and output that fits is returned +byte-identical. The launcher's own 512-byte window keeps both ends too +(`diagnostic_window`, `native/AGENTS.md`), so the head of a large blob now +survives the one place it used to be erased. Its elision is a cut, and a cut +can split a capability in half into a fragment exact redaction cannot match, so +`scrubCutFragments` matches that fragment here, where the turn's capabilities +are known — on both sides of every marker and at the window's outer ends. A +margin reserved in the launcher could not do this: there, what is kept is +exactly what is sent. It is an optional, control-character-free +member of the sealed terminal response's closed diagnostic — admitted by +`engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — +so it replays with the sealed record and reaches the operator through +`engineBrokerControlClient.ts`'s failure message. Nothing new is written to +disk: the reason travels inside the response the broker already seals. + +A turn whose worker said nothing still records what it spent. The launcher can +refuse to publish a worker's output (`DBL_MAX_OUTPUT`, `native/AGENTS.md`) and +the native transport can fail outright, and in both cases `result.text` never +exists, so there are no stream frames to read usage from. `streamOrMeterUsage` +then falls to the proxy's own per-request measurements — the meter admitted and +settled every forwarded request, so the broker knows the spend even when the +worker never speaks — and `finishBrokerTurnWithUsage` seals and appends it with +`outcome: "failed"`. That is the whole of the guarantee and it is pinned by +"a worker whose work succeeded but whose output crossed the launcher bound" +(`grokEngineBrokerUsage.test.ts`), which builds the launcher's own +output-limit frame at the ABI offsets and decodes it with the shipped client. +Deleting the meter fallback, or refusing that frame shape in +`decodeNativeBrokerResult`, both turn it red. The one window that stays open is +the documented one: a crash before the turn record's rename, which the next +boot seals `usage: null`. + +Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and +the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's +Grok credential without holding it. `request_inference_grant {model, +reasoningEffort, purpose: judge|optimizer}` is an additive control protocol v2 +verb (`engineBrokerInferenceProtocol.ts`); only the organization uid reaches it, +because the native relay admits only that `SO_PEERCRED` uid on `control.sock` +(the TS backend sees only the relay). The answer is a token +(`inference_` + 32 random bytes), the proxy base URL, an expiry (TTL ten +minutes) and the manifest limits; `release_inference_grant` frees one of the +eight live-grant slots early. Grants are their own kind: their own map keyed by +a random grant id, never the turn capability or turn meter maps, and the proxy +routes a bearer by its prefix to exactly one of the two lookups. A grant has no +worker isolation guard but the same spend gate as a turn (one request in +flight, request ceiling, between-requests token ceiling, estimate on missing +usage), so one grant is one sequential lane — parallel judges each hold one. +`grokInferenceProxyRequest.ts` accepts exactly what Grok 1.0.34 sends for the +Paideia judge argv (live stub capture): `stream: true` with +`stream_options.include_usage`, the declared `model`/`reasoning_effort`, plain +`{role, content}` messages, optional `response_format` json_schema, and **no +`tools` or `tool_choice` member at all** — the CLI's per-call `session_title` +request carries both and is refused locally. Every settled request appends one +`kind: "inference"` row (`purpose`, `grant`, `request`, model, usage, +`usage_source`) to `service.json` v2's optional `inferenceLedgerPath`, which +may never be a subject ledger; readers dedupe on `(grant, request)` +(`dedupeInferenceUsageRows`), and `wakeFuse.ts` skips inference rows. Without +that path every grant request is refused `unavailable`. Grants share the +subject's credential authority, so a stale realm fails both (accepted shared +fate): the grant request is refused `auth_stale`, and a proxied grant request +that meets a stale realm gets HTTP 401 `{"error":"auth_stale"}`, which the CLI +surfaces immediately as `Internal error: "Unauthorized (401) from …: +auth_stale …"`. `grokInferenceClientConfig.ts` renders the evaluator's private +`GROK_HOME` `config.toml` (pinned per model/effort in the manifest): the grant +token through `env_key = "DAIMON_INFERENCE_GRANT"`, the worker's lean settings, +no MCP, and `max_retries = 0` — with the default, Grok retries a refused (503) +request with backoff past 45 s instead of failing in ~0.35 s. Its init frame +reports `apiKeySource: "user"`, `tools: []`, `mcp_servers: []`, and the CLI +must be run with `--model daimon-inference-grok`. The inference ledger +directory must be provisioned setgid to the organization group (e.g. +`2100:2000 2750`) for uid 2000 to read rows the broker creates `0640`. + +Every model block the worker can reach carries `max_retries = 0`. Grok 1.0.34's +default retries a refused or failed request with backoff **past 45 s**, blindly: +one live turn emitted the same refusal fifteen times over five minutes, spent +$0 and died with no account of why. The session-title sink and the evaluator +client (`grokInferenceClientConfig.ts`) always pinned it; the worker's own +model — the single path that spends money — was left on the default, so the one +place a stall costs a wake was the only one that could idle for minutes after +its work was done, silently, because a retried request that never reaches +upstream writes no ledger row and prints no proxy line. Daimon owns the retry +decision here because the thing being retried is Daimon's own proxy: a +genuinely transient fault is already answered 503 and is the broker's to +retry, and everything else is a refusal that repeating cannot fix. The worker +fails fast instead and the turn reaches the host with a status. These bytes are +manifest-pinned per model and effort, so changing them rotates +`GROK_ENGINE_BROKER.worker.configSha256` and every deployment must re-vendor. + +`grokBrokerProjection.ts` is the public, I/O-free projection of one brokered +Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own +deny collectors plus the caller's evaluator paths, profile/config/prompt +digests, pinned executable, model, limits and ledger. A Grok agent must declare +`model` and `reasoningEffort` for it; nothing is defaulted, and a supplied +profile digest that differs is refused. Paths are never resolved: Spawnfile +must supply canonical non-symlink paths (its fixed tmpfs and workspace roots) +and verify that during provisioning. The projection also carries the seccomp +profile digest and the `bubblewrap` sandbox runtime a receipt must match. `grokSlotPreflightReceipt.ts` is the +zod schema a root slot supervisor's receipt must satisfy +(`noopolis.daimon.grok-slot-preflight.v2`, fixtures under +`fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to +the projection digest and requires a denied canary for exactly every deny path. +The projection digest does not change across recycles, so the receipt also +carries freshness: a supervisor-owned per-slot `generation` (strictly +increasing) and the caller's recycle `nonce` (32 random bytes, hex). The +verifier requires `{expectedNonce, minGeneration}` and refuses another nonce, a +lower generation, and any v1 receipt. + +`grokBrokerWorkerConfig.ts` is the only source of worker `config.toml` bytes; +the manifest pins the sha256 of every model/effort combination and the broker +refuses a turn whose worker config does not hash to the declared one. Three +1.0.34 facts shape it, each verified against a loopback stub model: +`[auth_provider.*]` helpers never run for a custom model, so the turn's proxy +capability reaches the model through `env_key = "DAIMON_PROVIDER_CAPABILITY"` +set by the native launcher (as exposed as `DAIMON_MCP_CAPABILITY`); the +per-turn `session_title` request cannot be disabled by any key, so +`[models] session_summary` points it at a hidden model +(`GROK_SESSION_TITLE_SINK_MODEL_ID`) whose `base_url` is the broker's own +provider proxy and whose `api_key` is a placeholder too short to ever be a turn +capability — so the request does reach the proxy and is refused there, before +any capability lookup, isolation guard, credential read or upstream call, and +Grok falls back to the truncated prompt as the title. That refusal and a bare +unauthenticated `GET /` probe are the two requests a healthy turn always makes +and the proxy never forwards; neither prints a `refused:` line, because for as +long as they did, every healthy turn read as broken. Every *other* refused +request does name itself on the broker's stderr, and a fault that is not a +`GrokBrokerProxyRefusal` names its own class and message beside +`broker_unavailable` — `[grok-proxy] refused: broker_unavailable (TypeError: +…)` — because the bare word carries no diagnostic content and is answered 503, +which Grok blind-retries: one live turn emitted it fifteen times over five +minutes, spent $0, and died with no account of why. That cause is the error's +class and message, plus one level of its own `cause` — every failed provider +`fetch` is `TypeError: fetch failed` and names nothing without it, so the line +reads `broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)`, an +errno cause with no message named by its `code`. Nothing else: never a body, +bearer, capability, session id or +header. It is redacted through `redactCredentialText` with that request's own +capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, +flattened to one line, exactly as the failed CLI child and the launcher's +worker diagnostic are. It is a log line only: the 503 is unchanged, because a +genuinely transient fault is still transient. + +One fault is *not* transient and no longer wears that shape: a fenced +credential realm. `isStale()` is checked on the turn path before the +credential read, and the request that discovers the fence (the authority's own +generic error) is promoted to the same refusal, so a stale realm is a named +400 `auth_stale` instead of one 503 plus fourteen blind retries — the training +login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning +nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single +name behind the turn failure code, this refusal reason and the grant path's +401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, +and the title sink keeps its 503 on a fenced realm like everywhere else. + +The sink keeps that 503 +shape because every live capture was taken with it: forcing 400 and 503 there +were both observed to end the turn `exit=0, result: success`, so a hard 4xx on +that request does *not* end Grok's session. And effort is only sent when the +model declares it, so the declared effort is +the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in +the image even for a loopback `http://` URL ("Failed to build HTTP client"). + +`engineBrokerMcpFacade.ts` is the worker's only route to its per-wake MCP mount +and rebuilds every header from a closed allowlist in both directions, so the +worker's bearer never reaches the mount and no mount header reaches the worker +uninvited. That allowlist must include the Streamable HTTP transport's own +routing headers or the route does not exist: forwarding only +`content-type`/`accept` destroyed `Mcp-Session-Id`, so `initialize` returned 200 +while every request after it — `notifications/initialized`, `tools/list`, +`tools/call` — came back HTTP 400 `Mcp-Session-Id header is required`, and the +model saw `search_tool` answer `{"results":[],"total_hidden_tools":0,"status": +"partial"}`. Client to mount: `content-type`, `accept`, `mcp-session-id`, +`mcp-protocol-version`, `last-event-id`. Mount to client: `content-type`, +`mcp-session-id`, `mcp-protocol-version`, plus the facade's own +`cache-control: no-store`. The session id is an opaque routing value and is +never logged or ledgered. The facade also carries the three methods the +transport uses — POST, the standalone `GET` SSE stream that is the only route a +server notification or progress frame can take, and the `DELETE` that ends a +session — and streams each body rather than buffering it, because a GET tunnel +stays open for the whole session. Streaming means backpressure, and a +backpressured tunnel must never park: `awaitMcpTunnelDrain` races the client's +`drain` against its `close`/`error` and the turn's abort, because a bare +`once("drain")` cannot fire for a client that hung up mid-write and left the +handler — and the upstream call it was relaying — awaiting for the life of the +process, with no status, no refusal and no line anywhere to read. Every +outcome but a real drain rejects, so the relay tears the tunnel down instead +of writing into a socket that is gone. Never widen it into a transparent proxy: the +whole point of the boundary is that the allowlist is closed. + +The facade is also the only place an MCP tool call is observable *while it is +still running*. Daimon writes a tool receipt on completion, so a call that +started and never returned is byte-identical, in every artifact, to a call that +was never made — and that was the last unlit path under a live hang where the +worker stopped acting after its eighth provider response, the per-request +ledger published `open: 0`, and the trial deadline killed it seven minutes +later. `engineBrokerMcpCallLog.ts` records each relayed `tools/call` POST and +whether the facade ever answered it, and the observation rides the *sealed +terminal response* of a failed turn (`mcpCalls`, optional and v2-only) — +the seam the worker's redacted last words and the sealed usage already take, +because the slot's control root is tmpfs that dies with the container. It +replays with the record and reaches the operator through +`engineBrokerControlClient.ts` as `mcp=/ answered` plus +`mcp_outstanding=@ms`. Its rules are the per-request ledger's: names +and timings only (never arguments, never a result, never a session id or +bearer; a name that is not a plain short identifier is ``, and the +list is bounded with a `` last entry); absence stays absence (a turn +the facade never registered observes as *nothing*, a turn that called nothing +observes `started: 0`, and a POST body the facade could not read counts in +`undecoded` rather than inventing a name); and it can never fail, delay or +refuse a turn. "Answered" means one thing and it is load bearing: the relay +reached its own `end()`. A tunnel torn down when the worker dies did not +answer, so the call it was blocked on stays outstanding with the elapsed time +it had reached — otherwise the turn's death would erase the evidence the +instrument exists to keep. + +The facade relays one more thing, for the whole session, and until now wrote +nothing about it. A `tools/call` is a POST that answers; the standalone `GET` +SSE tunnel is the route a server notification or progress frame takes, and it +stays open from `initialize` to the worker's own shutdown. A worker parked +reading it was, in every artifact the broker wrote, identical to a worker doing +nothing: every provider request closed, every tool call answered, idle to the +deadline. `EngineBrokerMcpCallLog.openTunnel` records that lifecycle on the same +observation — `tunnels: {opened, closed, delivered, open: [{openMs, delivered}]}` +— so a turn sealed with one still open says so and says how long it had been +open, and `delivered` separates a tunnel actively carrying frames from one held +open having received nothing, which is the difference that decides whether it is +the blocker. Bounded at `ENGINE_BROKER_MCP_TUNNEL_MAX` open entries (a session +opens one), counts and elapsed milliseconds only, never a frame, an event +payload or a session id. It is *observation only*: nothing here closes, times +out or refuses a tunnel, because an instrument that tore the stream down would +destroy the evidence it exists to gather. The member is optional on the wire for +one reason — a turn sealed before it existed must still replay — so its absence +means "not measured" and never zero, exactly as `mcp`'s own absence does. +A request the facade *refuses* is the sharpest form of the same silence, and +it used to observe as nothing at all: `route()` threw before `calls.begin`, so +a turn 403'd on every request sealed `answered == started, outstanding: []` — +byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts +each one by a closed reason class (`route`, `expired`, `exhausted`, +`unrouted`, `oversized`), because the classes call for opposite fixes: an +exhausted per-turn capability is a budget, an unserved route is a worker +asking for something that does not exist. That budget is *derived*, not +picked: `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` is `GROK_WORKER_MAX_TURNS` +times `ENGINE_BROKER_MCP_ROUND_REQUESTS` (3 — a round's `search_tool`, its +`use_tool`, and one spare for a retry or a second discovery) plus +`ENGINE_BROKER_MCP_SESSION_REQUESTS` (5 — `initialize`, +`notifications/initialized`, `tools/list`, the GET tunnel, the DELETE). It was +a literal 128 against a bound of 48 rounds whose legitimate traffic is ~101, so +the first round that also retried met a mid-turn 403 storm; the two numbers +that must agree now live in one place, and raising the turn bound can no longer +silently exhaust the budget. It stays a bound rather than a comfortable number +because the derivation is exact: the request *after* the worst-case legitimate +session is refused, so a compromised worker gets three MCP calls per round it +was compiled to take and not one more. `engineBrokerMcpObservation.test.ts` +drives that worst case through the real facade, computed from the turn bound +alone. Attribution comes from +`EngineBrokerCapabilities.classifyToken`, which names the token's turn and why +it would be refused *without spending its budget*; a bearer no grant matches +names no turn and stays unattributed, because guessing an owner would be +inventing the measurement. Counts only: never the token, the capability, the +URL or the body. The member is optional on the wire for `tunnels`' one reason, +and reaches the operator as `mcp_refused=exhausted:41` and the seal row's +`mcp.refusals`. + +Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the +tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and +**closes 16 ms before the worker exits** — the close is the worker's own +shutdown, not the facade's. A turn that never reaches that shutdown is the one +that seals with it open; a deliberately stalled `tools/call` sealed +`open: [{openMs: 14652, delivered: false}]` beside its outstanding call. + +That seam is enough for a turn that *fails with a reply* and not for the turn +the instrument was built for. A worker that crashes still produces a terminal +response; a worker that HANGS is cancelled by its client's deadline, and a +cancelled turn has no client left to answer, so the sealed response — with +`mcpCalls` and the worker's redacted last words riding on it — is sealed into a +turn record in the broker's own `0700` turn store and dies with the slot's +tmpfs. Six live runs reproduced that exactly. What *does* survive a slot is the +broker's ledger directory, which Paideia already recovers `usage.jsonl` and +`requests.jsonl` from on the failure path, so `engineBrokerSealLedger.ts` writes +a third stream beside them: one `noopolis.daimon.turn-seal.v1` row per sealed +terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from +the sealed response and nothing else. Its members are the accounting, the +failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with +the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the +facade's `mcp` observation — names, counts, refusals by reason class, +GET-tunnel lifecycle and elapsed milliseconds. That projection is a closed +allow-list and `engineBrokerSealLedger.test.ts` asserts the *exact key set* of +a written row for a completed turn: replacing it with `...terminal` writes +`usage`, `diagnostic`, `mcpCalls` and the model's entire reply into the +ledger, and that mutation is what the assertion exists to catch. Never a +prompt, body, reply, bearer, capability or session id; the terminal response +carries none of those in the first place, and the projection is an allow-list +rather than a spread, so a future additive member of the response cannot become +a ledger field by accident. + +Two invariants make it worth having. The row is rendered for *every* terminal +turn including one whose `usage` is `null` — a turn cancelled before any spend +could be attributed writes no usage row at all, and is precisely the turn whose +outstanding call has no other route out. And absence stays absence three ways: +no `mcp` member when the facade never observed the turn, `started: 0` when it +observed a turn that called nothing, and no row when nothing sealed. Reading +any of those three as another is the failure this stream exists to prevent. The +line is sealed into the turn record's ledger bytes with the other two and +appended last, so a replay completes an interrupted append the same way and +readers dedupe on `turn`; `seal` is optional in `parseBrokerTurnLedgerLines`, so +a record written before the stream existed still replays. It is advisory +throughout: `recordLedgerLines` swallows every I/O fault, and nothing here can +refuse, delay or fail a turn. + +Worker `GROK_HOME` layout the deployment must provision (attested before every +turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): +`$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, +`sandbox.toml`, `trusted_folders.toml` (empty), `managed_config.toml` (empty) +and `requirements.toml` (empty) `root:root 0444`; and +`sessions/sandbox-events.jsonl` `: 0640`. Grok 1.0.34 writes its +sandbox events there (the root `sandbox-events.jsonl` stays empty) and runs +every profile inside bubblewrap, where a non-empty `deny` list is enforced; +`grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process +can neither write, rename, nor unlink any of the root-owned files. + +Deny-path placement (`grokWorkerDenyPlacement.ts`). Grok 1.0.34 materializes +every `deny` entry inside bubblewrap **as the worker uid**, bind-mounting +`$GROK_HOME/sandbox-blocked-{file,dir}` over the target, so an entry is +placeable only when every ancestor directory is searchable by that uid and the +target already exists and is not a symlink. One unplaceable entry makes Grok +refuse the *whole* profile (`bwrap: Can't create file at …: Permission +denied`), so every turn of that worker fails, not just that path. Matrix: +`.runtime/grok-deny-placement/EVIDENCE.md` in the ecosystem folder. The rule +therefore has two halves: +- shape, decidable without a filesystem and asserted by the renderer: canonical, + and strictly below every base-profile grant (`GROK_WORKER_BASE_PROFILE_GRANTS`); +- placement, asserted by whoever provisions the paths — root provisioning and + every slot recycle on the Spawnfile side, `prepareGrokWorkerAttestation` + before every brokered turn, and `prepareAndVerifyGrokSandbox` on the direct + path, which runs as the worker uid itself. The broker (uid 2100) cannot + descend into a `2000: 0710` runtime home, so an `EACCES` below an + ancestor the worker *can* search is left undecided there; root, which holds + `CAP_DAC_READ_SEARCH`, decides every entry. + +When a protected path is not placeable, the deny entry is **lifted** to the +nearest ancestor that is — never adding `o+x` to a private directory, because a +lift masks a superset and never widens the worker's reach. The durable +wake-acceptance store is exactly that case: it lives under the organization's +`state` directory, which the ownership guard secures `2000:2000 0700`, so the +mask goes on that directory (`acceptanceStoreDenyPath` in +`grokBrokerProjection.ts`, which refuses a mask that does not contain the +store). + +Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every +turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). +Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write +and refuses to start if either, or any path equal to or above a base grant, is +in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; +`/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: +- the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its + read-write grants; Python, Node and `mktemp` use it); provision it + `: 0700`. Every registered worker's private temp is attested + before any turn, so one misprovisioned sibling refuses all turns; +- `/tmp` and `/var/tmp` must be `root: 1774`: + Grok needs to open the directory, but without search or write a worker can + only list names — `cat`/`read_file` get EACCES and it cannot create files. + `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root + process outside that group that needs temp space must get its own `TMPDIR`; +- the organization runtime home of a brokered Grok agent is `2000: + 0710` — traverse-only, so the worker can reach `tool-output/` and nothing + else. `physicalReadiness.ts` accepts exactly that shape for a `grok` agent + (owner the runtime user, mode `0710`, group a worker group that is not the + runtime's own) and keeps the plain `0700` rule for every other engine; wider + (`0711`, `0730`, `0750`, `0770`, any world bit, setgid) is refused, and so is + a `0700` home for a Grok agent, because its worker could not read its own + spills. Everything Daimon creates inside a runtime home is `0700` + (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, + `tool-state`, the engine XDG directories, `.tmp`), so a traversable home + still exposes nothing but `tool-output/`. A deployment-provisioned memory + home under that runtime home must stay `0700` for the same reason. That mode + is *asserted and corrected*, not merely passed to `mkdir`, because `mkdir`'s + `mode` decides nothing for a directory that already exists: a `telemetry/` + left at `0755` by a pre-branch Daimon or pre-created by a deployment stayed + `0755` forever, and under a `0710` home that is the worker reading its own + agent's prompts, replies and causal history. `ensureRuntimeHomeDirectory` + walks every level below the home, opens each through + `O_DIRECTORY|O_NOFOLLOW` and `fchmod`s the directory it stat'd; one owned by + another uid is **refused**, never widened, and a symlink planted where a + directory belongs is refused rather than followed. The home itself is + create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok + agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The + mode constant lives only in that module, a test fails the build if any writer + imports it again, and the same test refuses any `mkdir` that names a runtime + home outside the layout — a `mode:` argument covers only the install where + the directory is new. `wakeAcceptanceFs.ts` is the one exception and closes + the hole the other way, by asserting the directory it found and refusing a + wider one; +- spills (`toolResultSpill.ts`) are written `0640`; provision + `/tool-output` as `2000: 2750` (setgid) under a + runtime home the worker can traverse, so each spill carries that agent's + worker group and no other worker can read it. The writer pins the directory + (`O_DIRECTORY|O_NOFOLLOW`, dev/ino re-checked before publishing) and refuses + one that is a symlink, not owned by the runtime, wider than `2750`, or + group-open without setgid or in the runtime's own group; it cannot tell + *which* worker gid belongs to the agent, so that mapping stays the + deployment's. A spill is published by rename, replacing any existing entry + (a planted symlink included) without following it. +- registered workspace and home paths must be canonical (no `.`, `..`, empty + components or trailing slash) in both `service.json` and `registrations.bin`; + the launcher refuses the slot otherwise. `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. @@ -28,10 +514,13 @@ runtime-writable home without clobbering a newer CLI-refreshed credential. `grokSubscriptionRealm.ts` owns the single durable rotating Grok credential, the lifetime lease, crash journal, stale fence, and serialized per-turn stage/promote cycle while each agent retains private non-auth home state. -`../pi/grokSandbox.ts` owns the production Grok process boundary: it replaces -the provider's fail-open built-in profile with an exact custom profile denying -the realm, bootstrap, and peer roots, and requires a kernel-enforcement event -before every Grok setup, turn, and cleanup process. +`../pi/grokSandbox.ts` owns the direct (non-broker) Grok process boundary: it +replaces the provider's fail-open built-in profile with an exact custom profile +denying the realm, bootstrap, and peer roots, and requires a kernel-enforcement +event (read from `$GROK_HOME/sessions/sandbox-events.jsonl`) before every Grok +turn. The direct path registers its per-wake MCP endpoint in the agent's +Daimon-owned `GROK_HOME` config (`../pi/grokHomeMcpRegistration.ts`), because +1.0.34 skips project-scoped MCP servers in untrusted workspaces. Strict Codex uses its native permission profile only for model-run local commands: the profile denies current `.codex/auth.json`, current `.daimon-inbound`, `/proc`, `/run`, shared protected stores, and peer roots @@ -55,8 +544,10 @@ the only place its per-wake tool-call bound is decided. `maxToolTurns` only mediates daimon-MCP tool calls; Codex's own shell (`exec_command`) is never routed through it, so Codex gets its own bounds instead — `DEFAULT_CODEX_WAKE_TIMEOUT_MS` (wall clock) and -`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/cliSession.ts`, overridable -via `DAIMON_CODEX_WAKE_TIMEOUT_MS`/`DAIMON_CODEX_WAKE_TOKEN_CEILING`. The token +`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/engineWakeLimits.ts`, overridable +via the engine-neutral `DAIMON_ENGINE_WAKE_TIMEOUT_MS`/`DAIMON_ENGINE_WAKE_TOKEN_CEILING` +(the `DAIMON_CODEX_*` names are aliases; conflicting values are refused), which +the dispatcher also passes to the Grok broker as lowering limits. The token ceiling can only be checked when Codex reports it: its `--json` stream carries usage exactly once, on the turn's own `turn.completed`, so crossing it kills the child immediately and fails the wake instead of letting an over-budget @@ -97,7 +588,10 @@ request count) look like the first without proving it. `../pi/cliChildOutput.ts` carries the thread id off Codex's own `thread.started` frame, and `../pi/codexRolloutUsage.ts` reads that thread's rollout under `$CODEX_HOME/sessions/**` for the per-request `token_usage_record` frames the -`--json` stream never emits. Rows go to `requests.jsonl` beside `usage.jsonl` +`--json` stream never emits. Each Codex row carries its own `started_at`/`ended_at` +from the rollout frame timestamps (end = the usage frame; start = the first +non-usage frame after the previous request's usage frame, else that request's +end), absent rather than substituted when a frame has no valid timestamp. Rows go to `requests.jsonl` beside `usage.jsonl` (`DAIMON_TURN_REQUESTS_LEDGER_PATH` relocates it) under the same invariants: a wake whose rollout is absent, unreadable, or undecodable writes *nothing*, because a fabricated zero is byte-identical to a measured one; and every failure @@ -105,6 +599,34 @@ is swallowed, because instrumentation must never fail a wake. The existing ledger's version, path, and field list are untouched, so Spawnfile's `v`-pinned reader is unaffected. +Each Grok row also carries `tool_calls`: the tool-call NAMES that request's +response carried, read by the proxy from the body it already buffers for usage +(`parseGrokResponseToolNames` in `grokBrokerTurnMeter.ts`). Timings and tokens +alone cannot answer "did the model ever *try* to call `use_tool` or +`search_tool`", which is exactly the question two live turns left open. Names +only — never arguments, never message content, never a bearer; a `name` that is +not a plain short identifier is recorded as `` rather than passed +through, and the list is bounded at `GROK_REQUEST_TOOL_CALLS_MAX` (16) entries +with a `` last entry, so a pathological response cannot write an +unbounded row. Absence stays absence, as everywhere in these ledgers: a decoded +response that called nothing records `[]`, and a response that could not be +decoded records *no field at all*, because a fabricated empty list is +byte-identical to a measured one. On the stream row path the names are attached +only when the proxy's timings and the worker's stream requests are aligned +request-for-request, since an unaligned index would credit one request's attempt +to another. The *usage* decode beside it is wrapped the same way, and for a +sharper reason: the upstream call has already succeeded, so a decoder fault +that failed the request would throw away a response the broker paid for and +have Grok buy it again. A fault there falls through to the documented estimate +(`ceil(bodyBytes/2) + 4096`, `usage_source: "estimated"`, counted in +`estimated_requests`) — never to zero and never to absence, because the +ceiling must still count what was spent. It is an additive field inside the unchanged +`noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: +Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia +only relocates this stream's path. The whole path is advisory — the parse is +wrapped, and nothing it does can refuse, delay, or fail a turn, or reach the +spend gate. + `testRuntimeSubprocess.ts` is an unexported, explicit-test-only JSONL process surface for exercising the real control, schedule, and acceptance paths with a controlled clock and deterministic scripted cognition. Its ephemeral loopback @@ -203,3 +725,24 @@ unmarked/deferred deliveries wait for new input without a self-wake loop. Live turn authority is `activity.executions`, independent of receipt completion; its execution id must equal the engine wake id. Budget pauses retain acceptance, and operator stop remains a hard latch. + +That authority has to outlive the host, because the caller who needs it reads it +last. `activityV2` used to answer `undefined` once `stop()` closed the acceptance +store — HTTP 503 `native_host_unavailable` through a caller's route — and the one +caller that must prove an execution closed asks *after* the runtime stopped: a +harness worker stops its host the moment a delivery closes its execution and +stays deferred awaiting external input. So a trial whose subject really ran, +spent its budget and simply did not do the work could not be told from a hung or +crashed one, and reported as an unscorable infrastructure failure. `stop()` now +seals the projection between the dispatcher's own shutdown — which awaits every +admitted turn, so `executions` is settled rather than momentary — and the store's +close, and `activityV2` serves that seal afterwards with `state: "stopped"`. A +stopped host has *more* certainty about quiescence than a live poll, not less, +because nothing can be admitted after the seal. Three things it is not: a bypass +of the control token, a fabricated idle runtime (a host that never started and +one whose seal could not be read both still answer nothing, because absence must +stay absence), and a claim about the store-backed routes beside it — +`availability` and `wakeReceipt` keep answering `undefined` after a stop, since +neither settles a closure proof. `state` is optional on the wire for the reason +every additive member here is: a projection published before the seal existed +must still parse, and its absence means "not stated", never "running". diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 4774ae2..6e1c0b6 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; -import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { AttentionDispatcher, inboxPrompt } from "./attentionDispatcher.js"; +import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; import { parseWakeAcceptanceRequest } from "./wakeAcceptanceTypes.js"; @@ -14,7 +15,7 @@ import type { OrganizationRuntimeHost, OrganizationRuntimeWakeRequest, Organizat const token = "attention-test"; const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; -const config = (maxExecutions = 20) => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: "codex" }, attention: { maxBatchMessages: 3, maxExecutions } })) }); +const config = (maxExecutions = 20, engine: "codex" | "grok" = "codex") => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: engine }, attention: { maxBatchMessages: 3, maxExecutions } })) }); const request = (id: string, agent_id = "alpha") => ({ token, agent_id, delivery_id: id, event: { version: "noopolis.daimon.wake.v2", kind: "message", text: `Handle ${id}`, occurred_at: "2026-09-11T00:00:00.000Z" } }); const pause = (ms = 5) => new Promise((resolve) => setTimeout(resolve, ms)); async function until(test: () => boolean | Promise): Promise { for (let n = 0; n < 200; n++) { if (await test()) return; await pause(); } throw new Error("expected side effect did not appear"); } @@ -29,13 +30,13 @@ class Core implements OrganizationRuntimeHost { async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; } async stop() { this.stops++; this.releases.forEach((release, index) => release({ version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: this.wakes[index]!.agentId, wakeId: this.wakes[index]!.event.id, code: "active_wake_aborted" })); return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } } -async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000) { +async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000, engine: "codex" | "grok" = "codex") { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-")); await chmod(root, 0o700); const usage = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-usage-")); await writeFile(path.join(usage, "usage.jsonl"), ""); const registry: AttentionRegistry = new Map(); const core = new Core(); const options = { acceptanceStorePath: root, controlToken: token, storeOptions: { ...storeOptions, claimTtlMs }, attentionRegistryForTest: registry, fuseEnvironment: { DAIMON_WAKE_FUSE_DIRECTORY: usage, DAIMON_WAKE_FUSE_EPOCH: "attention", DAIMON_WAKE_FUSE_MAX_WAKES: String(maxWakes), DAIMON_WAKE_FUSE_MAX_TOKENS: "10000", DAIMON_TURN_USAGE_LEDGER_PATH: path.join(usage, "usage.jsonl") } }; - const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit), core, options); await control.start(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit, engine), core, options); await control.start(); return { root, usage, registry, core, options, control, cleanup: async () => { await control.stop(); await rm(root, { recursive: true, force: true }); await rm(usage, { recursive: true, force: true }); } }; } @@ -200,3 +201,65 @@ test("claim-renewal failure revokes execution authority, stops cognition, and la const accepted = await f.control.accept(request("after-fence")); assert.equal(accepted.state, "stopped"); assert.equal(accepted.blocked!.reason, "ledger_unavailable"); } finally { WakeAcceptanceStore.prototype.renewClaim = original; await f.cleanup(); } }); + +test("an inbox turn leads with each delivery's own text and keeps the accounting after the work", async () => { + const f = await fixture(); + try { + await f.control.accept(request("d-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text; + // The task comes first: a delivery's text is the work, not a JSON payload to account for. + assert.match(text, /^Carry out this delivery\./u); + assert.match(text, //u); + const task = text.indexOf("Handle d-1"), accounting = text.indexOf("daimon_inbox_disposition"); + assert.ok(task >= 0 && accounting > task, "accounting must follow the delivery text"); + assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); + } finally { await f.cleanup(); } +}); + +/** + * `daimon_inbox_disposition` is the tool that records a finished wake as + * complete; an agent that cannot name it leaves its work recorded as deferred. + * On Grok the bare name reaches nothing, so the inbox prompt must name the + * `daimon__` form the engine can actually invoke. + */ +test("a Grok inbox turn names both inbox tools the way use_tool can call them", async () => { + const f = await fixture(20, 100, 240000, "grok"); + try { + await f.control.accept(request("g-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes(grokDaimonToolName("daimon_inbox_disposition")), "disposition tool carries the daimon__ prefix"); + assert.ok(text.includes(grokDaimonToolName("daimon_inbox")), "inbox tool carries the daimon__ prefix"); + // No bare occurrence survives: every mention is the prefixed one. + assert.equal(text.split("daimon_inbox").length - 1, text.split(DAIMON_GROK_TOOL_PREFIX).length - 1); + } finally { await f.cleanup(); } +}); + +test("every other engine's inbox turn keeps the bare tool names", async () => { + const f = await fixture(); + try { + await f.control.accept(request("c-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes("with daimon_inbox_disposition (complete)")); + assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); + } finally { await f.cleanup(); } +}); + +/** + * Every branch of the inbox prompt, not only the one a small delivery takes: + * the oversized-payload fallback is the branch a busy agent meets, and it names + * `daimon_inbox` too. + */ +test("every branch of the inbox prompt names its tools the way the engine can call them", () => { + const delivery = { acceptance_id: "a-1", delivery_id: "d-1", kind: "message", text: "Do the thing", occurred_at: "2026-09-11T00:00:00.000Z" }; + const oversized = { ...delivery, text: "x".repeat(2_000) }; + for (const [messages, budget] of [[[delivery], 12_000], [[oversized], 64], [[oversized], 8]] as const) { + const grok = inboxPrompt(messages, "grok", budget), codex = inboxPrompt(messages, "codex", budget); + // Mutation guard: an unprefixed branch leaves a bare name in the Grok text. + assert.equal(grok.split("daimon_inbox").length - 1, grok.split(DAIMON_GROK_TOOL_PREFIX).length - 1, grok); + assert.ok(grok.includes(grokDaimonToolName("daimon_inbox")), grok); + assert.equal(codex.includes(DAIMON_GROK_TOOL_PREFIX), false, codex); + assert.ok(codex.includes("daimon_inbox"), codex); + } + // The smallest budget is the fallback that only points at the tool. + assert.match(inboxPrompt([oversized], "grok", 8), new RegExp(`exceeds the prompt budget; read it with ${grokDaimonToolName("daimon_inbox")}\\.$`, "u")); +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index b1a8f77..ad204fc 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -6,6 +6,7 @@ import { WakeAcceptanceStore, WakeExecutionClaimLostError, type WakeExecutionCla import type { StoredWakeAcceptanceRecord } from "./wakeAcceptanceRecord.js"; import { WakeFuse } from "./wakeFuse.js"; import { ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS, ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES } from "../contracts/organizationRuntimeContract.js"; +import { grokDaimonToolName } from "../contracts/grokWorkerContract.js"; type Claimed = { record: StoredWakeAcceptanceRecord; claim: WakeExecutionClaim; done: boolean }; type Options = Readonly<{ store: WakeAcceptanceStore; host: OrganizationRuntimeHost; fuse: WakeFuse; agents: readonly OrganizationRuntimeAgentConfig[]; registry: AttentionRegistry; token: string | undefined; onIdle(agentId: string): void }>; @@ -111,7 +112,7 @@ export class AttentionDispatcher { result = await host.wake({ token, agentId: agent.id, event: { version: "noopolis.daimon.wake.v1", id: agent.attention === undefined ? first.delivery_id : executionId, kind: first.event.kind, occurredAt: first.event.occurred_at, - text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.attention.maxBatchBytes) + text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.engine.kind, agent.attention.maxBatchBytes) } }); } catch (error) { result = { version: "noopolis.daimon.wake-result.v1", status: "failed", agentId: agent.id, wakeId: executionId, code: "engine_failed", detail: engineFailureDetail(error) }; @@ -171,15 +172,56 @@ export function selectBatch(records: readonly StoredWakeAcceptanceRecord[], agen return selected.length ? selected : [first]; } -function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { +/** One claimed delivery, rendered as the task it is. */ +function deliveryBlock(message: unknown, index: number): string | undefined { + if (message === null || typeof message !== "object") return undefined; + const row = message as Record; + const text = typeof row.text === "string" ? row.text : undefined; + if (text === undefined) return undefined; + const from = typeof row.from === "string" ? row.from : undefined; + const kind = typeof row.kind === "string" ? row.kind : "delivery"; + const id = typeof row.delivery_id === "string" ? row.delivery_id : `#${index + 1}`; + return [``, text, ""].join("\n"); +} + +/** + * The inbox turn, task first. + * + * A delivery's own text *is* the work. Leading with bookkeeping and handing the + * model `JSON.stringify(messages)` buried the task: an agent read the JSON, did + * the accounting and deferred without doing the job (observed on Grok: nine + * model requests, no tool calls, nothing filed). The deliveries are therefore + * rendered as labelled blocks and the `daimon_inbox` accounting follows them as + * what to do *after* the work, with the machine-readable payload kept as a + * trailing appendix while it fits the same budget. + * + * Both tools are named the way the agent's own engine can call them. On Grok a + * Daimon tool is an MCP tool of server `daimon` and its bare name reaches + * nothing (`grokDaimonToolName`, the same contract module the worker's system + * prompt and identity envelope render from), so an agent handed the bare name + * cannot mark its work complete — and an unmarked, finished wake is recorded as + * deferred. + */ +export function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); - const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; + const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); + const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; + const accounting = `\nWhen the work above is done, record each delivery with ${tool("daimon_inbox_disposition")} (complete), or defer the ones you could not finish; use ${tool("daimon_inbox")} for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n`; + const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; + const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES + && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; + if (blocks.length > 0 && Buffer.byteLength(blocks.join("\n\n")) <= maxBytes) { + const task = header + blocks.join("\n\n") + accounting; + const withPayload = `${task}\nMachine-readable payload: ${body}`; + // The inbox budget bounds selection; the v1 execution boundary independently + // bounds the complete prompt, including metadata, escaping, and instructions. + if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; + if (fits(task)) return task; + } + const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; - // The inbox budget bounds selection; the v1 execution boundary independently - // bounds the complete prompt, including metadata, escaping, and instructions. - if (Buffer.byteLength(body) > maxBytes || Buffer.byteLength(prompt) > ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES - || [...prompt].length > ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS) { - return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; + if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { + return prefix + `The selected payload exceeds the prompt budget; read it with ${tool("daimon_inbox")}.`; } return prompt; } diff --git a/src/runtime/codexSandbox.linux.test.ts b/src/runtime/codexSandbox.linux.test.ts new file mode 100644 index 0000000..e9f3afd --- /dev/null +++ b/src/runtime/codexSandbox.linux.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const image = process.env.DAIMON_CODEX_SANDBOX_TEST_IMAGE; +const script = ` +const fs=require('node:fs'),path=require('node:path'),cp=require('node:child_process'),net=require('node:net'); +const input=JSON.parse(process.argv[1]); +for(const p of [input.workspace,input.home,...input.denied.filter(p=>p!=='/proc'&&p!=='/run').map(p=>p.endsWith('/auth.json')?path.dirname(p):p),...input.readable])fs.mkdirSync(p,{recursive:true,mode:448}); +const canaries=input.denied.filter(p=>p!=='/proc').map(p=>p.endsWith('/auth.json')?p:path.join(p,'canary')); +for(const p of canaries)fs.writeFileSync(p,'non-secret-denied-canary',{mode:384}); +for(const p of canaries)if(fs.readFileSync(p,'utf8')!=='non-secret-denied-canary')throw Error('outside-sandbox canary missing'); +for(const p of input.readable)fs.writeFileSync(path.join(p,'canary'),'readable-canary'); +fs.writeFileSync(path.join(input.workspace,'input'),'workspace-read'); +const server=net.createServer(s=>s.destroy()); +server.listen(0,'127.0.0.1',()=>{ +const command=input.workspaceDenied?\`const fs=require('node:fs'); +for(const p of \${JSON.stringify([input.workspace+'/input',input.workspace+'/secret/canary'])}){let denied=false;try{fs.readFileSync(p);}catch{denied=true;}if(!denied)throw Error('workspace deny lost to implicit write');} +let denied=false;try{fs.writeFileSync(\${JSON.stringify(input.workspace+'/output')},'forbidden');}catch{denied=true;}if(!denied)throw Error('denied workspace writable'); +process.stdout.write('EXACT_WORKSPACE_DENIED');\`:\`const fs=require('node:fs'),net=require('node:net'); +if(fs.readFileSync('input','utf8')!=='workspace-read')throw Error('workspace read failed'); +fs.writeFileSync('output','workspace-write'); +for(const p of \${JSON.stringify(canaries)}){let denied=false;try{fs.readFileSync(p);}catch{denied=true;}if(!denied)throw Error('canary became readable: '+p);} +for(const p of \${JSON.stringify(input.readable)})if(fs.readFileSync(p+'/canary','utf8')!=='readable-canary')throw Error('readable exception missing'); +const socket=net.connect({host:'127.0.0.1',port:\${server.address().port}});let done=false; +socket.once('connect',()=>{done=true;socket.destroy();throw Error('network reached outside sandbox');}); +socket.once('error',()=>{if(!done){done=true;fs.writeFileSync('complete','CODEX_SANDBOX_ENFORCED');}}); +socket.setTimeout(1500,()=>{socket.destroy();if(!done){done=true;throw Error('network probe timed out');}});\`; +const env={PATH:process.env.PATH,HOME:input.home,CODEX_HOME:input.home+'/.codex',TMPDIR:'/tmp',LANG:'C.UTF-8'}; +const executable=input.workspaceDenied?['/bin/sh','-c','cd /tmp && exec /usr/local/bin/node -e "$1"','probe',command]:['/usr/local/bin/node','-e',command]; +cp.execFile('codex',['sandbox','-P','daimon-strict','-C',input.workspace,'-c',input.profile,'--',...executable],{env,timeout:15000,maxBuffer:65536},(error,stdout,stderr)=>{ +server.close(); +if(error){process.stderr.write(stderr);process.exitCode=1;return;} +if(input.workspaceDenied){if(stdout!=='EXACT_WORKSPACE_DENIED'||fs.existsSync(path.join(input.workspace,'output')))throw Error('workspace deny not enforced: '+JSON.stringify({stdout,stderr,outputExists:fs.existsSync(path.join(input.workspace,'output'))}));process.stdout.write('CODEX_SANDBOX_ENFORCED');return;} +if(fs.readFileSync(path.join(input.workspace,'complete'),'utf8')!=='CODEX_SANDBOX_ENFORCED'||fs.readFileSync(path.join(input.workspace,'output'),'utf8')!=='workspace-write')throw Error('missing sandbox side effect'); +process.stdout.write('CODEX_SANDBOX_ENFORCED'); +});});`; + +async function runSandbox(input: { workspace: string; home: string; denied: readonly string[]; readable: readonly string[]; profile: string; workspaceDenied?: boolean }): Promise { + return (await promisify(execFile)("docker", [ + "run", "--rm", "--network", "none", "--read-only", "--user", "501:20", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--security-opt", "seccomp=unconfined", "--security-opt", "apparmor=unconfined", + ...["/tmp", "/run", "/var/lib/daimon", "/var/lib/spawnfile"].flatMap(target => ["--tmpfs", `${target}:rw,uid=501,gid=20,mode=0700`]), + "--entrypoint", "/usr/local/bin/node", image!, "-e", script, JSON.stringify(input) + ], { timeout: 25000, maxBuffer: 131072 })).stdout; +} + +test("actual Codex sandbox executes the production policy with overlapping control denies", { skip: !image, timeout: 60000 }, async () => { + const configFile = process.env.DAIMON_CODEX_SANDBOX_TEST_CONFIG; + const agent: OrganizationRuntimeAgentConfig = configFile + ? (JSON.parse(await readFile(configFile, "utf8")) as { agents: OrganizationRuntimeAgentConfig[] }).agents[0]! + : { id: "agent:current", name: "Current", instructions: "Unused in this mechanical probe.", + workspacePath: "/var/lib/daimon/workspace", runtimeHomePath: "/var/lib/daimon/home", + engine: { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } }; + const control = "/run/paideia/control"; + const denied = codexSandboxProtectedPaths(agent.id, agent, path.join(agent.runtimeHomePath, ".codex"), [agent], [control]); + const readable = codexSandboxReadablePaths(agent); + const profile = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: denied, codexSandboxReadablePaths: readable }, agent.workspacePath, "http://127.0.0.1:1/mcp") + .find(arg => arg.startsWith("permissions="))!; + assert.ok(denied.includes("/run") && denied.includes(control)); + assert.ok(!profile.includes(`"${control}"="deny"`)); + const run = (permissionConfig: string): Promise => runSandbox({ workspace: agent.workspacePath, + home: agent.runtimeHomePath, denied, readable, profile: permissionConfig }); + assert.equal(await run(profile), "CODEX_SANDBOX_ENFORCED"); + // Restore the exact redundant deny emitted before the fix. The real sandbox + // must reproduce the setup failure instead of merely executing an empty shell. + await assert.rejects(run(profile.replace('"/run"="deny"', `"/run"="deny","${control}"="deny"`)), /Can't mkdir parents.*Read-only file system/u); +}); + +test("a denied workspace fails readiness when Codex exits zero without executing the command", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home"; + const denied = [workspace, `${workspace}/secret`, "/run", "/proc", `${home}/.codex/auth.json`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied }, workspace, "http://127.0.0.1:1/mcp").find(arg => arg.startsWith("permissions="))!; + assert.ok(!profile.includes(`"${workspace}/secret"="deny"`)); + // Codex 0.142.3 executes neither Node nor the shell wrapper in this geometry. + // Exit zero is not readiness: the missing sentinel must reject it. + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], profile, workspaceDenied: true }), + /workspace deny not enforced: \{"stdout":"","stderr":"","outputExists":false\}/u); + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], workspaceDenied: true, + profile: profile.replace(`,"${workspace}"="deny"`, "") }), /workspace deny lost to implicit write/u); +}); + +test("a shared protected path containing .. cannot inherit the wrong ancestor deny", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home"; + const denied = ["/run", "/run/../var/lib/daimon/control/", "/proc", `${home}/.codex/auth.json`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied }, workspace, "http://127.0.0.1:1/mcp").find(arg => arg.startsWith("permissions="))!; + assert.ok(profile.includes('"/var/lib/daimon/control"="deny"')); + assert.equal(await runSandbox({ workspace, home, denied, readable: [], profile }), "CODEX_SANDBOX_ENFORCED"); + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], + profile: profile.replace(',"/var/lib/daimon/control"="deny"', "") }), /canary became readable/u); +}); + +test("unsupported nested exception geometry fails closed instead of dropping its protected descendant", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home", vault = "/var/lib/daimon/vault"; + const denied = ["/run", "/proc", `${home}/.codex/auth.json`, `${home}/.daimon-inbound`, vault, `${vault}/open/secret`]; + const readable = [`${vault}/open`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied, codexSandboxReadablePaths: readable }, workspace, "http://127.0.0.1:1/mcp") + .find(arg => arg.startsWith("permissions="))!; + assert.ok(profile.includes(`"${vault}/open/secret"="deny"`)); + // Codex 0.142.3 cannot mount this nonredundant geometry. Keep the deny and + // require the actual-profile readiness probe to refuse it before cognition. + await assert.rejects(runSandbox({ workspace, home, denied, readable, profile }), /Can't mkdir parents.*Read-only file system/u); + await assert.rejects(runSandbox({ workspace, home, denied, readable, + profile: profile.replace(`,"${vault}/open/secret"="deny"`, "") }), /canary became readable/u); +}); diff --git a/src/runtime/codexSandboxProjection.test.ts b/src/runtime/codexSandboxProjection.test.ts new file mode 100644 index 0000000..b3ba256 --- /dev/null +++ b/src/runtime/codexSandboxProjection.test.ts @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { resolveOrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; + +test("public projection uses actual canonical policy and never reads auth or runs cognition", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-projection-"))); + const priorPath = process.env.PATH; + const agent = { id: "agent:writer", name: "Writer", instructions: "Unused", workspacePath: path.join(root, "workspace"), + runtimeHomePath: path.join(root, "home"), schedule: { kind: "disabled" }, engine: { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } } as const; + const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, agents: [agent] }; + try { + for (const directory of [agent.workspacePath, agent.runtimeHomePath]) await mkdir(directory, { mode: 0o700 }); + const executable = path.join(root, "codex"), log = path.join(root, "calls.jsonl"); + await writeFile(executable, `#!${process.execPath}\nrequire('node:fs').appendFileSync(${JSON.stringify(log)},JSON.stringify(process.argv.slice(2))+'\\n');if(process.argv[2]!=='--version')process.exit(9);process.stdout.write('unit-codex-version');`, { mode: 0o700 }); + process.env.PATH = `${root}${path.delimiter}${priorPath ?? ""}`; + const projection = await resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "/run/paideia/control" }); + const expected = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: codexSandboxProtectedPaths(agent.id, agent, path.join(agent.runtimeHomePath, ".codex"), [agent], ["/run/paideia/control"]), + codexSandboxReadablePaths: codexSandboxReadablePaths(agent) }, agent.workspacePath, undefined).find(arg => arg.startsWith("permissions=")); + assert.equal(projection.permissionConfig, expected); + assert.equal(projection.executablePath, executable); + assert.equal(projection.engineHomePath, path.join(agent.runtimeHomePath, ".codex")); + assert.deepEqual(projection.sandboxArgs, ["sandbox", "-P", "daimon-strict", "-C", agent.workspacePath, "-c", expected]); + assert.ok(!projection.permissionConfig.includes('"/run/paideia/control"="deny"')); + assert.deepEqual((await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line)), [["--version"], ["--version"]]); + await assert.rejects(readFile(path.join(agent.runtimeHomePath, ".codex", "auth.json")), /ENOENT/); + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, "missing", { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection({ ...config, agents: [{ ...agent, engine: { kind: "grok" } }] }, agent.id, { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection({ ...config, agents: [{ ...agent, engine: { kind: "codex" } }] }, agent.id, { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "relative" }), /absolute/); + await chmod(executable, 0o600); + process.env.PATH = root; + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "/run/control" }), /unavailable/); + } finally { if (priorPath === undefined) delete process.env.PATH; else process.env.PATH = priorPath; await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/codexSandboxProjection.ts b/src/runtime/codexSandboxProjection.ts new file mode 100644 index 0000000..b8d62b8 --- /dev/null +++ b/src/runtime/codexSandboxProjection.ts @@ -0,0 +1,50 @@ +import path from "node:path"; +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; +import { engineHomeName, prepareEngineExecutable } from "./engineReadiness.js"; +import { parseOrganizationRuntimeConfig } from "./organizationRuntime.js"; +import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; + +export const CODEX_SANDBOX_PROJECTION_VERSION = "noopolis.daimon.codex-sandbox-projection.v1"; +export type OrganizationCodexSandboxProjection = Readonly<{ + version: typeof CODEX_SANDBOX_PROJECTION_VERSION; + agentId: string; + workspacePath: string; + runtimeHomePath: string; + engineHomePath: string; + executablePath: string; + profileName: string; + permissionConfig: string; + sandboxArgs: readonly string[]; +}>; + +/** Resolves the production command policy without importing auth or accepting a wake. */ +export async function resolveOrganizationCodexSandboxProjection(config: unknown, agentId: string, + options: Readonly<{ acceptanceStorePath: string }>): Promise { + const parsed = parseOrganizationRuntimeConfig(config); + const selected = parsed.agents.find(agent => agent.id === agentId); + if (!selected || selected.engine.kind !== "codex" || selected.engine.codexSandbox === undefined) { + throw new Error("Sandbox projection requires a known strict Codex agent"); + } + if (!path.isAbsolute(options.acceptanceStorePath)) throw new Error("Sandbox projection requires an absolute acceptance store path"); + const authority = await prepareOrganizationRuntimePaths(parsed.agents); + try { + const paths = authority.forAgent(selected); + await paths.verify(); + const agent = { ...selected, workspacePath: paths.workspacePath, runtimeHomePath: paths.runtimeHomePath }; + const engineHomePath = path.join(agent.runtimeHomePath, engineHomeName("codex")); + const executable = await prepareEngineExecutable(agent.id, "codex"); + // These are the same collectors, canonical roots and renderer used by + // startOrganizationRuntimeEngine and its strict production invocation. + const args = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: codexSandboxProtectedPaths(agent.id, agent, engineHomePath, parsed.agents, [options.acceptanceStorePath]), + codexSandboxReadablePaths: codexSandboxReadablePaths(agent) }, agent.workspacePath, undefined); + const permissionConfig = args.find(arg => arg.startsWith("permissions="))!; + const profileName = JSON.parse(args.find(arg => arg.startsWith("default_permissions="))!.slice("default_permissions=".length)) as string; + await paths.verify(); + await executable.verify(); + return { version: CODEX_SANDBOX_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, + runtimeHomePath: agent.runtimeHomePath, engineHomePath, executablePath: executable.executablePath, profileName, permissionConfig, + sandboxArgs: ["sandbox", "-P", profileName, "-C", agent.workspacePath, "-c", permissionConfig] }; + } finally { await authority.close(); } +} diff --git a/src/runtime/engineBrokerCapabilities.test.ts b/src/runtime/engineBrokerCapabilities.test.ts index af9835f..b29fc54 100644 --- a/src/runtime/engineBrokerCapabilities.test.ts +++ b/src/runtime/engineBrokerCapabilities.test.ts @@ -13,3 +13,11 @@ test("proxy resolves scope from opaque token without caller identity", () => { const capabilities = new EngineBrokerCapabilities(); const token = capabilities.issue("agent-a", "turn-a"); assert.deepEqual(capabilities.authorizeToken(token), { agentId: "agent-a", turnId: "turn-a" }); }); + +test("the live proxy lookups refuse an expired capability", async () => { + const capabilities = new EngineBrokerCapabilities(); const token = capabilities.issue("agent-a", "turn-a", 20, 64); + assert.deepEqual(capabilities.inspectToken(token), { agentId: "agent-a", turnId: "turn-a" }); + await new Promise((resolve) => setTimeout(resolve, 40)); + assert.equal(capabilities.inspectToken(token), undefined); + assert.equal(capabilities.authorizeToken(token), undefined); +}); diff --git a/src/runtime/engineBrokerCapabilities.ts b/src/runtime/engineBrokerCapabilities.ts index 0bbd102..1f99f8a 100644 --- a/src/runtime/engineBrokerCapabilities.ts +++ b/src/runtime/engineBrokerCapabilities.ts @@ -23,6 +23,28 @@ export class EngineBrokerCapabilities { } return undefined; } + /** + * Which grant a token names and why it would be refused, *without* spending + * it. + * + * The facade needs this on its refusal path alone. A 403 it cannot attribute + * to a turn is a 403 that seals as nothing at all, and a turn whose every + * request was refused then reads exactly like a healthy one — the silence + * `engineBrokerMcpCallLog.ts` exists to end. A token no grant matches names + * no turn and stays unattributed; nothing here returns the token, the grant + * or the agent's capability, only the turn id and a closed reason. + */ + classifyToken(token: string): Readonly<{ turnId: string; state: "live" | "expired" | "exhausted" }> | undefined { + const candidate = hash(token); + for (const grant of this.grants.values()) { + if (!timingSafeEqual(grant.digest, candidate)) continue; + // Budget before expiry: the TTL outlives every declared turn limit, so an + // exhausted grant is the reachable refusal and the actionable answer. + if (grant.requests >= grant.maxRequests) return { turnId: grant.turnId, state: "exhausted" }; + return { turnId: grant.turnId, state: grant.expiresAt <= Date.now() ? "expired" : "live" }; + } + return undefined; + } inspectToken(token:string):Readonly<{agentId:string;turnId:string}>|undefined{const candidate=hash(token);for(const grant of this.grants.values()){if(grant.expiresAt>Date.now()&&grant.requests { + assert.equal(GROK_ENGINE_BROKER.controlProtocolVersion, ENGINE_BROKER_VERSION); + assert.deepEqual(GROK_ENGINE_BROKER.turnRecordVersions, [ENGINE_BROKER_TURN_RECORD_V1, ENGINE_BROKER_TURN_RECORD_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.serviceConfigVersions, [ENGINE_BROKER_SERVICE_V1, ENGINE_BROKER_SERVICE_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.wakeLimitEnvironment, { timeoutMs: DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, maxTokens: DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV }); + assert.deepEqual([GROK_ENGINE_BROKER.turnLimits.v1Defaults.timeoutMs, GROK_ENGINE_BROKER.turnLimits.v1Defaults.maxTokens], [DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING]); + assert.ok(GROK_ENGINE_BROKER.turnLimits.bounds.maxRequests[1] <= GROK_ENGINE_BROKER.worker.maxTurns, "the broker request ceiling fires before the launcher --max-turns backstop"); +}); diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 7136132..b9d3a76 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -1,13 +1,64 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; -import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import { ENGINE_BROKER_MCP_REFUSAL_REASONS, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; -export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise; } +/** + * What the broker saw of the worker's MCP tool calls on a failed turn + * (`engineBrokerMcpCallLog.ts`). Absent for a turn with no observation at all; + * `outstanding` names every call that started and was never answered, with how + * long it had been waiting — the one thing a completion-only tool receipt can + * never say. + */ +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpRefusals(calls.refusals)}${renderMcpTunnels(calls.tunnels)}`; +/** The refusals the facade never relayed, by reason class, and only the classes that happened: a turn refused 403 must not read as a turn that was served. */ +const renderMcpRefusals=(refusals:EngineBrokerMcpRefusalObservation|undefined):string=>{if(refusals===undefined)return"";const named=ENGINE_BROKER_MCP_REFUSAL_REASONS.filter((reason)=>refusals[reason]>0);return named.length===0?"":`; mcp_refused=${named.map((reason)=>`${reason}:${refusals[reason]}`).join(",")}`;}; +/** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ +const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; + +export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; +/** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ +export class EngineBrokerInferenceGrantRefused extends Error { + constructor(readonly code: EngineBrokerInferenceFailureCode) { super(`engine broker inference grant refused (${code})`); } +} + +/** + * What the organization runtime asks of a brokered turn beyond the prompt: + * `limits` may only lower the registration's declared limits, and `model`, + * when the agent declared one, must equal the model the broker sealed. + */ +export type EngineBrokerTurnOptions = Readonly<{ limits?: EngineBrokerTurnLimitOverrides; model?: string }>; +export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options?:EngineBrokerTurnOptions):Promise; } export class EngineBrokerControlClient implements EngineBrokerTurnClient { constructor(private readonly socketPath="/run/daimon-engine-broker/control.sock"){} - async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:"noopolis.daimon.engine-broker.v1",kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} - async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise{ - const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:"noopolis.daimon.engine-broker.v1",kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); - return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup();if(response.kind==="completed")resolve(response.text);else reject(new Error(response.diagnostic ? `engine broker turn failed (${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal})` : "engine broker turn failed"));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:ENGINE_BROKER_VERSION,kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} + /** + * Evaluator side (organization uid only; the native relay enforces it): + * borrow the broker credential for one sequential lane of judge or optimizer + * requests. Refusals reject with {@link EngineBrokerInferenceGrantRefused}; + * a broker that cannot answer rejects with `engine broker unavailable`. + */ + async requestInferenceGrant(request:Readonly<{model:GrokBrokerModel;reasoningEffort:GrokBrokerReasoningEffort;purpose:GrokInferencePurpose}>):Promise{ + const response=await this.exchange({version:ENGINE_BROKER_VERSION,kind:"request_inference_grant",requestId:randomUUID(),model:request.model,reasoningEffort:request.reasoningEffort,purpose:request.purpose}); + if(response.kind==="inference_grant_refused")throw new EngineBrokerInferenceGrantRefused(response.code); + if(response.kind!=="inference_grant"||response.model!==request.model||response.reasoningEffort!==request.reasoningEffort||response.purpose!==request.purpose)throw new Error("engine broker unavailable"); + const {version:_version,kind:_kind,requestId:_requestId,...grant}=response;return grant; + } + async releaseInferenceGrant(grantId:string):Promise{ + const response=await this.exchange({version:ENGINE_BROKER_VERSION,kind:"release_inference_grant",requestId:randomUUID(),grantId}); + if(response.kind==="inference_grant_refused")throw new EngineBrokerInferenceGrantRefused(response.code); + if(response.kind!=="inference_grant_released"||response.grantId!==grantId)throw new Error("engine broker unavailable"); + return response.released; + } + private exchange(request:EngineBrokerInferenceRequest):Promise{const socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();return new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(settled||response.requestId!==request.requestId||(response.kind!=="inference_grant"&&response.kind!=="inference_grant_released"&&response.kind!=="inference_grant_refused"))throw new Error();settled=true;socket.destroy();resolve(response);}}catch{fail();}});});} + async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options:EngineBrokerTurnOptions={}):Promise{ + const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); + return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); + if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""}${renderMcpCalls(response.mcpCalls)})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerInferenceProtocol.ts b/src/runtime/engineBrokerInferenceProtocol.ts new file mode 100644 index 0000000..02310b9 --- /dev/null +++ b/src/runtime/engineBrokerInferenceProtocol.ts @@ -0,0 +1,77 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; + +/** + * Evaluator inference grant frames, additive kinds of control protocol v2. + * + * Both ends ship in this package (the organization-side client and the broker + * service), so the kinds join v2 without a version bump; a broker that + * predates them refuses the unknown kind and closes the connection, which the + * client reports as `unavailable`. The native relay forwards frames opaquely + * and admits only the organization uid on `control.sock` (`SO_PEERCRED`), so + * that check is what limits grants to the evaluator side. + * + * The grant `token` is the bearer the evaluator's Grok CLI presents to the + * provider proxy (`env_key`). It never carries the broker credential. + */ +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; +export const ENGINE_BROKER_INFERENCE_FAILURE_CODES = SPEC.failureCodes; +export type EngineBrokerInferenceFailureCode = (typeof ENGINE_BROKER_INFERENCE_FAILURE_CODES)[number]; +export const GROK_INFERENCE_PROXY_BASE_URL = `http://${GROK_ENGINE_BROKER.providerProxy.host}:${GROK_ENGINE_BROKER.providerProxy.port}/v1` as const; + +type V = "noopolis.daimon.engine-broker.v2"; +export type EngineBrokerInferenceRequest = + | Readonly<{ version: V; kind: "request_inference_grant"; requestId: string; model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort; purpose: GrokInferencePurpose }> + | Readonly<{ version: V; kind: "release_inference_grant"; requestId: string; grantId: string }>; +export type EngineBrokerInferenceResponse = + | Readonly<{ version: V; kind: "inference_grant"; requestId: string; grantId: string; token: string; baseUrl: typeof GROK_INFERENCE_PROXY_BASE_URL; model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort; purpose: GrokInferencePurpose; expiresAt: string; limits: EngineBrokerTurnLimits }> + | Readonly<{ version: V; kind: "inference_grant_released"; requestId: string; grantId: string; released: boolean }> + | Readonly<{ version: V; kind: "inference_grant_refused"; requestId: string; code: EngineBrokerInferenceFailureCode }>; + +type JsonRecord = Record; +const invalid = (): TypeError => new TypeError("invalid broker frame"); +const exact = (value: JsonRecord, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; +const member = (list: readonly T[], value: unknown): T => { if (!(list as readonly unknown[]).includes(value)) throw invalid(); return value as T; }; +const grantId = (value: unknown): string => { if (typeof value !== "string" || !/^[a-f0-9]{32}$/u.test(value)) throw invalid(); return value; }; +const TOKEN = new RegExp(`^${SPEC.tokenPrefix}[A-Za-z0-9_-]{43}$`, "u"); + +/** `input` has already passed the v2 envelope checks; `requestId` is the parsed id. */ +export function parseEngineBrokerInferenceRequest(input: JsonRecord, requestId: string, version: V): EngineBrokerInferenceRequest { + if (input.kind === "request_inference_grant") { + exact(input, ["version", "kind", "requestId", "model", "reasoningEffort", "purpose"]); + return { version, kind: "request_inference_grant", requestId, model: member(GROK_BROKER_MODELS, input.model), reasoningEffort: member(GROK_BROKER_REASONING_EFFORTS, input.reasoningEffort), purpose: member(SPEC.purposes, input.purpose) }; + } + if (input.kind === "release_inference_grant") { + exact(input, ["version", "kind", "requestId", "grantId"]); + return { version, kind: "release_inference_grant", requestId, grantId: grantId(input.grantId) }; + } + throw invalid(); +} + +export function parseEngineBrokerInferenceResponse(input: JsonRecord, requestId: string, version: V): EngineBrokerInferenceResponse { + if (input.kind === "inference_grant") { + exact(input, ["version", "kind", "requestId", "grantId", "token", "baseUrl", "model", "reasoningEffort", "purpose", "expiresAt", "limits"]); + if (typeof input.token !== "string" || !TOKEN.test(input.token) || input.baseUrl !== GROK_INFERENCE_PROXY_BASE_URL || typeof input.expiresAt !== "string" || Number.isNaN(Date.parse(input.expiresAt)) || new Date(input.expiresAt).toISOString() !== input.expiresAt) throw invalid(); + const limits = input.limits as JsonRecord; + if (limits === null || typeof limits !== "object" || Array.isArray(limits)) throw invalid(); + exact(limits, ["maxRequests", "maxTokens", "timeoutMs"]); + if (!(Number.isSafeInteger(limits.maxRequests) && (limits.maxRequests as number) >= 1 && (limits.maxRequests as number) <= SPEC.limits.maxRequests && Number.isSafeInteger(limits.maxTokens) && (limits.maxTokens as number) >= 1 && (limits.maxTokens as number) <= SPEC.limits.maxTokens && Number.isSafeInteger(limits.timeoutMs) && (limits.timeoutMs as number) >= 1 && (limits.timeoutMs as number) <= SPEC.ttlMs)) throw invalid(); + return { version, kind: "inference_grant", requestId, grantId: grantId(input.grantId), token: input.token, baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: member(GROK_BROKER_MODELS, input.model), reasoningEffort: member(GROK_BROKER_REASONING_EFFORTS, input.reasoningEffort), purpose: member(SPEC.purposes, input.purpose), expiresAt: input.expiresAt, limits: { maxRequests: limits.maxRequests as number, maxTokens: limits.maxTokens as number, timeoutMs: limits.timeoutMs as number } }; + } + if (input.kind === "inference_grant_released") { + exact(input, ["version", "kind", "requestId", "grantId", "released"]); + if (typeof input.released !== "boolean") throw invalid(); + return { version, kind: "inference_grant_released", requestId, grantId: grantId(input.grantId), released: input.released }; + } + if (input.kind === "inference_grant_refused") { + exact(input, ["version", "kind", "requestId", "code"]); + return { version, kind: "inference_grant_refused", requestId, code: member(ENGINE_BROKER_INFERENCE_FAILURE_CODES, input.code) }; + } + throw invalid(); +} + +export const isEngineBrokerInferenceRequestKind = (kind: unknown): boolean => kind === "request_inference_grant" || kind === "release_inference_grant"; +export const isEngineBrokerInferenceResponseKind = (kind: unknown): boolean => kind === "inference_grant" || kind === "inference_grant_released" || kind === "inference_grant_refused"; diff --git a/src/runtime/engineBrokerInferenceService.test.ts b/src/runtime/engineBrokerInferenceService.test.ts new file mode 100644 index 0000000..27eed5a --- /dev/null +++ b/src/runtime/engineBrokerInferenceService.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { EngineBrokerControlClient, EngineBrokerInferenceGrantRefused } from "./engineBrokerControlClient.js"; +import { GROK_INFERENCE_PROXY_BASE_URL } from "./engineBrokerInferenceProtocol.js"; +import { parseEngineBrokerRequest, parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { startEngineBrokerServiceWithIdentity, type EngineBrokerServiceEngine } from "./engineBrokerService.js"; +import { GrokInferenceGrantRefused, GrokInferenceGrants } from "./grokInferenceGrants.js"; + +const V = "noopolis.daimon.engine-broker.v2"; +const baseEngine = (): EngineBrokerServiceEngine => ({ turn: async () => { throw new Error("no turns"); }, readiness: () => ({ providerProxyPort: 43123, mcpFacadePort: 43124, registrations: 1, credentialStale: false, realmLease: true, workerIsolation: true }), close: async () => undefined }); + +async function withService(engine: EngineBrokerServiceEngine, run: (client: EngineBrokerControlClient) => Promise): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "daimon-broker-grants-")), socketPath = path.join(directory, "broker.sock"); + const service = await startEngineBrokerServiceWithIdentity(engine, socketPath, process.getuid!()); + try { await run(new EngineBrokerControlClient(socketPath)); } finally { await service.close(); await rm(directory, { recursive: true, force: true }); } +} +const refusedWith = (code: string) => (error: unknown) => error instanceof EngineBrokerInferenceGrantRefused && error.code === code; + +test("the control socket issues, refuses and releases inference grants", async () => { + const grants = new GrokInferenceGrants({ maxLiveGrants: 1 }); + let stale = false; + const engine: EngineBrokerServiceEngine = { ...baseEngine(), requestInferenceGrant: (request) => { if (stale) throw new GrokInferenceGrantRefused("auth_stale"); return grants.issue(request); }, releaseInferenceGrant: (grantId) => grants.release(grantId) }; + try { + await withService(engine, async (client) => { + const grant = await client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + assert.equal(grant.baseUrl, GROK_INFERENCE_PROXY_BASE_URL); assert.equal(grant.baseUrl, "http://127.0.0.1:43123/v1"); + assert.match(grant.token, /^inference_/u); assert.deepEqual(grant.limits, { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 }); + assert.ok(Date.parse(grant.expiresAt) - Date.now() <= 600_000); + assert.ok(grants.authorize(grant.token)); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "optimizer" }), refusedWith("grant_limit")); + assert.equal(await client.releaseInferenceGrant(grant.grantId), true); + assert.equal(await client.releaseInferenceGrant(grant.grantId), false); + assert.equal(grants.authorize(grant.token), undefined); + stale = true; + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }), refusedWith("auth_stale")); + }); + } finally { grants.close(); } +}); + +test("an engine without grants refuses them as unavailable, and an off-list model never reaches the engine", async () => { + await withService(baseEngine(), async (client) => { + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }), refusedWith("unavailable")); + }); + let called = false; + await withService({ ...baseEngine(), requestInferenceGrant: () => { called = true; throw new Error("unreachable"); } }, async (client) => { + await assert.rejects(client.requestInferenceGrant({ model: "grok-3" as "grok-4.6", reasoningEffort: "low", purpose: "judge" }), /unavailable/u); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "xhigh" as "low", purpose: "judge" }), /unavailable/u); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "subject" as "judge" }), /unavailable/u); + }); + assert.equal(called, false); +}); + +test("grant frames are closed and never carry tools or undeclared members", () => { + const request = { version: V, kind: "request_inference_grant", requestId: "r1", model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }; + assert.deepEqual(parseEngineBrokerRequest(request), request); + for (const bad of [{ ...request, tools: [] }, { ...request, purpose: "subject" }, { ...request, model: "grok-3" }, { ...request, limits: { maxRequests: 1 } }, { ...request, version: "noopolis.daimon.engine-broker.v1" }]) assert.throws(() => parseEngineBrokerRequest(bad), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerRequest({ version: V, kind: "release_inference_grant", requestId: "r1", grantId: "not-hex" }), /invalid broker frame/u); + const grant = { version: V, kind: "inference_grant", requestId: "r1", grantId: "a".repeat(32), token: `inference_${"A".repeat(43)}`, baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: "grok-4.6", reasoningEffort: "low", purpose: "judge", expiresAt: "2026-09-17T05:00:00.000Z", limits: { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 } }; + assert.deepEqual(parseEngineBrokerResponse(grant), grant); + for (const bad of [{ ...grant, baseUrl: "http://evil:43123/v1" }, { ...grant, token: "A".repeat(53) }, { ...grant, limits: { ...grant.limits, timeoutMs: 600_001 } }, { ...grant, limits: { ...grant.limits, maxRequests: 65 } }, { ...grant, credential: "x" }]) assert.throws(() => parseEngineBrokerResponse(bad), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerResponse({ version: V, kind: "inference_grant_refused", requestId: "r1", code: "because" }), /invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts new file mode 100644 index 0000000..dbf51ac --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; + +const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); +const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); + +/** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +/** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ +const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; +/** An observed turn the facade never refused — also a measurement, and not the same statement as a turn it never observed. */ +const NO_REFUSALS = { route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }; + +const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { + let at = 1_000; + return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; +}; + +test("an unanswered tool call is outstanding with its name and its elapsed time; an answered one is neither", () => { + const { log, advance } = clock(); + log.open("turn"); + const answered = log.begin("turn", call("daimon__moltnet_send")); + advance(20); answered.answer(); answered.close(); + const hung = log.begin("turn", call("daimon__moltnet_read", 2)); + advance(420_000); + const observed = log.observe("turn"); + assert.deepEqual(observed?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1); assert.equal(observed?.undecoded, 0); + // A relay torn down by the turn's death answered nothing, and the elapsed + // time freezes where it stopped rather than growing with the report. + hung.close(); advance(5_000); + assert.deepEqual(log.observe("turn")?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.ok(!JSON.stringify(log.observe("turn")).includes("argument-bytes"), "names and timings only"); +}); + +test("absence stays absence: an unopened turn observes undefined, a turn that called nothing observes zero", () => { + const { log } = clock(); + assert.equal(log.observe("turn"), undefined); + log.open("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + // Everything that is not a tool call records nothing at all, so `started` + // stays a count of tool calls and not of traffic. + for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); + log.begin("turn", undefined).answer(); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + log.close("turn"); + assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); +}); + +test("a body the facade could not read counts as undecoded, never as a call with a name", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", Buffer.from("{not json", "utf8")); + log.undecodable("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); + log.undecodable("absent"); +}); + +test("a tool name that is not a plain identifier is recorded as invalid, and a batch names each of its calls", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", call("moltnet read\nBearer sk-live-000")); + log.begin("turn", body([])); + log.begin("turn", Buffer.from(`[${call("memory_recall", 2).toString("utf8")},${call("world_probe", 3).toString("utf8")}]`, "utf8")); + assert.deepEqual(log.observe("turn")?.outstanding.map((entry) => entry.name), [ENGINE_BROKER_MCP_CALL_INVALID, "memory_recall", "world_probe"]); +}); + +test("the outstanding list is bounded, and the earliest calls are the ones kept", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4; index += 1) { log.begin("turn", call(`tool_${index}`, index)); advance(1); } + const observed = log.observe("turn"); + assert.equal(observed?.started, ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4); + assert.equal(observed?.outstanding.length, ENGINE_BROKER_MCP_OUTSTANDING_MAX); + assert.equal(observed?.outstanding[0]?.name, "tool_0"); + assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); +}); + +/** + * The channel a brokered turn had no light on at all. + * + * Seven live runs closed every provider request, answered every tool call, and + * still sat idle to the deadline. The one thing none of them could say is + * whether the worker was parked on the session's standalone GET SSE tunnel, + * because the facade relayed it and recorded nothing. The boundary these + * assertions straddle is exactly that: a tunnel still open at observation + * against one that ended before it — one is a worker that may still be reading, + * the other is a channel already closed and therefore not the blocker. + * + * Mutation: drop `openTunnels.delete(record)` from `close`, and the closed + * tunnel keeps reporting itself open; drop `tunnelsOpened += 1`, and an open + * tunnel becomes indistinguishable from a turn that never opened one. + */ +test("a GET tunnel still open reports its age; one that closed first reports closed and nothing open", () => { + const { log, advance } = clock(); + log.open("turn"); + const parked = log.openTunnel("turn"); + advance(430_000); + const open = log.observe("turn")?.tunnels; + assert.deepEqual(open, { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 430_000, delivered: false }] }); + + parked.close(); + advance(5_000); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 1, closed: 1, delivered: 0, open: [] }); +}); + +test("a turn that never opened a GET tunnel is not a turn that opened one, and neither is an unobserved turn", () => { + const { log, advance } = clock(); + log.open("turn"); + // Observed, and it measured zero: every count is a measurement. + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }); + log.begin("turn", call("daimon__moltnet_read")).answer(); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, "a POST is not a tunnel"); + const tunnel = log.openTunnel("turn"); + advance(1_000); + assert.equal(log.observe("turn")?.tunnels?.open.length, 1); + tunnel.close(); + // And the third state, which is not zero: a turn the facade never registered. + assert.equal(log.observe("absent"), undefined); + log.openTunnel("absent").deliver(); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); +}); + +test("a tunnel the mount pushed through is a different fact from one held open in silence", () => { + const { log, advance } = clock(); + log.open("turn"); + const silent = log.openTunnel("turn"); + const pushing = log.openTunnel("turn"); + advance(90_000); + pushing.deliver(); pushing.deliver(); + assert.deepEqual(log.observe("turn")?.tunnels, { + opened: 2, closed: 0, delivered: 1, + open: [{ openMs: 90_000, delivered: false }, { openMs: 90_000, delivered: true }] + }); + silent.close(); silent.close(); + assert.deepEqual(log.observe("turn")?.tunnels?.closed, 1, "closing twice closes one tunnel"); +}); + +test("the open-tunnel list is bounded, and the counts still name every tunnel beyond it", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_TUNNEL_MAX + 3; index += 1) { log.openTunnel("turn"); advance(1); } + const tunnels = log.observe("turn")?.tunnels; + assert.equal(tunnels?.opened, ENGINE_BROKER_MCP_TUNNEL_MAX + 3); + assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); + assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); +}); + +/** + * The refusal is the sharpest form of the silence this log exists to end. + * + * A refused request never reaches the relay, so a turn every one of whose + * requests was 403'd observed as `started: 0, answered: 0, outstanding: []` — + * the same three numbers a turn that simply had nothing to call observes. The + * boundary these assertions straddle is that one: a turn refused against a + * turn served, and an exhausted capability budget against a route the facade + * does not serve, because the two call for opposite fixes. + * + * Mutation: drop the `refusals` member from `observe`, or make `refuse` a + * no-op, and a refused turn reads as an idle one again. + */ +test("a refused request is counted by reason class, and refusing is not calling", () => { + const { log } = clock(); + log.open("turn"); + assert.deepEqual(log.observe("turn")?.refusals, NO_REFUSALS, "an observed turn that was never refused measured zero"); + log.refuse("turn", "exhausted"); log.refuse("turn", "exhausted"); log.refuse("turn", "route"); + const observed = log.observe("turn"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + // The reading the instrument has to keep honest: a refused turn is not a + // turn that called nothing and answered everything. + assert.deepEqual([observed?.started, observed?.answered, observed?.outstanding], [0, 0, []]); + // Every reason class is its own count, and the observation is a copy: a + // later refusal cannot rewrite a report already handed out. + log.refuse("turn", "unrouted"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + assert.deepEqual(log.observe("turn")?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1, unrouted: 1 }); +}); + +test("a refusal the facade cannot attribute is recorded against no turn at all", () => { + const { log } = clock(); + log.refuse("absent", "expired"); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "a refusal before the turn existed is not this turn's"); + log.refuse("absent", "oversized"); + log.close("absent"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "re-opening an id resets its refusals with everything else"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts new file mode 100644 index 0000000..f8d8b8d --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -0,0 +1,234 @@ +/** + * The in-flight MCP tool calls of one brokered turn. + * + * Daimon writes a tool receipt only when a call *completes*, so a call that + * started and never returned is byte-identical, in every artifact, to a call + * that was never made. A live Grok turn stopped acting after its eighth + * provider response and was killed by the trial deadline seven minutes later + * with the proxy's per-request ledger reporting `open: 0` — every provider + * request closed — which leaves exactly one unlit path: a tool call the worker + * issued and the facade never answered. + * + * This is that light, and it follows the per-request ledger's rules rather + * than inventing its own: + * + * - **names and timings only.** The tool name off the JSON-RPC envelope and + * two clocks. Never arguments, never a result, never a session id, never a + * capability or bearer. A name that is not a plain short identifier is + * recorded as {@link ENGINE_BROKER_MCP_CALL_INVALID} rather than passed + * through, and the list is bounded at + * {@link ENGINE_BROKER_MCP_OUTSTANDING_MAX} with + * {@link ENGINE_BROKER_MCP_CALL_TRUNCATED} as its last entry. + * - **absence stays absence.** A turn the log never opened observes as + * `undefined`; a turn that made no call observes `started: 0`, which is not + * the same statement as an answered call. A POST whose body the facade could + * not read counts in `undecoded` and never as a call with a name, because a + * fabricated name is byte-identical to a measured one — and because "zero + * calls started" is exactly the reading this instrument exists to make + * trustworthy. + * - **it cannot fail a turn.** Every operation here is arithmetic over a map, + * the one parse is wrapped, and the facade treats a missing handle as a + * no-op. + * + * The same map carries the facade's *other* channel, and for the same reason. + * A `tools/call` is a POST that answers; the standalone `GET` SSE tunnel the + * Streamable HTTP transport opens once per session is the route a server + * notification or progress frame takes, and it stays open for the whole + * session by design. A worker parked reading that tunnel is, in every artifact + * the broker writes, indistinguishable from a worker doing nothing at all: + * every provider request closed, every tool call answered, and the turn idle + * until its deadline. {@link EngineBrokerMcpCallLog.openTunnel} records the + * lifecycle — how many the facade relayed, how many ended, and for the ones + * still open at seal time how long each has been open and whether the mount + * ever pushed a single byte through it. A tunnel held open having delivered + * nothing is a different fact from one actively carrying frames, and it is the + * difference that decides whether the tunnel is the blocker. + * + * Observing is all it does. The facade's behaviour is unchanged: nothing here + * closes, times out or refuses a tunnel, because an instrument that tore down + * the stream would destroy the evidence it exists to gather. + * + * "Answered" means the facade wrote a complete response back to the worker — + * the relay reached its own `end()`. A relay that was torn down (the worker + * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay + * outstanding with the elapsed time they had reached. That is the whole point: + * the turn's death must not retroactively mark the call it was blocked on as + * finished. + */ +/** + * The same map carries the facade's *refusals*, for the sharpest form of the + * same problem. A refused request never reaches the relay at all, so a turn + * every one of whose requests was 403'd sealed as `answered == started, + * outstanding: []` — byte-identical to a healthy turn, which is precisely the + * reading this instrument exists to make trustworthy. The reason class is what + * makes it actionable: an exhausted per-turn capability budget — the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`, derived from the compiled turn + * bound times what a round may spend — is a different fault from a mount that + * was never registered, and both differ from a worker asking for a route the + * facade does not serve. Counts only, keyed by a closed vocabulary: + * never the token, the capability, the URL or the body. A refusal the facade + * cannot attribute to a turn — a bearer no live grant matches — is recorded + * nowhere, because attributing it to a turn would be inventing the fact. + */ +export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; +export const ENGINE_BROKER_MCP_CALL_INVALID = ""; +export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; +/** A plain short identifier, or one of the two sentinels above. */ +export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; +const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ +export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; + +/** + * Why the facade refused one relayed request, as a closed vocabulary: + * - `route`: a path or method the facade does not serve. + * - `expired`: the turn capability's TTL had passed. + * - `exhausted`: the turn capability's request budget was spent. + * - `unrouted`: a live capability whose turn has no registered mount. + * - `oversized`: a POST body past the facade's own request bound. + */ +export const ENGINE_BROKER_MCP_REFUSAL_REASONS = ["route", "expired", "exhausted", "unrouted", "oversized"] as const; +export type EngineBrokerMcpRefusalReason = (typeof ENGINE_BROKER_MCP_REFUSAL_REASONS)[number]; +/** How many requests of this turn the facade refused, by reason class. Every member is a measurement; the whole member is absent only where it was never measured. */ +export type EngineBrokerMcpRefusalObservation = Readonly>; + +export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ +export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; +/** + * What the facade saw of one turn's standalone GET SSE tunnels: how many it + * relayed, how many ended, how many ever carried a byte from the mount, and + * the ones still open with the age of each. + */ +export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed: number; delivered: number; open: readonly EngineBrokerOpenMcpTunnel[] }>; +/** + * What the facade saw of one turn's tool calls: how many started, how many the + * facade answered, how many POST bodies it could not read, and the ones still + * unanswered with the time each has been outstanding. + */ +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation; refusals?: EngineBrokerMcpRefusalObservation }>; + +/** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ +export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } +const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; +/** One relayed GET tunnel. `deliver` marks the first byte the mount pushed; `close` ends it. Both are idempotent. */ +export interface EngineBrokerMcpTunnelHandle { deliver(): void; close(): void } +const INERT_TUNNEL: EngineBrokerMcpTunnelHandle = { deliver: () => undefined, close: () => undefined }; + +type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; +type TunnelRecord = { readonly openedAt: number; delivered: boolean }; +type TurnLog = { + started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; + tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; + readonly refusals: Record; +}; +const noRefusals = (): Record => ({ route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + +export class EngineBrokerMcpCallLog { + private readonly turns = new Map(); + constructor(private readonly now: () => number = Date.now) {} + + /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set(), refusals: noRefusals() }); } + close(turnId: string): void { this.turns.delete(turnId); } + + /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ + begin(turnId: string, body: Uint8Array | undefined): EngineBrokerMcpCallHandle { + const log = this.turns.get(turnId); + if (log === undefined || body === undefined || body.byteLength === 0) return INERT; + const names = toolCallNames(body); + if (names === undefined) { log.undecoded += 1; return INERT; } + if (names.length === 0) return INERT; + const startedAt = this.now(); + const records = names.map((name): CallRecord => ({ name, startedAt })); + log.started += records.length; + for (const record of records) log.live.add(record); + let settled = false; + return { + answer: (): void => { + if (settled) return; settled = true; + log.answered += records.length; + for (const record of records) log.live.delete(record); + }, + close: (): void => { + if (settled) return; settled = true; + const endedAt = this.now(); + for (const record of records) { + log.live.delete(record); record.endedAt = endedAt; + // Retain only as many as can be reported; the earliest are the ones + // a hang is about, so a later flood cannot displace them. + if (log.ended.length < ENGINE_BROKER_MCP_OUTSTANDING_MAX) log.ended.push(record); + } + } + }; + } + + /** + * A GET SSE tunnel the facade is about to relay. Counted when it opens, not + * when it succeeds: a tunnel the mount refused still ends, so `opened` and + * `closed` stay a pair and an open one is exactly `opened - closed`. + */ + openTunnel(turnId: string): EngineBrokerMcpTunnelHandle { + const log = this.turns.get(turnId); + if (log === undefined) return INERT_TUNNEL; + const record: TunnelRecord = { openedAt: this.now(), delivered: false }; + log.tunnelsOpened += 1; + log.openTunnels.add(record); + let ended = false; + return { + deliver: (): void => { if (record.delivered) return; record.delivered = true; log.tunnelsDelivered += 1; }, + close: (): void => { if (ended) return; ended = true; log.tunnelsClosed += 1; log.openTunnels.delete(record); } + }; + } + + /** + * A request the facade refused before it could ever be relayed. A refusal it + * cannot attribute to a turn is never recorded against one, so an unknown + * turn is a no-op here exactly as every other operation is. + */ + refuse(turnId: string, reason: EngineBrokerMcpRefusalReason): void { const log = this.turns.get(turnId); if (log !== undefined) log.refusals[reason] += 1; } + + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ + undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } + + observe(turnId: string): EngineBrokerMcpCallObservation | undefined { + const log = this.turns.get(turnId); + if (log === undefined) return undefined; + const at = this.now(); + const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); + const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + const open = [...log.openTunnels] + .sort((left, right) => left.openedAt - right.openedAt) + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); + return { + started: log.started, answered: log.answered, undecoded: log.undecoded, + refusals: { ...log.refusals }, + tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, + outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX + ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] + : outstanding + }; + } +} + +/** + * The tool names one JSON-RPC POST asks for: `undefined` when the body did not + * decode at all (which is a fact of its own, not zero calls), `[]` when it + * decoded and asked for no tool. A batch names each of its calls. + */ +function toolCallNames(body: Uint8Array): readonly string[] | undefined { + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8")); } catch { return undefined; } + const entries: readonly unknown[] = Array.isArray(parsed) ? parsed : [parsed]; + const names: string[] = []; + for (const entry of entries) { + if (!isRecord(entry) || entry.method !== "tools/call") continue; + const name = isRecord(entry.params) ? entry.params.name : undefined; + names.push(typeof name === "string" && TOOL_NAME.test(name) ? name : ENGINE_BROKER_MCP_CALL_INVALID); + } + return names; +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 5244b4f..6ad19ca 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,10 +1,193 @@ import assert from "node:assert/strict"; import { createServer } from "node:http"; + import test from "node:test"; -import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import { ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; + +import { connectClient, FACADE_URL, releaseShared, sharedFacade, startFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; test("MCP facade routes only valid active capabilities to the registered mount", async () => { + const facade=await sharedFacade(); let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch("http://127.0.0.1:43124/mcp",{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); - try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} + const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} +}); + +test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { + const rig = await startRig(); + try { + // connect() is initialize + notifications/initialized: before the session + // header was forwarded the notification came back HTTP 400. + const { client, transport } = await connectClient(rig.capability); + try { + assert.equal(typeof transport.sessionId, "string", "the mount's session id must reach the client"); + const listed = await client.listTools(); + assert.deepEqual(listed.tools.map((tool) => tool.name), ["moltnet_read"]); + const called = await client.callTool({ name: "moltnet_read", arguments: { target: "room:desk" } }); + assert.deepEqual(called.structuredContent, { target: "room:desk" }); + assert.equal(called.isError, undefined); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the mount's server-initiated SSE stream, which only the GET route provides", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + // The standalone GET stream is the only route a server notification can + // take; a POST-only facade answers it 403 and this never arrives. + await new Promise((resolve) => setTimeout(resolve, 150)); + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); + assert.equal(typeof transport.sessionId, "string"); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const sessionId = transport.sessionId; + assert.equal(typeof sessionId, "string"); + await transport.terminateSession(); + assert.equal(transport.sessionId, undefined, "DELETE must be accepted, not 403ed"); + assert.ok(rig.observed.some((request) => request.method === "DELETE"), "the mount must have seen the DELETE"); + const stale = await fetch(FACADE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${rig.capability}`, + "content-type": "application/json", + accept: "application/json, text/event-stream", + "mcp-session-id": sessionId! + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 9, method: "tools/list", params: {} }) + }); + assert.ok(stale.status >= 400, `a terminated session must not still route (got ${stale.status})`); + await stale.body?.cancel(); + await client.close().catch(() => undefined); + } finally { + await rig.close(); + } +}); + +test("the facade forwards a closed header allowlist and never the worker's bearer", async () => { + const rig = await startRig(); + try { + const { client } = await connectClient(rig.capability); + await client.listTools(); + await client.close(); + const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); + assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); + assert.equal(forwarded.includes("cookie"), false); + const allowed = new Set([ + "host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", + "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound header, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode" + ]); + const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); + assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); + } finally { + await rig.close(); + } +}); + +test("the facade withholds a mount response header that is not on the allowlist", async () => { + // The facade comes first: nothing must be listening while the fixed port is + // still in doubt, or a refused start leaks this mount and parks the runner. + const facade = await sharedFacade(); + const target = createServer((_request, response) => { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-from-mount", + "set-cookie": "leak=1", + "www-authenticate": "Bearer realm=\"mount\"", + "x-mount-internal": "private" + }); + response.end('{"ok":true}'); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); + if (address === null || typeof address === "string") throw new Error("target address unavailable"); + const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); + try { + const answered = await fetch(FACADE_URL, { + method: "POST", + headers: { authorization: `Bearer ${capability}`, "content-type": "application/json" }, + body: "{}" + }); + assert.equal(answered.status, 200); + assert.equal(answered.headers.get("mcp-session-id"), "session-from-mount"); + assert.equal(answered.headers.get("cache-control"), "no-store"); + assert.equal(answered.headers.get("set-cookie"), null); + assert.equal(answered.headers.get("www-authenticate"), null); + assert.equal(answered.headers.get("x-mount-internal"), null); + await answered.body?.cancel(); + } finally { + facade.revoke("turn-response-headers"); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +test("the facade still refuses every route and method outside the MCP surface", async () => { + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-refusals", "http://127.0.0.1:1/mcp"); + const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { + method, headers: { authorization: `Bearer ${capability}` } + }); + try { + assert.equal((await call("GET", "/")).status, 403); + assert.equal((await call("GET", "/mcp?probe=1")).status, 403); + assert.equal((await call("PUT", "/mcp")).status, 403); + assert.equal((await call("PATCH", "/mcp")).status, 403); + assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); + } finally { + facade.revoke("turn-refusals"); + } +}); + +/** + * Last, because both cases take the fixed port for themselves: an open GET + * tunnel must not survive its own capability, and must not stall shutdown + * either — a server-to-client stream stays open for the whole session, so + * before it existed nothing could hold the listener open. + */ +test("revoking a turn tears down its open server-to-client stream, and closing never stalls on one", async () => { + await releaseShared(); + const facade = await startFacade(); + const rig = await startRig(facade); + const { client } = await connectClient(rig.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the GET stream must be open before revoking"); + + // The client's stream survives its own capability unless the facade ends the + // tunnel: the mount's GET response is the side that has to close. + facade.revoke(rig.turnId); + await withDeadline(rig.getStreamClosed, 4_000, "a revoked capability left its SSE tunnel open"); + + // A second turn's tunnel, deliberately left open, is what shutdown must not wait on. + const second = await startRig(facade); + const held = await connectClient(second.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + await withDeadline(facade.close(), 3_000, "closing the facade stalled on an open SSE tunnel"); + + await held.client.close().catch(() => undefined); + await client.close().catch(() => undefined); + await second.close().catch(() => undefined); + await rig.close().catch(() => undefined); }); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 156506b..68cd0d3 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,10 +1,325 @@ -import { createServer } from "node:http"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { Readable } from "node:stream"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; + +/** + * The brokered worker's only route to its own per-wake Daimon MCP mount. The + * worker holds a turn capability and nothing else: it never learns the mount's + * address, and the mount never learns the capability. Every header crossing + * either way is rebuilt from a closed allowlist, so this stays a boundary and + * not a transparent proxy — a blanket passthrough would hand the mount the + * worker's bearer and hand the worker whatever the mount chose to say. + * + * The allowlists are exactly the Streamable HTTP transport's own routing + * headers. Anything outside them (authorization above all, cookies, auth + * challenges, forwarding and tracing headers) is dropped in both directions. + * + * Client -> mount: + * - `content-type`: the JSON-RPC body's media type; the mount refuses a POST + * without it. + * - `accept`: the transport negotiates `application/json, text/event-stream` + * per request and the mount answers 406 when a POST does not accept both. + * - `mcp-session-id`: the opaque session the mount issued on `initialize`. + * Dropping it made the mount answer every later request with HTTP 400 + * `Mcp-Session-Id header is required`, so no tool was ever reachable. It is + * a routing value, not a secret — and not a value to log either. + * - `mcp-protocol-version`: the version the handshake settled on. The mount + * validates it and otherwise assumes a default that can disagree with what + * the client negotiated. + * - `last-event-id`: SSE resumability. A reconnecting stream replays from the + * last event it saw; without it the mount cannot tell where to resume. + * + * Mount -> client: + * - `content-type`: tells the client whether it got JSON or an SSE stream. + * - `mcp-session-id`: the id minted on `initialize`. The client must learn it + * or it can never make a second request. + * - `mcp-protocol-version`: the version the mount confirms for the session. + * - `cache-control: no-store` is the facade's own, not the mount's. + * + * Methods are the three the transport uses: POST for JSON-RPC, GET for the + * server-to-client SSE stream (notifications and progress arrive only there), + * and DELETE to end a session. POST alone left the GET stream answering 403. + */ +const FORWARDED_REQUEST_HEADERS = ["content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id"] as const; +const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-protocol-version"] as const; +const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); +const MAX_REQUEST_BYTES = 1024 * 1024; +export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +/** Requests one worker round may legitimately make: its `search_tool`, its `use_tool`, and one spare for a retry or a second discovery. */ +export const ENGINE_BROKER_MCP_ROUND_REQUESTS = 3; +/** The session's fixed cost, once per turn: `initialize`, `notifications/initialized`, `tools/list`, the standalone GET tunnel, the closing DELETE. */ +export const ENGINE_BROKER_MCP_SESSION_REQUESTS = 5; +/** + * Requests one turn capability may spend, across all three methods — *derived* + * from the compiled turn bound rather than chosen. + * + * It was 128, and a legitimate 48-round wake needs ~101 of them: a worker's + * round is a `search_tool` and a `use_tool`, so the first round that also + * retries, or looks something up twice, eats the margin. A bound that can be + * predicted to bite mid-turn is not a bound, it is a 403 storm waiting for a + * real wake — and a number raised until it feels comfortable is not one + * either, because the budget exists to cap a *compromised* worker. + * + * So the two numbers that must agree are kept in one place: the launcher's + * `--max-turns` backstop ({@link GROK_WORKER_MAX_TURNS}) times what a round + * may legitimately spend, plus the session's fixed cost. Raising the turn + * bound raises this with it, and a compromised worker still gets exactly three + * MCP calls per round it was compiled to take and not one more. The arithmetic + * is spelled out rather than folded into a literal so it can be audited: the + * two multiplicands above each say what they are. + */ +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; +export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; + +class FacadeRefusal extends Error {} export async function startEngineBrokerMcpFacade() { - const capabilities=new EngineBrokerCapabilities();const targets=new Map(); - const server=createServer((request,response)=>{void(async()=>{try{if(request.url!=="/mcp"||request.method!=="POST")throw new Error();const match=request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u);if(!match)throw new Error();const scope=capabilities.authorizeToken(match[1]!);if(!scope)throw new Error();const target=targets.get(scope.turnId);if(!target)throw new Error();const body=await bounded(request);const payload=body.buffer.slice(body.byteOffset,body.byteOffset+body.byteLength) as ArrayBuffer;const upstream=await fetch(target,{method:"POST",headers:{"content-type":request.headers["content-type"]??"application/json","accept":request.headers.accept??"application/json, text/event-stream"},body:payload});response.writeHead(upstream.status,{"content-type":upstream.headers.get("content-type")??"application/json","cache-control":"no-store"});response.end(Buffer.from(await upstream.arrayBuffer()));}catch{response.writeHead(403,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"forbidden"}');}})();}); - await new Promise((resolve,reject)=>{server.once("error",reject);server.listen(43_124,"127.0.0.1",()=>{server.off("error",reject);resolve();});}); - return {register(agentId:string,turnId:string,endpoint:string){const url=new URL(endpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid scoped MCP mount");if(targets.has(turnId))throw new Error("MCP turn already registered");targets.set(turnId,url.href);return capabilities.issue(agentId,turnId,15*60_000,128);},revoke(turnId:string){targets.delete(turnId);capabilities.revoke(turnId);},close:()=>new Promise((resolve,reject)=>server.close((error)=>error?reject(error):resolve()))}; + const capabilities = new EngineBrokerCapabilities(); + const targets = new Map(); + /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ + const inflight = new Map>(); + /** + * What the facade saw of each turn's tool calls. A tool receipt is written + * only on completion, so without this a call that started and never returned + * and a call never made are the same absence (`engineBrokerMcpCallLog.ts`). + */ + const calls = new EngineBrokerMcpCallLog(); + + const server = createServer((request, response) => { + void route(request, response).catch((error: unknown) => { + if (response.headersSent || response.destroyed) { response.destroy(); return; } + const status = error instanceof FacadeRefusal ? 403 : 502; + const body = error instanceof FacadeRefusal ? '{"error":"forbidden"}' : '{"error":"bad_gateway"}'; + response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(body); + }); + }); + + /** + * A 403 the call log can read, whenever the bearer names a turn. + * + * The refusal itself is unchanged — same status, same body, same silence + * towards the worker — but it is attributed first, through a lookup that + * does not spend the capability's budget. A bearer no grant matches names no + * turn and stays unattributed, because guessing whose it was would be + * inventing the measurement. A capability that is *also* spent or expired + * reports that instead of the route it asked for: it is the fault the + * operator can act on. + */ + function refuse(bearer: string | undefined, live: EngineBrokerMcpRefusalReason): FacadeRefusal { + const classified = bearer === undefined ? undefined : capabilities.classifyToken(bearer); + if (classified !== undefined) calls.refuse(classified.turnId, classified.state === "live" ? live : classified.state); + return new FacadeRefusal(); + } + + async function route(request: IncomingMessage, response: ServerResponse): Promise { + const method = request.method ?? ""; + const bearer = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u)?.[1]; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw refuse(bearer, "route"); + if (bearer === undefined) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(bearer); + // A live grant that authorizes is the only way past here; anything else is + // classified so an exhausted budget and an expired TTL reach the turn's + // sealed row as themselves. `live` cannot be the answer on this path — + // `authorizeToken` read the same grant a moment ago, expiry only moves + // forward and a budget only spends — so it is the unreachable arm. + if (!scope) throw refuse(bearer, "expired"); + const target = targets.get(scope.turnId); + if (target === undefined) { calls.refuse(scope.turnId, "unrouted"); throw new FacadeRefusal(); } + + // Only POST carries a JSON-RPC body; drain anything else so the socket + // never stalls waiting for a body the facade will not forward. + let body: Buffer | undefined; + // A body refused for size is a call the log can never name, and counting + // it keeps "no tool call started" an honest reading rather than a gap. + try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } + catch (error) { + calls.undecodable(scope.turnId); + // A body past the bound is refused, not merely unreadable: the worker + // gets a 403 for it, so it is counted as one too. + if (error instanceof FacadeRefusal) calls.refuse(scope.turnId, "oversized"); + throw error; + } + const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + const call = calls.begin(scope.turnId, body); + // The GET SSE tunnel is the session's other channel and the one that + // outlives every request: it stays open until the worker or the mount ends + // it, so a turn sealed with one still open is a turn whose worker may be + // parked on it. Observed only — never closed, timed out or refused here. + const tunnel = method === "GET" ? calls.openTunnel(scope.turnId) : undefined; + + const controller = new AbortController(); + const open = inflight.get(scope.turnId) ?? new Set(); + open.add(controller); + inflight.set(scope.turnId, open); + const abort = (): void => controller.abort(); + response.on("close", abort); + try { + // Answered only on a relay that reached its own end: a tunnel torn down + // by the worker's death must not mark the call it was blocked on as + // finished. + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response, tunnel)) call.answer(); + } finally { + call.close(); + tunnel?.close(); + response.off("close", abort); + open.delete(controller); + if (open.size === 0) inflight.delete(scope.turnId); + } + } + + function endTurnStreams(turnId: string): void { + for (const controller of inflight.get(turnId) ?? []) controller.abort(); + inflight.delete(turnId); + } + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(ENGINE_BROKER_MCP_FACADE_PORT, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + + return { + register(agentId: string, turnId: string, endpoint: string): string { + const url = new URL(endpoint); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); + if (targets.has(turnId)) throw new Error("MCP turn already registered"); + targets.set(turnId, url.href); + calls.open(turnId); + return capabilities.issue(agentId, turnId, ENGINE_BROKER_MCP_CAPABILITY_TTL_MS, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + }, + revoke(turnId: string): void { + targets.delete(turnId); + capabilities.revoke(turnId); + endTurnStreams(turnId); + calls.close(turnId); + }, + /** + * What the facade saw of this turn's tool calls, GET tunnels and refusals, + * or `undefined` for a turn + * it never registered. Read on the failure path, before `revoke`. + */ + observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), + close: async (): Promise => { + for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + // A GET SSE tunnel keeps its socket open indefinitely, and `close` + // only stops accepting; without this a shutdown would hang on it. + server.closeAllConnections(); + }); + } + }; +} + +/** The client -> mount allowlist, with the two defaults the mount requires of a POST. */ +function headersFor(method: string, request: IncomingMessage): Record { + const headers: Record = {}; + for (const name of FORWARDED_REQUEST_HEADERS) { + const value = request.headers[name]; + if (typeof value === "string" && value.length > 0) headers[name] = value; + } + if (method === "POST") { + headers["content-type"] ??= "application/json"; + headers["accept"] ??= "application/json, text/event-stream"; + } else { + delete headers["content-type"]; + } + return headers; +} + +/** + * Streams the exchange rather than buffering it: a GET stream stays open for + * the whole session, and a buffered POST would withhold progress + * notifications until the call had already finished. + */ +async function forward( + target: string, + method: string, + headers: Record, + body: ArrayBuffer | undefined, + signal: AbortSignal, + response: ServerResponse, + tunnel?: EngineBrokerMcpTunnelHandle +): Promise { + const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); + // MCP never redirects, and following one would let the mount aim the facade + // at a host the capability was never scoped to. `manual` also reports an + // opaque redirect as status 0, which is not a status to relay at all. + const relayable = (upstream.status >= 200 && upstream.status < 300) || (upstream.status >= 400 && upstream.status <= 599); + if (!relayable) throw new Error("unexpected MCP mount status"); + const outbound: Record = { "cache-control": "no-store" }; + for (const name of FORWARDED_RESPONSE_HEADERS) { + const value = upstream.headers.get(name); + if (value !== null && value.length > 0) outbound[name] = value; + } + outbound["content-type"] ??= "application/json"; + response.writeHead(upstream.status, outbound); + if (upstream.body === null) { response.end(); return true; } + const stream = Readable.fromWeb(upstream.body as Parameters[0]); + let delivered = false; + try { + for await (const chunk of stream) { + // The first byte the mount pushes: a tunnel that carried frames is a + // different fact from one held open having delivered nothing. + if (!delivered) { delivered = true; tunnel?.deliver(); } + if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); + if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); + } + response.end(); + return true; + } catch { + // The client hung up or the mount's stream broke: tear the tunnel down + // rather than leaving a half-written response open. + response.destroy(); + return false; + } finally { + stream.destroy(); + } +} + +/** + * Waits for a backpressured tunnel to drain, or for the tunnel to end — + * whichever happens first, but always one of them. + * + * A bare `once("drain")` never settles for a client that hung up mid-write: + * `drain` cannot fire on a socket nobody is reading, and neither the + * response's own `close` nor the turn's abort woke that await. The GET SSE + * tunnel stays open for a whole session, so the handler — and the upstream + * call it was relaying — leaked for the life of the broker process, with no + * refusal, no status and no line anywhere to read. Every outcome settles this + * now, and every outcome but an actual drain *rejects*, so the caller tears + * the tunnel down instead of writing into a socket that is gone. + * + * Exported for its own test: a hang is only observable from inside. + */ +export function awaitMcpTunnelDrain(response: ServerResponse, signal: AbortSignal): Promise { + if (response.destroyed || response.writableEnded) return Promise.reject(new Error("MCP tunnel closed")); + if (signal.aborted) return Promise.reject(new Error("MCP tunnel aborted")); + return new Promise((resolve, reject) => { + const settle = (finish: () => void) => (): void => { + response.off("drain", onDrain); response.off("close", onClosed); response.off("error", onClosed); + signal.removeEventListener("abort", onAborted); + finish(); + }; + const onDrain = settle(resolve); + const onClosed = settle(() => reject(new Error("MCP tunnel closed"))); + const onAborted = settle(() => reject(new Error("MCP tunnel aborted"))); + response.once("drain", onDrain); response.once("close", onClosed); response.once("error", onClosed); + signal.addEventListener("abort", onAborted, { once: true }); + }); +} + +async function bounded(request: AsyncIterable): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const value = Buffer.from(chunk as Uint8Array); + bytes += value.length; + if (bytes > MAX_REQUEST_BYTES) throw new FacadeRefusal(); + chunks.push(value); + } + return Buffer.concat(chunks); } -async function bounded(request:AsyncIterable):Promise{const chunks:Buffer[]=[];let bytes=0;for await(const chunk of request){const value=Buffer.from(chunk as Uint8Array);bytes+=value.length;if(bytes>1024*1024)throw new Error();chunks.push(value);}return Buffer.concat(chunks);} diff --git a/src/runtime/engineBrokerMcpFacadeRig.test.ts b/src/runtime/engineBrokerMcpFacadeRig.test.ts new file mode 100644 index 0000000..fd336a4 --- /dev/null +++ b/src/runtime/engineBrokerMcpFacadeRig.test.ts @@ -0,0 +1,164 @@ +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; + +import { randomUUID } from "node:crypto"; +import test from "node:test"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +/** + * The facade's shared test rig, and not a suite of its own. + * + * Two suites drive the same boundary — the facade's routing and header + * contract, and the observations it records while relaying — and both need one + * facade on the protocol's fixed port plus a real Daimon MCP mount behind a + * real Streamable HTTP transport. Splitting them into one file each kept both + * readable; duplicating the rig into each would have left two copies of the + * thing every assertion depends on. It carries a `.test.ts` name so it never + * reaches production `dist` (`tsconfig.build.json` excludes exactly that), and + * running it on its own asserts nothing, which is what it is. + * + * Each suite is its own process, so each holds its own shared facade and each + * takes the fixed port for the length of its file. + */ +export const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +export type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +export const sharedFacade = async (): Promise => (shared ??= await startFacade()); +export const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +/** + * The facade's port is the control protocol's own, so the two suites that + * drive it cannot each hold one at the same time. Whichever binds first runs; + * the other waits for it to release the port rather than failing on the + * collision, which is the whole cost of splitting this boundary in two. + */ +const bindFacade = async (): Promise => { + for (let attempt = 0; attempt < 240; attempt += 1) { + try { return await startEngineBrokerMcpFacade(); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + throw new Error("the MCP facade port never came free"); +}; + +export const startFacade = async (): Promise => { + const facade = await bindFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +export type Rig = Readonly<{ + facade: Facade; + turnId: string; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; + close: () => Promise; +}>; + +export const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +let turns = 0; + +export const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); + const mount = createServer((request, response) => { + observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); + return { + facade: host, turnId, server, capability, observed, getStreamClosed, + close: async () => { + host.revoke(turnId); + await closeMount(mount, transport, server); + } + }; +}; + +export const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + +export const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +export const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts new file mode 100644 index 0000000..86fdfeb --- /dev/null +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; +import { createServer, type ServerResponse } from "node:http"; + +import test from "node:test"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, ENGINE_BROKER_MCP_ROUND_REQUESTS, ENGINE_BROKER_MCP_SESSION_REQUESTS } from "./engineBrokerMcpFacade.js"; +import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; + +/** + * What the facade saw while it was relaying, which is the only place a call or + * a stream is observable *while it is still running*. + * + * Daimon writes a tool receipt on completion and nothing at all for the + * session's GET tunnel, so a call that never returned, a worker parked on an + * open stream, and a worker doing nothing were one indistinguishable silence. + * Both instruments are driven here through the real facade. + */ +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); + +/** + * The refusal, which was the one relay outcome that observed as nothing. + * + * `route()` refuses before the call log is ever touched, so a request the + * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn + * whose every request was refused sealed as `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals as. + * + * The middle of this test is a second guarantee and it is a *relationship*, + * not a number. The demand side is computed from the launcher's compiled + * `--max-turns` backstop alone — every round spending its full MCP allowance, + * plus the session's fixed cost — and the whole of it must be served on one + * capability, with the very next request refused. Two numbers that must agree + * and live apart drift: the budget was a literal 128 against a bound of 48 + * rounds, ~101 requests of legitimate traffic, and the first round that also + * retried would have met a mid-turn 403 storm. Deriving one from the other is + * what makes raising `GROK_WORKER_MAX_TURNS` unable to silently exhaust the + * budget — and asserting the refusal one past the worst case is what keeps the + * derivation a bound rather than a comfortable number. + * + * The boundary these assertions straddle: a turn served against a turn + * refused; within the refusals, a spent capability against a route the facade + * does not serve; and, for the budget, legitimate worst-case traffic against + * the first request beyond it. + * + * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` + * call in `route()`, and a 403'd turn reads as an idle one again. Pin + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` back to a literal `128` and this is + * already red at today's turn bound; raise `GROK_WORKER_MAX_TURNS` beside that + * literal and it stays red, which is the drift the derivation removes. Raising + * `GROK_WORKER_MAX_TURNS` *with* the derivation keeps it green, which is the + * guarantee itself. + */ +test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { + const facade = await sharedFacade(); + let served = 0; + const target = createServer((request, response) => { + served += 1; + request.resume(); + request.on("end", () => { response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-refused", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const listed = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }); + const send = async (url: string, bearer: string, method = "POST"): Promise => { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, ...(method === "POST" ? { body: listed } : {}) }); + await response.text(); + return response.status; + }; + try { + // A route the facade does not serve, asked for with a live capability. + assert.equal(await send(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/nope`, token), 403); + assert.equal(await send(FACADE_URL, token, "PUT"), 403); + assert.deepEqual(facade.observe(turnId)?.refusals, { route: 2, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + + // A bearer no live grant matches names no turn, so it is recorded against + // none: inventing an owner would be worse than the silence. + assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); + assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); + + // Neither refusal spent the capability, so what follows is the whole of it. + // The demand is read off the compiled turn bound and nothing else: every + // round the launcher admits, each spending its full MCP allowance, plus the + // one-off session cost. All of it must be served. + const legitimate = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; + for (let spent = 0; spent < legitimate; spent += 1) assert.equal(await send(FACADE_URL, token), 200, `request ${spent + 1} of ${legitimate} was refused`); + assert.equal(served, legitimate, "every legitimate request must reach the mount"); + // And it is still a bound: the first request past the worst case is refused, + // so the budget caps a compromised worker at exactly the traffic the turn + // bound compiles for. + assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); + assert.equal(served, legitimate, "an exhausted capability never reaches the mount"); + + const observed = facade.observe(turnId); + assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); + // And the reading the seal row used to publish for all of it: nothing. + assert.deepEqual([observed?.started, observed?.answered, observed?.undecoded, observed?.outstanding], [0, 0, 0, []]); + assert.ok(!JSON.stringify(observed).includes(token), "counts and reason classes only: never the capability"); + } finally { + facade.revoke(turnId); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerMcpTunnel.test.ts b/src/runtime/engineBrokerMcpTunnel.test.ts new file mode 100644 index 0000000..b22bf36 --- /dev/null +++ b/src/runtime/engineBrokerMcpTunnel.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { connect, type Socket } from "node:net"; +import test from "node:test"; + +import { awaitMcpTunnelDrain } from "./engineBrokerMcpFacade.js"; + +const withDeadline = async (work: Promise, ms: number, message: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(message)), ms); })]); } + finally { if (timer) clearTimeout(timer); } +}; + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 754dcfa..104150d 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;reserved?:number;text?:string}>={}):Buffer{ - const text=Buffer.from(values.text??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.reserved??0,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);return out; +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string|Buffer;text?:string}>={}):Buffer{ + const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } test("encodes ABI v2 and decodes a closed successful result",()=>{ @@ -21,7 +22,111 @@ test("returns bounded typed diagnostics for closed native failures",()=>{ ])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.stage!=="none"); }); -test("rejects unknown, reserved, output-bearing, and cross-class failure frames",()=>{ - for(const value of [{status:9},{reserved:1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); +test("rejects unknown, diagnostic-bearing success, output-bearing, and cross-class failure frames",()=>{ + for(const value of [{status:9},{diagnostic:"late words"},{status:1,stage:4,failure:4,pid:0,uid:0,ticks:0n,exit:-1,diagnostic:"no worker ran"},{status:2,stage:6,failure:5,exit:1,diagnosticLength:ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES+1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); for(const offset of [28,31,105,107,124,127]){const hostile=frame({text:"done"});hostile[offset]=1;assert.throws(()=>decodeNativeBrokerResult(hostile,turnId),/^Error: engine broker turn failed$/u);} }); + +test("a failed worker's own last words cross as a redacted, bounded reason",()=>{ + const words=`{"type":"error","message":"session store unwritable"}\nBearer provider-cap-secret-value\ngrok: exiting 1\n`; + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1,diagnostic:words}),turnId,["provider-cap-secret-value"]),(error:unknown)=>{ + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason=error.diagnostic.reason; + assert.ok(reason!==undefined,"the worker's own reason must reach the diagnostic"); + assert.match(reason,/session store unwritable/u); + assert.doesNotMatch(reason,/provider-cap-secret-value/u,"the turn capability must never reach a diagnostic"); + assert.doesNotMatch(reason,/[\n\r\u0000-\u001f]/u,"the reason is one bounded line"); + assert.ok(Buffer.byteLength(reason,"utf8")<=CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + return true; + }); + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); +}); + +/** + * The assertion is the SENTENCE. A live turn reported its worker's last words + * as `reason=108,111,110,101,46,32,87,104,101,110,...` — the bytes of "lone. + * When ..." rendered as decimals, because a byte view that is not a Node + * `Buffer` answers `toString("utf8")` with a comma-separated list and every + * check the diagnostic passed on the way out (a string, no control bytes, + * under the bound) is satisfied by digits. Nothing weaker than the decoded + * text can catch that. + */ +const words = "lone. When assigned, read `room:assignment`, open my row in the desk index."; + +test("a worker's last words cross as decoded text, not as the decimals of their bytes", () => { + for (const [shape, view] of [["a Buffer", (bytes: Buffer): Uint8Array => bytes], ["a plain Uint8Array", (bytes: Buffer): Uint8Array => new Uint8Array(bytes)]] as const) { + assert.throws(() => decodeNativeBrokerResult(view(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words })), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, `${shape}: the reason must be the worker's sentence, byte-identical`); + assert.doesNotMatch(error.diagnostic.reason ?? "", /^[0-9,]+$/u, `${shape}: a decimal byte list is what this regressed to before`); + return true; + }); + } +}); + +test("a window that cut a multi-byte sequence in half decodes with replacement instead of throwing", () => { + // The launcher's window is a byte count: these first two bytes are the tail + // of a three-byte sequence whose leading byte the window already dropped. + const cut = Buffer.concat([Buffer.from([0x9c, 0xa8]), Buffer.from(" grok: exiting 1")]); + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.match(reason, /grok: exiting 1$/u, "the legible remainder survives the cut sequence"); + assert.match(reason, /\uFFFD/u, "the cut sequence is replaced, not thrown on"); + return true; + }); +}); + +/** + * The window keeps both ends. A pure tail is what turned the one diagnostic + * this project has ever got out of a failed worker into 512 bytes of the + * worker's own prompt echoed back, with the error itself off the front. + */ +test("the bounded diagnostic window keeps the head, the tail, and a marker naming what it dropped", () => { + const blob = `START-OF-ERROR ${"m".repeat(40_000)} END-OF-ECHO`; + const window = boundedDiagnosticWindow(blob, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + assert.ok(window.startsWith("START-OF-ERROR "), `the head must survive: ${window.slice(0, 40)}`); + assert.ok(window.endsWith(" END-OF-ECHO"), "the tail must survive too"); + const marker = /\[… (\d+) bytes elided …\]/u.exec(window); + assert.ok(marker !== null, "the elision is named, not silent"); + assert.equal(Number(marker[1]) + Buffer.byteLength(window.replace(marker[0], ""), "utf8"), Buffer.byteLength(blob, "utf8"), "the marker's count is exactly what was dropped"); + assert.ok(Buffer.byteLength(window, "utf8") <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES, `the marker is paid for out of the same budget: ${Buffer.byteLength(window, "utf8")} bytes`); +}); + +test("output that fits the window is returned byte-identical, with no marker", () => { + for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) + assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); +}); + +/** + * The cut the launcher makes is the one place exact redaction cannot reach on + * its own, and it is the cut this test straddles: the capability begins inside + * the retained head and ends inside the elided middle, so the redactor never + * sees it whole and, without the scrub, its first characters travel verbatim. + * The mirror case is the tail's leading edge, where the capability's last + * characters survive instead. + */ +test("a capability the launcher's window cut in half never crosses as a fragment", () => { + const provider = `provider-${"A".repeat(34)}`, mcp = `mcp-${"B".repeat(39)}`; + const elision = "[… 9000 bytes elided …]"; + const cut = `grok: refused ${provider.slice(0, 20)}${elision}${mcp.slice(mcp.length - 20)} exiting 1`; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, [provider, mcp]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.doesNotMatch(reason, /A{12}|B{12}|provider-A|BBB-?mcp/u, `no credential fragment may cross: ${reason}`); + assert.match(reason, /grok: refused \[REDACTED\]/u, "the head's cut fragment is marked where it was"); + assert.match(reason, /\[REDACTED\] exiting 1$/u, "and so is the tail's"); + assert.ok(reason.includes(elision), "the launcher's own elision marker is left alone"); + return true; + }); +}); + +test("ordinary words at a cut are not eaten by the fragment scrub", () => { + const provider = `provider-${"A".repeat(34)}`; + const words = "grok: profile refused, exiting 1"; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words }), turnId, [provider]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, "text that is not a credential fragment is untouched"); + return true; + }); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 8d905bd..6a612e1 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,14 +1,19 @@ import { spawn } from "node:child_process"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; -const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; +/** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ +export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; +/** `DBL_MAX_OUTPUT`: the launcher's bound on a whole turn's stdout, and the control protocol's own `text` bound, which is the next boundary this output crosses. */ +const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 262_144; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; -export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } +export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;reason?:string;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } export class NativeBrokerTurnFailure extends Error { constructor(readonly diagnostic:NativeBrokerDiagnostic){super("engine broker turn failed");} } export type NativeBrokerTurn = Readonly<{slot:number;requestId:string;turnId:string;agentId:string;wakeId:string;prompt:string;providerCapability:string;mcpCapability:string}>; export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid:number;startTicks:bigint;diagnostic:NativeBrokerDiagnostic} @@ -16,22 +21,99 @@ export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid: export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTurn,signal?:AbortSignal):Promise>{ const frame=encodeNativeBrokerTurn(input),child=trackCliChild(spawn(executable,["--client"],{detached:process.platform!=="win32",env:{LANG:"C",LC_ALL:"C",TZ:"UTC"},stdio:["pipe","pipe","ignore"],...(signal===undefined?{}:{signal})}));const chunks:Buffer[]=[];let bytes=0; child.stdout!.on("data",(chunk:Buffer)=>{bytes+=chunk.length;if(bytes<=ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)chunks.push(chunk);});child.stdin!.end(frame);frame.fill(0); - try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} + try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId,[input.providerCapability,input.mcpCapability]);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string):NativeBrokerTurnResult{ +/** + * `output` is accepted as any byte view, and normalized to a `Buffer` once + * here: `Uint8Array.prototype.toString("utf8")` ignores its argument and + * renders the bytes as comma-separated decimals, which is how a live worker's + * last words reached an operator as `reason=108,111,110,101,...` instead of a + * sentence. Decoding is explicit from here on, never a stringification. + */ +export function decodeNativeBrokerResult(input:Uint8Array,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ + const output=Buffer.isBuffer(input)?input:Buffer.from(input.buffer,input.byteOffset,input.byteLength); if(output.lengthbytes.every((byte)=>byte===0)); - if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||reserved!==0||!paddingZero||observed!==turnId||length>MAX_OUTPUT)throw new Error("engine broker turn failed"); - const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; - const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0; - const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n; + if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); + // Decoded from the caller's own view, not from the normalized frame: the + // decode is what must be correct for any byte view, and it is the step the + // live `reason=108,111,110,...` failure came from. + const reason=workerReason(input.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; + const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; + const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; const worker=status===2&&stage===6&&(failure===5||failure===6)&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const outputFailure=status===3&&stage===7&&failure===7&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const cancelled=status===4&&stage===6&&failure===8&&profile===0&&pid>0&&uid>=2200&&ticks>0n; if(!success){if(length!==0||(!prelaunch&&!worker&&!outputFailure&&!cancelled))throw new Error("engine broker turn failed");throw new NativeBrokerTurnFailure(diagnostic);} return{text:output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES).toString("utf8"),workerUid:uid,workerPid:pid,startTicks:ticks,diagnostic}; } +/** + * The worker's own last words, fit to cross a boundary. + * + * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher + * merges the worker's stdout and stderr into one pipe and publishes no output + * for a failure, so this bounded window is the only account of why it failed. + * It keeps both ends of what it is given (`boundedDiagnosticWindow`), because + * a worker that dies early prints its error before it echoes anything. + * It is worker-controlled text, so it is redacted exactly as the CLI child + * path redacts a failed engine child (`redactCredentialText` with the turn's + * own capabilities as exact secrets, the same diagnostic bound) and flattened + * to one line, because it travels inside a failure message. + */ +function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefined{ + if(tail.byteLength===0)return undefined; + // Decoded explicitly, and with replacement rather than a throw: the window + // is a byte count, so it can cut a multi-byte sequence in half at either + // end, and a worker's last words must not be lost to its own encoding. + const flattened=UTF8.decode(tail).replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + // Redact first, unbounded, then window: redaction can lengthen the text + // ([REDACTED] is longer than a short secret), so bounding before it could + // hand back more bytes than the boundary admits. + const redacted=redactCredentialText(scrubCutFragments(flattened,secrets),secrets,Number.MAX_SAFE_INTEGER).trim(); + const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return reason.length===0?undefined:reason; +} +/** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ +const UTF8=new TextDecoder("utf-8"); + +/** + * A credential the launcher's window cut in half, at either side of a cut. + * + * Exact redaction matches a secret whole, so a secret a cut split survives as + * a fragment it can never match: the piece before a cut can end with a + * secret's prefix, and the piece after it can begin with a secret's suffix. + * The trick that answers this where Daimon owns both ends — retain one whole + * secret more than is reported (`cliChildOutput.ts`) — cannot work at this + * boundary, because the launcher's window *is* what it sends: a margin + * reserved there would be reported along with everything else. So the fragment + * is matched here, where the turn's own capabilities are known, and every cut + * the window can make is covered: the two sides of each elision marker, and + * the outer ends, where the launcher's capture itself stopped reading. + * + * Only a fragment long enough to be a credential is scrubbed. Below + * {@link MIN_CREDENTIAL_FRAGMENT} characters a piece of a random token is + * indistinguishable from ordinary words and carries nothing usable, and + * scrubbing it would eat real text. + */ +const MIN_CREDENTIAL_FRAGMENT=12; +const ELISION_MARKER=/(\[… \d+ bytes elided …\])/u; +const scrubCutFragments=(value:string,secrets:readonly string[]):string=> + value.split(ELISION_MARKER).map((part)=>ELISION_MARKER.test(part)?part:scrubEnds(part,secrets)).join(""); +function scrubEnds(part:string,secrets:readonly string[]):string{ + let result=part; + for(const secret of secrets){ + if(secret.length<=MIN_CREDENTIAL_FRAGMENT)continue; + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.endsWith(secret.slice(0,length))){result=`${result.slice(0,result.length-length)}[REDACTED]`;break;} + } + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.startsWith(secret.slice(secret.length-length))){result=`[REDACTED]${result.slice(length)}`;break;} + } + } + return result; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 1be3d53..8d59256 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse } from "./engineBrokerProtocol.js"; -const start = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; +const accounting = { outcome: "completed", usage: { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }, model: "grok-4.6", requests: 2, limitReason: "none" } as const; +const start = { version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; test("broker frames survive arbitrary chunking and validate closed requests", () => { const encoded = encodeEngineBrokerFrame(start); const decoder = new EngineBrokerFrameDecoder(); const values: unknown[] = []; @@ -13,15 +14,133 @@ test("broker frames survive arbitrary chunking and validate closed requests", () }); test("broker response attestation is mandatory and bounded", () => { - const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; assert.deepEqual(parseEngineBrokerResponse(value), value); assert.throws(() => parseEngineBrokerResponse({ ...value, workerUid: 0 }), /invalid broker frame/); const decoder = new EngineBrokerFrameDecoder(); assert.throws(() => decoder.push(Uint8Array.from([0, 16, 0, 1])), /invalid broker frame/); }); test("broker failure diagnostics are closed and contain no raw worker output",()=>{ - const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"}} as const; + const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const; assert.deepEqual(parseEngineBrokerResponse(value),value); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,rawOutput:"secret"}}),/invalid broker frame/u); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,failureClass:"secret"}}),/invalid broker frame/u); }); + +test("v2 terminal frames carry closed numeric accounting and refuse anything else", () => { + const completed = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; + assert.deepEqual(parseEngineBrokerResponse(completed), completed); + for (const bad of [ + { ...completed, usage: { ...accounting.usage, total: 31 } }, + { ...completed, usage: { ...accounting.usage, note: "text" } }, + { ...completed, usage: { ...accounting.usage, input: "20" } }, + { ...completed, model: "grok-4.6-build" }, + { ...completed, outcome: "failed" }, + { ...completed, limitReason: "tokens" }, + { ...completed, limitReason: "budget" }, + { ...completed, requests: -1 }, + { ...completed, extra: 1 }, + (({ limitReason: _omit, ...rest }) => rest)(completed) + ]) assert.throws(() => parseEngineBrokerResponse(bad), /invalid broker frame/u); + const limit = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", outcome: "failed", usage: { input: 5, cacheRead: 5, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 3, limitReason: "requests" } as const; + assert.deepEqual(parseEngineBrokerResponse(limit), limit); + assert.throws(() => parseEngineBrokerResponse({ ...limit, limitReason: "none" }), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerResponse({ ...limit, code: "engine_failed" }), /invalid broker frame/u); +}); + +test("v1 frames are refused on the wire but a v1 terminal record still parses", () => { + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + assert.throws(() => parseEngineBrokerResponse(v1), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerRequest({ ...start, version: "noopolis.daimon.engine-broker.v1" }), /invalid broker frame/u); + assert.deepEqual(parseEngineBrokerV1TerminalResponse(v1), v1); + assert.throws(() => parseEngineBrokerV1TerminalResponse({ ...v1, ...accounting }), /invalid broker frame/u); +}); + +test("start_turn limits are an optional closed subset inside their bounds", () => { + assert.deepEqual(parseEngineBrokerRequest({ ...start, limits: { maxTokens: 1_000 } }), { ...start, limits: { maxTokens: 1_000 } }); + for (const limits of [{}, { maxTokens: 0 }, { maxRequests: 49 }, { timeoutMs: 999 }, { maxTokens: 1, raise: true }, { maxTokens: 1.5 }]) { + assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); + } +}); + +test("a failed worker's redacted reason is an optional bounded member of its diagnostic",()=>{ + const worker={status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9"} as const; + const failed={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"} as const; + const named={...failed,diagnostic:{...worker,reason:"grok: session store unwritable"}} as const; + assert.deepEqual(parseEngineBrokerResponse(named),named); + const prelaunch={status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"} as const; + for(const bad of [ + {...failed,diagnostic:{...worker,reason:""}}, + {...failed,diagnostic:{...worker,reason:"x".repeat(769)}}, + {...failed,diagnostic:{...worker,reason:"line\nbreak"}}, + {...failed,diagnostic:{...worker,reason:7}}, + {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} + ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); +}); + +/** + * The in-flight tool-call observation (`engineBrokerMcpCallLog.ts`) rides the + * sealed failed frame, so the seam that carries the worker's last words and + * its accounting carries this too — nothing new on a tmpfs that dies with the + * container. It is names and timings, bounded, and internally consistent: a + * frame that claims more answered than started, or more outstanding than + * started minus answered, is a fabrication and is refused rather than clamped. + */ +test("a failed frame carries the broker's in-flight MCP tool-call observation, bounded and consistent", () => { + const value = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", mcpCalls: { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }, outcome: "failed", usage: null, model: "grok-4.6", requests: 8, limitReason: "timeout" } as const; + assert.deepEqual(parseEngineBrokerResponse(value), value); + for (const mcpCalls of [ + { ...value.mcpCalls, answered: 4 }, + { ...value.mcpCalls, started: 2 }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1 }, { name: "memory_recall", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: -1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "moltnet read; Bearer sk-live", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1, arguments: { text: "secret" } }] }, + { ...value.mcpCalls, outstanding: Array.from({ length: 17 }, () => ({ name: "tool", outstandingMs: 1 })) }, + { started: 3, answered: 2, outstanding: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); + // A v1 record predates the instrument; a v1 frame that carries it is forged. + assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); + + /** + * The session's standalone GET SSE tunnel rides the same member, under the + * same rules. It is optional for exactly one reason — a turn sealed before + * the facade observed that channel replays without it — so its absence means + * "not measured" and never zero, and a record that carries it must still be + * a measurement: nothing closes before it opens, nothing delivers without + * opening, and no more can be open than `opened - closed`. + */ + const tunnels = { opened: 2, closed: 1, delivered: 1, open: [{ openMs: 428_004, delivered: false }] } as const; + const observed = { ...value, mcpCalls: { ...value.mcpCalls, tunnels } } as const; + assert.deepEqual(parseEngineBrokerResponse(observed), observed); + assert.deepEqual(parseEngineBrokerResponse(value), value, "a frame sealed before the tunnel was observed still replays, without the member"); + for (const forged of [ + { ...tunnels, closed: 3 }, + { ...tunnels, delivered: 3 }, + { ...tunnels, opened: 1, closed: 1, open: [{ openMs: 1, delivered: false }] }, + { ...tunnels, open: Array.from({ length: 9 }, () => ({ openMs: 1, delivered: false })), opened: 12, closed: 0 }, + { ...tunnels, open: [{ openMs: -1, delivered: false }] }, + { ...tunnels, open: [{ openMs: 1, delivered: "yes" }] }, + { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, + { opened: 1, closed: 0, open: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); + + /** + * The refusals ride the same member under the same rules, and they are the + * counts that make a 403'd turn readable: without them a turn the facade + * refused every request of publishes `started: 0, answered: 0` — an idle + * turn's numbers. Every reason class is its own measurement, so a partial + * member is refused rather than zero-filled, and the whole member is optional + * for the one reason `tunnels` is. + */ + const refusals = { route: 1, expired: 0, exhausted: 41, unrouted: 0, oversized: 2 } as const; + const refused = { ...value, mcpCalls: { ...value.mcpCalls, refusals } } as const; + assert.deepEqual(parseEngineBrokerResponse(refused), refused); + for (const forged of [ + { ...refusals, exhausted: -1 }, + { ...refusals, exhausted: 1.5 }, + { ...refusals, exhausted: "41" }, + { route: 1, expired: 0, unrouted: 0, oversized: 0 }, + { ...refusals, capability: 3 } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, refusals: forged } }), /invalid broker frame/u, JSON.stringify(forged)); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index f8c79aa..f5e37b4 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,19 +1,56 @@ -const VERSION = "noopolis.daimon.engine-broker.v1" as const; +import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; + +/** + * Control protocol v2. Both ends ship in the same Daimon package and image + * (organization runtime client, broker service), so the wire moved to v2 in + * one step: v1 frames are refused. v1 survives only as a *durable record* + * shape, which {@link parseEngineBrokerV1TerminalResponse} still reads so + * turns sealed before the upgrade keep replaying. + */ +const VERSION = "noopolis.daimon.engine-broker.v2" as const; +export const ENGINE_BROKER_VERSION = VERSION; +const V1 = "noopolis.daimon.engine-broker.v1" as const; export const ENGINE_BROKER_MAX_FRAME_BYTES = 1_048_576; const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "health"; requestId: string }> - | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string }> - | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }>; + | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string; limits?: EngineBrokerTurnLimitOverrides }> + | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }> + | EngineBrokerInferenceRequest; -export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +/** + * `reason` is the worker's own last words (`engineBrokerNativeClient.ts`), + * already redacted and flattened to one bounded line by the broker. It is the + * only field a failed turn carries that the worker itself wrote, so it is + * optional, bounded, control-character free, and admitted only for the + * statuses where a worker actually ran and spoke. + */ +export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;reason?:string;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +export const ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES = 768; export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> - | Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> - | Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: "auth_stale" | "cancelled" | "engine_failed" | "invalid_request" | "turn_conflict" | "unavailable"; diagnostic?: EngineBrokerFailureDiagnostic }>; + | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic; mcpCalls?: EngineBrokerMcpCallObservation }> & EngineBrokerTurnAccounting) + | EngineBrokerInferenceResponse; +/** + * The one name for a fenced credential realm. The turn's failure code, the + * proxy's refusal reason on a worker request, and the grant path's 401 body + * (`GROK_INFERENCE_AUTH_STALE_BODY`) all say this same word, so an operator + * greps one string across every surface instead of three spellings of it. + */ +export const ENGINE_BROKER_AUTH_STALE = "auth_stale" as const; +export const ENGINE_BROKER_FAILURE_CODES = [ENGINE_BROKER_AUTH_STALE, "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; +export type EngineBrokerTerminalResponse = Extract; +type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; +type V1Failed = Readonly<{ version: typeof V1; kind: "failed"; requestId: string; turnId: string; code: Exclude; diagnostic?: EngineBrokerFailureDiagnostic }>; +export type EngineBrokerV1TerminalResponse = V1Completed | V1Failed; +const ACCOUNTING = ["outcome", "usage", "model", "requests", "limitReason"] as const; type JsonRecord = Record; const record = (value: unknown): JsonRecord => { @@ -34,14 +71,18 @@ export function parseEngineBrokerRequest(value: unknown): EngineBrokerRequest { const input = record(value); version(input.version); if(input.kind==="health"){exact(input,["version","kind","requestId"]);return {version:VERSION,kind:"health",requestId:id(input.requestId)};} if (input.kind === "start_turn") { - exact(input, ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]); + const fields = ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]; + exact(input, input.limits === undefined ? fields : [...fields, "limits"]); const mcpEndpoint=text(input.mcpEndpoint,2048);const url=new URL(mcpEndpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint }; + let limits: EngineBrokerTurnLimitOverrides | undefined; + if (input.limits !== undefined) { try { limits = parseEngineBrokerTurnLimitOverrides(input.limits); } catch { throw new TypeError("invalid broker frame"); } } + return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint,...(limits === undefined ? {} : { limits }) }; } if (input.kind === "cancel_turn") { exact(input, ["version", "kind", "requestId", "turnId"]); return { version: VERSION, kind: "cancel_turn", requestId: id(input.requestId), turnId: id(input.turnId) }; } + if (isEngineBrokerInferenceRequestKind(input.kind)) return parseEngineBrokerInferenceRequest(input, id(input.requestId), VERSION); throw new TypeError("invalid broker frame"); } @@ -52,24 +93,120 @@ export function parseEngineBrokerResponse(value: unknown): EngineBrokerResponse exact(input, ["version", "kind", "requestId", "turnId"]); return { version: VERSION, kind: "accepted", requestId: id(input.requestId), turnId: id(input.turnId) }; } + if (input.kind === "completed" || input.kind === "failed") return parseTerminal(input, VERSION) as EngineBrokerTerminalResponse; + if (isEngineBrokerInferenceResponseKind(input.kind)) return parseEngineBrokerInferenceResponse(input, id(input.requestId), VERSION); + throw new TypeError("invalid broker frame"); +} + +/** + * A terminal response persisted by a pre-v2 broker: the v1 field sets exactly, + * with no accounting. Accepted only from the durable turn registry, never from + * the wire. + */ +export function parseEngineBrokerV1TerminalResponse(value: unknown): EngineBrokerV1TerminalResponse { + const input = record(value); if (input.version !== V1 || (input.kind !== "completed" && input.kind !== "failed")) throw new TypeError("invalid broker frame"); + return parseTerminal(input, V1) as EngineBrokerV1TerminalResponse; +} + +function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): EngineBrokerTerminalResponse | EngineBrokerV1TerminalResponse { + const accounting = expected === VERSION ? ACCOUNTING : []; if (input.kind === "completed") { - exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime"]); + exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime", ...accounting]); if (!Number.isSafeInteger(input.workerPid) || (input.workerPid as number) < 1 || !Number.isSafeInteger(input.workerUid) || (input.workerUid as number) < 1) throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) }; + const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; + return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - if (input.kind === "failed") { - exact(input, input.diagnostic === undefined ? ["version", "kind", "requestId", "turnId", "code"] : ["version", "kind", "requestId", "turnId", "code", "diagnostic"]); - const codes = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "turn_conflict", "unavailable"] as const; - if (!codes.includes(input.code as typeof codes[number])) throw new TypeError("invalid broker frame"); - let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} - return { version: VERSION, kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as typeof codes[number],...(diagnostic?{diagnostic}:{}) }; + // `mcpCalls` is the broker's own observation of the worker's tool calls + // (`engineBrokerMcpCallLog.ts`), additive in v2 and never part of a v1 + // record, which predates the instrument entirely. + if (expected === V1 && input.mcpCalls !== undefined) throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting, ...(input.diagnostic === undefined ? [] : ["diagnostic"]), ...(input.mcpCalls === undefined ? [] : ["mcpCalls"])]; + exact(input, fields); + const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); + if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); + let diagnostic:EngineBrokerFailureDiagnostic|undefined; + if(input.diagnostic!==undefined){const value=record(input.diagnostic);const fields=["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"];exact(value,value.reason===undefined?fields:[...fields,"reason"]);if(value.reason!==undefined&&(typeof value.reason!=="string"||value.reason.length===0||Buffer.byteLength(value.reason,"utf8")>ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES||/[\u0000-\u001f\u007f]/u.test(value.reason)))throw new TypeError("invalid broker frame");const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; + if (expected === V1) return { version: V1, ...base } as V1Failed; + const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); + if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); + const mcpCalls = input.mcpCalls === undefined ? undefined : parseMcpCallObservation(input.mcpCalls); + return { version: VERSION, ...base, ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accountingValue }; +} + +/** + * Names and timings, bounded, and internally consistent: a report can never + * claim more answered calls than started ones, nor more outstanding ones than + * started minus answered. Every count is its own measurement, so a missing + * field is refused rather than defaulted — a zero the broker did not measure + * would read exactly like one it did. + */ +function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { + const input = record(value); + // `tunnels` is optional for one reason only: a turn sealed before the GET + // tunnel was observed carries no such member, and its record must still + // replay. Absence there means "the instrument did not exist", never zero. + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"]), ...(input.refusals === undefined ? [] : ["refusals"])]); + const started = input.started, answered = input.answered, undecoded = input.undecoded; + if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); + const outstanding = input.outstanding.map((entry) => { + const call = record(entry); + exact(call, ["name", "outstandingMs"]); + if (typeof call.name !== "string" || !ENGINE_BROKER_MCP_CALL_NAME.test(call.name)) throw new TypeError("invalid broker frame"); + if (!Number.isSafeInteger(call.outstandingMs) || (call.outstandingMs as number) < 0) throw new TypeError("invalid broker frame"); + return { name: call.name, outstandingMs: call.outstandingMs as number }; + }); + if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); + const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); + const refusals = input.refusals === undefined ? undefined : parseMcpRefusalObservation(input.refusals); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }), ...(refusals === undefined ? {} : { refusals }) }; +} + +/** + * The refused requests, by reason class: one count per closed reason, all of + * them required once the member is present. Optional for the same single + * reason `tunnels` is — a turn sealed before the facade counted its refusals + * must still replay — so its absence means "not measured" and never zero, and + * a partial member is refused rather than zero-filled. + */ +function parseMcpRefusalObservation(value: unknown): EngineBrokerMcpRefusalObservation { + const input = record(value); + exact(input, ENGINE_BROKER_MCP_REFUSAL_REASONS); + for (const reason of ENGINE_BROKER_MCP_REFUSAL_REASONS) { + if (!Number.isSafeInteger(input[reason]) || (input[reason] as number) < 0) throw new TypeError("invalid broker frame"); } - throw new TypeError("invalid broker frame"); + return Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, input[reason] as number])) as EngineBrokerMcpRefusalObservation; +} + +/** + * The GET SSE tunnels, under the call observation's rules: counts and elapsed + * milliseconds, bounded, and internally consistent. A tunnel cannot close + * before it opened, cannot deliver without having opened, and no more can be + * reported open than `opened - closed` — a report claiming otherwise is a + * frame, not a measurement. + */ +function parseMcpTunnelObservation(value: unknown): EngineBrokerMcpTunnelObservation { + const input = record(value); + exact(input, ["opened", "closed", "delivered", "open"]); + const opened = input.opened, closed = input.closed, delivered = input.delivered; + if (![opened, closed, delivered].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if ((closed as number) > (opened as number) || (delivered as number) > (opened as number)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.open) || input.open.length > ENGINE_BROKER_MCP_TUNNEL_MAX || input.open.length > (opened as number) - (closed as number)) throw new TypeError("invalid broker frame"); + const open = input.open.map((entry) => { + const tunnel = record(entry); + exact(tunnel, ["openMs", "delivered"]); + if (!Number.isSafeInteger(tunnel.openMs) || (tunnel.openMs as number) < 0 || typeof tunnel.delivered !== "boolean") throw new TypeError("invalid broker frame"); + return { openMs: tunnel.openMs as number, delivered: tunnel.delivered }; + }); + return { opened: opened as number, closed: closed as number, delivered: delivered as number, open }; } function closedDiagnostic(value:JsonRecord):boolean{ if(value.profileApplied!==false||(value.workerPid as number)<0||(value.workerUid as number)<0)return false; + // Only a worker that ran and wrote something can have said why it failed: + // no prelaunch failure and no attestation refusal carries worker words. + if(value.reason!==undefined&&!((value.status==="worker_failed"&&value.stage==="wait")||value.status==="output_failed"||value.status==="cancelled"))return false; const noWorker=value.workerPid===0&&value.workerUid===0&&value.startTicks==="0"; const worker=(value.workerPid as number)>0&&(value.workerUid as number)>=2200&&value.startTicks!=="0"; if(value.status==="prelaunch_failed")return noWorker&&({peer:"peer",request:"protocol",registration:"registration",executable:"executable",exec:"exec"} as Record)[value.stage as string]===value.failureClass; diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts new file mode 100644 index 0000000..5639d94 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -0,0 +1,246 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; + +/** + * The hung turn's evidence, across the one boundary it has to cross. + * + * A cancelled or timed-out turn never answers its client, so the sealed + * terminal response — the only carrier of `mcpCalls` and the worker's redacted + * last words — dies with the slot. These tests pin the other route: the same + * sealed response, projected into the broker's own ledger directory, which + * Paideia already recovers `usage.jsonl` and `requests.jsonl` from. + * + * The turn runs through its real registry, meter, seal and ledger path; only + * the launcher, the proxy registrations and the MCP facade's observation are + * stubbed, exactly as the live hang presented them. + */ +const registration = (usageLedgerPath: string): EngineBrokerServiceRegistration => ({ + agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", + profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", + profileSha256: "a".repeat(64), usageLedgerPath, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + model: { model: "grok-4.6", reasoningEffort: "low" } +}); + +const proxy: GrokEngineBrokerTurnDependencies["proxy"] = { + capabilities: { issue: () => "provider-capability-0123456789ab", revoke: () => undefined }, + registerIsolationGuard: () => undefined, revokeIsolationGuard: () => undefined, + registerTurn: () => undefined, revokeTurn: () => undefined +}; + +/** A worker that does real work and then stops acting, until the deadline aborts it. */ +const hangs = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { + const fail = (): void => reject(new Error("engine broker turn failed")); + if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); +}); + +async function cancelledTurn(observe: () => EngineBrokerMcpCallObservation | undefined): Promise | undefined; usage: string; failure: EngineBrokerTurnFailure }>> { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe }, + prepareIsolation: async () => async () => undefined, + runNative: async (_input, signal) => hangs(signal) + }; + const controller = new AbortController(); + const running = runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-hung", "prompt", "http://127.0.0.1:43124/mcp", controller.signal); + setTimeout(() => controller.abort(), 5); + const failure = await running.then(() => { throw new Error("the hung turn resolved"); }, (error: unknown) => error as EngineBrokerTurnFailure); + const text = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8").catch(() => ""); + const lines = text.split("\n").filter((line) => line.length > 0); + assert.ok(lines.length <= 1, "one sealed turn writes at most one seal row"); + return { seal: lines[0] === undefined ? undefined : JSON.parse(lines[0]) as Record, usage: await readFile(usageLedgerPath, "utf8").catch(() => ""), failure }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +/** + * The finding this whole instrument exists for, on the host side of the seam. + * + * Live: ten provider requests, all closed, one tool receipt, then 430 s of + * silence and an abort at 489 s. The facade's log says the worker was blocked + * on `use_tool` the whole time, the sealed response carries it — and the + * sealed response never travels, because a cancelled turn has no client left to + * answer. The row in `turns.jsonl` is that evidence on a durable file the + * evaluator already reads. + * + * Mutation: drop the `seal` append from `appendBrokerTurnLedger`, or the `mcp` + * member from `renderBrokerTurnSealLine`, and this goes red. So does reverting + * `renderBrokerTurnLedger` to return `EMPTY_BROKER_TURN_LEDGER` for a turn with + * no attributable usage — which is exactly this turn. + */ +test("a cancelled turn's outstanding MCP call reaches the host on the broker's own ledger directory", async () => { + const { seal, usage, failure } = await cancelledTurn(() => ({ started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstandingMs: 430_112 }] })); + assert.equal(failure.code, "cancelled"); + // The usage ledger stays silent for a turn with nothing to attribute, so the + // seal row is the only host-visible account this turn has. + assert.equal(usage, ""); + assert.ok(seal, "a cancelled turn seals a row"); + assert.equal(seal.v, TURN_SEAL_LEDGER_VERSION); + assert.equal(seal.agent, "foreman"); + assert.equal(seal.wake, "wake-hung"); + assert.equal(seal.outcome, "failed"); + assert.equal(seal.code, "cancelled"); + assert.equal(seal.limit_reason, "none"); + assert.equal(seal.model, "grok-4.6"); + assert.deepEqual(seal.mcp, { started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstanding_ms: 430_112 }] }); +}); + +/** + * Absence stays absence, and the two absences are not the same statement. + * + * A turn that called no tool measured `started: 0`; a turn the facade never + * registered measured nothing at all and writes no `mcp` member; a turn that + * never sealed writes no row. Reading them as one another is precisely the + * mistake this session kept making. + * + * Mutation: render `mcp` unconditionally (as `{}` or as zeros) when the + * observation is `undefined`, and the second assertion goes red. + */ +test("a cancelled turn that called no tool is distinguishable from one the facade never observed, and both from no row at all", async () => { + const called = await cancelledTurn(() => ({ started: 0, answered: 0, undecoded: 0, outstanding: [] })); + assert.deepEqual(called.seal?.mcp, { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + + const unobserved = await cancelledTurn(() => undefined); + assert.ok(unobserved.seal, "an unobserved turn still seals its row"); + assert.equal(Object.hasOwn(unobserved.seal, "mcp"), false); + + // And the third state: no seal row at all, which is what every one of the six + // live runs published before this stream existed. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-none-")); + try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } + finally { await rm(root, { recursive: true, force: true }); } +}); + +/** + * The channel the seal row could not see, on the same durable route. + * + * Seven live runs sealed with every provider request closed and every tool call + * answered, and still went idle to the deadline. The facade relays one more + * thing for the whole session — the standalone GET SSE tunnel — and recorded + * nothing about it, so a worker parked reading that stream and a worker doing + * nothing wrote identical rows. These three seal the boundary that separates + * them: still open at seal time, closed before it, and never opened at all. + * + * Mutation: drop the `tunnels` member from `renderBrokerTurnSealLine` and the + * first three go red; render it unconditionally as zeros when the observation + * carries none, and the fourth does — a zero nobody measured reads exactly + * like a zero somebody did. + */ +test("a cancelled turn's GET tunnel is sealed open with its age, closed, or never opened — three distinct rows", async () => { + const mcp = (tunnels: Record): EngineBrokerMcpCallObservation => + ({ started: 1, answered: 1, undecoded: 0, outstanding: [], ...tunnels } as EngineBrokerMcpCallObservation); + + const parked = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 428_004, delivered: false }] } })); + assert.deepEqual((parked.seal?.mcp as Record).tunnels, { + opened: 1, closed: 0, delivered: 0, open: [{ open_ms: 428_004, delivered: false }] + }, "a turn sealed with a tunnel still open must say so, and say how long it had been open"); + + const ended = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 1, delivered: 2, open: [] } })); + assert.deepEqual((ended.seal?.mcp as Record).tunnels, { opened: 1, closed: 1, delivered: 2, open: [] }, + "a tunnel that closed before the seal is not an open one"); + + const never = await cancelledTurn(() => mcp({ tunnels: { opened: 0, closed: 0, delivered: 0, open: [] } })); + assert.deepEqual((never.seal?.mcp as Record).tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, + "a turn whose facade never relayed a GET measured zero, which is not the same as not having looked"); + + // And the fourth state, which is the absence: a turn sealed before this + // channel was observed at all carries no `tunnels` member. + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); +}); + +/** + * The refusal, on the same durable route as the hang it looks like. + * + * A request the facade 403'd never reached the relay, so before it was counted + * a turn whose capability was spent sealed `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals. The row has to carry the reason class, because an + * exhausted budget and an unserved route are opposite fixes. + * + * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the + * first assertion goes red; render it unconditionally as zeros for an + * observation that carries none, and the second does — a zero nobody measured + * reads exactly like a zero somebody did. + */ +test("a turn whose MCP requests were refused seals the refusals by reason, and a turn sealed before they were counted seals none", async () => { + const refused = await cancelledTurn(() => ({ + started: 0, answered: 0, undecoded: 0, outstanding: [], + refusals: { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 } + })); + assert.deepEqual((refused.seal?.mcp as Record).refusals, { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 }); + // Without it this row is `started: 0, answered: 0` — an idle turn's row. + assert.deepEqual([(refused.seal?.mcp as Record).started, (refused.seal?.mcp as Record).answered], [0, 0]); + + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); +}); + +/** + * The projection is an allow-list, and this is the assertion that makes it one. + * + * `renderBrokerTurnSealLine` copies a closed field set out of the sealed + * terminal response. Replacing that copy with `...terminal` passed every other + * test in this suite while writing `usage`, `diagnostic`, `mcpCalls` — and, for + * a completed turn, `text`: the model's entire reply, into a ledger whose whole + * rule is that it carries no prompt, body or reply. Nothing sealed a completed + * turn and read the file back, so nothing was watching the one row that + * carries a reply at all. + * + * The boundary: the exact key set of a written row, for the turn kind that has + * the most to leak. + * + * Mutation: spread the terminal into the row (`...terminal, v: ..., agent: ...`) + * and this goes red on both halves — the key set gains `text`, `kind`, + * `version`, `requestId`, `workerPid`, `workerUid`, `workerStartTime` and + * `usage`, and the reply itself appears in the file's bytes. + */ +const reply = "TANGERINE-7-IS-THE-MODELS-OWN-REPLY"; +const answered = (text: string): string => { + const session = "01a0ad21-a90f-7f71-8054-93fdb4334d6a"; + const usage = { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }; + return [ + { type: "system", subtype: "init", session_id: session }, + { type: "assistant", message: { id: "msg_0", type: "message", role: "assistant", model: "daimon-broker-grok", content: [{ type: "text", text }], stop_reason: "end_turn", usage }, parent_tool_use_id: null, session_id: session }, + { type: "result", subtype: "success", is_error: false, num_turns: 1, result: text, stop_reason: "end_turn", total_cost_usd: 0.0024, usage, modelUsage: { "grok-4.6-build": {} }, session_id: session } + ].map((frame) => JSON.stringify(frame)).join("\n"); +}; + +test("a completed turn's seal row carries exactly its declared fields, and never the model's reply", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-completed-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, + prepareIsolation: async () => async () => undefined, + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }) + }; + const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); + assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); + + const bytes = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8"); + const lines = bytes.split("\n").filter((line) => line.length > 0); + assert.equal(lines.length, 1); + const row = JSON.parse(lines[0]!) as Record; + assert.deepEqual(Object.keys(row).sort(), ["agent", "at", "engine", "limit_reason", "model", "outcome", "requests", "turn", "v", "wake"]); + assert.deepEqual([row.v, row.agent, row.wake, row.engine, row.outcome, row.model, row.limit_reason, row.requests], [TURN_SEAL_LEDGER_VERSION, "foreman", "wake-done", "grok", "completed", "grok-4.6", "none", 1]); + // The second half of the same guarantee, on the bytes rather than the keys: + // a reply that reached the ledger under any name is the failure. + assert.ok(!bytes.includes(reply), "the model's reply must never reach the ledger"); + // A completed turn carries no failure members at all, and the facade's + // observation is a failed turn's member: neither may appear here. + for (const absent of ["text", "code", "diagnostic", "mcp", "usage", "workerPid", "workerUid", "kind", "version"]) { + assert.equal(Object.hasOwn(row, absent), false, absent); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts new file mode 100644 index 0000000..4d64893 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.ts @@ -0,0 +1,131 @@ +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; + +/** + * The operator-visible half of a sealed terminal turn, as a durable row. + * + * The seal itself already carries everything an operator needs to read a turn + * that stopped acting — the failure code, the worker's own redacted last words, + * and `engineBrokerMcpCallLog.ts`'s observation of the tool call the worker was + * blocked on. All of it travels in the control-protocol terminal response, and + * that response is the one artifact a *hung* turn never produces: the client is + * gone before the broker answers, the turn registry record lives in the + * broker's own `0700` turn store, and a training slot's control root is tmpfs + * that dies with the container. Six live runs reproduced the same signature and + * none of them could read the instrument built for it. + * + * What does survive a slot is the broker's ledger directory: Paideia already + * recovers `usage.jsonl` and `requests.jsonl` from it on the failure path. So + * this is a third stream beside those two, written by the broker — still the + * single sealed usage writer — from the sealed response and nothing else, at + * the moment that response is sealed. + * + * Its rules are the two ledgers' rules: + * + * - **Numbers, names, timings and closed vocabularies only.** The failure code, + * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` + * and its already-redacted, already-bounded, control-character-free `reason` + * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), + * plus tool-call names, GET-tunnel counts and elapsed milliseconds. Never a + * prompt, a body, a + * reply, a bearer, a capability or a session id — none of which the terminal + * response carries in the first place. + * - **Absence stays absence.** `mcp` is written only when the facade actually + * observed the turn, `diagnostic` only when the sealed response carried one, + * and `code` only for a failure. A turn that called no tool publishes + * `started: 0`, which is a measurement; a turn the facade never registered + * publishes no `mcp` member at all, which is not. + * - **It can never fail a turn.** The row is rendered from an + * already-validated frame and appended through `recordLedgerLines`, which + * swallows every I/O fault. + * + * A separate stream and a separate `v`, for `turnRequestLedger.ts`'s reason: + * Spawnfile's reader pins `noopolis.daimon.turn-usage.v1` and drops any other + * `v` outright, and Paideia's request reader refuses a row it cannot type. A + * row in a new file is invisible to both. + */ +export const TURN_SEAL_LEDGER_VERSION = "noopolis.daimon.turn-seal.v1" as const; + +/** Default location: beside `usage.jsonl` and `requests.jsonl`, no new mount. */ +export const TURN_SEAL_LEDGER = { + version: TURN_SEAL_LEDGER_VERSION, + directoryPath: TURN_USAGE_LEDGER.directoryPath, + filePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl`, + rotatedFilePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl.1`, + fileMode: TURN_USAGE_LEDGER.fileMode +} as const; + +/** A rendered seal line is bounded by its own contents: a 768-byte reason, 16 bounded names and 8 tunnel timings. */ +export const TURN_SEAL_MAX_LINE_BYTES = 8_192; + +export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; + +const bounded = (value: string): string => [...value].slice(0, TURN_USAGE_MAX_IDENTIFIER_CHARS).join(""); + +/** + * One newline-terminated row for one sealed terminal turn. + * + * Every member is copied from the terminal response the protocol already + * validated, so this projection cannot widen what the response admits; it is an + * allow-list rather than a spread, so a future additive member of the response + * does not silently become a ledger field. + */ +export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, entry: BrokerTurnSealEntry): string => `${JSON.stringify({ + v: TURN_SEAL_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at: entry.at, + turn: terminal.turnId, + outcome: terminal.outcome, + requests: terminal.requests, + model: terminal.model, + limit_reason: terminal.limitReason, + ...(terminal.kind === "failed" ? { code: terminal.code } : {}), + ...(terminal.kind === "failed" && terminal.diagnostic !== undefined + ? { + diagnostic: { + status: terminal.diagnostic.status, + stage: terminal.diagnostic.stage, + failure_class: terminal.diagnostic.failureClass, + exit_code: terminal.diagnostic.exitCode, + term_signal: terminal.diagnostic.termSignal, + ...(terminal.diagnostic.reason === undefined ? {} : { reason: terminal.diagnostic.reason }) + } + } + : {}), + ...(terminal.kind === "failed" && terminal.mcpCalls !== undefined + ? { + mcp: { + started: terminal.mcpCalls.started, + answered: terminal.mcpCalls.answered, + undecoded: terminal.mcpCalls.undecoded, + outstanding: terminal.mcpCalls.outstanding + .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // Every request the facade refused before it could relay it, by reason + // class. Without it a turn whose capability was spent seals as + // `answered == started, outstanding: []`, which is what a healthy turn + // seals as. + ...(terminal.mcpCalls.refusals === undefined + ? {} + : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), + // The session's standalone GET tunnel, absent for a turn sealed before + // the facade observed that channel at all. + ...(terminal.mcpCalls.tunnels === undefined + ? {} + : { + tunnels: { + opened: terminal.mcpCalls.tunnels.opened, + closed: terminal.mcpCalls.tunnels.closed, + delivered: terminal.mcpCalls.tunnels.delivered, + open: terminal.mcpCalls.tunnels.open + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((tunnel) => ({ open_ms: tunnel.openMs, delivered: tunnel.delivered })) + } + }) + } + } + : {}) +})}\n`; diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index e4fe882..1dac5d5 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -5,6 +5,9 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerControlClient } from "./engineBrokerControlClient.js"; import { startEngineBrokerServiceWithIdentity, type EngineBrokerServiceEngine } from "./engineBrokerService.js"; +import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; + +const completedAccounting = { outcome: "completed", usage: { input: 8, cacheRead: 2, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; test("broker backend serves a turn and preserves worker attestation", async () => { await withService(async (client) => {await client.ready();assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),"answer");}); @@ -33,8 +36,40 @@ test("service shutdown aborts turns and closes connected clients", async () => { await started;await service.close();await rejected;assert.equal(aborted,true);await rm(directory,{recursive:true,force:true}); }); -async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123"}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ +async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ const directory=await mkdtemp(path.join(tmpdir(),"daimon-broker-service-")),socketPath=path.join(directory,"broker.sock"),engine=makeEngine(turn,readiness);const service=await startEngineBrokerServiceWithIdentity(engine,socketPath,process.getuid!()); try{await run(new EngineBrokerControlClient(socketPath));}finally{await service.close();await rm(directory,{recursive:true,force:true});} } function makeEngine(turn:EngineBrokerServiceEngine["turn"],readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):EngineBrokerServiceEngine{return {turn,readiness,close:async()=>undefined};} + +test("the wake's lowering limits reach the broker, and the client verifies the declared model", async () => { + let seen: unknown; + await withService(async (client) => { + assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp",undefined,{limits:{maxTokens:1_000,timeoutMs:5_000},model:"grok-4.6"}),"answer"); + assert.deepEqual(seen,{maxTokens:1_000,timeoutMs:5_000}); + await assert.rejects(client.turn("agent-a","wake-b","hello","http://127.0.0.1:44001/mcp",undefined,{model:"grok-4.5"}),/model grok-4.6, not the declared grok-4.5/u); + },async(_agent,_wake,_prompt,_endpoint,_signal,limits)=>{seen=limits;return {text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting};}); +}); + +test("a limit failure reaches the client with its code and limit reason", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); +}); + +test("a failed worker's own reason reaches the client instead of a bare exit code", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); + },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); +}); + +/** + * The end of the seam: an outstanding tool call has to be readable by whoever + * reads the failure, not just sealed. The live hang would have read + * `mcp=1/2 answered; mcp_outstanding=daimon__moltnet_read@419000ms`. + */ +test("an outstanding MCP tool call reaches the client by name, with how long it waited", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=timeout; mcp=1\/2 answered; mcp_undecoded=1; mcp_outstanding=daimon__moltnet_read@419000ms\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:null,model:"grok-4.6",requests:8,limitReason:"timeout"},{started:2,answered:1,undecoded:1,outstanding:[{name:"daimon__moltnet_read",outstandingMs:419_000}]});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index 9faa5c4..79f55f0 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -1,11 +1,17 @@ import { chmod, lstat, unlink } from "node:fs/promises"; import { createServer, type Server, type Socket } from "node:net"; import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerRequest,type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnAccounting, EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; +import { GROK_INFERENCE_PROXY_BASE_URL, type EngineBrokerInferenceRequest } from "./engineBrokerInferenceProtocol.js"; +import { GrokInferenceGrantRefused, type GrokInferenceGrantIssued, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export interface EngineBrokerServiceEngine { - turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise>; + turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,limits?:EngineBrokerTurnLimitOverrides):Promise&EngineBrokerTurnAccounting>; readiness():Readonly<{providerProxyPort:number;mcpFacadePort:number;registrations:number;credentialStale:boolean;realmLease:boolean;workerIsolation:boolean}>; close():Promise; + /** Evaluator inference grants; an engine without them refuses every grant request as `unavailable`. Refusals throw {@link GrokInferenceGrantRefused}. */ + requestInferenceGrant?(request:GrokInferenceGrantRequest):GrokInferenceGrantIssued; + releaseInferenceGrant?(grantId:string):boolean; } export function startEngineBrokerService(broker:EngineBrokerServiceEngine,socketPath="/run/daimon-engine-broker/backend.sock"){ @@ -36,7 +42,30 @@ export async function startEngineBrokerServiceWithIdentity(broker:EngineBrokerSe * reader has disconnected. */ function handleSocketError(socket:Socket,owned:()=>Readonly<{turnId:string;controller:AbortController}>|undefined):void{socket.on("error",()=>{owned()?.controller.abort();socket.destroy();});} -function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,...result}),(error:unknown)=>send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:error instanceof EngineBrokerTurnFailure?error.code:controller.signal.aborted?"cancelled":"engine_failed",...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{})})).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(request.kind==="request_inference_grant"||request.kind==="release_inference_grant"){if(started)throw new Error();started=true;serveInferenceGrant(socket,broker,request);continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal,request.limits).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,text:result.text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.workerStartTime,outcome:"completed",usage:result.usage,model:result.model,requests:result.requests,limitReason:"none"}),(error:unknown)=>failed(socket,request,error,controller.signal.aborted)).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +/** + * Every failure the broker raises for a known registration carries its + * accounting (a wake that tried to raise a limit: `usage: null`, zero + * requests). A failure with no registration behind it (unknown agent, closed + * broker) has no declared model to report, so it is refused as a bare + * connection close and the client reports the broker unavailable. + */ +function failed(socket:Socket,request:Extract,{kind:"start_turn"}>,error:unknown,aborted:boolean):void{ + const accounting=error instanceof EngineBrokerTurnFailure?error.accounting:undefined; + if(accounting===undefined){socket.destroy();return;} + const code=error instanceof EngineBrokerTurnFailure?error.code:aborted?"cancelled":"engine_failed"; + send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code,...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{}),...(error instanceof EngineBrokerTurnFailure&&error.mcpCalls?{mcpCalls:error.mcpCalls}:{}),outcome:"failed",usage:accounting.usage,model:accounting.model,requests:accounting.requests,limitReason:accounting.limitReason}); +} +/** One grant verb per connection, answered and closed; any failure becomes a closed-code refusal and never touches turn state. */ +function serveInferenceGrant(socket:Socket,broker:EngineBrokerServiceEngine,request:EngineBrokerInferenceRequest):void{ + const refuse=(code:GrokInferenceGrantRefused["code"])=>send(socket,{version:request.version,kind:"inference_grant_refused",requestId:request.requestId,code}); + try{ + if(request.kind==="release_inference_grant"){if(!broker.releaseInferenceGrant)return refuse("unavailable");return send(socket,{version:request.version,kind:"inference_grant_released",requestId:request.requestId,grantId:request.grantId,released:broker.releaseInferenceGrant(request.grantId)});} + if(!broker.requestInferenceGrant)return refuse("unavailable"); + const grant=broker.requestInferenceGrant({model:request.model,reasoningEffort:request.reasoningEffort,purpose:request.purpose}); + send(socket,{version:request.version,kind:"inference_grant",requestId:request.requestId,grantId:grant.grantId,token:grant.token,baseUrl:GROK_INFERENCE_PROXY_BASE_URL,model:grant.policy.model,reasoningEffort:grant.policy.reasoningEffort,purpose:grant.purpose,expiresAt:new Date(grant.expiresAt).toISOString(),limits:grant.limits}); + }catch(error){refuse(error instanceof GrokInferenceGrantRefused?error.code:"unavailable");} +} function send(socket:Socket,response:EngineBrokerResponse):void{if(!socket.destroyed)socket.end(encodeEngineBrokerFrame(response));} async function removeOwnedSocket(file:string,uid:number):Promise{try{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid)throw new Error("unsafe broker socket");await unlink(file);}catch(error){if((error as NodeJS.ErrnoException).code!=="ENOENT")throw error;}} async function verifySocket(file:string,uid:number):Promise{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid||(Number(entry.mode)&0o777)!==0o600)throw new Error("unsafe broker socket");} diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 7e5ff97..5522f10 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -1,15 +1,78 @@ import assert from "node:assert/strict"; import test from "node:test"; import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceCli.js"; +import { engineBrokerRequestLedgerPathFor } from "./engineBrokerServiceConfig.js"; -test("parses the closed broker service configuration",()=>{ - const registration=reg("agent-a",0);assert.deepEqual(parseEngineBrokerServiceConfig({version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}),{credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}); +const paths = { credentialHome: "/var/lib/daimon-engine-broker/credential", turnStore: "/var/lib/daimon-engine-broker/turns" }; +const reg = (agentId: string, slot: number) => ({ agentId, slot, workerUid: 2200 + slot, workspace: `/workspace/${slot}`, profilePath: `/workers/${slot}/.grok/sandbox.toml`, eventsPath: `/workers/${slot}/.grok/sessions/sandbox-events.jsonl`, profileSha256: "a".repeat(64) }); +const v2 = (agentId: string, slot: number) => ({ ...reg(agentId, slot), usageLedgerPath: `/run/slots/${slot}/usage/usage.jsonl`, limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { id: "grok-4.6", reasoningEffort: "low" } }); +const config = (version: string, registrations: readonly unknown[]) => ({ version: `noopolis.daimon.engine-broker-service.${version}`, ...paths, registrations }); + +test("v1 service config is still accepted and receives today's defaults", () => { + assert.deepEqual(parseEngineBrokerServiceConfig(config("v1", [reg("agent-a", 0)])), { ...paths, registrations: [{ + ...reg("agent-a", 0), usageLedgerPath: "/var/lib/spawnfile/daimon/usage/usage.jsonl", + limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } + }] }); +}); + +test("v2 declares a per-slot ledger, limits and a closed-list model per registration", () => { + const parsed = parseEngineBrokerServiceConfig(config("v2", [v2("agent-a", 0), v2("agent-b", 1)])); + assert.deepEqual(parsed.registrations[1], { ...reg("agent-b", 1), usageLedgerPath: "/run/slots/1/usage/usage.jsonl", limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } }); + assert.equal(engineBrokerRequestLedgerPathFor(parsed.registrations[0]!.usageLedgerPath), "/run/slots/0/usage/requests.jsonl"); +}); + +test("v2 rejects unknown keys at every level, off-list models, missing fields and out-of-bound limits", () => { + const base = v2("agent-a", 0); + // Mutation guard: loosening any exact-member check accepts one of these. + for (const registration of [ + { ...base, extra: true }, + { ...base, limits: { ...base.limits, maxWakes: 1 } }, + { ...base, model: { ...base.model, provider: "xai" } }, + { ...base, model: { id: "grok-4.6-build", reasoningEffort: "low" } }, + { ...base, model: { id: "grok-4.6", reasoningEffort: "xhigh" } }, + { ...base, limits: { ...base.limits, maxRequests: 49 } }, + { ...base, limits: { ...base.limits, maxTokens: 0 } }, + { ...base, usageLedgerPath: "relative/usage.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/requests.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/usage" }, + (({ model: _omit, ...rest }) => rest)(base), + reg("agent-a", 0) + ]) assert.throws(() => parseEngineBrokerServiceConfig(config("v2", [registration])), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v2", [base]), grokCommand: "evil" }), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [base])), /invalid engine broker service config/u, "v2 members are not accepted under v1"); + assert.throws(() => parseEngineBrokerServiceConfig(config("v3", [base])), /invalid engine broker service config/u); +}); + +test("rejects caller-selected commands, duplicate identities, and traversal", () => { + const base = config("v1", [reg("agent-a", 0)]); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, grokCommand: "evil" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, turnStore: "/var/lib/../secret" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-a", 1)] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-b", 0)] }), /invalid/u, "one slot is one worker"); + // Grok 1.0.34 logs sandbox events under $GROK_HOME/sessions/; the 1.0.13 root path stays empty and must not be attested. + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/0/.grok/sandbox-events.jsonl" }] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/1/.grok/sessions/sandbox-events.jsonl" }] })); +}); + +test("v2 may declare an evaluator inference ledger that is never a subject ledger", () => { + const base = config("v2", [v2("agent-a", 0), v2("agent-b", 1)]); + assert.equal(parseEngineBrokerServiceConfig(base).inferenceLedgerPath, undefined); + assert.equal(parseEngineBrokerServiceConfig({ ...base, inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }).inferenceLedgerPath, "/run/paideia-inference/inference.jsonl"); + for (const inferenceLedgerPath of ["/run/slots/0/usage/usage.jsonl", "/run/slots/1/usage/requests.jsonl", "/var/lib/spawnfile/daimon/usage/usage.jsonl", "/var/lib/spawnfile/daimon/usage/requests.jsonl", "relative.jsonl", "/run/x/../inference.jsonl", "/run/inference.json", 7]) { + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, inferenceLedgerPath }), /invalid engine broker service config/u, String(inferenceLedgerPath)); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [reg("agent-a", 0)]), inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }), /invalid engine broker service config/u); }); -test("rejects caller-selected commands, duplicate identities, and traversal",()=>{ - const base={version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[reg("agent-a",0)]}; - assert.throws(()=>parseEngineBrokerServiceConfig({...base,grokCommand:"evil"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,turnStore:"/var/lib/../secret"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[reg("agent-a",0),reg("agent-a",1)]})); +test("registration paths must be canonical, matching the native launcher's registration check", () => { + const good = reg("agent-a", 0); + for (const [field, value] of [ + ["workspace", "/workspace/0/../1"], ["workspace", "/workspace//0"], ["workspace", "/workspace/0/"], ["workspace", "/workspace/./0"], + ["profilePath", "/workers/0/../1/.grok/sandbox.toml"], ["profilePath", "/workers//0/.grok/sandbox.toml"] + ] as const) { + const registration = { ...good, [field]: value, ...(field === "profilePath" ? { eventsPath: value.replace(/sandbox\.toml$/u, "sessions/sandbox-events.jsonl") } : {}) }; + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [registration])), /invalid engine broker service config/u, `${field}=${value}`); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [good]), turnStore: "/var/lib/turns/" })); + assert.doesNotThrow(() => parseEngineBrokerServiceConfig(config("v1", [good]))); }); -const reg=(agentId:string,slot:number)=>({agentId,slot,workerUid:2200+slot,workspace:`/workspace/${slot}`,profilePath:`/workers/${slot}/.grok/sandbox.toml`,eventsPath:`/workers/${slot}/.grok/sandbox-events.jsonl`,profileSha256:"a".repeat(64)}); diff --git a/src/runtime/engineBrokerServiceCli.ts b/src/runtime/engineBrokerServiceCli.ts index bbc89cd..8661b02 100644 --- a/src/runtime/engineBrokerServiceCli.ts +++ b/src/runtime/engineBrokerServiceCli.ts @@ -1,7 +1,10 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import { startEngineBrokerService } from "./engineBrokerService.js"; -import { startGrokEngineBroker, type GrokEngineBrokerRegistration } from "./grokEngineBroker.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { startGrokEngineBroker } from "./grokEngineBroker.js"; + +export { parseEngineBrokerServiceConfig }; export const ENGINE_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; const MAX_CONFIG_BYTES=65_536; @@ -9,18 +12,10 @@ const MAX_CONFIG_BYTES=65_536; export async function runEngineBrokerServiceCli():Promise{ if(process.getuid?.()!==2100)throw new Error("engine broker service requires broker identity"); const config=parseEngineBrokerServiceConfig(await readRootConfig(ENGINE_BROKER_SERVICE_CONFIG)); - const broker=await startGrokEngineBroker({grokCommand:"/usr/local/bin/grok",nativeClient:"/opt/daimon/bin/daimon-engine-broker",credentialHome:config.credentialHome,turnStore:config.turnStore,registrations:config.registrations}); + const broker=await startGrokEngineBroker({grokCommand:"/usr/local/bin/grok",nativeClient:"/opt/daimon/bin/daimon-engine-broker",credentialHome:config.credentialHome,turnStore:config.turnStore,registrations:config.registrations,...(config.inferenceLedgerPath===undefined?{}:{inferenceLedgerPath:config.inferenceLedgerPath})}); const service=await startEngineBrokerService(broker);let stopping:Promise|undefined; const stop=()=>{stopping??=service.close();return stopping;}; const onSignal=()=>{void stop().catch(()=>{process.exitCode=1;});};process.once("SIGINT",onSignal);process.once("SIGTERM",onSignal); } -export function parseEngineBrokerServiceConfig(value:unknown):Readonly<{credentialHome:string;turnStore:string;registrations:readonly GrokEngineBrokerRegistration[]}>{ - if(value===null||typeof value!=="object"||Array.isArray(value))throw new TypeError("invalid engine broker service config");const input=value as Record; - if(Object.keys(input).length!==4||input.version!=="noopolis.daimon.engine-broker-service.v1"||typeof input.credentialHome!=="string"||typeof input.turnStore!=="string"||!Array.isArray(input.registrations))throw new TypeError("invalid engine broker service config"); - const absolute=(item:string)=>item.startsWith("/")&&!item.includes("/../")&&!item.endsWith("/..");if(!absolute(input.credentialHome)||!absolute(input.turnStore))throw new TypeError("invalid engine broker service config"); - const seen=new Set();const registrations=input.registrations.map((entry)=>{if(entry===null||typeof entry!=="object"||Array.isArray(entry))throw new TypeError("invalid engine broker service config");const item=entry as Record;if(Object.keys(item).length!==7||typeof item.agentId!=="string"||!item.agentId.trim()||!Number.isSafeInteger(item.slot)||(item.slot as number)<0||!Number.isSafeInteger(item.workerUid)||(item.workerUid as number)<2200||typeof item.workspace!=="string"||!absolute(item.workspace)||typeof item.profilePath!=="string"||!absolute(item.profilePath)||typeof item.eventsPath!=="string"||!absolute(item.eventsPath)||typeof item.profileSha256!=="string"||!/^[a-f0-9]{64}$/u.test(item.profileSha256)||item.profilePath!==`${item.eventsPath.replace(/\/sandbox-events\.jsonl$/u,"")}/sandbox.toml`||seen.has(item.agentId))throw new TypeError("invalid engine broker service config");seen.add(item.agentId);return {agentId:item.agentId,slot:item.slot as number,workerUid:item.workerUid as number,workspace:item.workspace,profilePath:item.profilePath,eventsPath:item.eventsPath,profileSha256:item.profileSha256};}); - if(registrations.length===0)throw new TypeError("invalid engine broker service config");return {credentialHome:input.credentialHome,turnStore:input.turnStore,registrations}; -} - async function readRootConfig(file:string):Promise{const handle=await open(file,constants.O_RDONLY|constants.O_NOFOLLOW);try{const stat=await handle.stat();if(!stat.isFile()||stat.uid!==0||stat.gid!==2100||(stat.mode&0o777)!==0o440||stat.size<2||stat.size>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");const bytes=await handle.readFile();if(bytes.length>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");return JSON.parse(bytes.toString("utf8"));}finally{await handle.close();}} diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts new file mode 100644 index 0000000..eb6d274 --- /dev/null +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -0,0 +1,98 @@ +import path from "node:path"; + +import { DEFAULT_GROK_BROKER_TURN_LIMITS, parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { TURN_REQUEST_LEDGER } from "./turnRequestLedger.js"; +import { TURN_USAGE_LEDGER } from "./turnUsageLedger.js"; + +export const ENGINE_BROKER_SERVICE_V1 = "noopolis.daimon.engine-broker-service.v1" as const; +export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v2" as const; + +/** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ +export type EngineBrokerServiceRegistration = Readonly<{ + agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` and per-turn seals to `turns.jsonl` beside it. */ + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + model: GrokBrokerModelPolicy; +}>; +/** + * `inferenceLedgerPath` (v2, optional) is where evaluator inference grants + * append their rows (`inferenceUsageLedger.ts`). Without it the broker refuses + * every grant request. It can never be a subject ledger: not any + * registration's usage ledger or its `requests.jsonl`, and not the container + * ledger the wake fuse sums. + */ +export type EngineBrokerServiceConfig = Readonly<{ credentialHome: string; turnStore: string; registrations: readonly EngineBrokerServiceRegistration[]; inferenceLedgerPath?: string }>; + +const V1_REGISTRATION = ["agentId", "slot", "workerUid", "workspace", "profilePath", "eventsPath", "profileSha256"] as const; +const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model"] as const; +const invalid = (): TypeError => new TypeError("invalid engine broker service config"); +const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; +/** + * Absolute and canonical: no `.`/`..`/empty components and no trailing slash. + * The native launcher derives HOME, GROK_HOME and TMPDIR from the registered + * home and refuses a non-canonical one, so the broker's view must match it. + */ +const absolute = (item: unknown): item is string => typeof item === "string" && item.length > 1 && item.startsWith("/") && !item.endsWith("/") && path.posix.normalize(item) === item && !item.split("/").slice(1).some((part) => part === "." || part === "..") && !item.includes("\0"); + +/** The per-request stream written beside a registration's usage ledger. */ +export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); + +/** + * The per-turn seal stream written beside the other two + * (`engineBrokerSealLedger.ts`): the operator-visible half of a sealed terminal + * response, for the turn whose response never reaches a client. + */ +export const engineBrokerSealLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "turns.jsonl"); + +/** + * Strict `service.json` parser. + * + * v2 requires every registration to declare its usage ledger, limits and model + * (`model.id`/`model.reasoningEffort` from the closed lists); unknown keys at + * any level are refused. v1 is still accepted and receives today's defaults: + * the container ledger, {@link DEFAULT_GROK_BROKER_TURN_LIMITS}, and + * `grok-4.6`/`low`. + */ +export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServiceConfig { + if (!plain(value)) throw invalid(); + const v2 = value.version === ENGINE_BROKER_SERVICE_V2; + if (!v2 && value.version !== ENGINE_BROKER_SERVICE_V1) throw invalid(); + const top = ["version", "credentialHome", "turnStore", "registrations"]; + exact(value, v2 && Object.hasOwn(value, "inferenceLedgerPath") ? [...top, "inferenceLedgerPath"] : top); + if (!absolute(value.credentialHome) || !absolute(value.turnStore) || !Array.isArray(value.registrations) || value.registrations.length === 0) throw invalid(); + const seen = new Set(), slots = new Set(); + const registrations = value.registrations.map((entry: unknown): EngineBrokerServiceRegistration => { + if (!plain(entry)) throw invalid(); + exact(entry, v2 ? V2_REGISTRATION : V1_REGISTRATION); + const { agentId, slot, workerUid, workspace, profilePath, eventsPath, profileSha256 } = entry; + if (typeof agentId !== "string" || !agentId.trim() || seen.has(agentId) || !Number.isSafeInteger(slot) || (slot as number) < 0 || slots.has(slot as number) || !Number.isSafeInteger(workerUid) || (workerUid as number) < 2200 || !absolute(workspace) || !absolute(profilePath) || !absolute(eventsPath) || typeof profileSha256 !== "string" || !/^[a-f0-9]{64}$/u.test(profileSha256) || eventsPath !== grokWorkerEventsPathFor(profilePath) || !profilePath.endsWith("/sandbox.toml")) throw invalid(); + seen.add(agentId); slots.add(slot as number); + const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; + if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; + const usageLedgerPath = entry.usageLedgerPath; + if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || usageLedgerPath === engineBrokerSealLedgerPathFor(usageLedgerPath)) throw invalid(); + let limits: EngineBrokerTurnLimits; + try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } + return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; + }); + const base = { credentialHome: value.credentialHome, turnStore: value.turnStore, registrations }; + if (!Object.hasOwn(value, "inferenceLedgerPath")) return base; + const inferenceLedgerPath = value.inferenceLedgerPath; + if (!ledgerPath(inferenceLedgerPath)) throw invalid(); + const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath), engineBrokerSealLedgerPathFor(entry.usageLedgerPath)])]); + if (subject.has(inferenceLedgerPath)) throw invalid(); + return { ...base, inferenceLedgerPath }; +} + +const ledgerPath = (item: unknown): item is string => absolute(item) && item.endsWith(".jsonl") && path.posix.normalize(item) === item; + +function parseServiceModel(value: unknown): GrokBrokerModelPolicy { + if (!plain(value)) throw invalid(); + exact(value, ["id", "reasoningEffort"]); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.id) || !(GROK_BROKER_REASONING_EFFORTS as readonly unknown[]).includes(value.reasoningEffort)) throw invalid(); + return Object.freeze({ model: value.id as GrokBrokerModelPolicy["model"], reasoningEffort: value.reasoningEffort as GrokBrokerModelPolicy["reasoningEffort"] }); +} diff --git a/src/runtime/engineBrokerTurnAccounting.ts b/src/runtime/engineBrokerTurnAccounting.ts new file mode 100644 index 0000000..7f7f346 --- /dev/null +++ b/src/runtime/engineBrokerTurnAccounting.ts @@ -0,0 +1,123 @@ +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + +/** + * Numeric-only accounting the Grok engine broker seals beside every terminal + * turn response, and the per-turn limits it enforces. + * + * The broker is the single writer of this data (turn registry record, control + * response, usage ledger row). Nothing engine-controlled and non-numeric is + * persisted: `model` is a member of the closed declared list, never the + * provider's own string, and `limitReason` is a closed vocabulary. + * + * Buckets are disjoint and `total = input + cacheRead + cacheWrite + output`. + * `reasoning` is reported only when the source separates it; it is already + * inside `output` and never added to `total`. + */ +export type EngineBrokerTurnUsage = Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: number }>; +export const ENGINE_BROKER_LIMIT_REASONS = GROK_ENGINE_BROKER.turnLimits.limitReasons; +export type EngineBrokerLimitReason = (typeof ENGINE_BROKER_LIMIT_REASONS)[number]; +export type EngineBrokerTurnLimits = Readonly<{ maxRequests: number; maxTokens: number; timeoutMs: number }>; +export type EngineBrokerTurnLimitOverrides = Readonly>; +export type EngineBrokerTurnAccounting = Readonly<{ + outcome: "completed" | "failed"; + usage: EngineBrokerTurnUsage | null; + model: GrokBrokerModel; + requests: number; + limitReason: EngineBrokerLimitReason; +}>; + +/** + * Bounds every declared limit must sit inside, and the v1 defaults, both from + * the runtime contract manifest. `maxRequests` stays at or below the + * launcher's compiled `--max-turns` backstop, so the broker ceiling is the one + * that fires first. + */ +export const ENGINE_BROKER_LIMIT_BOUNDS = GROK_ENGINE_BROKER.turnLimits.bounds; + +/** What a v1 `service.json` registration gets; equal to the Codex per-wake defaults. */ +export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ ...GROK_ENGINE_BROKER.turnLimits.v1Defaults }); + +const LIMIT_KEYS = ["maxRequests", "maxTokens", "timeoutMs"] as const; +type JsonRecord = Record; +const plain = (value: unknown): value is JsonRecord => + value !== null && typeof value === "object" && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null); +const count = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +const invalid = (label: string): TypeError => new TypeError(`invalid ${label}`); + +const limitValue = (key: (typeof LIMIT_KEYS)[number], value: unknown, label: string): number => { + const [minimum, maximum] = ENGINE_BROKER_LIMIT_BOUNDS[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) throw invalid(label); + return value; +}; + +/** Exactly `{maxRequests, maxTokens, timeoutMs}`, each inside its bound. */ +export function parseEngineBrokerTurnLimits(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimits { + if (!plain(value) || Object.keys(value).length !== LIMIT_KEYS.length || LIMIT_KEYS.some((key) => !Object.hasOwn(value, key))) throw invalid(label); + return Object.freeze({ maxRequests: limitValue("maxRequests", value.maxRequests, label), maxTokens: limitValue("maxTokens", value.maxTokens, label), timeoutMs: limitValue("timeoutMs", value.timeoutMs, label) }); +} + +/** A non-empty subset of the limit keys, each inside its bound. */ +export function parseEngineBrokerTurnLimitOverrides(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimitOverrides { + if (!plain(value) || Object.keys(value).length === 0 || Object.keys(value).some((key) => !(LIMIT_KEYS as readonly string[]).includes(key))) throw invalid(label); + const result: Partial> = {}; + for (const key of LIMIT_KEYS) if (Object.hasOwn(value, key)) result[key] = limitValue(key, value[key], label); + return Object.freeze(result); +} + +/** + * The limits one turn runs under: the registration's, lowered by the wake. + * A wake can never raise a declared limit; asking to is refused rather than + * clamped, so a misconfigured caller learns it instead of silently getting less. + */ +export function lowerEngineBrokerTurnLimits(declared: EngineBrokerTurnLimits, overrides: EngineBrokerTurnLimitOverrides | undefined): EngineBrokerTurnLimits { + if (overrides === undefined) return declared; + for (const key of LIMIT_KEYS) { + const requested = overrides[key]; + if (requested !== undefined && requested > declared[key]) throw new RangeError(`engine broker turn limit ${key} may only be lowered`); + } + return Object.freeze({ maxRequests: overrides.maxRequests ?? declared.maxRequests, maxTokens: overrides.maxTokens ?? declared.maxTokens, timeoutMs: overrides.timeoutMs ?? declared.timeoutMs }); +} + +/** Four disjoint buckets plus an optional reasoning split; the total invariant is re-checked. */ +export function parseEngineBrokerTurnUsage(value: unknown, label = "engine broker turn usage"): EngineBrokerTurnUsage { + const required = ["input", "cacheRead", "cacheWrite", "output", "total"]; + if (!plain(value)) throw invalid(label); + const keys = Object.keys(value); + if (required.some((key) => !Object.hasOwn(value, key)) || keys.some((key) => !required.includes(key) && key !== "reasoning")) throw invalid(label); + if (![value.input, value.cacheRead, value.cacheWrite, value.output, value.total].every(count)) throw invalid(label); + const usage = value as { input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: unknown }; + if (usage.total !== usage.input + usage.cacheRead + usage.cacheWrite + usage.output) throw invalid(label); + if (usage.reasoning !== undefined && (!count(usage.reasoning) || usage.reasoning > usage.output)) throw invalid(label); + return Object.freeze({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total, ...(usage.reasoning === undefined ? {} : { reasoning: usage.reasoning as number }) }); +} + +export const sumEngineBrokerTurnUsage = (items: readonly EngineBrokerTurnUsage[]): EngineBrokerTurnUsage | null => { + if (items.length === 0) return null; + const sum = (pick: (usage: EngineBrokerTurnUsage) => number): number => items.reduce((total, usage) => total + pick(usage), 0); + const reasoning = items.every((usage) => usage.reasoning !== undefined) ? { reasoning: sum((usage) => usage.reasoning ?? 0) } : {}; + return Object.freeze({ input: sum((usage) => usage.input), cacheRead: sum((usage) => usage.cacheRead), cacheWrite: sum((usage) => usage.cacheWrite), output: sum((usage) => usage.output), total: sum((usage) => usage.total), ...reasoning }); +}; + +/** Validates the accounting members of a v2 terminal response against its kind. */ +export function parseEngineBrokerTurnAccounting(value: JsonRecord, kind: "completed" | "failed"): EngineBrokerTurnAccounting { + if (value.outcome !== kind) throw invalid("broker frame"); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.model)) throw invalid("broker frame"); + if (!count(value.requests) || value.requests > 1_024) throw invalid("broker frame"); + if (!(ENGINE_BROKER_LIMIT_REASONS as readonly unknown[]).includes(value.limitReason)) throw invalid("broker frame"); + if (kind === "completed" && value.limitReason !== "none") throw invalid("broker frame"); + let usage: EngineBrokerTurnUsage | null = null; + if (value.usage !== null) { try { usage = parseEngineBrokerTurnUsage(value.usage); } catch { throw invalid("broker frame"); } } + return { outcome: kind, usage, model: value.model as GrokBrokerModel, requests: value.requests, limitReason: value.limitReason as EngineBrokerLimitReason }; +} + +/** + * Maps a provider-reported model key onto the declared closed-list model. + * + * Grok 1.0.34 reports `grok-4.6` usage under `grok-4.6-build` (P0). The exact + * declared id and its `-build` alias are accepted; anything else is a + * different model and yields `undefined`. + */ +export const mapGrokReportedModel = (reported: string, declared: GrokBrokerModel): GrokBrokerModel | undefined => + reported === declared || reported === `${declared}-build` ? declared : undefined; diff --git a/src/runtime/engineBrokerTurnRegistry.test.ts b/src/runtime/engineBrokerTurnRegistry.test.ts index 16172c7..249cbd6 100644 --- a/src/runtime/engineBrokerTurnRegistry.test.ts +++ b/src/runtime/engineBrokerTurnRegistry.test.ts @@ -5,18 +5,67 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); +const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); test("turn registry replays terminal results across restart and rejects conflicts", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { - const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start()), "start"); - await assert.rejects(first.begin(start()), /already active/); - const response = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" } as const; + const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start(),"grok-4.6"), "start"); + await assert.rejects(first.begin(start(),"grok-4.6"), /already active/); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 1, limitReason: "none" } as const; await first.finish(start(), response); - assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()), { replay: response }); - await assert.rejects(first.begin(start("different")), /conflict/); + assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response, ledger: { usage: null, requests: "" } }); + await assert.rejects(first.begin(start("different"),"grok-4.6"), /conflict/); } finally { await rm(root, { recursive: true, force: true }); } }); -test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start()),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start());assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start()),replay);}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"}} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),/invalid broker frame/u);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start(),"grok-4.6"),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6");assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start(),"grok-4.6"),replay);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response,ledger:{usage:null,requests:""}});}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret",outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),/registry unavailable/u);}finally{await rm(root,{recursive:true,force:true});}}); + +const withRoot = async (run: (root: string) => Promise): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { await run(root); } finally { await rm(root, { recursive: true, force: true }); } }; +const recordFile = async (root: string): Promise => { const [name] = await readdir(root); return path.join(root, name!); }; + +test("a v1 record sealed before the upgrade still replays, upgraded with no usage and never re-metered", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.5"), "start"); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" }; + await writeFile(file, JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: record.digest, state: "terminal", bootId: "boot-a", response: v1 })); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" }, ledger: { usage: null, requests: "" } }); + }); +}); + +test("the v2 record parser is strict: an unknown member or a v1 frame inside a v2 record is refused", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 0, limitReason: "none" } as const; + await registry.finish(start(), response); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + assert.equal(record.version, "noopolis.daimon.engine-broker-turn.v2"); + // Mutation guard: dropping the exact-member check accepts this record. + assert.deepEqual(record.ledger, { usage: null, requests: "" }); + await writeFile(file, JSON.stringify({ ...record, usageRow: "extra" })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + const { outcome: _o, usage: _u, model: _m, requests: _r, limitReason: _l, ...legacy } = response; + await writeFile(file, JSON.stringify({ ...record, response: { ...legacy, version: "noopolis.daimon.engine-broker.v1" } })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + }); +}); + +test("sealed ledger bytes must be this turn's own rows and agree with the sealed usage", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const turnId = "turn-1"; + const usage = { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }; + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId, text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage, model: "grok-4.6", requests: 1, limitReason: "none" } as const; + const line = (turn: string) => `${JSON.stringify({ v: "noopolis.daimon.turn-usage.v1", turn, total: 6 })}\n`; + await registry.finish(start(), response, { usage: line(turnId), requests: "" }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), { replay: response, ledger: { usage: line(turnId), requests: "" } }); + for (const ledger of [{ usage: line("other-turn"), requests: "" }, { usage: null, requests: "" }, { usage: line(turnId), requests: "not json\n" }, { usage: line(turnId), requests: "", extra: 1 }]) { + await registry.finish(start(), response, ledger as never); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-c").begin(start(), "grok-4.6"), /registry unavailable/u, JSON.stringify(ledger)); + } + }); +}); diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index bdee3c3..43ff351 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -2,33 +2,91 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; import path from "node:path"; -import { parseEngineBrokerResponse,type EngineBrokerRequest, type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse, type EngineBrokerRequest, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; +import { EMPTY_BROKER_TURN_LEDGER, parseBrokerTurnLedgerLines, type BrokerTurnLedgerLines } from "./grokEngineBrokerLedger.js"; type Start = Extract; -type Terminal = Extract; -type Record = { version: "noopolis.daimon.engine-broker-turn.v1"; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }; +type Terminal = EngineBrokerTerminalResponse; +export const ENGINE_BROKER_TURN_RECORD_V1 = "noopolis.daimon.engine-broker-turn.v1" as const; +export const ENGINE_BROKER_TURN_RECORD_V2 = "noopolis.daimon.engine-broker-turn.v2" as const; +// The digest deliberately excludes `limits` and the protocol version, so a v1 +// record written before the upgrade still identifies the same turn. const digest = (request: Start): string => createHash("sha256").update(JSON.stringify([request.turnId, request.agentId, request.wakeId, request.prompt,request.mcpEndpoint])).digest("hex"); const safe = (turnId: string): string => `${createHash("sha256").update(turnId).digest("hex")}.json`; +type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal; ledger?: BrokerTurnLedgerLines }>; +/** + * Durable per-turn state. Record v2 stores the terminal response *with* its + * sealed accounting (usage, outcome, model, requests, limitReason), so a replay + * returns exactly what was metered and never meters again: metering happens + * only on the path that returned `"start"`. + */ export class EngineBrokerTurnRegistry { - constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} - async begin(request: Start): Promise<"start" | { replay: Terminal }> { + /** `syncDirectoryOf` is injectable only so the post-publish failure path can be exercised under test. */ + constructor(private readonly root: string,private readonly bootId:string=randomUUID(),private readonly syncDirectoryOf:(directory:string)=>Promise=syncDirectory) {} + /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ + async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal; ledger: BrokerTurnLedgerLines }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); - try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } + try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await this.syncDirectoryOf(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } - const observed = JSON.parse(await readFile(file, "utf8")) as Record; - if (observed.version !== "noopolis.daimon.engine-broker-turn.v1" || observed.digest !== expected) throw new Error("broker turn conflict"); - if (observed.state === "terminal" && observed.response !== undefined) { const response=parseEngineBrokerResponse(observed.response);if(response.kind!=="completed"&&response.kind!=="failed")throw new Error("broker turn registry unavailable");return { replay: response }; } - if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed"} as const;await this.finish(request,response);return {replay:response};} + const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); + if (observed.digest !== expected) throw new Error("broker turn conflict"); + if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response, ledger: observed.ledger ?? EMPTY_BROKER_TURN_LEDGER }; + if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response,ledger:EMPTY_BROKER_TURN_LEDGER};} throw new Error("broker turn already active"); } - async finish(request: Start, response: Terminal): Promise { + /** + * `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. + * + * The rename is the publish point. A failure before it rejects (nothing was + * published); a failure after it — the directory fsync — must not, because + * the terminal record is already the visible truth and a caller that saw a + * rejection would believe the turn unsealed and write a contradicting + * record over it. That durability gap is reported as `directorySynced: false` + * instead: the record is published but may not survive a power loss. + */ + async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise> { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); - try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } + try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response, ledger })}\n`); await handle.sync(); } finally { await handle.close(); } + try { await rename(temporary, file); } finally { await unlink(temporary).catch(() => undefined); } + try { await this.syncDirectoryOf(this.root); return { directorySynced: true }; } catch { return { directorySynced: false }; } } } +/** + * Strict record parser. v2 accepts exactly `{version,digest,state,bootId}` + * plus `response` and its sealed `ledger` bytes when terminal, and the response must be a v2 terminal frame. + * v1 records keep their historical looser shape and are upgraded on read: no + * usage (`null`), zero requests, `limitReason: "none"`, the declared model. + */ +export function parseEngineBrokerTurnRecord(text: string, model: GrokBrokerModel): Observed { + let value: unknown; + try { value = JSON.parse(text); } catch { throw new Error("broker turn registry unavailable"); } + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error("broker turn registry unavailable"); + const input = value as Record; + if (typeof input.digest !== "string" || !/^[a-f0-9]{64}$/u.test(input.digest) || typeof input.bootId !== "string" || (input.state !== "active" && input.state !== "terminal")) throw new Error("broker turn registry unavailable"); + const base = { digest: input.digest, state: input.state, bootId: input.bootId } as const; + if (input.version === ENGINE_BROKER_TURN_RECORD_V1) { + if (input.state !== "terminal" || input.response === undefined) return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base }; + let legacy; + try { legacy = parseEngineBrokerV1TerminalResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + const accounting = { outcome: legacy.kind, usage: null, model, requests: 0, limitReason: "none" } as const; + const response: Terminal = { ...legacy, ...accounting, version: "noopolis.daimon.engine-broker.v2" } as Terminal; + return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base, response }; + } + if (input.version !== ENGINE_BROKER_TURN_RECORD_V2) throw new Error("broker turn conflict"); + const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response", "ledger"] : ["version", "digest", "state", "bootId"]; + if (Object.keys(input).length !== fields.length || fields.some((field) => !Object.hasOwn(input, field))) throw new Error("broker turn registry unavailable"); + if (input.state === "active") return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base }; + let response; + try { response = parseEngineBrokerResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + if (response.kind !== "completed" && response.kind !== "failed") throw new Error("broker turn registry unavailable"); + const ledger = parseBrokerTurnLedgerLines(input.ledger, response.turnId); + if ((response.usage === null) !== (ledger.usage === null)) throw new Error("broker turn registry unavailable"); + return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response, ledger }; +} + async function syncDirectory(directory: string): Promise { const handle = await open(directory, constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } } diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index ef80d32..ca1b5e1 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -243,9 +243,10 @@ test("production Grok dispatcher routes every wake through the broker without ag process.env.PATH = `${root}${path.delimiter}${priorPath ?? ""}`; process.env.NOOPOLIS_RUN_ID = "dispatcher-grok-realm-test"; const broker: EngineBrokerTurnClient = { - async turn(agentId,wakeId,prompt,endpoint,signal) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false);return "brokered"; } + async turn(agentId,wakeId,prompt,endpoint,signal,options) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false); + assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } // the engine-neutral wake bound reaches the broker as a lowering limit }; - const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); + const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker).finally(() => { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; }); assert.equal((await handle.wake({ id: "first", kind: "manual", text: "work" })).text, "brokered"); assert.equal((await handle.wake({ id: "second", kind: "manual", text: "work" })).text, "brokered"); assert.equal(turns, 2); @@ -289,7 +290,7 @@ test("Daimon frames one escaped identity envelope for every production engine", const envelope = JSON.stringify({ id: config.id, name: identity.name, instructions: identity.instructions }); assert.equal(result.text.split(envelope).length - 1, 1); assert.match(result.text, //); - assert.match(result.text, /Colleagues only hear you when you call moltnet_send/u); + assert.ok(result.text.includes(`Colleagues only hear you when you call ${kind === "grok" ? "daimon__moltnet_send" : "moltnet_send"};`), `${kind} must name the send tool the way it can call it`); assert.match(result.text, /Do not seek transport credentials or invoke a transport CLI/u); assert.match(result.text, /payload/); await handle.stop(); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 9414752..9b72149 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,8 +1,9 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; +import { grokDaimonToolName, grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; -import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling } from "../pi/cliSession.js"; +import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "../pi/cliSession.js"; import { GROK_DAIMON_SANDBOX_PROFILE, prepareAndVerifyGrokSandbox @@ -32,6 +33,9 @@ export async function startOrganizationRuntimeEngine( sharedProtectedPaths: readonly string[] = [], attention?: AttentionRegistry ): Promise { + // A declared Grok model is enforced by the broker proxy and worker config; + // the direct path has neither, so it refuses rather than silently ignoring it. + if (agent.engine.kind === "grok" && agent.engine.model !== undefined && grokBroker === undefined) throw new Error(`Agent ${agent.id} declares a Grok model, which requires the engine broker`); await paths?.verify(); const canonicalAgent = paths === undefined ? agent : { ...agent, workspacePath: paths.workspacePath, runtimeHomePath: paths.runtimeHomePath }; const readiness = canonicalAgent.engine.kind === "grok" && grokBroker !== undefined @@ -53,7 +57,8 @@ export async function startOrganizationRuntimeEngine( readablePaths: codexSandboxReadablePaths(canonicalAgent) } : undefined; - const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])], wakeContext, grokSandbox,grokBroker,codexSandboxPaths); + const mountedTools = [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])]; + const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, mountedTools, wakeContext, grokSandbox,grokBroker,codexSandboxPaths, mountedTools.map((tool) => tool.name)); const handle = await adapter.startAgent({ id: canonicalAgent.id, name: canonicalAgent.name, @@ -115,16 +120,16 @@ export function codexSandboxReadablePaths( return [path.join(currentAgent.runtimeHomePath, "tool-output")]; } -function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }): PiHarnessAdapter { +function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }, mountedToolNames: readonly string[] = []): PiHarnessAdapter { const engine = agent.engine.kind; const sessionFactory = createCliSessionFactory( engine === "agy" - ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, // AGY has no broker to meter it, so the session hands its decoded // terminal-frame usage straight to the same ledger the Grok broker // appends to. `recordTurnUsage` is advisory and never rejects. onTurnUsage: (usage, outcome) => recordTurnUsage(resolveTurnUsageLedgerPath(), { agent: agent.id, wake: wakeEnvironmentContext.current ?? "wake", engine: "agy", usage, outcome }) } - : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, ...(engine === "codex" ? { // Codex has no broker to meter it, so publish terminal-frame usage // to the shared advisory ledger — on the wake that published and on @@ -151,7 +156,10 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri }) } : {}), ...(engine==="grok"&&grokBroker!==undefined?{}:{credentialSecretValues: () => readPortableEngineCredentialSecrets(agent.id, engine, engineHomePath)}), - ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:(prompt:string,endpoint:string,signal:AbortSignal)=>grokBroker.turn(agent.id,wakeEnvironmentContext.current??"wake",prompt,endpoint,signal)}:{}), + // The broker seals usage and enforces its registration's limits; the + // wake may only lower them (DAIMON_ENGINE_WAKE_*), and a declared model + // must be the one the broker reports it ran. + ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:grokBrokerTurnFor(agent,grokBroker,wakeEnvironmentContext)}:{}), ...(engine === "grok" && verifyGrokSandbox ? { grokSandboxProfile: GROK_DAIMON_SANDBOX_PROFILE, verifyGrokSandbox @@ -160,21 +168,65 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri return cliHarness(agent, sessionFactory, [controlTokenEnv], productionTools, wakeEnvironmentContext); } +function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: EngineBrokerTurnClient, wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef) { + const limits = resolveEngineWakeLimitOverrides(); + const options = { ...(limits === undefined ? {} : { limits }), ...(agent.engine.model === undefined ? {} : { model: agent.engine.model }) }; + return (prompt: string, endpoint: string, signal: AbortSignal) => grokBroker.turn(agent.id, wakeEnvironmentContext.current ?? "wake", prompt, endpoint, signal, options); +} + /** * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig): string { +/** + * The caller-owned prompt preamble. + * + * It names the mounted tools explicitly, because a CLI engine reaches Daimon's + * tools over MCP and an agent whose instructions name another engine's tool + * spelling can finish a turn having called nothing. The declared names are the + * caller's own configuration, not engine-supplied text. + * + * On Grok the bare names are not the callable ones: every Daimon tool is a + * deferred MCP tool of server `daimon`, and Grok 1.0.34 refuses an unqualified + * name before any HTTP ("Tool names must be qualified as `server__tool`"). This + * envelope used to instruct exactly that refused form. The correct rule is not + * restated here — it is rendered by `grokMountedToolNamingRule` in the same + * contract module that renders the worker's pinned system prompt, so the two + * texts cannot contradict each other again. Every other engine's sentence is + * unchanged, byte for byte. + */ +export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", - "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + ...(mountedToolNames.length === 0 ? [] : [ + (agent.engine.kind === "grok" + ? grokMountedToolNamingRule(mountedToolNames) + : `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently.") + + " No other tool reaches the newsroom." + ]), + // The one tool that reaches colleagues is named the way this engine can + // call it. Meaning and prohibition are unchanged; only the spelling is. + `Colleagues only hear you when you call ${engineToolName(agent, "moltnet_send")}; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. ` + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." ].join("\n") + "\n"; } +/** + * One Daimon tool name, spelled the way this agent's engine accepts it. + * + * On Grok the bare form is refused as an invalid MCP tool name, so any + * engine-facing sentence that *names* a tool renders it through the contract's + * `grokDaimonToolName`; every other engine keeps the bare name byte for byte. + * Grok's own native tools (`read_file`, `search_tool`, `use_tool`) are not + * Daimon tools and never take the prefix. + */ +const engineToolName = (agent: OrganizationRuntimeAgentConfig, tool: string): string => + agent.engine.kind === "grok" ? grokDaimonToolName(tool) : tool; + function cliHarness( agent: OrganizationRuntimeAgentConfig, sessionFactory: ReturnType, diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts new file mode 100644 index 0000000..13b1607 --- /dev/null +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import test from "node:test"; + +import { identityEnvelope } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const rootConfig = (root: string, kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): OrganizationRuntimeAgentConfig => ({ + id: `${kind}-agent`, name: kind, instructions: "Reply.", + workspacePath: path.join(root, "workspace", kind), runtimeHomePath: path.join(root, "runtime", kind), + engine: { kind } +}); + +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope names the prefixed form as the only valid one, once", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +/** + * The transport sentence names the one tool an agent needs to reach its + * colleagues. A bare `moltnet_send` one line under "a bare name is not a valid + * MCP tool name and reaches nothing" is the same self-contradiction, on the + * tool that matters most. + */ +test("the transport sentence names the send tool the way Grok can call it, prohibition unchanged", () => { + const grok = identityEnvelope(rootConfig("/private/org", "grok"), mounted); + const transport = grok.split("\n").find((line) => line.startsWith("Colleagues only hear you"))!; + // Mutation guard: un-prefixing this name leaves the bare form the line above declares invalid. + assert.ok(transport.includes(`you call ${grokDaimonToolName("moltnet_send")};`), transport); + assert.equal(new RegExp(`(? { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); + // The transport sentence keeps its bare spelling on every other engine. + const transport = "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool."; + for (const kind of ["codex", "agy"] as const) { + assert.ok(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(`\n${transport}\n`), kind); + assert.equal(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(DAIMON_GROK_TOOL_PREFIX), false, kind); + } +}); diff --git a/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl b/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl new file mode 100644 index 0000000..a728cef --- /dev/null +++ b/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl @@ -0,0 +1,2 @@ +{"timestamp":"2026-09-17T02:33:59.302698678Z","event_type":"ProfileApplied","profile":"daimon-strict","workspace":"/var/lib/spawnfile/instance/workspace/agents/a1","platform":"linux/landlock","enforced":true,"restrict_network":true,"read_write_paths":["/var/lib/spawnfile/instance/workspace/agents/a1","/var/lib/daimon-workers/2200/.grok/sessions","/tmp","/var/tmp"],"read_only_paths":["/usr","/lib","/bin","/sbin","/etc","/dev","/proc","/sys","/tmp","/run","/var","/var/lib/spawnfile/instance/workspace/agents/a1","/var/lib/daimon-workers/2200/.grok"],"deny_paths":["/run/paideia"]} +{"timestamp":"2026-09-17T02:33:59.913077091Z","event_type":"FsViolation","profile":"daimon-strict","operation":"read","target":"/run/paideia/context.json"} diff --git a/src/runtime/fixtures/grok-slot-preflight/projection-input.json b/src/runtime/fixtures/grok-slot-preflight/projection-input.json new file mode 100644 index 0000000..303abd0 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/projection-input.json @@ -0,0 +1,46 @@ +{ + "config": { + "version": "noopolis.daimon.organization-runtime.v2", + "host": { + "bindHost": "127.0.0.1", + "port": 19700, + "controlTokenEnv": "DAIMON_CONTROL_TOKEN" + }, + "agents": [ + { + "id": "foreman", + "name": "Foreman", + "instructions": "Fixture agent.", + "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", + "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", + "schedule": { + "kind": "disabled" + }, + "engine": { + "kind": "grok", + "model": "grok-4.6", + "reasoningEffort": "low" + } + } + ] + }, + "agentId": "foreman", + "options": { + "slot": 0, + "workerUid": 2200, + "workerHomePath": "/var/lib/daimon-workers/2200", + "architecture": "arm64", + "usageLedgerPath": "/run/daimon-slots/0/usage/usage.jsonl", + "limits": { + "maxRequests": 24, + "maxTokens": 400000, + "timeoutMs": 480000 + }, + "acceptanceStorePath": "/run/paideia/control", + "denyPaths": [ + "/run/paideia", + "/run/training/inputs" + ], + "seccompProfileSha256": "7777777777777777777777777777777777777777777777777777777777777777" + } +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json new file mode 100644 index 0000000..b28307c --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -0,0 +1,45 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/not-projected", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json new file mode 100644 index 0000000..fd1e2de --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json @@ -0,0 +1,38 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json new file mode 100644 index 0000000..03a3c9d --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -0,0 +1,35 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json new file mode 100644 index 0000000..fd520ed --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -0,0 +1,40 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json new file mode 100644 index 0000000..c2f4908 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -0,0 +1,40 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "readable" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json new file mode 100644 index 0000000..d5978c9 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -0,0 +1,41 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z", + "operator": "root" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json new file mode 100644 index 0000000..bf3e344 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json @@ -0,0 +1,40 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokBrokerModelPolicy.ts b/src/runtime/grokBrokerModelPolicy.ts new file mode 100644 index 0000000..266a1d7 --- /dev/null +++ b/src/runtime/grokBrokerModelPolicy.ts @@ -0,0 +1,32 @@ +/** + * The closed model and reasoning-effort vocabulary a Grok broker worker may be + * declared with. + * + * Both halves of one declaration are consumed from this single parser: the + * worker `config.toml` renderer (`grokBrokerWorkerConfig.ts`) writes them into + * the worker's only custom model, and the provider proxy + * (`grokBrokerProxyRequest.ts`) refuses any request body that does not carry + * exactly them. Nothing is inherited: Grok 1.0.34 silently drops an effort for + * a model that does not declare effort support, and its embedded catalog + * default for `grok-4.6` is `high`. + */ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; + +export { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS }; +export type GrokBrokerModel = (typeof GROK_BROKER_MODELS)[number]; +export type GrokBrokerReasoningEffort = (typeof GROK_BROKER_REASONING_EFFORTS)[number]; +export type GrokBrokerModelPolicy = Readonly<{ model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort }>; + +export const DEFAULT_GROK_BROKER_MODEL_POLICY: GrokBrokerModelPolicy = Object.freeze({ model: "grok-4.6", reasoningEffort: "low" }); + +export function parseGrokBrokerModelPolicy(value: unknown = {}): GrokBrokerModelPolicy { + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new TypeError("invalid Grok broker model policy"); + const input = value as Record; + if (Object.keys(input).some((key) => key !== "model" && key !== "reasoningEffort")) throw new TypeError("invalid Grok broker model policy"); + const model = input.model ?? DEFAULT_GROK_BROKER_MODEL_POLICY.model; + const reasoningEffort = input.reasoningEffort ?? DEFAULT_GROK_BROKER_MODEL_POLICY.reasoningEffort; + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(model) || !(GROK_BROKER_REASONING_EFFORTS as readonly unknown[]).includes(reasoningEffort)) { + throw new TypeError("invalid Grok broker model policy"); + } + return Object.freeze({ model: model as GrokBrokerModel, reasoningEffort: reasoningEffort as GrokBrokerReasoningEffort }); +} diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts new file mode 100644 index 0000000..960e917 --- /dev/null +++ b/src/runtime/grokBrokerProjection.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER, GROK_SUBSCRIPTION_REALM } from "../contracts/runtimeContractManifest.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, verifyGrokBrokerRegistrationMatchesProjection } from "./grokBrokerProjection.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import { grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; + +const agent = (id: string, engine: Record) => ({ id, name: id, instructions: "Unused", workspacePath: `/var/lib/spawnfile/instance/workspace/agents/${id}`, runtimeHomePath: `/var/lib/spawnfile/instance/homes/${id}`, schedule: { kind: "disabled" }, engine }); +const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, + agents: [agent("foreman", { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }), agent("peer", { kind: "codex" })] }; +const options = { slot: 0, workerUid: 2_200, workerHomePath: "/var/lib/daimon-workers/2200", architecture: "arm64", usageLedgerPath: "/run/slots/0/usage/usage.jsonl", + limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"], seccompProfileSha256: "7".repeat(64) } as const; + +test("the projection is Daimon's own renderers and collectors, fully declared and deterministic", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const denyPaths = [GROK_SUBSCRIPTION_REALM.bootstrapMountPath, GROK_SUBSCRIPTION_REALM.durableMountPath, "/run/paideia/control", "/var/lib/spawnfile/instance/homes/peer", "/var/lib/spawnfile/instance/workspace/agents/peer", "/run/paideia", "/run/training/inputs"].sort(); + assert.deepEqual(projection, { + version: "noopolis.daimon.grok-broker-projection.v1", agentId: "foreman", + workspacePath: "/var/lib/spawnfile/instance/workspace/agents/foreman", runtimeHomePath: "/var/lib/spawnfile/instance/homes/foreman", + workerUid: 2_200, slot: 0, profilePath: "/var/lib/daimon-workers/2200/.grok/sandbox.toml", profileSha256: grokWorkerSandboxProfileSha256(denyPaths), denyPaths, + workerConfigSha256: grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "low" }), systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, + grokCliVersion: "1.0.34", grokExecutableSha256: GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, + model: "grok-4.6", reasoningEffort: "low", limits: options.limits, usageLedgerPath: options.usageLedgerPath, + seccompProfileSha256: "7".repeat(64), + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", sandboxRuntime: "bubblewrap", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } + }); + assert.equal(grokBrokerProjectionSha256(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: [...options.denyPaths].reverse() })), grokBrokerProjectionSha256(projection)); + assert.match(grokBrokerProjectionSha256(projection), /^[a-f0-9]{64}$/u); +}); + +test("the projection refuses undeclared models, non-Grok agents, and a profile digest it did not render", () => { + assert.throws(() => resolveOrganizationGrokBrokerProjection({ ...config, agents: [agent("foreman", { kind: "grok" })] }, "foreman", options), /declared model/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "peer", options), /known Grok agent/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, seccompProfileSha256: "not-a-digest" }), /seccomp profile sha256/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "missing", options), /known Grok agent/u); + // Mutation guard: skipping the digest comparison accepts a weaker profile's digest. + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: grokWorkerSandboxProfileSha256([]) }), /profile digest mismatch/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: ["relative"] }), /deny path/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, limits: { ...options.limits, maxRequests: 49 } }), /invalid/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, usageLedgerPath: "/run/slots/0/usage/requests.jsonl" }), /invalid engine broker service config/u); + assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: resolveOrganizationGrokBrokerProjection(config, "foreman", options).profileSha256 }).agentId, "foreman"); +}); + +test("a provisioned registration must describe its projection exactly", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const parse = (registration: Record) => parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: "/c", turnStore: "/t", registrations: [registration] }).registrations[0]!; + const registration = grokBrokerServiceRegistrationFor(projection); + verifyGrokBrokerRegistrationMatchesProjection(parse(registration), projection); + for (const drift of [{ profileSha256: grokWorkerSandboxProfileSha256([]) }, { model: { id: "grok-4.5", reasoningEffort: "low" } }, { limits: { ...options.limits, maxTokens: 400_001 } }, { usageLedgerPath: "/run/slots/1/usage/usage.jsonl" }]) { + assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); + } +}); + +test("the acceptance store mask may be the directory that covers the store", () => { + // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a + // deployment that secures `/state` to `2000:2000 0700` declares that directory here. + const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; + const state = "/var/lib/spawnfile/instance/state"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: state }); + assert.equal(lifted.denyPaths.includes(state), true); + assert.equal(lifted.denyPaths.includes(store), false); + // The store itself still works where its parent is traversable, and produces a different digest — + // a recomputation that disagrees with the deployment fails closed rather than certifying the slot. + const leaf = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }); + assert.equal(leaf.denyPaths.includes(store), true); + assert.notEqual(grokBrokerProjectionSha256(leaf), grokBrokerProjectionSha256(lifted)); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: "/var" }), + /base profile grant/u); +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts new file mode 100644 index 0000000..298b433 --- /dev/null +++ b/src/runtime/grokBrokerProjection.ts @@ -0,0 +1,156 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { canonicalJson } from "../contracts/canonicalJson.js"; +import { DAIMON_GROK_SYSTEM_PROMPT } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokSandboxProtectedPaths } from "./engineDispatcher.js"; +import { parseEngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import { GROK_WORKER_SANDBOX_PROFILE, grokWorkerEventsPathFor, renderGrokWorkerSandboxProfile, grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; +import { parseOrganizationRuntimeConfig } from "./organizationRuntime.js"; + +export const GROK_BROKER_PROJECTION_VERSION = GROK_ENGINE_BROKER.projectionVersion; + +/** + * Everything a consumer (Spawnfile provisioning, Paideia's native adapter, the + * root slot supervisor) needs to know about one brokered Grok agent's slot, + * computed by Daimon from the same renderers and collectors the broker attests + * against. It never reads credentials, runs a worker, or touches the realm. + */ +export type OrganizationGrokBrokerProjection = Readonly<{ + version: typeof GROK_BROKER_PROJECTION_VERSION; + agentId: string; + workspacePath: string; + runtimeHomePath: string; + workerUid: number; + slot: number; + profilePath: string; + profileSha256: string; + denyPaths: readonly string[]; + workerConfigSha256: string; + systemPromptSha256: string; + grokCliVersion: string; + grokExecutableSha256: string; + nativeAbiVersion: number; + model: GrokBrokerModel; + reasoningEffort: GrokBrokerReasoningEffort; + limits: EngineBrokerTurnLimits; + usageLedgerPath: string; + /** The container seccomp profile the worker must run under (the pinned default-plus-userns profile bubblewrap needs). */ + seccompProfileSha256: string; + attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; sandboxRuntime: "bubblewrap"; eventsPath: string }>; +}>; + +export type OrganizationGrokBrokerProjectionOptions = Readonly<{ + /** Deployment-assigned slot identity. */ + slot: number; + workerUid: number; + /** The worker's home; its `GROK_HOME` is `/.grok`. */ + workerHomePath: string; + architecture: "arm64" | "x64"; + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + /** + * The deny entry that protects the durable wake-acceptance store: the store + * itself, or a directory containing it. + * + * Grok 1.0.34 materializes every deny target inside bubblewrap **as the + * worker uid**, so a target whose parent directory the worker cannot search + * is unplaceable and makes Grok refuse the whole profile — every turn of that + * worker then fails, not just that path. Deployments that keep the store + * under a private `state` directory (`2000:2000 0700`) therefore declare that + * directory here: it covers the store, nothing else lives there, and lifting + * the mask adds the worker no reach, where opening the parent with `o+x` + * would. The caller is the one party that knows both the layout and the + * modes; whoever recomputes this projection must pass the same value or the + * digests will not match, which is the intended fail-closed outcome. + */ + acceptanceStorePath: string; + /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ + denyPaths?: readonly string[]; + /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ + seccompProfileSha256: string; + /** When the caller already holds a rendered profile digest, it must equal Daimon's. */ + profileSha256?: string; +}>; + +/** + * Resolve the public Grok broker projection for one agent. + * + * Paths are taken as given, never resolved: the caller (Spawnfile provisioning) + * must supply canonical, non-symlink paths — the fixed tmpfs/workspace roots it + * creates — and its provisioning must verify they are not symlinks before a + * slot is used; the broker's own attestation re-checks the worker home at + * every turn. + * + * Deterministic and I/O-free on purpose: its digest + * ({@link grokBrokerProjectionSha256}) is what the slot preflight receipt + * binds, so the supervisor that writes the receipt and the evaluator that reads + * it must compute byte-equal projections from the same inputs. + * + * The agent must be a Grok agent that *declares* its model and reasoning + * effort; nothing is defaulted. The deny list is Daimon's own protected set for + * this agent (realm, bootstrap, peers, and the mask covering the acceptance + * store) plus the caller's evaluator paths, sorted and deduplicated exactly as + * the profile renderer does. A supplied `profileSha256` that differs is + * refused. + */ +export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { + const parsed = parseOrganizationRuntimeConfig(config); + const agent = parsed.agents.find((entry) => entry.id === agentId); + if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); + if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { + if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); + } + if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); + const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); + renderGrokWorkerSandboxProfile(denyPaths); + const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); + if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); + const workerConfigSha256 = grokBrokerWorkerConfigSha256({ model, reasoningEffort }); + if (workerConfigSha256 !== GROK_ENGINE_BROKER.worker.configSha256[model][reasoningEffort]) throw new Error("Grok broker projection worker config drifted from the manifest"); + if (createHash("sha256").update(DAIMON_GROK_SYSTEM_PROMPT).digest("hex") !== GROK_ENGINE_BROKER.worker.systemPromptSha256) throw new Error("Grok broker projection system prompt drifted from the manifest"); + const artifact = GROK_ENGINE_BROKER.grokCliArtifacts[options.architecture]; + if (artifact === undefined) throw new Error("Grok broker projection requires a pinned architecture"); + const profilePath = path.posix.join(options.workerHomePath, ".grok", "sandbox.toml"); + const projection: OrganizationGrokBrokerProjection = { + version: GROK_BROKER_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, runtimeHomePath: agent.runtimeHomePath, + workerUid: options.workerUid, slot: options.slot, profilePath, profileSha256, denyPaths, workerConfigSha256, + systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: GROK_ENGINE_BROKER.grokCliVersion, grokExecutableSha256: artifact.sha256, + nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, seccompProfileSha256: options.seccompProfileSha256, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, sandboxRuntime: "bubblewrap", eventsPath: grokWorkerEventsPathFor(profilePath) } + }; + // The registration this projection implies must itself be a valid v2 service.json entry. + grokBrokerServiceRegistrationFor(projection); + return projection; +} + +/** sha256 over the projection's canonical JSON; what a slot preflight receipt binds. */ +export const grokBrokerProjectionSha256 = (projection: OrganizationGrokBrokerProjection): string => + createHash("sha256").update(canonicalJson(projection)).digest("hex"); + +/** The `service.json` v2 registration a deployment provisions for this projection, validated by the broker's own parser. */ +export function grokBrokerServiceRegistrationFor(projection: OrganizationGrokBrokerProjection): Readonly> { + const registration = { + agentId: projection.agentId, slot: projection.slot, workerUid: projection.workerUid, workspace: projection.workspacePath, + profilePath: projection.profilePath, eventsPath: projection.attestation.eventsPath, profileSha256: projection.profileSha256, + usageLedgerPath: projection.usageLedgerPath, limits: projection.limits, model: { id: projection.model, reasoningEffort: projection.reasoningEffort } + }; + parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [registration] }); + return registration; +} + +/** + * Refuses a provisioned registration that does not describe this projection: + * any differing member — a weaker profile digest, another model, raised + * limits, a different ledger — is a mismatch, never a merge. + */ +export function verifyGrokBrokerRegistrationMatchesProjection(registration: EngineBrokerServiceRegistration, projection: OrganizationGrokBrokerProjection): void { + const expected = parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [grokBrokerServiceRegistrationFor(projection)] }).registrations[0]!; + if (canonicalJson(expected) !== canonicalJson(registration)) throw new Error("Grok broker registration does not match its projection"); +} diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 5f2a362..93d4a9f 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -1,6 +1,24 @@ import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import { connect } from "node:net"; import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "./grokBrokerModelPolicy.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; + +type Proxy = Awaited>; +const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { + proxy.registerIsolationGuard("turn", guard); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + return meter; +}; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = (overrides: Record = {}): string => JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean, ...overrides }); test("proxy retries one 401 with refreshed broker bearer and shuts down", async () => { const calls: string[] = []; let refreshes = 0; @@ -8,20 +26,221 @@ test("proxy retries one 401 with refreshed broker bearer and shuts down", async calls.push(request.headers.authorization); return calls.length === 1 ? { status: 401, headers: { "content-type": "application/json" }, body: new Uint8Array() } : { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }; }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); - const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.25" }, body: JSON.stringify({ stream: true, messages: [] }) }); + arm(proxy, async () => undefined); + const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 200); assert.equal(await result.text(), "data: done\n\n"); assert.deepEqual(calls, ["Bearer first", "Bearer second"]); assert.equal(refreshes, 0); await proxy.close(); await assert.rejects(fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`)); }); -test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.25"},body:JSON.stringify({stream:true,messages:[]})});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); +test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); test("proxy failures expose only a fixed diagnostic", async () => { const proxy = await startGrokBrokerProxy({ accessToken: async () => { throw new Error("secret-token"); }, markRejected: async () => undefined }, async () => { throw new Error("unreachable"); }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); - const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.25" }, body: JSON.stringify({ stream: true, messages: [] }) }); + arm(proxy, async () => undefined); + const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 503); const body = await result.text(); assert.equal(body, '{"error":"broker unavailable"}'); assert.doesNotMatch(body, /secret/u); await proxy.close(); }); -test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.25"},body:JSON.stringify({stream:true,messages:[]})});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); +test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); + +test("proxy refuses a fail-open tool set or an undeclared effort without calling upstream", async () => { + let calls = 0; let accessed = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }, { model: "grok-4.6", reasoningEffort: "low" }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; + for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { + // A policy miss is non-retryable: 400, so Grok fails fast instead of retrying a 503. + assert.equal(await post(proxy.port, token, payload), 400); + } + assert.equal(calls, 0); + assert.equal(await post(proxy.port, token, leanBody()), 200); assert.equal(calls, 1); assert.ok(accessed >= 1); + } finally { await proxy.close(); } +}); + +// One unpooled connection per request: the proxy listens on a fixed port that the +// previous test just closed, and a pooled keep-alive socket to it would be stale. +function post(port: number, token: string, body: string): Promise { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(body); + }); +} + +test("the session-title sink is refused before capability, guard, credential, or upstream use", async () => { + let calls = 0, accessed = 0, guarded = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); + const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); + // The title sink keeps its transient 503 shape: a 4xx there ends Grok's session. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); + assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); + // The turn capability (budget 1 request) is untouched and still serves the real request. + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.equal(calls, 1); + } finally { await proxy.close(); } +}); + +test("the isolation guard is awaited before the first upstream call, and a failing guard makes no upstream call", async () => { + const order: string[] = []; let upstreamCalls = 0; let fail = true; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { order.push("credential"); return "provider-token"; }, markRejected: async () => undefined }, async () => { upstreamCalls++; order.push("upstream"); return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); + arm(proxy, async () => { + order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); + if (fail) throw new Error("no enforcement evidence"); + }); + assert.equal(await post(proxy.port, token, leanBody()), 400); + assert.equal(upstreamCalls, 0); + assert.deepEqual(order, ["guard-start", "guard-end"]); + fail = false; order.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual(order, ["guard-start", "guard-end", "credential", "upstream"]); + } finally { await proxy.close(); } +}); + +test("the two requests every healthy turn makes are not named as refusals", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => { lines.push(String(chunk)); return original(chunk as string, ...rest as []); }) as typeof process.stderr.write; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + arm(proxy, async () => undefined); + const title = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(title.status, 503, "the title sink keeps its transient shape"); + const probe = await fetch(`http://127.0.0.1:${proxy.port}/`); + assert.equal(probe.status, 400, "the unauthenticated probe keeps its non-retryable shape"); + assert.deepEqual(lines, [], "expected per-turn traffic must not read as a refusal on the broker's stderr"); + const miss = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${"z".repeat(48)}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(miss.status, 400); + assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("a non-refusal fault names its own class and message on one bounded line, credentials withheld", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + const provider = "provider-vqmxdfhlzptgnbwc"; let capability = ""; + // The fault's own words carry both credentials verbatim — the worker's turn + // capability and the broker's provider bearer, neither in a shape any generic + // pattern recognises — plus a newline, a control character, and far more text + // than the bound admits. + const proxy = await startGrokBrokerProxy( + { accessToken: async () => provider, markRejected: async () => undefined }, + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider}\u0007 ${"pad ".repeat(400)}`); }); + try { + capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); + assert.equal(lines.length, 1, "one line per fault"); + const line = lines[0]!; + assert.match(line, /^\[grok-proxy\] refused: broker_unavailable \(RangeError: socket hang up forwarding /u, "the fault names its own class and message"); + assert.ok(!line.includes(capability), `the worker's own capability is withheld: ${line}`); + assert.ok(!line.includes(provider), `the broker's provider bearer is withheld: ${line}`); + assert.match(line, /\[REDACTED\]/u, "the withheld values are marked, not silently dropped"); + assert.match(line, /^[^\n]+\n$/u, "one line: newlines and control characters are flattened"); + assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("a fault's own cause is named too, because `fetch failed` on its own names nothing", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // Exactly the shape undici throws when the provider is unreachable. + const fault = new TypeError("fetch failed"); (fault as { cause?: unknown }).cause = Object.assign(new Error(""), { code: "ENOTFOUND" }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { throw fault; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +/** + * A fenced realm is the one fault this proxy answered worst: the credential is + * gone until an operator logs in again, and 503 made Grok retry it fifteen + * times over five minutes for nothing. It is a named, non-retryable refusal + * now — and it wears the same name as the turn failure code and the grant + * path's 401 body, so one word finds it on every surface. + */ +test("a fenced credential realm is a named 400 auth_stale, before any credential read or upstream call", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + let reads = 0, calls = 0, stale = true; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { reads += 1; return "provider-token"; }, markRejected: async () => undefined, isStale: () => stale }, + async () => { calls += 1; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400, "a stale realm is not transient, so it must not be retryable"); + assert.deepEqual({ reads, calls }, { reads: 0, calls: 0 }, "no credential is read and nothing is forwarded for a fenced realm"); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + // The title sink keeps the 503 it has always had, fenced realm or not. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, leanBody()), 503); + // And the same capability serves the real request once the realm is healthy. + stale = false; lines.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual({ reads, calls, lines }, { reads: 1, calls: 1, lines: [] }); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the request that discovers the fence is named auth_stale too, not one transient fault", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // The live shape: `accessToken` fences the realm and throws the authority's + // own generic error, which on its own reads as a transient fault. + let stale = false; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { stale = true; throw new Error("Grok broker credential authority unavailable"); }, markRejected: async () => undefined, isStale: () => stale }, + async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the turn path and the grant path name a fenced realm the same way", () => { + assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); +}); + +/** + * A response the usage decoder cannot read is the worst case to get wrong: the + * upstream call already succeeded, so the money is spent whatever happens + * next. It must reach the worker anyway, and it must be charged — a fabricated + * zero is byte-identical to a measured one, so the documented estimate is the + * only honest landing place. (The fault is synthetic: the real one is a + * response body past the maximum string length, which is not a thing to + * allocate in a test.) + */ +test("a response whose usage cannot be decoded is still delivered, and charged the documented estimate", async () => { + const contentType = { toString: () => "application/json" } as unknown as string; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => "provider-token", markRejected: async () => undefined }, + async () => ({ status: 200, headers: { "content-type": contentType }, body: Buffer.from('{"usage":{"prompt_tokens":11,"completion_tokens":3}}') })); + try { + const token = proxy.capabilities.issue("agent", "turn"); + const meter = arm(proxy, async () => undefined); + const payload = leanBody(); + assert.equal(await post(proxy.port, token, payload), 200, "a paid response must not be thrown away over its own instrumentation"); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 1); + assert.equal(snapshot.estimatedRequests, 1, "the row says the charge was estimated, not measured"); + assert.deepEqual(snapshot.usage, estimateGrokRequestUsage(Buffer.byteLength(payload)), "charged the documented conservative estimate, never zero and never nothing"); + assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); + } finally { await proxy.close(); } +}); + +test("proxy shutdown ends a worker's leftover keep-alive socket instead of waiting for it", async () => { + const proxy = await startGrokBrokerProxy({ accessToken: async () => "token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }), DEFAULT_GROK_BROKER_MODEL_POLICY, 0); + // A worker's HTTP client keeps its connection to the proxy pooled; nothing + // in the proxy's own state accounts for it, so a shutdown that waits for the + // client to release it has no bound. + const socket = connect({ host: "127.0.0.1", port: proxy.port }); + socket.on("error", () => undefined); + await new Promise((resolve, reject) => { socket.once("connect", resolve); socket.once("error", reject); }); + try { + const outcome = await Promise.race([proxy.close().then(() => "closed" as const), delay(5_000).then(() => "parked" as const)]); + assert.equal(outcome, "closed", "proxy shutdown parked on a socket the proxy does not track"); + } finally { socket.destroy(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4dee112..7ee024e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -1,29 +1,205 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { parseGrokResponseToolNames, parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; +import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; -export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; -export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; +/** One running turn as the proxy sees it: its declared model/effort and its spend gate. */ +export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; -export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream): PromisePromise):void; revokeIsolationGuard(turnId:string):void; close(): Promise }>> { - const capabilities = new EngineBrokerCapabilities(); - const guards=new MapPromise>();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards); }); - await new Promise((resolve, reject) => { server.once("error", reject); server.listen(43_123, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); +export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise; isStale?(): boolean }>; +export type GrokBrokerUpstream = (request: ReturnType, signal?: AbortSignal) => Promise>; body: Uint8Array }>>; + +/** + * `policy` is the fallback declared model/effort (closed list); a registered + * turn's own policy wins. A request whose turn has no registered meter is + * refused like one without an isolation guard: nothing is forwarded unmetered. + * `listenPort` exists for tests that must not contend for the production port. + * + * `grants` are evaluator inference grants (`grokInferenceGrants.ts`): a bearer + * carrying the grant prefix is looked up only there and served by + * `grokInferenceProxy.ts`; every other bearer is looked up only among turn + * capabilities. Without `grants` a prefixed bearer is simply refused. + */ +export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY, listenPort = 43_123, grants?: GrokInferenceGrants): PromisePromise):void; revokeIsolationGuard(turnId:string):void; registerTurn(turnId:string,turn:GrokBrokerProxyTurn):void; revokeTurn(turnId:string):void; close(): Promise }>> { + const declared = parseGrokBrokerModelPolicy(policy); const capabilities = new EngineBrokerCapabilities(); + const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared,grants); }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => { server.close((error) => error === undefined ? resolve() : reject(error)); /* `close` waits for every open connection, and a worker keeps its client pool's socket to this proxy open with nothing here accounting for it — so that wait has no bound. Ending them is this listener's to do, exactly as the MCP facade does. */ server.closeAllConnections(); }) }; +} + +/** + * A refusal the caller must not retry. + * + * Every internal refusal used to collapse into one bare 503. Grok treats 503 as + * transient and blind-retries the same request (observed: 14 retries, ~141k + * estimated tokens, then `exit 1`), so a policy miss burned a turn's budget and + * reported itself as an engine crash. Policy refusals now answer 400 with a + * reason, and only genuinely transient faults keep 503. + */ +export class GrokBrokerProxyRefusal extends Error { + constructor(readonly reason: string) { super(`grok broker proxy refused: ${reason}`); } } -async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>): Promise { +async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { + let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; + let titleSink = false; + // Every credential this request holds, kept only for this request and only so + // that a fault's own words can be redacted against them exactly as the CLI + // child and launcher diagnostics are. Nothing reads them but {@link brokerFaultCause}. + const secrets: string[] = []; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); - const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId);if(!guard)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token); token = ""; - let result = await upstream(prepared); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } + titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; + const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if(match)secrets.push(match[1]!); + if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} + const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); + try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + // A fenced realm is not a transient fault: the credential is gone until an + // operator re-logs in, and 503 made Grok blind-retry it (observed: fifteen + // retries over five minutes, $0 spent, nothing learned). Named, 400, and + // checked before the credential read, so the miss costs one round trip. + if (authority.isStale?.() === true) throw new GrokBrokerProxyRefusal(ENGINE_BROKER_AUTH_STALE); + let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + // The spend gate runs after the body is proven a real lean worker request + // (a refused session-title body never counts) and before any upstream call. + const admission=turn.meter.admit(); + if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} + if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} + settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; + let result = await upstream(prepared,admission.signal); + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);secrets.push(token);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } + // Names only, bounded, and never a reason to fail the request: the response + // is already buffered here for its usage block, so what the model tried to + // call is in hand. A decoder fault records no attempt rather than a false + // empty one, and never disturbs the turn. + settle?.(usageOrEstimate(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch (error) { + settle?.(undefined); + // Name the refusal on the broker's own stderr (reason code only, never a body + // or a token) so a failing turn is diagnosable without a stub harness — + // except for the two requests every healthy turn makes anyway. + // The request that *discovers* the fence throws an ordinary error from the + // credential authority, so it is promoted to the same named refusal: one + // stale realm must not read as one transient fault plus fourteen retries. + const fenced = !(error instanceof GrokBrokerProxyRefusal) && authority.isStale?.() === true; + const refused = error instanceof GrokBrokerProxyRefusal || fenced; + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : fenced ? ENGINE_BROKER_AUTH_STALE : "broker_unavailable"; + // A named refusal is its own account; anything else used to reach the log as + // the bare word `broker_unavailable`, which names nothing — so it carries the + // fault's own class and message, and nothing else, beside it. + const named = refused ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); + // Grok's own session-title call is refused by design, and keeps the transient + // 503 shape it has always had. Forcing 400 and 503 on it were both observed + // to end the turn `exit=0, result: success`, so the shape is kept because it + // is the one every live capture was taken with, not because a 4xx there ends + // Grok's session — it does not. + if (titleSink) { + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + return; + } + if (refused) { + response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); + return; + } + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + } finally { secrets.length = 0; } +} + +/** + * A non-refusal fault, named on one bounded line. + * + * `broker_unavailable` on its own carries no diagnostic content at all, and it + * is answered 503, which Grok blind-retries: one live turn emitted it fifteen + * times over five minutes, spent $0 — so no upstream call ever succeeded — and + * died with no account of why. The error's own class and message are the whole + * of what is logged: never a request body, bearer, capability, session id or + * header. It is redacted exactly as the failed CLI child and the launcher's + * worker diagnostic are — `redactCredentialText` with this request's own + * capabilities as exact secrets and the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` + * bound — and flattened to one line, because it travels on a log line. Naming + * a fault must never be able to fail the response that reports it, so a value + * that cannot even be described degrades to a marker. + * + * One level of `cause` is named too, because the fault this exists for names + * nothing without it: every failed `fetch` to the provider is `TypeError: + * fetch failed`, and which fault it was — `ENOTFOUND`, `ECONNREFUSED`, a TLS + * refusal, an abort — is only in the cause. An errno error whose message is + * empty is named by its `code`. + */ +const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { + try { + const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; + const flattened = described.replace(/[\u0000-\u001f\u007f]+/gu, " ").replace(/\s+/gu, " ").trim(); + const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return named.length === 0 ? "unnamed" : named; + } catch { return "unnameable"; } +}; + +/** One value's class and words: an error's own, or the type of whatever else was thrown. */ +const describeFault = (error: unknown): string => { + if (!(error instanceof Error)) return `${typeof error}: ${String(error)}`; + const code = (error as NodeJS.ErrnoException).code; + const words = error.message.length > 0 ? error.message : typeof code === "string" ? code : "(no message)"; + return `${error.constructor?.name ?? error.name}: ${words}`; +}; + +/** + * The unauthenticated connectivity probe Grok sends before its own requests: a + * bare `GET /` with no Authorization header, which has no capability to look up + * and answers 400. + * + * It and the per-turn `session_title` POST are the only two requests a healthy + * turn makes that this proxy does not forward, and both used to print the same + * `refused: unknown_capability` line as a real policy miss — so every healthy + * turn read as two refusals and cost a live investigation. They answer exactly + * as before; they simply stop claiming a refusal on the broker's stderr, which + * is left for the misses that are actually worth reading. + */ +const expectedWorkerProbe = (request: IncomingMessage): boolean => + request.headers.authorization === undefined && (request.method ?? "") === "GET" && + new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; + +/** + * Upstream-reported usage, or the documented conservative estimate. + * + * The decoder faulting must not fail the request that already cost real + * money: the upstream call succeeded, the worker is owed its answer, and a + * 503 here would throw away a paid response and have Grok buy it again. The + * `undefined` this returns is not "no usage" — `GrokBrokerTurnMeter.settle` + * charges it `ceil(bodyBytes/2) + 4096` and marks the row + * `usage_source: "estimated"`, so the token ceiling still counts it and no + * fabricated zero ever reaches a ledger. + */ +const usageOrEstimate = (body: Uint8Array, contentType: string | undefined): ReturnType => { + try { return parseGrokUpstreamUsage(body, contentType); } catch { return undefined; } +}; + +/** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ +const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { + try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } +}; + +/** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ +function authorizeRequestOrRefuse(...args: Parameters): ReturnType { + try { return authorizeGrokBrokerProxyRequest(...args); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } -const defaultUpstream: GrokBrokerUpstream = async (request) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; +const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerProxyRequest.test.ts b/src/runtime/grokBrokerProxyRequest.test.ts index 462c140..7cdcdfb 100644 --- a/src/runtime/grokBrokerProxyRequest.test.ts +++ b/src/runtime/grokBrokerProxyRequest.test.ts @@ -3,28 +3,107 @@ import test from "node:test"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; -const body = Buffer.from(JSON.stringify({ stream: true, messages: [] })); - -test("proxy substitutes broker bearer, forwards bounded Grok client version, and rejects arbitrary routes and headers", () => { +const leanTools = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"]; +const tool = (name: string) => ({ type: "function", function: { name, parameters: { type: "object" } } }); +const leanBody = (overrides: Record = {}): Buffer => Buffer.from(JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: leanTools.map(tool), ...overrides })); +const request = (body: Uint8Array, headers: Record = {}) => { const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - const request = authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, cookie: "forbidden", "x-grok-client-version": "1.0.25", "x-grok-client-identifier": "attacker" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"); - assert.equal(request.url, "https://cli-chat-proxy.grok.com/v1/chat/completions"); - assert.equal(request.headers.authorization, "Bearer real-bearer"); - assert.equal(request.headers["x-grok-client-version"], "1.0.25"); - assert.equal(request.headers["x-grok-client-identifier"], "grok-shell"); - assert.equal("cookie" in request.headers, false); - assert.throws(() => authorizeGrokBrokerProxyRequest({ method: "GET", pathname: "/", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.25" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"), /rejected/); -}); - -test("proxy fails closed when the worker omits or malforms the Grok client version", () => { - for (const version of [undefined, "", "1", "1.0", "v1.0.25", "1.0.25\nInjected: yes", "1.0.25+build", `1.2.3-${"a".repeat(65)}`] as const) { - const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - assert.throws(() => authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, ...(version === undefined ? {} : { "x-grok-client-version": version }) }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"), /rejected/); + return { caps, input: { method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.34", ...headers }, body, agentId: "a", turnId: "t" } }; +}; + +test("proxy substitutes broker bearer, forwards the pinned client version, and rejects arbitrary routes and headers", () => { + const { caps, input } = request(leanBody(), { cookie: "forbidden", "x-grok-client-identifier": "attacker", "x-grok-model-override": "grok-build" }); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"); + assert.equal(upstream.url, "https://cli-chat-proxy.grok.com/v1/chat/completions"); + assert.equal(upstream.headers.authorization, "Bearer real-bearer"); + assert.equal(upstream.headers["x-grok-client-version"], "1.0.34"); + assert.equal(upstream.headers["x-grok-client-identifier"], "grok-shell"); + assert.equal(upstream.headers["x-grok-model-override"], "grok-4.6"); + assert.equal("cookie" in upstream.headers, false); + assert.throws(() => authorizeGrokBrokerProxyRequest({ ...input, method: "GET", pathname: "/" }, caps, "real-bearer"), /rejected/); +}); + +test("proxy fails closed on any client version other than the pinned Grok CLI", () => { + for (const version of [undefined, "", "1", "1.0", "v1.0.34", "1.0.13", "1.0.25", "1.0.33", "1.0.35", "1.0.34-beta.1", "1.0.34\nInjected: yes", "1.0.34+build"] as const) { + const { caps, input } = request(leanBody(), { "x-grok-client-version": version }); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, String(version)); } }); -test("proxy accepts prerelease Grok client versions", () => { - const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - const request = authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.25-beta.1" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"); - assert.equal(request.headers["x-grok-client-version"], "1.0.25-beta.1"); +test("proxy refuses a request carrying Grok's fail-open full tool set before any upstream call", () => { + const full = [...leanTools, "search_replace", "kill_command_or_subagent", "todo_write", "get_command_or_subagent_output", "spawn_subagent", "scheduler_create", "scheduler_delete", "scheduler_list", "monitor", "workflow", "enter_plan_mode", "exit_plan_mode", "write"]; + for (const tools of [full.map(tool), [tool("session_title")], leanTools.slice(1).map(tool), [...leanTools, "use_tool"].map(tool), [...leanTools.slice(1), "run_terminal_cmd"].map(tool), undefined, [], leanTools.map((name) => ({ type: "custom", function: { name } }))]) { + const { caps, input } = request(leanBody({ tools })); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/); + } +}); + +test("proxy refuses a reasoning effort or model other than the declared policy", () => { + for (const overrides of [{ reasoning_effort: "high" }, { reasoning_effort: undefined }, { reasoning_effort: "xhigh" }, { model: "grok-build" }, { model: "daimon-broker-grok" }]) { + const { caps, input } = request(leanBody(overrides)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, JSON.stringify(overrides)); + } + const { caps, input } = request(leanBody({ model: "grok-build", reasoning_effort: "medium" })); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer", { model: "grok-build", reasoningEffort: "medium" }); + assert.equal(upstream.headers["x-grok-model-override"], "grok-build"); + const other = request(leanBody()); + assert.throws(() => authorizeGrokBrokerProxyRequest(other.input, other.caps, "real-bearer", { model: "grok-3" as never, reasoningEffort: "low" }), /model policy/); +}); + +test("proxy forwards exactly the validated object, so duplicate members cannot smuggle a different tool set upstream", () => { + const full = [...leanTools, "search_replace", "kill_command_or_subagent", "todo_write", "get_command_or_subagent_output", "spawn_subagent", "scheduler_create", "scheduler_delete", "scheduler_list", "monitor", "workflow", "enter_plan_mode", "exit_plan_mode", "write"]; + const lean = leanTools.map(tool); + // First `tools`/`reasoning_effort`/`model` are the fail-open values; JSON.parse keeps the last (lean) ones. + const smuggled = `{"model":"grok-build","reasoning_effort":"high","tools":${JSON.stringify(full.map(tool))},"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[],"stream_options":{"include_usage":true},"tools":${JSON.stringify(lean)}}`; + const { caps, input } = request(Buffer.from(smuggled)); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"); + const canonical = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", tools: lean, stream: true, messages: [], stream_options: { include_usage: true } }); + assert.equal(Buffer.from(upstream.body).toString("utf8"), canonical); + assert.equal(Buffer.from(upstream.body).toString("utf8").split('"tools"').length, 2); + assert.doesNotMatch(Buffer.from(upstream.body).toString("utf8"), /search_replace|grok-build|"high"/u); +}); + +test("proxy refuses top-level members a lean Grok 1.0.34 worker never sends", () => { + for (const overrides of [{ functions: [{ name: "write" }] }, { n: 2 }, { tool_choice: "required" }, { max_tokens: 100 }, { temperature: 0 }, { stream_options: { include_usage: true, extra: 1 } }, { stream_options: "yes" }]) { + const { caps, input } = request(leanBody(overrides)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, JSON.stringify(overrides)); + } + const { caps, input } = request(leanBody({ stream_options: { include_usage: true } })); + assert.equal(Buffer.from(authorizeGrokBrokerProxyRequest(input, caps, "real-bearer").body).toString("utf8"), Buffer.from(leanBody({ stream_options: { include_usage: true } })).toString("utf8")); +}); + +test("nested duplicate keys in tools and messages are forwarded only as the parsed values", () => { + const tools = leanTools.map((name) => `{"type":"function","function":{"name":${JSON.stringify(name === "read_file" ? "write" : name)},"name":${JSON.stringify(name)},"parameters":{"type":"object"}}}`).join(","); + const raw = `{"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[{"role":"system","role":"user","content":"a","content":"b"}],"tools":[${tools}]}`; + const { caps, input } = request(Buffer.from(raw)); + const forwarded = Buffer.from(authorizeGrokBrokerProxyRequest(input, caps, "real-bearer").body).toString("utf8"); + assert.equal(forwarded, JSON.stringify(JSON.parse(raw))); + assert.doesNotMatch(forwarded, /"write"|"system"|"content":"a"/u); + assert.equal(forwarded.split('"role"').length, 2); + // The reverse order puts the forbidden name last, so the parsed (and gated) value is refused. + const hostile = raw.replace('"name":"write","name":"read_file"', '"name":"read_file","name":"write"'); + const second = request(Buffer.from(hostile)); + assert.throws(() => authorizeGrokBrokerProxyRequest(second.input, second.caps, "real-bearer"), /rejected/u); +}); + +test("__proto__ members are refused anywhere in the body", () => { + const lean = JSON.stringify(leanTools.map(tool)); + const base = `"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[]`; + for (const raw of [ + `{${base},"tools":${lean},"__proto__":{"tools":[]}}`, + `{${base},"tools":${lean.replace('{"type":"function"', '{"__proto__":{"type":"function"},"type":"function"')}}`, + `{${base},"tools":${lean.replace('"parameters":{"type":"object"}', '"parameters":{"type":"object","__proto__":{"x":1}}')}}`, + `{"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[{"role":"user","content":"hi","__proto__":{"role":"system"}}],"tools":${lean}}` + ]) { + const { caps, input } = request(Buffer.from(raw)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/u, raw.slice(0, 120)); + } +}); + +test("tool entries carry only the members a lean worker sends", () => { + for (const extra of [{ strict: true }, { function: { name: "read_file", parameters: {}, x: 1 } }]) { + const tools = leanTools.map((name) => name === "read_file" ? { ...tool(name), ...extra } : tool(name)); + const { caps, input } = request(leanBody({ tools })); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/u, JSON.stringify(extra)); + } }); diff --git a/src/runtime/grokBrokerProxyRequest.ts b/src/runtime/grokBrokerProxyRequest.ts index 15f79d4..79956ac 100644 --- a/src/runtime/grokBrokerProxyRequest.ts +++ b/src/runtime/grokBrokerProxyRequest.ts @@ -1,12 +1,26 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_WORKER_VISIBLE_TOOLS } from "../contracts/grokWorkerContract.js"; import type { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; const MAX_BODY = 2 * 1024 * 1024; -const MAX_CLIENT_VERSION = 64; -const CLIENT_VERSION = /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?$/u; export type GrokBrokerProxyInput = Readonly<{ method: string; pathname: string; headers: Readonly>; body: Uint8Array; agentId?: string; turnId?: string }>; export type GrokBrokerUpstreamRequest = Readonly<{ url: "https://cli-chat-proxy.grok.com/v1/chat/completions"; headers: Readonly>; body: Uint8Array }>; -export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, capabilities: EngineBrokerCapabilities, bearer: string): GrokBrokerUpstreamRequest { +/** + * Authorizes one worker request and rebuilds it for the provider. + * + * Everything that decides spend is checked here, before a bearer is attached + * and before any upstream call: + * - the client version is exactly the pinned Grok CLI (`GROK_ENGINE_BROKER.grokCliVersion`); + * - the body model and `reasoning_effort` are exactly the declared policy, and + * the model override header follows that declaration instead of a constant; + * - the offered tool names are exactly the lean visible set. Grok 1.0.34 turns + * an unmappable `--tools` entry into its full 19-tool set, and its per-turn + * `session_title` request carries a single forced tool; both are refused. + */ +export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, capabilities: EngineBrokerCapabilities, bearer: string, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY): GrokBrokerUpstreamRequest { + const declared = parseGrokBrokerModelPolicy(policy); if (input.method !== "POST" || input.pathname !== "/v1/chat/completions" || input.body.byteLength < 2 || input.body.byteLength > MAX_BODY) throw new Error("broker proxy request rejected"); const authorization = input.headers.authorization; const match = authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); if (match === null || match === undefined) throw new Error("broker proxy request rejected"); @@ -14,7 +28,35 @@ export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, cap if (scope === undefined || (input.agentId !== undefined && scope.agentId !== input.agentId) || (input.turnId !== undefined && scope.turnId !== input.turnId)) throw new Error("broker proxy request rejected"); if (!bearer || /[\r\n]/u.test(bearer)) throw new Error("broker credential authority unavailable"); const clientVersion = input.headers["x-grok-client-version"]; - if (clientVersion === undefined || clientVersion.length > MAX_CLIENT_VERSION || !CLIENT_VERSION.test(clientVersion)) throw new Error("broker proxy request rejected"); - try { const parsed = JSON.parse(Buffer.from(input.body).toString("utf8")) as Record; if (parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error(); } catch { throw new Error("broker proxy request rejected"); } - return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": "grok-build", "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: input.body }; + if (clientVersion !== GROK_ENGINE_BROKER.grokCliVersion) throw new Error("broker proxy request rejected"); + let parsed: Record; + // `__proto__` is refused at any depth: JSON.parse makes it an ordinary own + // member, but an upstream JavaScript parser may treat it as a prototype. + try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8"), (key, value: unknown) => { if (key === "__proto__") throw new Error(); return value; }) as Record; } catch { throw new Error("broker proxy request rejected"); } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error("broker proxy request rejected"); + if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort) throw new Error("broker proxy request rejected"); + if (!exactLeanTools(parsed.tools)) throw new Error("broker proxy request rejected"); + if (Object.keys(parsed).some((key) => !LEAN_BODY_MEMBERS.has(key)) || !validStreamOptions(parsed.stream_options)) throw new Error("broker proxy request rejected"); + // Forward what was validated, never the worker's bytes: JSON.parse keeps the + // last of duplicate keys, and an upstream that keeps the first would + // otherwise see a different `tools`/`model`/`reasoning_effort` than the gate. + return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: Buffer.from(JSON.stringify(parsed)) }; +} + +/** Top-level members of a Grok 1.0.34 lean worker chat-completions body (live stub capture). */ +const LEAN_BODY_MEMBERS: ReadonlySet = new Set(["messages", "model", "reasoning_effort", "stream", "stream_options", "tools"]); +/** Members of each lean tool `function` entry (live stub capture). */ +const LEAN_FUNCTION_MEMBERS: ReadonlySet = new Set(["description", "name", "parameters"]); +const validStreamOptions = (value: unknown): boolean => value === undefined + || (value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).every((key) => key === "include_usage") && typeof (value as { include_usage?: unknown }).include_usage === "boolean"); + +export function exactLeanTools(tools: unknown): boolean { + if (!Array.isArray(tools) || tools.length !== GROK_WORKER_VISIBLE_TOOLS.length) return false; + const names = tools.map((tool) => { + if (tool === null || typeof tool !== "object" || Array.isArray(tool) || (tool as { type?: unknown }).type !== "function" || Object.keys(tool).some((key) => key !== "type" && key !== "function")) return undefined; + const fn = (tool as { function?: unknown }).function; + if (fn === null || typeof fn !== "object" || Array.isArray(fn) || Object.keys(fn).some((key) => !LEAN_FUNCTION_MEMBERS.has(key))) return undefined; + return (fn as { name?: unknown }).name; + }); + return JSON.stringify([...names].sort()) === JSON.stringify(GROK_WORKER_VISIBLE_TOOLS); } diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts new file mode 100644 index 0000000..852b514 --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -0,0 +1,249 @@ +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import test from "node:test"; + +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { GROK_REQUEST_TOOL_CALLS_MAX, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_TRUNCATED, GrokBrokerTurnMeter, parseGrokResponseToolNames, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const sse = (usage: Record): Uint8Array => Buffer.from([{ choices: [{ index: 0, delta: { content: "x" } }] }, { choices: [], usage }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); + +function post(port: number, token: string): Promise> { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { + const chunks: Buffer[] = []; response.on("data", (chunk: Buffer) => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }); + req.on("error", reject); req.end(body); + }); +} + +const withProxy = async (usage: Record | undefined, meter: GrokBrokerTurnMeter, run: (post: () => Promise>, calls: () => number) => Promise): Promise => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usage === undefined ? Buffer.from("data: [DONE]\n\n") : sse(usage) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + await run(() => post(proxy.port, token), () => calls); + } finally { await proxy.close(); } +}; + +test("maxRequests is hard: request N+1 is refused with 429 before any upstream call, and the limit trips once", async () => { + const tripped: string[] = []; + const meter = new GrokBrokerTurnMeter({ maxRequests: 2, maxTokens: 1_000_000, timeoutMs: 60_000 }, (reason) => tripped.push(reason)); + await withProxy({ prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + assert.equal((await send()).status, 200); + const refused = await send(); + assert.equal(refused.status, 429); + assert.deepEqual(JSON.parse(refused.text), { error: "turn limit reached", limit: "requests" }); + assert.equal((await send()).status, 429); + assert.equal(calls(), 2, "upstream never sees a request past maxRequests"); + }); + assert.deepEqual(tripped, ["requests"]); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 2); + assert.equal(snapshot.limitReason, "requests"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }); +}); + +test("the token ceiling overshoots by at most one request, counting cached input", async () => { + // 60 tokens per request (40 cached) against a 100-token ceiling: requests 1 and + // 2 are admitted (0 and 60 < 100 before each), request 3 is refused at 120. + // Mutation guard: checking the ceiling after forwarding, or ignoring cached + // tokens, admits a third request and this goes red. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 50, completion_tokens: 10, total_tokens: 60, prompt_tokens_details: { cached_tokens: 40 } }, meter, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 429, 429, 429]); + assert.equal(calls(), 2); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "tokens"); + assert.equal(snapshot.tokens, 120); + assert.ok(snapshot.tokens - meter.limits.maxTokens <= 60, "overshoot is bounded by the last admitted request"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 80, cacheWrite: 0, output: 20, total: 120 }); +}); + +test("a request after the elapsed deadline is refused, and every admitted request is timed", async () => { + let now = 1_000_000; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 5_000 }, undefined, () => now); + await withProxy(undefined, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + now += 5_000; + assert.equal((await send()).status, 429); + assert.equal(calls(), 1); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "timeout"); + // A body without usage is never a zero: it is charged the conservative estimate (402-byte body). + const estimate = { input: 201, cacheRead: 0, cacheWrite: 0, output: 4_096, total: 4_297 }; + assert.deepEqual(snapshot.usage, estimate); + assert.equal(snapshot.estimatedRequests, 1); + assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); +}); + +// A live capability with no registered meter means the turn is already over: the +// launcher registers a turn before it starts the worker, so nothing can arrive +// before the meter exists, and nothing can make a finished turn live again. The +// answer is therefore 400 (a named, non-retryable refusal) rather than the 503 it +// once was — a retryable shape here bought only Grok's blind retry storm, which +// spent ~141k tokens re-asking a question that could never start being answerable. +test("a turn without a registered meter is refused non-retryably and never forwarded", async () => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + const answer = await post(proxy.port, token); + assert.equal(answer.status, 400); + assert.equal(JSON.parse(answer.text).reason, "no_active_turn"); + assert.equal(calls, 0); + } finally { await proxy.close(); } +}); + +test("upstream usage parsing takes the last usage block and never zero-fills", () => { + assert.deepEqual(parseGrokUpstreamUsage(sse({ prompt_tokens: 100, completion_tokens: 7, total_tokens: 120, prompt_tokens_details: { cached_tokens: 30 }, completion_tokens_details: { reasoning_tokens: 13 } }), "text/event-stream"), + { input: 70, cacheRead: 30, cacheWrite: 0, output: 20, total: 120, reasoning: 13 }); + assert.deepEqual(parseGrokUpstreamUsage(Buffer.from(JSON.stringify({ usage: { prompt_tokens: 4, completion_tokens: 1 } })), "application/json"), { input: 4, cacheRead: 0, cacheWrite: 0, output: 1, total: 5 }); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: "4", completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); + // Mutation guard: keeping the earlier valid block under-reports a request whose final usage is implausible. + const twoBlocks = Buffer.from([{ choices: [], usage: { prompt_tokens: 5, completion_tokens: 1 } }, { choices: [], usage: { prompt_tokens: 900_000, completion_tokens: 1 } }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")); + assert.equal(parseGrokUpstreamUsage(twoBlocks, "text/event-stream"), undefined); +}); + +test("at most one upstream request is in flight per turn: an overlapping request is refused, uncounted", async () => { + // Mutation guard: without the in-flight gate both overlapping requests pass on + // the same pre-settle token total and the one-request overshoot bound is gone. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + const first = meter.admit(); + assert.ok("index" in first); + assert.deepEqual(meter.admit(), { busy: true }); + assert.equal(meter.snapshot().requests, 1); + meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }, 0); + assert.deepEqual(meter.admit(), { refused: "tokens" }, "once settled, the next request sees the reported total"); + + let release!: () => void; let calls = 0; + const gate = new Promise((resolve) => { release = resolve; }); + const live = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: sse({ prompt_tokens: 1, completion_tokens: 1 }) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: live }); + const pending = post(proxy.port, token); + while (calls === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const overlapping = await post(proxy.port, token); + assert.deepEqual([overlapping.status, JSON.parse(overlapping.text)], [429, { error: "turn request in flight" }]); + assert.equal(calls, 1); + release(); + assert.equal((await pending).status, 200); + assert.equal((await post(proxy.port, token)).status, 200); + assert.deepEqual([calls, live.snapshot().requests, live.snapshot().limitReason], [2, 2, "none"]); + } finally { release(); await proxy.close(); } +}); + +test("tripping a limit aborts the in-flight upstream call instead of letting it run", async () => { + let observed: AbortSignal | undefined; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { + observed = signal; + await new Promise((_resolve, reject) => signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true })); + throw new Error("unreachable"); + }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + const pending = post(proxy.port, token); + while (observed === undefined) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(observed.aborted, false); + // Mutation guard: a trip that leaves the upstream signal alone hangs this request. + meter.trip("timeout"); + assert.equal(observed.aborted, true); + assert.equal((await pending).status, 503); + assert.deepEqual(meter.admit(), { refused: "timeout" }); + } finally { await proxy.close(); } +}); + +test("an implausible per-request usage block is never added, and missing usage still trips the token ceiling", async () => { + // Mutation guard: without the plausibility bound this adds 400 billion tokens to the total. + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 11 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 10 }), "text/event-stream")?.total, 500_000); + const huge = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }, huge, async (send) => { assert.equal((await send()).status, 200); }); + assert.deepEqual([huge.snapshot().tokens, huge.snapshot().estimatedRequests], [4_297, 1]); + + // Mutation guard: settling a usage-less response as zero lets this turn run to maxRequests. + const blind = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 10_000, timeoutMs: 60_000 }); + await withProxy(undefined, blind, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 200, 429, 429]); + assert.equal(calls(), 3); + }); + assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); +}); + +const events = (chunks: readonly unknown[]): Uint8Array => + Buffer.from(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); +/** One streaming tool call: the name arrives in one delta, the arguments in the next. */ +const callDeltas = (names: readonly string[]): unknown[] => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"query":"secret"}' } }] } }] })) +]; + +/** + * Two live turns ended with every tool correctly mounted and no way to tell + * whether the model had tried to call anything. These names are that answer, + * and nothing more than that answer. + */ +test("a response's tool-call names are read, bounded, and stripped of everything but the names", async () => { + // Mutation guard: passing the response through instead of the names leaks arguments here. + const names = parseGrokResponseToolNames(events([...callDeltas(["use_tool", "search_tool"]), { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }]), "text/event-stream"); + assert.deepEqual(names, ["use_tool", "search_tool"]); + assert.equal(JSON.stringify(names).includes("secret"), false); + + // A non-streaming body carries its calls on the message; one tool called + // twice is two attempts, because neither carries a streaming call index. + assert.deepEqual(parseGrokResponseToolNames(Buffer.from(JSON.stringify({ choices: [{ message: { tool_calls: [{ function: { name: "use_tool" } }, { function: { name: "use_tool" } }] } }] })), "application/json"), ["use_tool", "use_tool"]); + + // Absence stays absence: a decoded response that called nothing is `[]`, and + // an undecodable one is nothing at all. A zero-length list must never be + // invented for a response nobody could read. + assert.deepEqual(parseGrokResponseToolNames(events([{ choices: [{ index: 0, delta: { content: "x" } }] }]), "text/event-stream"), []); + assert.equal(parseGrokResponseToolNames(Buffer.from("gateway"), "text/html"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from(""), "text/event-stream"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from("data: not-json\n\n"), "text/event-stream"), undefined); + + // A name that is not a plain short identifier is counted, never passed through. + assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + + // A hostile but decodable shape yields no attempt instead of throwing: + // instrumentation must never be able to fail the turn it observes. + assert.deepEqual(parseGrokResponseToolNames(events([ + { choices: "not-an-array" }, { choices: [null, 7, { delta: { tool_calls: "no" } }, { message: { tool_calls: [null, { function: null }, { function: { name: 42 } }, { function: { name: "" } }] } }] } + ]), "text/event-stream"), []); + + // Mutation guard: unbounded, a pathological response writes 400 names into one row. + const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); + assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); + assert.equal(many!.at(-1), GROK_TOOL_CALL_TRUNCATED); + assert.deepEqual(many!.slice(0, 2), ["tool_0", "tool_1"]); + // Exactly the bound is not truncated. + assert.equal(parseGrokResponseToolNames(events(callDeltas(Array.from({ length: GROK_REQUEST_TOOL_CALLS_MAX }, (_value, index) => `tool_${index}`))), "text/event-stream")!.includes(GROK_TOOL_CALL_TRUNCATED), false); +}); + +test("the meter carries each request's tool-call names without letting them touch the spend gate", async () => { + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 11, completion_tokens: 2 }, meter, async (send) => { assert.equal((await send()).status, 200); }); + const [timing] = meter.snapshot().timings; + // The stub response carries no tool call, and says so rather than staying silent. + assert.deepEqual(timing!.toolCalls, []); + assert.deepEqual([meter.snapshot().tokens, meter.snapshot().limitReason, meter.snapshot().estimatedRequests], [13, "none", 0]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts new file mode 100644 index 0000000..f266a16 --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -0,0 +1,233 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; + +/** + * `estimated` marks a request whose response carried no valid usage and was + * charged {@link estimateGrokRequestUsage}. `toolCalls` are the tool-call names + * that request's response carried, names only ({@link parseGrokResponseToolNames}); + * absent means the response could not be decoded, `[]` that it called nothing. + */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }>; +export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; + +/** + * The proxy's per-turn spend gate. + * + * Every forwarded model request of one turn passes through {@link admit} + * *before* a bearer is attached, so the checks are hard for requests and + * elapsed time and between-requests for tokens: + * + * - `maxRequests`: request `maxRequests + 1` is refused; upstream never sees it. + * - `timeoutMs`: a request arriving after the deadline is refused (the broker's + * own timer additionally kills a worker that is mid-request). + * - `maxTokens`: checked against the running total of upstream-reported usage + * of the requests already answered. A request is admitted while that total is + * still below the ceiling, so the overshoot is bounded by exactly one + * request's usage — the last admitted one. Usage counts total input + * *including* cached tokens (P0 observed an uncached replay at +55%). + * + * The first limit that fires is sticky: every later request is refused with + * the same reason, and `onLimit` runs once. + * + * The token bound is only a bound if no request can be admitted on a total + * that an in-flight request has not yet reported into. So a turn has at most + * ONE upstream request in flight: a second request arriving before the first + * settled is refused (`busy`, HTTP 429) without being counted or tripping a + * limit. Grok's headless loop is sequential — every live capture (P1 + * live-round1, P2 live) shows each request ending before the next starts — so + * this refuses only a worker that is not behaving like Grok. Tripping a limit + * (including the broker's timer) aborts that in-flight upstream call through + * its own `AbortSignal` rather than letting it run to completion. + */ +export class GrokBrokerTurnMeter { + private readonly startedAt: number; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }[] = []; + private tokens = 0; + private reason: EngineBrokerLimitReason = "none"; + private inFlight: { index: number; controller: AbortController } | undefined; + constructor(readonly limits: EngineBrokerTurnLimits, private readonly onLimit: (reason: Exclude) => void = () => undefined, private readonly now: () => number = Date.now) { + this.startedAt = now(); + } + + /** Returns the request index and its upstream abort signal when admitted, the limit that refused it, or `busy` while another request is in flight. */ + admit(): Readonly<{ index: number; signal: AbortSignal } | { refused: Exclude } | { busy: true }> { + if (this.reason === "none") { + if (this.now() - this.startedAt >= this.limits.timeoutMs) this.trip("timeout"); + else if (this.timings.length >= this.limits.maxRequests) this.trip("requests"); + else if (this.tokens >= this.limits.maxTokens) this.trip("tokens"); + } + if (this.reason !== "none") return { refused: this.reason }; + if (this.inFlight !== undefined) return { busy: true }; + this.timings.push({ startedAt: new Date(this.now()).toISOString() }); + const controller = new AbortController(); + this.inFlight = { index: this.timings.length - 1, controller }; + return { index: this.inFlight.index, signal: controller.signal }; + } + + /** + * Records one admitted request's end, its usage, and the tool-call names its + * response carried. A response without valid usage (absent, malformed, + * implausible, or a failed/aborted call) is charged a conservative estimate + * from the request body size, so a missing `usage` can never silently disable + * the token ceiling. `toolCalls` is observation only: it never affects + * admission, the running total, or any limit. + */ + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number, toolCalls?: readonly string[]): void { + const timing = this.timings[index]; + if (timing === undefined || timing.endedAt !== undefined) return; + if (this.inFlight?.index === index) this.inFlight = undefined; + timing.endedAt = new Date(this.now()).toISOString(); + if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } + else timing.usage = usage; + if (toolCalls !== undefined) timing.toolCalls = toolCalls; + this.tokens += timing.usage.total; + } + + /** Trips a limit from outside the request path (the broker's wall-clock timer). */ + trip(reason: Exclude): void { + if (this.reason !== "none") return; + this.reason = reason; + this.abortInFlight(); + this.onLimit(reason); + } + + /** Aborts the in-flight upstream call, if any (limit trip, or the broker ending the turn). */ + abortInFlight(): void { this.inFlight?.controller.abort(); } + + snapshot(): GrokBrokerTurnMeterSnapshot { + const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); + return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), estimatedRequests: this.timings.filter((timing) => timing.estimated === true).length, timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; + } +} + +const { requestUsageMaxTokens, missingUsageEstimate } = GROK_ENGINE_BROKER.turnLimits; + +/** The charge for a request without valid usage: `ceil(bodyBytes / 2)` input plus a fixed output allowance. */ +export const estimateGrokRequestUsage = (requestBytes: number): EngineBrokerTurnUsage => { + const input = Math.ceil(Math.max(0, requestBytes) / missingUsageEstimate.inputBytesPerToken), output = missingUsageEstimate.outputTokens; + return { input, cacheRead: 0, cacheWrite: 0, output, total: input + output }; +}; + +type JsonRecord = Record; +const isRecord = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); +const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +/** + * Upstream-reported usage of one chat-completions response, or `undefined`. + * + * The proxy buffers the whole upstream body, so the last `usage` object of an + * SSE stream (`stream_options.include_usage`) or of a JSON body is available + * before the body is returned to the worker. OpenAI-shaped `prompt_tokens` + * include cached tokens; they are split into disjoint buckets here, and any + * reasoning tokens reported outside `completion_tokens` (visible as + * `total_tokens` above prompt + completion) are folded into `output` so the + * total invariant holds. A malformed block, or one whose total exceeds + * `GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens`, is invalid: never + * zero-filled and never added — the meter charges an estimate instead. + */ +export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of decodeUpstreamResponse(body, contentType) ?? []) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); + } + return found; +} + +/** + * Every decodable JSON object of one upstream response: each `data:` event of + * an SSE stream, or the single body of a JSON response. + * + * `undefined` means *nothing* decoded — an unparseable or non-JSON response. + * Callers must keep that distinct from a decoded response that said nothing, + * because the ledger never fabricates an observation it did not make. + */ +function decodeUpstreamResponse(body: Uint8Array, contentType: string | undefined): unknown[] | undefined { + const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); + const candidates: unknown[] = []; + if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { + for (const line of text.split(/\r?\n/u)) { + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (payload === "[DONE]" || payload.length === 0) continue; + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries neither usage nor a tool call */ } + } + } else { + try { candidates.push(JSON.parse(text)); } catch { return undefined; } + } + return candidates.length === 0 ? undefined : candidates; +} + +/** At most this many names per request row; a longer list ends in {@link GROK_TOOL_CALL_TRUNCATED}. */ +export const GROK_REQUEST_TOOL_CALLS_MAX = 16; +/** A `name` that is not a plain short identifier is counted, never passed through. */ +export const GROK_TOOL_CALL_INVALID = ""; +export const GROK_TOOL_CALL_TRUNCATED = ""; +const TOOL_CALL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** + * The tool-call NAMES one upstream response carried, and nothing else. + * + * Two live turns could not answer "did the model ever try `use_tool` or + * `search_tool`", because the per-request rows recorded timings and tokens but + * never an attempt. This is that answer, under four rules: + * + * - names only. No arguments, no message content, no tokens, no header. A + * `name` that is not a plain short identifier is recorded as + * {@link GROK_TOOL_CALL_INVALID} rather than passing provider bytes through; + * - bounded. At most {@link GROK_REQUEST_TOOL_CALLS_MAX} entries, the last being + * {@link GROK_TOOL_CALL_TRUNCATED} when the response carried more, so a + * pathological response cannot write an unbounded row; + * - absence stays absence. A decoded response that called nothing returns `[]`; + * a response that could not be decoded returns `undefined` and the row records + * no field at all; + * - one streaming call names itself in one delta and streams its arguments in + * the rest, so a repeat of the same `(choice, call)` index is that same call, + * not a second attempt. + */ +export function parseGrokResponseToolNames(body: Uint8Array, contentType: string | undefined): readonly string[] | undefined { + const candidates = decodeUpstreamResponse(body, contentType); + if (candidates === undefined) return undefined; + const names: string[] = [], seen = new Set(); + scan: for (const candidate of candidates) { + if (!isRecord(candidate) || !Array.isArray(candidate.choices)) continue; + for (const choice of candidate.choices) { + if (!isRecord(choice)) continue; + for (const source of [choice.delta, choice.message]) { + if (!isRecord(source) || !Array.isArray(source.tool_calls)) continue; + for (const call of source.tool_calls) { + if (!isRecord(call) || !isRecord(call.function)) continue; + const name = call.function.name; + // An arguments-only delta names nothing; it is not an attempt of its own. + if (typeof name !== "string" || name.length === 0) continue; + if (typeof choice.index === "number" && typeof call.index === "number") { + const key = `${choice.index}:${call.index}`; + if (seen.has(key)) continue; + seen.add(key); + } + names.push(TOOL_CALL_NAME.test(name) ? name : GROK_TOOL_CALL_INVALID); + if (names.length > GROK_REQUEST_TOOL_CALLS_MAX) break scan; + } + } + } + } + return names.length > GROK_REQUEST_TOOL_CALLS_MAX ? [...names.slice(0, GROK_REQUEST_TOOL_CALLS_MAX - 1), GROK_TOOL_CALL_TRUNCATED] : names; +} + +function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { + const prompt = count(usage.prompt_tokens), completion = count(usage.completion_tokens); + if (prompt === undefined || completion === undefined) return undefined; + const details = isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : {}; + const cached = details.cached_tokens === undefined ? 0 : count(details.cached_tokens); + const reported = usage.total_tokens === undefined ? prompt + completion : count(usage.total_tokens); + if (cached === undefined || reported === undefined || cached > prompt) return undefined; + const total = Math.max(reported, prompt + completion); + const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}; + const reasoning = count(completionDetails.reasoning_tokens); + const output = total - prompt; + if (total > requestUsageMaxTokens) return undefined; + return { input: prompt - cached, cacheRead: cached, cacheWrite: 0, output, total, ...(reasoning === undefined || reasoning > output ? {} : { reasoning }) }; +} diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index 0be5567..0642b89 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -1,8 +1,82 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import test from "node:test"; -import { renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; -test("worker config uses only named in-memory auth and fixed loopback proxy", () => { - const config = renderGrokBrokerWorkerConfig("/opt/daimon/bin/grok-broker-auth", 43123); - assert.match(config, /auth_provider\.daimon/u); assert.match(config, /args = \["--auth-provider"\]/u);assert.match(config, /127\.0\.0\.1:43123/u);assert.match(config,/127\.0\.0\.1:43124\/mcp/u);assert.match(config,/DAIMON_MCP_CAPABILITY/u); assert.doesNotMatch(config, /access_token|refresh_token|auth\.json/u); - const args = renderGrokBrokerWorkerArgs("/run/worker/prompt", "/workspace"); assert.equal(args.includes("--prompt-file"), true); assert.equal(args.includes("--single"), false); + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokBrokerModelPolicy.js"; +import { GROK_1_0_34_BUNDLED_SKILLS, grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfig, renderGrokBrokerWorkerConfigWith } from "./grokBrokerWorkerConfig.js"; + +const section = (config: string, header: string): string => { + const start = config.indexOf(`${header}\n`); + assert.notEqual(start, -1, `missing ${header}`); + const end = config.indexOf("\n[", start + header.length); + return config.slice(start, end === -1 ? undefined : end); +}; + +test("worker config uses only the launcher-set turn capability, the fixed loopback proxy, and the capability-scoped MCP facade", () => { + const config = renderGrokBrokerWorkerConfig(); + assert.match(section(config, "[model.daimon-broker-grok]"), /base_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_PROVIDER_CAPABILITY"\n/u); + // Grok 1.0.34 ignores [auth_provider.*] for custom models; a helper table would silently send no bearer. + assert.doesNotMatch(config, /auth_provider/u); + assert.equal(section(config, "[mcp_servers.daimon]"), '[mcp_servers.daimon]\nurl = "http://127.0.0.1:43124/mcp"\nbearer_token_env_var = "DAIMON_MCP_CAPABILITY"\n'); + assert.doesNotMatch(config, /access_token|refresh_token|auth\.json/u); +}); + +test("worker config disables every bundled 1.0.34 skill, workflows, and the per-turn session title request", () => { + const config = renderGrokBrokerWorkerConfig(); + assert.equal(GROK_1_0_34_BUNDLED_SKILLS.length, 25); + assert.equal(section(config, "[skills]"), `[skills]\ndisabled = [${GROK_1_0_34_BUNDLED_SKILLS.map((name) => JSON.stringify(name)).join(", ")}]\n`); + assert.equal(section(config, "[workflows]"), "[workflows]\nenabled = false\n"); + assert.match(section(config, "[models]"), /\nsession_summary = "daimon-session-title-disabled"\n/u); + assert.equal(section(config, "[model.daimon-session-title-disabled]"), '[model.daimon-session-title-disabled]\nmodel = "disabled"\nbase_url = "http://127.0.0.1:43123/v1"\napi_key = "session-title-disabled"\nmax_retries = 0\nhidden = true\n'); + // The sink carries a static placeholder key only: no env_key, so it can never pick up the turn capability. + assert.doesNotMatch(section(config, "[model.daimon-session-title-disabled]"), /env_key|DAIMON_/u); + for (const toggle of ["title_refresh", "telemetry", "session_recap", "turn_summary", "backend_tools", "ask_user_question"]) assert.match(section(config, "[features]"), new RegExp(`\\n${toggle} = false\\n`, "u")); + assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); +}); + +test("every model the worker can reach fails fast rather than retrying a refusal blindly", () => { + // Grok 1.0.34's default retries a refused request with backoff past 45 s. + // The sink and the evaluator client always pinned this; the worker's own + // model — the one path that spends money — did not, so a refusal there could + // stall a turn for minutes after its work was done with nothing logged. + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const config = renderGrokBrokerWorkerConfig({ model, reasoningEffort }); + for (const block of ["[model.daimon-broker-grok]", "[model.daimon-session-title-disabled]"]) { + assert.match(section(config, block), /\nmax_retries = 0\n/u, `${block} ${model}/${reasoningEffort}`); + } + } + } +}); + +test("the declared model and effort reach the worker's only model as its sole allowed effort", () => { + const config = renderGrokBrokerWorkerConfig({ model: "grok-build", reasoningEffort: "medium" }); + assert.match(section(config, "[model.daimon-broker-grok]"), /\nmodel = "grok-build"\n/u); + assert.match(section(config, "[models]"), /\ndefault = "daimon-broker-grok"\ndefault_reasoning_effort = "medium"\n/u); + assert.equal(section(config, "[[model.daimon-broker-grok.reasoning_efforts]]"), '[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = "medium"\nlabel = "Medium"\ndefault = true\n'); + assert.equal(config.match(/reasoning_efforts\]\]/gu)?.length, 1); + assert.match(renderGrokBrokerWorkerConfig(), /\nmodel = "grok-4\.6"\n[\s\S]*value = "low"/u); + for (const invalid of [{ model: "grok-3" }, { reasoningEffort: "xhigh" }, { model: "grok-4.6", reasoningEffort: "low", extra: true }]) { + assert.throws(() => renderGrokBrokerWorkerConfig(invalid as never), /model policy/u); + } +}); + +test("the manifest pins the sha256 of every renderable worker config", () => { + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const digest = createHash("sha256").update(renderGrokBrokerWorkerConfig({ model, reasoningEffort })).digest("hex"); + assert.equal(grokBrokerWorkerConfigSha256({ model, reasoningEffort }), digest); + assert.equal(GROK_ENGINE_BROKER.worker.configSha256[model][reasoningEffort], digest, `${model}/${reasoningEffort}`); + } + } +}); + +test("the probe-only renderer refuses non-loopback or injected endpoints", () => { + const policy = { model: "grok-4.6", reasoningEffort: "low" } as const; + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 0, mcpUrl: "http://127.0.0.1:1/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 1, mcpUrl: "http://example.com/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 1, mcpUrl: "http://127.0.0.1:1/mcp\"\n[evil]" }), /invalid/u); + const args = renderGrokBrokerWorkerArgs("/run/worker/prompt", "/workspace"); + assert.equal(args.includes("--prompt-file"), true); assert.equal(args.includes("--single"), false); }); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index 4223490..fde8952 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -1,17 +1,148 @@ +import { createHash } from "node:crypto"; import path from "node:path"; -export function renderGrokBrokerWorkerConfig(helperPath: string, proxyPort: number): string { - if (!path.posix.isAbsolute(helperPath) || /[\r\n"']/u.test(helperPath) || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65_535) throw new TypeError("invalid Grok broker worker configuration"); +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { DAIMON_GROK_SYSTEM_PROMPT, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS } from "../contracts/grokWorkerContract.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; + +export { DAIMON_GROK_SYSTEM_PROMPT, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS, GROK_WORKER_VISIBLE_TOOLS } from "../contracts/grokWorkerContract.js"; + +/** + * Bundled skills shipped by Grok CLI 1.0.34. Names are version-specific: + * `[skills] disabled` removed all ~2.1k skill tokens in the P0 matrix, and a + * CLI bump must re-derive this list rather than inherit it. + */ +export const GROK_1_0_34_BUNDLED_SKILLS = Object.freeze([ + "build-with-ai", "code-review", "create-skill", "create-workflow", "design", "docx", "execute-plan", + "game-animation-frames", "game-asset-core", "game-character-consistency", "game-tilesets", "game-ui-icons", + "imagine", "implement", "learn", "long-running-background-tasks", "pdf", "pptx", "pr-babysit", + "resume-claude", "resume-codex", "resume-cursor", "review", "skill-design-principles", "statusline" +] as const); + +/** The worker's only model id; the argv selects it and the proxy never sees another. */ +export const GROK_BROKER_WORKER_MODEL_ID = "daimon-broker-grok" as const; + +/** + * Grok 1.0.34 sends a `session_title` model request before every headless + * turn, and no config key or environment variable disables it + * (`features.title_refresh` governs only the later refresh; verified against a + * loopback stub). `[models] session_summary` does select the model it uses, so + * the title goes to a hidden model whose endpoint is the broker's own provider + * proxy with a placeholder key that can never be a turn capability (shorter + * than the 40-character capability alphabet). The proxy refuses it before any + * capability lookup, isolation guard, credential read, or upstream call, and + * Grok falls back to the truncated prompt as the title. The endpoint is always + * listening while a worker runs, so the refusal is bounded by one loopback + * round trip rather than by a connect timeout, and a prompt-derived title is + * never delivered anywhere but Daimon's own proxy. + */ +export const GROK_SESSION_TITLE_SINK_MODEL_ID = "daimon-session-title-disabled" as const; +export const GROK_SESSION_TITLE_SINK_KEY = "session-title-disabled" as const; +const renderSessionTitleSink = (proxyPort: number): readonly string[] => [ + `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `api_key = "${GROK_SESSION_TITLE_SINK_KEY}"`, + "max_retries = 0", "hidden = true", "" +]; + +/** + * The turn-scoped proxy capability reaches the worker's only model through + * `env_key`, set by the native launcher. Grok 1.0.34 accepts + * `[auth_provider.]` tables but never runs the helper for a custom model + * (verified against a loopback stub: no helper invocation and no Authorization + * header, with and without `args`, `api_backend`, `model_providers`, or a + * passwd-home config), while `env_key` attaches the bearer on every request. + * The capability is exactly as exposed as `DAIMON_MCP_CAPABILITY`: visible to + * the worker's own tool children, which run network-restricted, and revoked + * when the turn ends. + */ +export const GROK_BROKER_PROVIDER_CAPABILITY_ENV = "DAIMON_PROVIDER_CAPABILITY" as const; + +type WorkerEndpoints = Readonly<{ proxyPort: number; mcpUrl: string }>; +const PRODUCTION_ENDPOINTS: WorkerEndpoints = Object.freeze({ + proxyPort: GROK_ENGINE_BROKER.providerProxy.port, + mcpUrl: `http://${GROK_ENGINE_BROKER.mcpFacade.host}:${GROK_ENGINE_BROKER.mcpFacade.port}${GROK_ENGINE_BROKER.mcpFacade.path}` +}); + +/** + * Sections shared by every Grok worker Daimon configures, broker or direct. + * + * Each toggle is either measured (skills −2.1k tokens, workflows −314, the + * per-turn `session_title` model request) or behavioural hardening that P0 + * showed to be token-neutral and warning-free on 1.0.34. + */ +export const renderGrokLeanBaseConfig = (): string => [ + "[cli]", "auto_update = false", "use_leader = false", "show_tips = false", "", + "[features]", "telemetry = false", "title_refresh = false", "session_recap = false", "turn_summary = false", + "repo_status_in_system_prompt = false", "codebase_indexing = false", "backend_tools = false", "ask_user_question = false", + "image_gen = false", "video_gen = false", "web_fetch = false", "campaigns = false", "managed_config = false", "", + "[managed_mcps]", "enabled = false", "", + "[skills]", `disabled = [${GROK_1_0_34_BUNDLED_SKILLS.map((name) => JSON.stringify(name)).join(", ")}]`, "", + "[workflows]", "enabled = false", "" +].join("\n"); + +/** + * `max_retries = 0` on the worker's own model, for the same reason its two + * siblings already carry it (the session-title sink above, + * `grokInferenceClientConfig.ts` for the evaluator): with Grok 1.0.34's + * default, a refused or failed request is retried with backoff **past 45 s** + * instead of failing in ~0.35 s, and the retries are blind — one live turn + * emitted the same refusal fifteen times over five minutes, spent $0, and died + * with no account of why. Only this model block was left on the default, so + * the one request path that spends money was also the only one that could + * stall a turn for minutes after its work was done. Daimon owns the retry + * decision here because the proxy is the thing being retried: a genuinely + * transient fault is already answered 503 and is the broker's to retry, and + * anything else is a refusal that repeating cannot fix. The worker instead + * fails fast and the turn reaches the host with a status. + */ +/** + * The only source of broker worker `config.toml` bytes. + * + * Effort is declared here, not on the compiled launcher argv: the argv is one + * constant for every registration while effort is declared per deployment, and + * Grok 1.0.34 drops both `--reasoning-effort` and `[models] + * default_reasoning_effort` unless the model advertises effort support. A + * one-entry `reasoning_efforts` table makes the declared effort the model's + * default *and* its closed enum, so it reaches the request body and nothing + * else can be selected; the proxy then re-verifies it on every body. + */ +export function renderGrokBrokerWorkerConfig(policy: Partial = {}): string { + return renderGrokBrokerWorkerConfigWith(parseGrokBrokerModelPolicy(policy), PRODUCTION_ENDPOINTS); +} + +/** Explicit-endpoint variant for the local live probe; production bytes come only from the function above. */ +export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, endpoints: WorkerEndpoints): string { + const declared = parseGrokBrokerModelPolicy(policy); + const { proxyPort, mcpUrl } = endpoints; + if (!Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65_535 || !/^http:\/\/127\.0\.0\.1:\d{1,5}\/mcp$/u.test(mcpUrl)) { + throw new TypeError("invalid Grok broker worker configuration"); + } + const label = `${declared.reasoningEffort[0]!.toUpperCase()}${declared.reasoningEffort.slice(1)}`; return [ - "[cli]", "auto_update = false", "use_leader = false", "", - "[features]", "telemetry = false", "", - "[auth_provider.daimon]", `command = ${JSON.stringify(helperPath)}`, 'args = ["--auth-provider"]', "timeout_secs = 5", "token_ttl_secs = 600", "", - "[model.daimon-broker-grok]", 'model = "grok-build"', `base_url = "http://127.0.0.1:${proxyPort}/v1"`, 'auth_provider = "daimon"', "context_window = 131072", "supports_backend_search = false", "", - "[mcp_servers.daimon]", 'url = "http://127.0.0.1:43124/mcp"', 'headers = { Authorization = "Bearer ${DAIMON_MCP_CAPABILITY}" }', "" + renderGrokLeanBaseConfig(), + "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", + ...renderSessionTitleSink(proxyPort), + `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", + `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", + "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" ].join("\n"); } +export const grokBrokerWorkerConfigSha256 = (policy: Partial = {}): string => + createHash("sha256").update(renderGrokBrokerWorkerConfig(policy)).digest("hex"); + +/** + * TypeScript mirror of the argv compiled into `native/engineBrokerLauncherCore.inc`. + * `native/launcherArgv.test.ts` parses the C source and fails on any divergence. + */ export const renderGrokBrokerWorkerArgs = (promptFile: string, cwd: string): readonly string[] => { if (!path.posix.isAbsolute(promptFile) || !path.posix.isAbsolute(cwd)) throw new TypeError("invalid Grok broker worker path"); - return ["--sandbox", "daimon-strict", "--always-approve", "--no-subagents", "--prompt-file", promptFile, "--no-memory", "--disable-web-search", "--cwd", cwd, "--output-format", "streaming-messages-json", "--model", "daimon-broker-grok"]; + return [ + "--sandbox", "daimon-strict", "--always-approve", "--no-subagents", "--prompt-file", promptFile, + "--no-memory", "--disable-web-search", "--no-plan", "--verbatim", + "--system-prompt-override", DAIMON_GROK_SYSTEM_PROMPT, + "--tools", GROK_WORKER_TOOL_IDS.join(","), + "--max-turns", String(GROK_WORKER_MAX_TURNS), + "--cwd", cwd, "--output-format", "streaming-messages-json", "--model", GROK_BROKER_WORKER_MODEL_ID + ]; }; diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index 6ae6eaf..35fd7cc 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -1,53 +1,62 @@ -import { createHash, randomUUID } from "node:crypto"; -import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; -import { recordTurnUsage, TURN_USAGE_LEDGER, type TurnUsageEntry } from "./turnUsageLedger.js"; import { DurableGrokBrokerCredentialAuthority } from "./grokBrokerCredentialAuthority.js"; -import { NativeBrokerTurnFailure, runNativeBrokerTurn, type NativeBrokerDiagnostic } from "./engineBrokerNativeClient.js"; +import { runNativeBrokerTurn } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; -import { GrokWorkerAttestationFailure,prepareGrokWorkerAttestation,verifyGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +import { createGrokWorkerIsolationGuard,grokBrokerAttestationInput,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createLedgeredGrokInferenceGrants, GrokInferenceGrantRefused, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; -export type GrokEngineBrokerRegistration = Readonly<{ agentId:string;slot:number;workerUid:number;workspace:string;profilePath:string;eventsPath:string;profileSha256:string }>; +export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +export { finishBrokerTurnWithUsage } from "./grokEngineBrokerMetering.js"; +export type GrokEngineBrokerRegistration = EngineBrokerServiceRegistration; export type GrokEngineBroker = Awaited>; -export class EngineBrokerTurnFailure extends Error{constructor(readonly code:"auth_stale"|"cancelled"|"engine_failed",readonly diagnostic?:NativeBrokerDiagnostic){super("engine broker turn failed");}} /** - * Seal a completed turn, then meter it. + * The Grok engine broker: one credential realm, one provider proxy, one MCP + * facade, and the root-provisioned registrations. Each registration declares + * its own model/effort (whose worker config bytes are attested), usage ledger, + * and turn limits (`engineBrokerServiceConfig.ts`). * - * Order is load-bearing. `turns.finish` publishes the durable *completed* - * record; only after that does the advisory usage line get appended. A replayed - * turn returns before the enclosing `try` block and never reaches here, so a - * crash-recovered turn cannot double-count. Usage is deliberately kept out of - * the `completed` frame itself: that record is re-validated by the strict wire - * parser on the next `begin()`, whose exact field set would reject an extra key - * and break crash-recovery replay permanently. - * - * `recordTurnUsage` never rejects, so an append failure cannot escape into the - * caller's `catch` and rewrite this already-completed turn as failed. + * With an `inferenceLedgerPath` the broker also issues evaluator inference + * grants (`grokInferenceGrants.ts`) over the same credential authority and + * proxy; their rows go only to that ledger. A stale realm refuses a grant as + * `auth_stale`, exactly as it fails subject turns. */ -export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Parameters[0], completed: Parameters[1], usageLedgerPath: string, usage: TurnUsageEntry["usage"] | undefined, agentId: string, wakeId: string): Promise { - await turns.finish(request, completed); - if (usage === undefined) return; - await recordTurnUsage(usageLedgerPath, { agent: agentId, wake: wakeId, engine: "grok", usage }); -} -export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; usageLedgerPath?: string }>) { - const usageLedgerPath = options.usageLedgerPath ?? TURN_USAGE_LEDGER.filePath; - const registrations = new Map(options.registrations.map((entry) => [entry.agentId, entry])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation({...registration,brokerGid:2100});}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map; resolve: () => void }>(); let closed = false; +export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; inferenceLedgerPath?: string }>) { + const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); + const attestationFor = (registration: GrokEngineBrokerRegistration) => grokBrokerAttestationInput(registration, [...registrations.values()], grokBrokerWorkerConfigSha256(registration.model)); + const inferenceLedgerPath=options.inferenceLedgerPath;const grants=inferenceLedgerPath===undefined?undefined:createLedgeredGrokInferenceGrants(inferenceLedgerPath); + const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,undefined,undefined,grants);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; + const facade = mcp; + const deps = { + turns, proxy, mcp: facade, credentialStale: () => authority.isStale(), + prepareIsolation: async (registration: GrokEngineBrokerRegistration) => { const attestation = attestationFor(registration); return createGrokWorkerIsolationGuard(attestation, await prepareGrokWorkerAttestation(attestation)); }, + runNative: (input: Parameters[1], signal: AbortSignal) => runNativeBrokerTurn(options.nativeClient, input, signal) + }; return { - async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal): Promise> { + async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, limits?: EngineBrokerTurnLimitOverrides): Promise { if (closed) throw new Error("engine broker unavailable"); const registration = registrations.get(agentId); if (registration === undefined) throw new Error("engine broker unavailable"); - const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt,mcpEndpoint } as const; - const begun = await turns.begin(request); if (begun !== "start") { if (begun.replay.kind === "completed") return {text:begun.replay.text,workerPid:begun.replay.workerPid,workerUid:begun.replay.workerUid,workerStartTime:begun.replay.workerStartTime};const code=begun.replay.code==="auth_stale"||begun.replay.code==="cancelled"?begun.replay.code:"engine_failed";throw new EngineBrokerTurnFailure(code,begun.replay.diagnostic as NativeBrokerDiagnostic|undefined); } - const isolation=await prepareGrokWorkerAttestation({...registration,brokerGid:2100});proxy.registerIsolationGuard(turnId,()=>verifyGrokWorkerAttestation({...registration,brokerGid:2100},isolation));const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); - let nativeDiagnostic:NativeBrokerDiagnostic|undefined,attested=false; - try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await verifyGrokWorkerAttestation({...registration,brokerGid:2100},isolation);attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } - catch(error) { const code=authority.isStale()?"auth_stale":controller.signal.aborted?"cancelled":"engine_failed";const diagnostic=error instanceof NativeBrokerTurnFailure?error.diagnostic:nativeDiagnostic&&!attested?{...nativeDiagnostic,status:"worker_failed" as const,stage:"attestation" as const,failureClass:error instanceof GrokWorkerAttestationFailure?error.failureClass:"profile_invalid" as const,profileApplied:false}:undefined;await turns.finish(request, { version: request.version, kind: "failed", requestId: request.requestId, turnId, code,...(diagnostic?{diagnostic}:{}) }); throw new EngineBrokerTurnFailure(code,diagnostic); } - finally { signal?.removeEventListener("abort",onAbort);active.get(turnId)?.resolve(); active.delete(turnId); proxy.revokeIsolationGuard(turnId);proxy.capabilities.revoke(turnId);mcp.revoke(turnId); } + const controller = new AbortController(); const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const key = `${agentId}\0${wakeId}`; const done = runGrokEngineBrokerTurn(deps, registration, wakeId, prompt, mcpEndpoint, controller.signal, limits); + active.set(key, { controller, done: done.then(() => undefined, () => undefined) }); + try { return await done; } finally { signal?.removeEventListener("abort", onAbort); active.delete(key); } + }, + requestInferenceGrant(request: GrokInferenceGrantRequest) { + if (closed || grants === undefined) throw new GrokInferenceGrantRefused("unavailable"); + if (authority.isStale()) throw new GrokInferenceGrantRefused("auth_stale"); + return grants.issue(request); + }, + releaseInferenceGrant(grantId: string): boolean { + if (grants === undefined) throw new GrokInferenceGrantRefused("unavailable"); + return grants.release(grantId); }, - async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([mcp.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, + async close(): Promise { if (closed) return; closed = true; grants?.close(); const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([facade.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, readiness: () => ({ providerProxyPort: proxy.port, mcpFacadePort:43_124, registrations: registrations.size,credentialStale:authority.isStale(),realmLease:true,workerIsolation:true }) }; } diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts new file mode 100644 index 0000000..3d21e47 --- /dev/null +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -0,0 +1,111 @@ +import { readFile } from "node:fs/promises"; + +import { renderBrokerTurnSealLine, TURN_SEAL_LEDGER_VERSION, TURN_SEAL_MAX_LINE_BYTES } from "./engineBrokerSealLedger.js"; +import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; +import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; + +/** + * The exact ledger bytes a terminal broker turn owes, sealed into its turn + * record *before* they are appended. + * + * The record is published first and the ledger appended second, so a crash in + * between used to leave a sealed turn whose spend never reached the ledger — + * and a replay never metered. Now a replay re-checks: if the ledger holds no + * row for this `turn`, it appends these same bytes (same `at`, same numbers). + * That is completing the original metering, not re-metering: a replay after a + * normal append finds the row and writes nothing, and readers dedupe on `turn` + * should two replays race. + */ +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string; seal?: string }>; +/** A v1 record, or a replay with no sealed bytes at all: nothing to append, including no seal. */ +export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); + +export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; + +export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { + const at = new Date().toISOString(); + // The seal row is rendered for *every* terminal turn, including one whose + // usage is null. That is the whole point: a turn cancelled before any usage + // could be attributed is exactly the turn whose outstanding MCP call and + // redacted last words have no other route to the host. + const seal = renderBrokerTurnSealLine(terminal, { agent: detail.agentId, wake: detail.wakeId, at }); + if (terminal.usage === null) return { usage: null, requests: "", seal }; + const { usage } = terminal; + return { + seal, + usage: renderTurnUsageLine({ + agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, + usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, + outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests + }), + requests: renderGrokTurnRequestLines({ agent: detail.agentId, wake: detail.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, at, ...(detail.session === undefined ? {} : { session: detail.session }) }) + }; +} + +const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; +const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); + +/** + * Strict check of stored ledger bytes: exactly the turn's own rows, + * newline-terminated, bounded. + * + * `seal` is optional so a record sealed before this stream existed still + * replays; its absence means the turn owes no seal row, never that one was + * lost. + */ +export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { + const invalid = () => new Error("broker turn registry unavailable"); + if (value === null || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const sealed = Object.hasOwn(value, "seal"); + if (Object.keys(value).length !== (sealed ? 3 : 2)) throw invalid(); + const { usage, requests, seal } = value as { usage: unknown; requests: unknown; seal?: unknown }; + try { + if (usage !== null) { + if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); + const parsed = rows(usage); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_USAGE_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } + if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); + if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + if (sealed) { + if (typeof seal !== "string" || !seal.endsWith("\n") || Buffer.byteLength(seal) > TURN_SEAL_MAX_LINE_BYTES) throw invalid(); + const parsed = rows(seal); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_SEAL_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } + } catch { throw invalid(); } + return { usage: usage as string | null, requests, ...(sealed ? { seal: seal as string } : {}) }; +} + +export type BrokerTurnLedgerPaths = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; sealLedgerPath: string }>; + +/** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: BrokerTurnLedgerPaths): Promise { + if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + await recordLedgerLines(paths.requestLedgerPath, lines.requests); + // Last, and never conditional on usage: a turn with no attributable spend is + // precisely the one whose seal row is its only account of itself. + if (lines.seal !== undefined) await recordLedgerLines(paths.sealLedgerPath, lines.seal); +} + +/** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: BrokerTurnLedgerPaths): Promise { + try { + if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + if (lines.seal !== undefined && !await ledgerHasTurn(paths.sealLedgerPath, turnId)) await recordLedgerLines(paths.sealLedgerPath, lines.seal); + } catch { /* advisory: a replay never fails on its ledger */ } +} + +async function ledgerHasTurn(file: string, turnId: string): Promise { + for (const candidate of [`${file}.1`, file]) { + let text: string; + try { text = await readFile(candidate, "utf8"); } catch { continue; } + for (const line of text.split("\n")) { + if (!line.includes(turnId)) continue; + try { if ((JSON.parse(line) as { turn?: unknown }).turn === turnId) return true; } catch { /* a torn line is not this turn's row */ } + } + } + return false; +} diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts new file mode 100644 index 0000000..d0df740 --- /dev/null +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -0,0 +1,49 @@ +import type { EngineBrokerRequest, EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { appendBrokerTurnLedger, renderBrokerTurnLedger } from "./grokEngineBrokerLedger.js"; +import type { GrokTurnRequest } from "./turnRequestLedger.js"; +import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; + +export type BrokerTurnMetering = Readonly<{ + usageLedgerPath: string; + requestLedgerPath: string; + /** Where the sealed response's operator-visible projection is appended (`engineBrokerSealLedger.ts`). */ + sealLedgerPath: string; + agentId: string; + wakeId: string; +}>; +export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; + +/** + * Seal a terminal turn, then meter it. The broker is the single writer. + * + * Order is load-bearing. The ledger bytes are rendered first and sealed into + * the durable terminal record together with its accounting + * (`grokEngineBrokerLedger.ts`); only after `turns.finish` published that + * record are the same bytes appended. A replayed turn returns before the + * broker's `try` block and never meters again — it only completes an append a + * crash interrupted (`ensureBrokerTurnLedgered`), and every row carries the + * turn id as `turn`, so a reader that sees one twice counts it once. + * + * Remaining window, documented rather than closed: a crash before the record's + * rename (while the turn is still `active`, including mid-turn) makes the next + * boot seal that turn `failed` with `usage: null`, so its spend is unmetered. + * Closing it needs the running proxy usage checkpointed into the active record + * on every request (an fsync'd rewrite per model request); not done here. + * + * Both terminal kinds meter: a failed turn spent real tokens, so its partial + * usage is written with `outcome: failed` and its closed `limitReason`. A turn + * with no usage at all (`usage: null`) writes no *usage* row — a zero row is + * byte-identical to a measured zero — but it still writes its seal row, which + * is a record of what the turn did rather than of what it spent. + * + * Appends never reject, so an append failure cannot escape into the caller's + * `catch` and rewrite a completed turn as failed; the caller also refuses to + * re-seal a turn this helper already sealed. + */ +export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail, onSealed: () => void = () => undefined): Promise { + const lines = renderBrokerTurnLedger(terminal, { ...detail, agentId: metering.agentId, wakeId: metering.wakeId }); + await turns.finish(request, terminal, lines); + onSealed(); + await appendBrokerTurnLedger(lines, metering); +} diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts new file mode 100644 index 0000000..61dfaa1 --- /dev/null +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -0,0 +1,147 @@ +import { createHash, randomUUID } from "node:crypto"; + +import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; +import { decodeGrokStreamUsage, type GrokStreamUsage } from "../pi/grokStreamUsage.js"; +import { ENGINE_BROKER_VERSION, type EngineBrokerFailureCode, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; +import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; +import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; +import { GrokBrokerTurnMeter, type GrokBrokerTurnMeterSnapshot } from "./grokBrokerTurnMeter.js"; +import { GrokWorkerAttestationFailure } from "./grokWorkerAttestation.js"; + +export type GrokEngineBrokerTurnResult = Readonly<{ text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting; +export class EngineBrokerTurnFailure extends Error { + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting, readonly mcpCalls?: EngineBrokerMcpCallObservation) { super("engine broker turn failed"); } +} + +/** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ +export type GrokEngineBrokerTurnDependencies = Readonly<{ + turns: EngineBrokerTurnRegistry; + proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void; observe?(turnId: string): EngineBrokerMcpCallObservation | undefined }>; + credentialStale(): boolean; + prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; + runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; +}>; + +const limitReasonFor = { tokens: "token_ceiling", requests: "request_ceiling", timeout: "wake_timeout" } as const; +const usageOf = (usage: Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>): EngineBrokerTurnUsage => ({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total }); + +/** + * One brokered Grok turn under its declared limits. + * + * The limits are the registration's, lowered (never raised) by the wake. The + * proxy meter refuses model requests past `maxRequests`, past the elapsed + * deadline, or once the upstream-reported running total reached `maxTokens`, + * and a tripped limit aborts the worker through the same cancel/kill path a + * client cancellation uses. A wall-clock timer trips `timeout` for a worker + * that is mid-request. + * + * Every terminal path — completed, failed, limit, cancelled — is sealed and + * metered through {@link finishBrokerTurnWithUsage}; a replayed turn returns its + * sealed accounting before any of this runs and never meters again. + */ +export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependencies, registration: EngineBrokerServiceRegistration, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, overrides?: EngineBrokerTurnLimitOverrides): Promise { + const { agentId } = registration, declared = registration.model.model; + let limits; + try { limits = lowerEngineBrokerTurnLimits(registration.limits, overrides); } catch { throw new EngineBrokerTurnFailure("invalid_request", undefined, { outcome: "failed", usage: null, model: declared, requests: 0, limitReason: "none" }); } + const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); + const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; + const begun = await deps.turns.begin(request, declared); + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), sealLedgerPath: engineBrokerSealLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } + const controller = new AbortController(); + const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); + const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs); timer.unref?.(); + + let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false, sealed: GrokEngineBrokerTurnResult | undefined; + try { + const isolationGuard = await deps.prepareIsolation(registration); + deps.proxy.registerIsolationGuard(turnId, isolationGuard); + deps.proxy.registerTurn(turnId, { policy: registration.model, meter }); + const providerCapability = deps.proxy.capabilities.issue(agentId, turnId), mcpCapability = deps.mcp.register(agentId, turnId, mcpEndpoint); + const result = await deps.runNative({ slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability, mcpCapability }, controller.signal); + nativeDiagnostic = result.diagnostic; output = result.text; + if (result.workerUid !== registration.workerUid) throw new Error("engine broker worker identity mismatch"); + await isolationGuard(); attested = true; + rejected = true; + const decoded = decodeGrokHeadlessTurn(result.text), stream = decodeGrokStreamUsage(result.text); + if (stream.reportedModels.some((reported) => mapGrokReportedModel(reported, declared) === undefined)) throw new Error("engine broker reported an undeclared model"); + rejected = false; + const snapshot = meter.snapshot(); + if (snapshot.limitReason !== "none") throw new Error("engine broker turn limit reached"); + const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); + const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; + const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; + const result_ = { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; }); + return result_; + } catch (error) { + // Once the completed record is published it is the durable truth: anything + // failing after that (metering) must neither re-seal the turn as failed nor + // append a second row. + if (sealed !== undefined) return sealed; + const snapshot = meter.snapshot(); + const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; + const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; + const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); + const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; + // Read before the `finally` revokes this turn's MCP registration, and + // swallowed like every other instrument here: it must never be the reason + // a turn reports something other than why it failed. + let mcpCalls: EngineBrokerMcpCallObservation | undefined; + try { mcpCalls = deps.mcp.observe?.(turnId); } catch { mcpCalls = undefined; } + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting }; + const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; + await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls); + } finally { + clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); + deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); + } +} + +function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnResult { + const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; + if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; + const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting, response.mcpCalls); +} + +/** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ +const requestCount = (stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): number => snapshot.requests > 0 ? snapshot.requests : stream?.requests.length ?? 0; + +/** Best partial usage: the worker's own per-request frames when any arrived, else what upstream reported to the proxy. */ +function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): EngineBrokerTurnUsage | null { + if (stream !== undefined && stream.requests.length > 0) { + const sum = (pick: (value: GrokStreamUsage["requests"][number]) => number) => stream.requests.reduce((total, value) => total + pick(value), 0); + return { input: sum((value) => value.input), cacheRead: sum((value) => value.cacheRead), cacheWrite: sum((value) => value.cacheWrite), output: sum((value) => value.output), total: sum((value) => value.total) }; + } + return snapshot.usage; +} + +/** Per-request rows: stream usage with the proxy's own observation when both describe the same requests, else the proxy's measured requests. */ +function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { + if (stream !== undefined && stream.requests.length > 0) { + const timed = snapshot.timings.length === stream.requests.length; + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? observed(snapshot.timings[index]!) : {}) })); + } + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...observed(timing) }]); +} +/** + * What only the proxy saw of one request: its clock, and the tool-call names the + * response carried. Both are attached on the stream path only when the two + * descriptions are request-for-request aligned, because an unaligned index would + * credit one request's attempt to another. + */ +const observed = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ + startedAt: timing.startedAt, + ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }), + ...(timing.toolCalls === undefined ? {} : { toolCalls: timing.toolCalls }) +}); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 7be6215..b8d6701 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,130 +1,382 @@ import assert from "node:assert/strict"; -import { createHash, randomUUID } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { finishBrokerTurnWithUsage } from "./grokEngineBroker.js"; -import { TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; +import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; +import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { WakeFuse } from "./wakeFuse.js"; -const usage = { input: 8_746, output: 29, cacheRead: 5_760, cacheWrite: 12, total: 14_547, calls: 1, notionalUsd: 0.0035, complete: true }; +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const upstreamUsage = { prompt_tokens: 2_696, completion_tokens: 79, total_tokens: 2_775, prompt_tokens_details: { cached_tokens: 128 } }; +const turnIdFor = (agentId: string, wakeId: string): string => createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); -const startRequest = (agentId: string, wakeId: string) => ({ - version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), - turnId: createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"), - agentId, wakeId, prompt: "prompt", mcpEndpoint: "http://127.0.0.1:43124/mcp" -} as const); +const assistant = (id: string, usage: Record, content: unknown[], stop: string) => ({ type: "assistant", message: { id, type: "message", role: "assistant", model: "daimon-broker-grok", content, stop_reason: stop, usage }, parent_tool_use_id: null, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }); +const first = { input_tokens: 2_568, output_tokens: 79, cache_read_input_tokens: 128, cache_creation_input_tokens: 0 }; +const second = { input_tokens: 109, output_tokens: 13, cache_read_input_tokens: 2_688, cache_creation_input_tokens: 0 }; +const stream = (modelKey = "grok-4.6-build"): string => [ + { type: "system", subtype: "init", session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }, + assistant("msg_0", first, [{ type: "tool_use", id: "call-0", name: "use_tool", input: {} }], "tool_use"), + assistant("msg_1", second, [{ type: "text", text: "TANGERINE-7" }], "end_turn"), + { type: "result", subtype: "success", is_error: false, num_turns: 2, result: "TANGERINE-7", stop_reason: "end_turn", total_cost_usd: 0.00248676, usage: { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }, modelUsage: { [modelKey]: {} }, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" } +].map((frame) => JSON.stringify(frame)).join("\n"); -const completedFor = (request: ReturnType) => ({ - version: request.version, kind: "completed", requestId: request.requestId, turnId: request.turnId, - text: "ACK", workerPid: 4_242, workerUid: 2_200, workerStartTime: "99" -} as const); +function post(port: number, token: string): Promise { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(leanBody); + }); +} + +type Worker = (post: () => Promise, signal: AbortSignal) => Promise; +const nativeResult = (text: string): NativeBrokerTurnResult => ({ text, workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }); +const untilAborted = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { const fail = () => reject(new Error("engine broker turn failed")); if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); }); /** - * Reproduces the broker's turn control flow around the registry: replayed turns - * return before any work, and a fresh turn seals the record and then meters it. - * Everything but the engine call itself is the real production code. + * The real turn registry, proxy, meter and ledgers around a scripted worker that + * talks to the proxy exactly as the native worker does (capability bearer, + * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const runTurn = async (turns: EngineBrokerTurnRegistry, ledger: string, agentId: string, wakeId: string): Promise<"start" | "replay"> => { - const request = startRequest(agentId, wakeId); - const begun = await turns.begin(request); - if (begun !== "start") return "replay"; - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, usage, agentId, wakeId); - return "start"; -}; +/** The stub provider's own SSE response: usage only, and no tool call, unless a test says otherwise. */ +const upstreamResponse = (): string => `data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`; +/** One streaming tool call per name, arguments in a following delta, then the usage event. */ +const upstreamToolCallResponse = (names: readonly string[]): string => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"tool_name":"daimon__moltnet_read"}' } }] } }] })), + { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: upstreamUsage } +].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; -const withStore = async (body: (turnStore: string, ledger: string, root: string) => Promise): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15, upstreamBody: (call: number) => string = upstreamResponse): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); - try { await body(path.join(root, "turns"), path.join(root, "usage.jsonl"), root); } finally { await rm(root, { recursive: true, force: true }); } + let calls = 0, aborted = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(upstreamBody(calls)) }; }, undefined, 0); + const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); + const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); + try { + await body({ + root, + turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns"), syncDirectory = undefined) => { + const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; + const deps: GrokEngineBrokerTurnDependencies = { + turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => mcpObservation }, + prepareIsolation: async () => async () => undefined, + runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) + }; + return runGrokEngineBrokerTurn(deps, registration, wakeId, "prompt", "http://127.0.0.1:43124/mcp", undefined, overrides); + }, + usageRows: () => rows(ledger), + requestRows: () => rows(path.join(path.dirname(ledger), "requests.jsonl")), + upstreamCalls: () => calls, + upstreamAborts: () => aborted + }); + } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; -const ledgerLines = async (file: string): Promise[]> => { - const text = await readFile(file, "utf8").catch(() => ""); - return text.split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); -}; +/** What the facade would have seen of this turn's tool calls; the turn only reads it. */ +let mcpObservation: EngineBrokerMcpCallObservation | undefined; + +const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; -test("a completed broker turn writes exactly one metered line, and a replayed turn writes no second one", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-1"), "start"); - assert.deepEqual(await runTurn(turns, ledger, "cogsworth", "wake-1"), "replay"); - - // Mutation guard: removing the replay suppression makes the same wake - // append a second line and double-count the subscription. - const written = await ledgerLines(ledger); - assert.equal(written.length, 1); - assert.deepEqual(written[0], { - v: TURN_USAGE_LEDGER_VERSION, agent: "cogsworth", wake: "wake-1", engine: "grok", - at: written[0]?.at, input: 8_746, output: 29, cache_read: 5_760, cache_write: 12, - total: 14_547, calls: 1, notional_usd: 0.0035, complete: true, - // The broker only ever appends for a turn it finished, so its rows are - // completed by construction; the field still states it explicitly. - outcome: "completed" +test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + const result = await turn("wake-1", twoRequests); + assert.deepEqual({ ...result, text: undefined }, { text: undefined, workerPid: 4_242, workerUid: 2_200, workerStartTime: "99", outcome: "completed", usage: { input: 2_677, cacheRead: 2_816, cacheWrite: 0, output: 92, total: 5_585 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + // Mutation guard: metering on the replay path appends a second row here. + let replayedWorker = false; + assert.deepEqual(await turn("wake-1", async () => { replayedWorker = true; return stream(); }), result); + assert.deepEqual(await turn("wake-1", twoRequests, undefined, undefined, path.join(root, "turns")), result, "a fresh registry boot replays the sealed accounting"); + assert.equal(replayedWorker, false); + const usage = await usageRows(); + assert.equal(usage.length, 1); + assert.deepEqual({ ...usage[0], at: undefined }, { v: TURN_USAGE_LEDGER_VERSION, agent: "foreman", wake: "wake-1", engine: "grok", at: undefined, input: 2_677, output: 92, cache_read: 2_816, cache_write: 0, total: 5_585, calls: 2, notional_usd: 0.00248676, complete: true, outcome: "completed", turn: turnIdFor("foreman", "wake-1"), limit_reason: "none", model: "grok-4.6" }); + const requests = await requestRows(); + assert.deepEqual(requests.map((row) => [row.v, row.engine, row.request, row.requests, row.input, row.fresh_input, row.cached_input, row.total, row.turn]), [ + [TURN_REQUEST_LEDGER_VERSION, "grok", 0, 2, 2_696, 2_568, 128, 2_775, turnIdFor("foreman", "wake-1")], + [TURN_REQUEST_LEDGER_VERSION, "grok", 1, 2, 2_797, 109, 2_688, 2_810, turnIdFor("foreman", "wake-1")] + ]); + // Mutation guard: stamping every request with the wake end collapses these. + // The upstream stub takes 15 ms per request, so each request has a measurable interval. + const [a, b] = requests.map((row) => [Date.parse(String(row.started_at)), Date.parse(String(row.ended_at))] as const); + assert.ok(a![0] < a![1] && a![1] <= b![0] && b![0] < b![1], JSON.stringify(requests.map((row) => [row.started_at, row.ended_at]))); + assert.ok(b![1] <= Date.parse(String(requests[1]!.at)), "every request ended before the rows were appended"); + }); +}); + +test("a turn past maxRequests is refused before upstream, killed, sealed as limit_exceeded, and its partial usage is metered", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + await assert.rejects(turn("wake-2", worker, undefined, { maxRequests: 3, maxTokens: 300_000, timeoutMs: 240_000 }), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.equal(error.code, "limit_exceeded"); + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 7_704, cacheRead: 384, cacheWrite: 0, output: 237, total: 8_325 }, model: "grok-4.6", requests: 3, limitReason: "requests" }); + return true; }); + assert.equal(upstreamCalls(), 3); + // Mutation guard: metering only completed turns leaves this ledger empty. + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.limit_reason, row?.total, row?.calls, row?.complete], ["failed", "request_ceiling", "requests", 8_325, 3, false]); + await assert.rejects(turn("wake-2", twoRequests), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "requests"); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); + }); +}); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-2"), "start"); - assert.equal((await ledgerLines(ledger)).length, 2); +test("the token ceiling stops a turn one request past the ceiling at most", async () => { + await withBroker(async ({ turn, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + // 2,775 tokens per request against 5,000: requests 1 and 2 are admitted, 3 is refused. + await assert.rejects(turn("wake-3", worker, { maxTokens: 5_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "tokens" && error.accounting.usage?.total === 5_550); + assert.equal(upstreamCalls(), 2); }); }); -test("crash recovery replays the same completed turn without metering it again", async () => { - await withStore(async (turnStore, ledger) => { - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "start"); - // A fresh boot id is what the broker gets after a crash. - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "replay"); - assert.equal((await ledgerLines(ledger)).length, 1); +test("the wall-clock limit aborts a worker that is mid-request", async () => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamAborts }) => { + const started = Date.now(); + // Request 2 is still upstream (1.5 s) when the 1 s wall clock fires. + const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); void send().catch(() => undefined); return untilAborted(signal); }; + await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); + assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); + assert.equal(upstreamAborts(), 1, "the stuck upstream call is aborted, not left running"); + // The aborted request reported nothing, so it is charged the estimate and says so. + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls, row.estimated_requests]), [["wake_timeout", "timeout", 2_775 + 4_297, 2, 1]]); + assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests, row.usage_source, row.total]), [[0, 2, "upstream", 2_775], [1, 2, "estimated", 4_297]]); + }, undefined, (call) => call === 2 ? 1_500 : 15); +}); + +test("a wake may only lower a declared limit: raising one is refused before any turn record or worker", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + let ran = false; + // Mutation guard: clamping or accepting the raise runs the worker. + await assert.rejects(turn("wake-5", async () => { ran = true; return stream(); }, { maxTokens: 300_001 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "invalid_request"); + assert.equal(ran, false); assert.equal(upstreamCalls(), 0); assert.deepEqual(await usageRows(), []); + assert.equal((await turn("wake-5", twoRequests, { maxTokens: 299_999, timeoutMs: 1_000 })).outcome, "completed", "a lowered limit is accepted and the turn was never recorded"); }); }); -test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { - // Mutation guard: deleting the advisory try/catch in recordTurnUsage makes - // finishBrokerTurnWithUsage reject. In the broker that rejection lands in the - // catch that calls finish(..., failed), which renames over this already - // completed record — turning a published turn into a failed one. - await withStore(async (turnStore, _ledger, root) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const unwritable = path.join(root, "not-provisioned", "usage.jsonl"); - const request = startRequest("brass", "wake-3"); - assert.equal(await turns.begin(request), "start"); - await assert.doesNotReject(finishBrokerTurnWithUsage(turns, request, completedFor(request), unwritable, usage, "brass", "wake-3")); - - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(request); - assert.notEqual(replayed, "start"); - assert.equal((replayed as { replay: { kind: string } }).replay.kind, "completed"); +test("a turn whose stream reports an undeclared model fails as rejected but is still metered", async () => { + await withBroker(async ({ turn, usageRows }) => { + await assert.rejects(turn("wake-6", async (send) => { await send(); await send(); return stream("grok-4.5-build"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "engine_failed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.reason, row.model, row.total]), [["failed", "turn_rejected", "grok-4.6", 5_585]]); + }); +}); + +test("a crash between sealing and appending is completed by the replay exactly once, with the sealed bytes", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + await turn("wake-8", twoRequests); + const [sealedUsage] = await usageRows(); const sealedRequests = await requestRows(); + // Simulate the crash window: the record is published but the append never happened. + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + // Mutation guard: a replay that never ensures its ledger leaves this spend unmetered. + assert.equal((await turn("wake-8", async () => { throw new Error("a replay runs no worker"); })).outcome, "completed"); + assert.deepEqual(await usageRows(), [sealedUsage]); + assert.deepEqual(await requestRows(), sealedRequests); + // A replay after the rows exist writes nothing further. + await turn("wake-8", async () => { throw new Error("a replay runs no worker"); }); + assert.equal((await usageRows()).length, 1); + assert.equal((await requestRows()).length, 2); }); }); -test("a turn whose usage could not be decoded is sealed but writes no line", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const request = startRequest("brass", "wake-4"); - assert.equal(await turns.begin(request), "start"); - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, undefined, "brass", "wake-4"); - assert.deepEqual(await ledgerLines(ledger), []); - assert.notEqual(await new EngineBrokerTurnRegistry(turnStore).begin(request), "start"); +test("a ledger append that fails after the turn was sealed leaves it completed and appends nothing twice", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + // The request stream cannot be written (its path is a directory); the usage stream can. + await mkdir(path.join(root, "requests.jsonl")); + assert.equal((await turn("wake-9", twoRequests)).outcome, "completed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-9")]]); + assert.equal((await turn("wake-9", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed", "the sealed record was never rewritten as failed"); + assert.equal((await usageRows()).length, 1); }); }); -test("usage is never written into the completed frame the strict wire parser re-validates", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-5"), "start"); - // The durable record is re-parsed on the next begin(); an extra field there - // makes it throw permanently and breaks crash-recovery replay for good. - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(startRequest("cogsworth", "wake-5")); - const response = (replayed as { replay: Record }).replay; - assert.deepEqual(Object.keys(response).sort(), ["kind", "requestId", "text", "turnId", "version", "workerPid", "workerStartTime", "workerUid"]); +test("a directory-sync failure after the completed record is published never re-seals the turn as failed", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + let syncs = 0; + // Sync 1 is begin()'s active record; sync 2 follows the completed record's rename. + const failAfterPublish = async (): Promise => { syncs += 1; if (syncs === 2) throw new Error("EIO"); }; + // Mutation guard: letting the post-rename failure reject makes the turn's catch write `failed` over the published record. + const result = await turn("wake-10", twoRequests, undefined, undefined, path.join(root, "turns"), failAfterPublish); + assert.equal(result.outcome, "completed"); + assert.equal(syncs, 2); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-10")]]); + assert.equal((await requestRows()).length, 2); + const replayed = await turn("wake-10", async () => { throw new Error("a replay runs no worker"); }); + assert.deepEqual(replayed, result); + assert.equal((await usageRows()).length, 1); }); }); -test("the broker meters only on the success path, through the single sealing helper", async () => { - const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBroker.ts"), "utf8"); - const body = source.slice(source.indexOf("async turn(")); - assert.equal(body.includes("recordTurnUsage("), false, "the broker must meter only through finishBrokerTurnWithUsage"); - assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 1, "exactly one metering call, in the success branch"); - assert.equal(body.includes("turns.finish(request,completed)"), false, "the success branch must seal through the metering helper"); - assert.ok(body.indexOf("finishBrokerTurnWithUsage(") < body.indexOf("catch(error)"), "metering belongs to the success branch"); +test("two concurrent replays of one sealed turn may both append, and every reader still counts the turn once", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + await turn("wake-11", twoRequests); + const [sealed] = await usageRows(); + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + const noWorker = async (): Promise => { throw new Error("a replay runs no worker"); }; + const replays = await Promise.all([turn("wake-11", noWorker), turn("wake-11", noWorker), turn("wake-11", noWorker)]); + assert.ok(replays.every((replayed) => replayed.outcome === "completed")); + const rows = await usageRows(); + assert.ok(rows.length >= 1 && rows.every((row) => row.turn === sealed!.turn && row.total === sealed!.total), "duplicates, if any, are byte-equal sealed rows"); + // Readers dedupe on `turn`: the ledger helper and the wake fuse's sum both count it once. + assert.deepEqual(dedupeTurnUsageRows(rows).map((row) => row.total), [sealed!.total]); + const fuseDirectory = path.join(root, "fuse"); await mkdir(fuseDirectory); + const fuse = await WakeFuse.open({ organizationKey: "org", now: () => new Date(Date.parse(String(sealed!.at)) - 1), environment: { + DAIMON_WAKE_FUSE_DIRECTORY: fuseDirectory, DAIMON_WAKE_FUSE_EPOCH: "replay", DAIMON_WAKE_FUSE_MAX_WAKES: "10", + DAIMON_WAKE_FUSE_MAX_TOKENS: String(Number(sealed!.total) + 1), DAIMON_TURN_USAGE_LEDGER_PATH: path.join(root, "usage.jsonl") + } }); + // Counted once the turn is below the ceiling by one token; counted twice it would trip. + const concurrentRows = [...rows, ...rows]; + await writeFile(path.join(root, "usage.jsonl"), concurrentRows.map((row) => JSON.stringify(row)).join("\n") + "\n"); + assert.deepEqual(await fuse.admit("foreman", "next"), { state: "admitted" }); + }); +}); + +test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { + await withBroker(async ({ root, turn }) => { + assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); + assert.equal((await turn("wake-7", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed"); + }, path.join(os.tmpdir(), `daimon-missing-${process.pid}`, "not-provisioned", "usage.jsonl")); +}); + +test("the broker meters only through the single sealing helper, on both terminal branches", async () => { + const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBrokerTurn.ts"), "utf8"); + const body = source.slice(source.indexOf("export async function runGrokEngineBrokerTurn"), source.indexOf("function replay(")); + assert.equal(body.includes("recordTurnUsage("), false); + assert.equal(body.includes("turns.finish("), false, "every terminal record is sealed through the metering helper"); + assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); + assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); +}); + +/** + * The question two live turns could not answer: did the model ever *try* to + * call a tool? The rows carried timings and tokens and nothing about an + * attempt, so a turn with zero tool calls and a turn whose calls all failed + * read identically after the fact. + */ +test("each per-request row records the tool-call names that request's response carried, names only", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-tools", twoRequests); + const rows = await requestRows(); + // Mutation guard: without the field these are `[undefined, undefined]`. + assert.deepEqual(rows.map((row) => row.tool_calls), [["use_tool", "search_tool"], ["use_tool", "search_tool"]]); + const text = JSON.stringify(rows); + // Names only: no arguments, no message content, no bearer. + assert.equal(text.includes("daimon__moltnet_read"), false, "an argument value must never reach the ledger"); + assert.equal(text.includes("arguments"), false); + assert.equal(text.includes("provider-token"), false); + }, undefined, () => 1, () => upstreamToolCallResponse(["use_tool", "search_tool"])); +}); + +test("a response that called nothing records an empty list, and one that cannot be decoded records no field at all", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-silent", twoRequests); + // Decoded, and it called nothing: that is an observation, not a gap. + assert.deepEqual((await requestRows()).map((row) => row.tool_calls), [[], []]); + }, undefined, () => 1); + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-undecodable", twoRequests); + const rows = await requestRows(); + // Mutation guard: a fabricated `[]` here would be byte-identical to the + // measured empty list above, and the ledger would claim an observation the + // proxy never made. + assert.deepEqual(rows.map((row) => Object.hasOwn(row, "tool_calls")), [false, false]); + assert.deepEqual(rows.map((row) => row.request), [0, 1], "the rows themselves are still written"); + }, undefined, () => 1, () => "bad gateway"); +}); + + +/** + * The exact 128-byte frame `supervise()` emits when a worker crosses + * `DBL_MAX_OUTPUT`: it stops reading, SIGKILLs the process group, and publishes + * `output_length = 0` with `DBL_STATUS_OUTPUT_FAILED`. Built here at the wire + * offsets the header's `_Static_assert`s pin, so the test drives the real + * decoder rather than a hand-made exception. + */ +function outputLimitFrame(turnId: string): Buffer { + const frame = Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES); + frame.writeUInt32LE(2, 0); frame.writeUInt32LE(3, 4); frame.writeUInt32LE(2_200, 8); frame.writeUInt32LE(0, 12); + frame.writeInt32LE(4_242, 16); frame.writeInt32LE(0, 20); frame.writeInt32LE(9, 24); + frame.writeBigUInt64LE(99n, 32); frame.write(turnId, 40, "utf8"); + frame.writeUInt32LE(7, 108); frame.writeUInt32LE(7, 112); frame.writeUInt32LE(0, 116); frame.writeUInt32LE(0, 120); + return frame; +} + +test("a worker whose work succeeded but whose output crossed the launcher bound still seals the spend the proxy measured", async () => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamCalls }) => { + // The worker does its real work through the real proxy — two admitted, + // metered upstream requests — and only then loses its whole output: the + // launcher refused to publish it and the turn's text never exists. The + // frame is decoded by the shipped client, so the failure reaches the turn + // exactly as the native transport delivers it. + const worker: Worker = async (send) => { + assert.equal(await send(), 200); + assert.equal(await send(), 200); + throw decodeNativeBrokerResult(outputLimitFrame(turnIdFor("foreman", "wake-output-limit")), turnIdFor("foreman", "wake-output-limit"), []) as never; + }; + await assert.rejects(turn("wake-output-limit", worker), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + // No limit tripped and the credential is live: this is the worker's + // transport failing, not the turn being refused. + assert.equal(error.code, "engine_failed"); + assert.equal(error.diagnostic?.failureClass, "output_limit"); + // Mutation guard: the turn has no stream to read usage from, so this can + // only come from the proxy's own per-request measurements. Falling back + // to `null` here would report a fabricated zero for real spend. + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 5_136, cacheRead: 256, cacheWrite: 0, output: 158, total: 5_550 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + return true; + }); + assert.equal(upstreamCalls(), 2); + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.total, row?.calls, row?.complete], ["failed", "unknown", 5_550, 2, false]); + assert.notEqual(row?.total, 0, "a zero row would be byte-identical to a measured zero"); + assert.deepEqual((await requestRows()).map((value) => value.request), [0, 1], "every request the proxy answered keeps its own row"); + // The sealed record is the durable truth: a replay returns that spend and never meters again. + await assert.rejects(turn("wake-output-limit", twoRequests), (error: unknown) => + error instanceof EngineBrokerTurnFailure && error.accounting?.usage?.total === 5_550); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); + }); +}); + +/** + * The hang this instrument was built for: the worker stops acting with every + * provider request closed, the deadline kills it, and the only remaining + * question is whether it was waiting on a tool call. The answer has to reach + * the host, and the slot's control root is tmpfs that dies with the container — + * so it rides the seam the worker's last words and the sealed usage already + * ride: the sealed terminal response, which a replay hands back unchanged. + */ +test("a failed turn carries the facade's in-flight tool-call observation, and its replay still does", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }; + const worker: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); throw new Error("engine broker turn failed"); }; + const carried = (error: unknown): boolean => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.deepEqual(error.mcpCalls, { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }); + return true; + }; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + // The replay reads the durable record back through the frame parser, so + // this is the sealed bytes answering, not the live facade. + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + }); +}); + +test("a turn whose facade observed nothing seals no observation at all", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-absent", async (send) => { await send(); throw new Error("engine broker turn failed"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.mcpCalls === undefined); + }); }); diff --git a/src/runtime/grokInferenceClientConfig.test.ts b/src/runtime/grokInferenceClientConfig.test.ts new file mode 100644 index 0000000..d9ee6df --- /dev/null +++ b/src/runtime/grokInferenceClientConfig.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokBrokerModelPolicy.js"; +import { GROK_1_0_34_BUNDLED_SKILLS, GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInferenceClientConfigSha256, renderGrokInferenceClientConfig, renderProductionGrokInferenceClientConfig } from "./grokInferenceClientConfig.js"; + +const production = { baseUrl: "http://127.0.0.1:43123/v1", model: "grok-4.6", reasoningEffort: "low", envKey: "DAIMON_INFERENCE_GRANT" } as const; + +test("the evaluator client config reaches only the grant proxy through env_key, with no MCP and no credential", () => { + const config = renderGrokInferenceClientConfig(production); + assert.match(config, /\[models\]\ndefault = "daimon-inference-grok"\ndefault_reasoning_effort = "low"\nsession_summary = "daimon-session-title-disabled"\n/u); + assert.match(config, /\[model\.daimon-inference-grok\]\nmodel = "grok-4\.6"\nbase_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_INFERENCE_GRANT"\napi_backend = "chat_completions"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n/u); + assert.match(config, /\[\[model\.daimon-inference-grok\.reasoning_efforts\]\]\nvalue = "low"\nlabel = "Low"\ndefault = true\n/u); + assert.equal(config.match(/reasoning_efforts\]\]/gu)?.length, 1); + assert.doesNotMatch(config, /mcp_servers|auth_provider|access_token|refresh_token/u); + assert.equal(GROK_INFERENCE_CLIENT_MODEL_ID, "daimon-inference-grok"); assert.equal(GROK_INFERENCE_GRANT_ENV, "DAIMON_INFERENCE_GRANT"); +}); + +test("the evaluator client config mirrors the worker's lean settings and refuses the session title locally", () => { + const config = renderGrokInferenceClientConfig(production); + for (const skill of GROK_1_0_34_BUNDLED_SKILLS) assert.ok(config.includes(JSON.stringify(skill)), skill); + assert.match(config, /\[workflows\]\nenabled = false\n/u); assert.match(config, /\[managed_mcps\]\nenabled = false\n/u); assert.match(config, /auto_update = false/u); + assert.match(config, new RegExp(`\\[model\\.daimon-session-title-disabled\\]\\nmodel = "disabled"\\nbase_url = "http://127\\.0\\.0\\.1:43123/v1"\\napi_key = "${GROK_SESSION_TITLE_SINK_KEY}"\\nmax_retries = 0\\nhidden = true\\n`, "u")); + assert.ok(GROK_SESSION_TITLE_SINK_KEY.length < 40, "the placeholder can never pass the proxy bearer shape"); +}); + +test("the manifest pins the sha256 of every production evaluator client config", () => { + for (const model of GROK_BROKER_MODELS) for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const digest = grokInferenceClientConfigSha256({ ...production, model, reasoningEffort }); + assert.equal(GROK_ENGINE_BROKER.inferenceGrants.client.configSha256[model][reasoningEffort], digest, `${model}/${reasoningEffort}`); + assert.equal(renderProductionGrokInferenceClientConfig({ model, reasoningEffort }), renderGrokInferenceClientConfig({ ...production, model, reasoningEffort })); + } +}); + +test("the evaluator client config refuses non-loopback endpoints, injected keys and undeclared models", () => { + for (const bad of [ + { ...production, baseUrl: "https://cli-chat-proxy.grok.com/v1" }, { ...production, baseUrl: "http://127.0.0.1:43123/v1\"\nx = 1" }, { ...production, baseUrl: "http://127.0.0.1:99999/v1" }, + { ...production, envKey: "X\"\n[mcp_servers.evil]" }, { ...production, envKey: "lower" }, + { ...production, model: "grok-3" }, { ...production, reasoningEffort: "xhigh" } + ]) assert.throws(() => renderGrokInferenceClientConfig(bad as typeof production), /invalid Grok inference client configuration/u); +}); diff --git a/src/runtime/grokInferenceClientConfig.ts b/src/runtime/grokInferenceClientConfig.ts new file mode 100644 index 0000000..d9a043c --- /dev/null +++ b/src/runtime/grokInferenceClientConfig.ts @@ -0,0 +1,63 @@ +import { createHash } from "node:crypto"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_INFERENCE_PROXY_BASE_URL } from "./engineBrokerInferenceProtocol.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { GROK_SESSION_TITLE_SINK_KEY, GROK_SESSION_TITLE_SINK_MODEL_ID, renderGrokLeanBaseConfig } from "./grokBrokerWorkerConfig.js"; + +const SPEC = GROK_ENGINE_BROKER.inferenceGrants.client; +/** The evaluator CLI's only model id: Paideia passes `--model daimon-inference-grok`, never a catalog id. */ +export const GROK_INFERENCE_CLIENT_MODEL_ID = SPEC.modelId; +/** The environment variable the evaluator CLI reads its grant token from. */ +export const GROK_INFERENCE_GRANT_ENV = SPEC.envKey; + +export type GrokInferenceClientConfigInput = Readonly<{ baseUrl: string; model: GrokBrokerModelPolicy["model"]; reasoningEffort: GrokBrokerModelPolicy["reasoningEffort"]; envKey: string }>; + +/** + * `config.toml` bytes for an evaluator Grok CLI (Paideia judge or optimizer, + * uid 2000) that reaches the broker proxy through an inference grant. + * + * Paideia writes it into a private `GROK_HOME` (`0700`, uid 2000) and sets the + * grant token in `envKey`; the CLI never holds the broker credential. Grok + * 1.0.34 ignores `[auth_provider.*]` helpers for a custom model, so the token + * travels through `env_key` exactly as the worker's turn capability does. + * + * It mirrors the worker renderer's lean settings: every bundled skill + * disabled, workflows off, the per-call `session_title` request pointed at a + * hidden model with a placeholder key the proxy refuses before any credential + * read or upstream call, and the declared effort as the model's single + * effort. There is no MCP server. `max_retries = 0`: with the default, a + * refused request (HTTP 503) is retried with backoff past a 45 s bound + * (live stub capture), so a gate refusal would hang the judge until its own + * timeout; with it the CLI fails in ~0.35 s and the caller's routed retry + * policy decides. HTTP 401 is never retried either way. `baseUrl` is the `baseUrl` of the grant (the + * loopback provider proxy); the manifest pins the sha256 for the production + * proxy URL and {@link GROK_INFERENCE_GRANT_ENV}. + * + * Paideia must also accept the init frame this produces: `apiKeySource` is + * `"user"` (not `"oauth"`), with `tools: []` and `mcp_servers: []` (live + * stub capture with the Paideia judge argv). + */ +export function renderGrokInferenceClientConfig(input: GrokInferenceClientConfigInput): string { + let declared: GrokBrokerModelPolicy; + try { declared = parseGrokBrokerModelPolicy({ model: input.model, reasoningEffort: input.reasoningEffort }); } catch { throw new TypeError("invalid Grok inference client configuration"); } + if (declared.model !== input.model || declared.reasoningEffort !== input.reasoningEffort) throw new TypeError("invalid Grok inference client configuration"); + if (typeof input.baseUrl !== "string" || !/^http:\/\/127\.0\.0\.1:([1-9][0-9]{0,4})\/v1$/u.test(input.baseUrl) || Number(input.baseUrl.slice(17, -3)) > 65_535) throw new TypeError("invalid Grok inference client configuration"); + if (typeof input.envKey !== "string" || !/^[A-Z][A-Z0-9_]{2,63}$/u.test(input.envKey)) throw new TypeError("invalid Grok inference client configuration"); + const label = `${declared.reasoningEffort[0]!.toUpperCase()}${declared.reasoningEffort.slice(1)}`; + return [ + renderGrokLeanBaseConfig(), + "[models]", `default = "${GROK_INFERENCE_CLIENT_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", + `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', `base_url = "${input.baseUrl}"`, `api_key = "${GROK_SESSION_TITLE_SINK_KEY}"`, "max_retries = 0", "hidden = true", "", + `[model.${GROK_INFERENCE_CLIENT_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "${input.baseUrl}"`, `env_key = "${input.envKey}"`, + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", + `[[model.${GROK_INFERENCE_CLIENT_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "" + ].join("\n"); +} + +export const grokInferenceClientConfigSha256 = (input: GrokInferenceClientConfigInput): string => + createHash("sha256").update(renderGrokInferenceClientConfig(input)).digest("hex"); + +/** The production bytes for a declared model/effort: the grant's proxy URL and the canonical env key. */ +export const renderProductionGrokInferenceClientConfig = (policy: GrokBrokerModelPolicy): string => + renderGrokInferenceClientConfig({ baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: policy.model, reasoningEffort: policy.reasoningEffort, envKey: GROK_INFERENCE_GRANT_ENV }); diff --git a/src/runtime/grokInferenceGrants.test.ts b/src/runtime/grokInferenceGrants.test.ts new file mode 100644 index 0000000..8fb97e1 --- /dev/null +++ b/src/runtime/grokInferenceGrants.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { GrokInferenceGrantRefused, GrokInferenceGrants } from "./grokInferenceGrants.js"; +import type { InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +const judge = { model: "grok-4.6", reasoningEffort: "low", purpose: "judge" } as const; +const refusedWith = (code: string) => (error: unknown) => error instanceof GrokInferenceGrantRefused && error.code === code; + +test("a grant is scoped to its declared model, effort and purpose and carries the manifest limits", () => { + const grants = new GrokInferenceGrants(); + try { + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "medium", purpose: "optimizer" }); + assert.match(issued.token, /^inference_[A-Za-z0-9_-]{43}$/u); assert.match(issued.grantId, /^[a-f0-9]{32}$/u); + assert.deepEqual(issued.limits, { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 }); + const grant = grants.authorize(issued.token); + assert.deepEqual(grant?.policy, { model: "grok-4.5", reasoningEffort: "medium" }); assert.equal(grant?.purpose, "optimizer"); + assert.equal(grants.authorize(issued.token.replace(/.$/u, (last) => last === "A" ? "B" : "A")), undefined); + } finally { grants.close(); } +}); + +test("a grant request naming an undeclared model, effort or purpose, or omitting one, is refused", () => { + const grants = new GrokInferenceGrants(); + for (const request of [{ ...judge, model: "grok-3" }, { ...judge, reasoningEffort: "xhigh" }, { ...judge, purpose: "subject" }, { ...judge, model: undefined }, { ...judge, reasoningEffort: undefined }]) { + assert.throws(() => grants.issue(request), refusedWith("invalid_request")); + } + assert.equal(grants.live(), 0); +}); + +test("the grant TTL can never exceed ten minutes", () => { + assert.equal(GROK_ENGINE_BROKER.inferenceGrants.ttlMs <= 600_000, true); + assert.throws(() => new GrokInferenceGrants({ ttlMs: 600_001 }), /invalid inference grant policy/u); +}); + +test("an expired grant is refused and frees its slot", () => { + let now = 1_000_000; + const grants = new GrokInferenceGrants({ now: () => now, maxLiveGrants: 1 }); + try { + const issued = grants.issue(judge); + now += 599_999; assert.ok(grants.authorize(issued.token)); + now += 1; assert.equal(grants.authorize(issued.token), undefined); + assert.equal(grants.live(), 0); assert.ok(grants.issue(judge)); + } finally { grants.close(); } +}); + +test("live grants are capped and a release frees a slot", () => { + const grants = new GrokInferenceGrants(); + try { + const issued = Array.from({ length: GROK_ENGINE_BROKER.inferenceGrants.maxLiveGrants }, () => grants.issue(judge)); + assert.throws(() => grants.issue(judge), refusedWith("grant_limit")); + assert.equal(grants.release(issued[3]!.grantId), true); assert.equal(grants.authorize(issued[3]!.token), undefined); + assert.ok(grants.issue(judge)); assert.throws(() => grants.issue(judge), refusedWith("grant_limit")); + } finally { grants.close(); } +}); + +test("grants never share a key space with turn capabilities", () => { + const turnId = "0123456789abcdef0123456789abcdef"; + const capabilities = new EngineBrokerCapabilities(); const turnToken = capabilities.issue("agent-a", turnId); + const grants = new GrokInferenceGrants({ grantId: () => turnId }); + try { + const issued = grants.issue(judge); + assert.equal(issued.grantId, turnId); + assert.deepEqual(capabilities.inspectToken(turnToken), { agentId: "agent-a", turnId }); + assert.equal(capabilities.inspectToken(issued.token), undefined); + assert.equal(grants.authorize(turnToken), undefined); + capabilities.revoke(turnId); assert.ok(grants.authorize(issued.token)); + grants.release(turnId); assert.equal(capabilities.inspectToken(turnToken), undefined); + } finally { grants.close(); } +}); + +test("a grant meters one request at a time and emits one row per settled request, estimated when usage is missing", () => { + const rows: InferenceUsageEntry[] = []; + const grants = new GrokInferenceGrants({ onSettled: (row) => rows.push(row) }); + try { + const grant = grants.authorize(grants.issue(judge).token)!; + const first = grant.meter.admit(); assert.ok("index" in first); + assert.deepEqual(grant.meter.admit(), { busy: true }); + grants.settle(grant, first.index, { input: 90, cacheRead: 10, cacheWrite: 0, output: 5, total: 105 }, 400); + grants.settle(grant, first.index, { input: 1, cacheRead: 0, cacheWrite: 0, output: 1, total: 2 }, 400); + const second = grant.meter.admit(); assert.ok("index" in second); + grants.settle(grant, second.index, undefined, 1_000); + assert.deepEqual(rows.map((row) => [row.request, row.usage.total, row.usageSource, row.purpose, row.model]), [[0, 105, "upstream", "judge", "grok-4.6"], [1, 4_596, "estimated", "judge", "grok-4.6"]]); + } finally { grants.close(); } +}); + +test("releasing a grant aborts its in-flight upstream request", () => { + const grants = new GrokInferenceGrants(); + const issued = grants.issue(judge); const grant = grants.authorize(issued.token)!; + const admission = grant.meter.admit(); assert.ok("signal" in admission); + grants.release(issued.grantId); + assert.equal(admission.signal.aborted, true); +}); diff --git a/src/runtime/grokInferenceGrants.ts b/src/runtime/grokInferenceGrants.ts new file mode 100644 index 0000000..a8c1b93 --- /dev/null +++ b/src/runtime/grokInferenceGrants.ts @@ -0,0 +1,119 @@ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; +import type { EngineBrokerTurnLimits, EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GROK_INFERENCE_PURPOSES, recordInferenceUsage, type GrokInferencePurpose, type InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; + +/** One live evaluator grant as the proxy sees it. */ +export type GrokInferenceGrant = Readonly<{ grantId: string; purpose: GrokInferencePurpose; policy: GrokBrokerModelPolicy; expiresAt: number; meter: GrokBrokerTurnMeter }>; +export type GrokInferenceGrantIssued = Readonly<{ grantId: string; token: string; purpose: GrokInferencePurpose; policy: GrokBrokerModelPolicy; expiresAt: number; limits: EngineBrokerTurnLimits }>; +export type GrokInferenceGrantRequest = Readonly<{ model: unknown; reasoningEffort: unknown; purpose: unknown }>; + +export class GrokInferenceGrantRefused extends Error { + constructor(readonly code: EngineBrokerInferenceFailureCode) { super(`inference grant refused (${code})`); } +} + +type Entry = { grant: GrokInferenceGrant; digest: Buffer; timer: NodeJS.Timeout }; +export type GrokInferenceGrantsOptions = Readonly<{ + now?: () => number; + /** Test seam: a fixed grant id proves grants never share a key space with turn capabilities. */ + grantId?: () => string; + onSettled?: (entry: InferenceUsageEntry) => void; + ttlMs?: number; + maxLiveGrants?: number; +}>; + +/** + * Evaluator inference grants: a distinct kind beside subject turn capabilities. + * + * Grants live in their own map keyed by a random grant id; turn capabilities + * (`engineBrokerCapabilities.ts`) are keyed by turn id and never consulted + * here, and grant tokens carry `inference_` so the proxy routes a bearer to + * exactly one of the two lookups. A grant has no worker isolation guard (it is + * issued only to the organization uid, the trusted evaluator side), but it + * carries the same spend gate as a subject turn: a {@link GrokBrokerTurnMeter} + * with one request in flight, the request ceiling, the between-requests token + * ceiling and the estimate for a response without usage. Its lifetime is the + * meter's time limit: past `expiresAt` it is gone from the map, and a request + * still in flight at expiry is aborted. + * + * At most `maxLiveGrants` grants exist at once; a caller frees a slot early + * with {@link release}. Nothing here is durable — a broker restart drops every + * grant, and callers request a new one. + */ +export class GrokInferenceGrants { + private readonly grants = new Map(); + private readonly now: () => number; + private readonly ttlMs: number; + private readonly maxLive: number; + constructor(private readonly options: GrokInferenceGrantsOptions = {}) { + this.now = options.now ?? Date.now; + this.ttlMs = options.ttlMs ?? SPEC.ttlMs; + this.maxLive = options.maxLiveGrants ?? SPEC.maxLiveGrants; + if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs < 1 || this.ttlMs > SPEC.ttlMs || !Number.isSafeInteger(this.maxLive) || this.maxLive < 1 || this.maxLive > SPEC.maxLiveGrants) throw new TypeError("invalid inference grant policy"); + } + + issue(request: GrokInferenceGrantRequest): GrokInferenceGrantIssued { + let policy: GrokBrokerModelPolicy; + try { policy = parseGrokBrokerModelPolicy({ model: request.model, reasoningEffort: request.reasoningEffort }); } catch { throw new GrokInferenceGrantRefused("invalid_request"); } + // Nothing is defaulted: the model parser fills an absent member, a grant must name both. + if (request.model !== policy.model || request.reasoningEffort !== policy.reasoningEffort || !(GROK_INFERENCE_PURPOSES as readonly unknown[]).includes(request.purpose)) throw new GrokInferenceGrantRefused("invalid_request"); + this.prune(); + if (this.grants.size >= this.maxLive) throw new GrokInferenceGrantRefused("grant_limit"); + const grantId = this.options.grantId?.() ?? randomBytes(16).toString("hex"); + if (!/^[a-f0-9]{32}$/u.test(grantId) || this.grants.has(grantId)) throw new GrokInferenceGrantRefused("grant_limit"); + const limits: EngineBrokerTurnLimits = Object.freeze({ maxRequests: SPEC.limits.maxRequests, maxTokens: SPEC.limits.maxTokens, timeoutMs: this.ttlMs }); + const token = `${SPEC.tokenPrefix}${randomBytes(32).toString("base64url")}`; + const grant: GrokInferenceGrant = Object.freeze({ grantId, purpose: request.purpose as GrokInferencePurpose, policy, expiresAt: this.now() + this.ttlMs, meter: new GrokBrokerTurnMeter(limits, () => undefined, this.now) }); + const timer = setTimeout(() => this.release(grantId), this.ttlMs); timer.unref?.(); + this.grants.set(grantId, { grant, digest: digest(token), timer }); + return Object.freeze({ grantId, token, purpose: grant.purpose, policy, expiresAt: grant.expiresAt, limits }); + } + + /** The live, unexpired grant a bearer names, or `undefined`. Never counts a request. */ + authorize(token: string): GrokInferenceGrant | undefined { + if (!token.startsWith(SPEC.tokenPrefix)) return undefined; + this.prune(); + const candidate = digest(token); + for (const entry of this.grants.values()) if (timingSafeEqual(entry.digest, candidate)) return entry.grant; + return undefined; + } + + /** Records one admitted request's end on the grant's meter and emits its ledger row. */ + settle(grant: GrokInferenceGrant, index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { + const before = grant.meter.snapshot().timings[index]; + if (before === undefined || before.endedAt !== undefined) return; + grant.meter.settle(index, usage, requestBytes); + const timing = grant.meter.snapshot().timings[index]; + if (timing?.usage === undefined || timing.endedAt === undefined) return; + this.options.onSettled?.({ grant: grant.grantId, purpose: grant.purpose, model: grant.policy.model, request: index, usage: timing.usage, usageSource: timing.estimated === true ? "estimated" : "upstream", startedAt: timing.startedAt, endedAt: timing.endedAt }); + } + + /** Revokes a grant and aborts its in-flight request. Returns whether it was live. */ + release(grantId: string): boolean { + const entry = this.grants.get(grantId); + if (entry === undefined) return false; + clearTimeout(entry.timer); entry.grant.meter.trip("timeout"); entry.digest.fill(0); this.grants.delete(grantId); + return true; + } + + live(): number { this.prune(); return this.grants.size; } + + close(): void { for (const grantId of [...this.grants.keys()]) this.release(grantId); } + + private prune(): void { + const now = this.now(); + for (const [grantId, entry] of this.grants) if (entry.grant.expiresAt <= now) this.release(grantId); + } +} + +/** The broker's grants: every settled request is appended to the evaluator inference ledger and nowhere else. */ +export const createLedgeredGrokInferenceGrants = (inferenceLedgerPath: string): GrokInferenceGrants => + new GrokInferenceGrants({ onSettled: (entry) => { void recordInferenceUsage(inferenceLedgerPath, entry); } }); + +const digest = (value: string): Buffer => createHash("sha256").update(value).digest(); diff --git a/src/runtime/grokInferenceLedger.test.ts b/src/runtime/grokInferenceLedger.test.ts new file mode 100644 index 0000000..26fcaab --- /dev/null +++ b/src/runtime/grokInferenceLedger.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { NativeBrokerTurn } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; +import { createLedgeredGrokInferenceGrants } from "./grokInferenceGrants.js"; +import { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION } from "./inferenceUsageLedger.js"; +import { WakeFuse } from "./wakeFuse.js"; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const judgeBody = JSON.stringify({ messages: [{ role: "system", content: "judge" }, { role: "user", content: "Rate." }], model: "grok-4.5", reasoning_effort: "medium", stream: true, stream_options: { include_usage: true } }); +const post = (port: number, bearer: string, body: string): Promise => new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${bearer}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(body); +}); +const rows = async (file: string): Promise[]> => (await readFile(file, "utf8").catch(() => "")).split("\n").filter(Boolean).map((line) => JSON.parse(line) as Record); +const eventually = async (check: () => Promise): Promise => { for (let attempt = 0; attempt < 100 && !await check(); attempt++) await new Promise((resolve) => setTimeout(resolve, 10)); }; + +test("a judge grant used while a subject turn runs meters only into the inference ledger and never into the subject turn, its ledger or the wake fuse", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-inference-ledger-")); + const usageLedger = path.join(root, "slot0", "usage.jsonl"), inferenceLedger = path.join(root, "evaluator", "inference.jsonl"); + await mkdir(path.dirname(usageLedger)); await mkdir(path.dirname(inferenceLedger)); + const grants = createLedgeredGrokInferenceGrants(inferenceLedger); + const upstreamUsage = (model: string) => ({ prompt_tokens: model === "grok-4.6" ? 1_000 : 40_000, completion_tokens: 50, total_tokens: model === "grok-4.6" ? 1_050 : 40_050 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (request) => { + const model = (JSON.parse(Buffer.from(request.body).toString("utf8")) as { model: string }).model; + return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage(model) })}\n\ndata: [DONE]\n\n`) }; + }, undefined, 0, grants); + try { + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "medium", purpose: "judge" }); + const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: usageLedger, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } }; + let judgeStatus = 0; + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + prepareIsolation: async () => async () => undefined, + runNative: async (input: NativeBrokerTurn) => { + assert.equal(await post(proxy.port, input.providerCapability, leanBody), 200); + judgeStatus = await post(proxy.port, issued.token, judgeBody); + assert.equal(await post(proxy.port, input.providerCapability, leanBody), 200); + return { text: "", workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }; + } + }; + await assert.rejects(runGrokEngineBrokerTurn(deps, registration, "wake-1", "prompt", "http://127.0.0.1:43124/mcp")); + assert.equal(judgeStatus, 200); + await eventually(async () => (await rows(inferenceLedger)).length > 0); + + const subject = await rows(usageLedger), subjectRequests = await rows(path.join(path.dirname(usageLedger), "requests.jsonl")), inference = await rows(inferenceLedger); + assert.deepEqual(subject.map((row) => [row.requests ?? row.calls, row.total, row.model]), [[2, 2_100, "grok-4.6"]], "the subject turn counts only its own two requests"); + assert.deepEqual(subjectRequests.map((row) => row.total), [1_050, 1_050]); + assert.deepEqual(inference.map((row) => [row.v, row.kind, row.purpose, row.grant, row.request, row.model, row.total, row.usage_source]), [[INFERENCE_USAGE_LEDGER_VERSION, "inference", "judge", issued.grantId, 0, "grok-4.5", 40_050, "upstream"]]); + + // Even if an inference row reached the subject ledger, the wake fuse would not count it: + // 2,100 subject tokens are under a 2,101 ceiling; counted, the 40,050-token row would trip it. + await writeFile(usageLedger, [...subject, ...inference].map((row) => JSON.stringify(row)).join("\n") + "\n"); + const fuseDirectory = path.join(root, "fuse"); await mkdir(fuseDirectory); + const fuse = await WakeFuse.open({ organizationKey: "org", now: () => new Date(Date.parse(String(subject[0]!.at)) - 1), environment: { DAIMON_WAKE_FUSE_DIRECTORY: fuseDirectory, DAIMON_WAKE_FUSE_EPOCH: "grants", DAIMON_WAKE_FUSE_MAX_WAKES: "10", DAIMON_WAKE_FUSE_MAX_TOKENS: "2101", DAIMON_TURN_USAGE_LEDGER_PATH: usageLedger } }); + assert.deepEqual(await fuse.admit("foreman", "next"), { state: "admitted" }); + } finally { grants.close(); await proxy.close(); await rm(root, { recursive: true, force: true }); } +}); + +test("inference readers count each (grant, request) once", () => { + type Row = Readonly<{ grant?: string; request?: number; total: number }>; + const row = (grant: string, request: number, total: number): Row => ({ grant, request, total }); + assert.deepEqual(dedupeInferenceUsageRows([row("a", 0, 1), row("a", 1, 2), row("a", 0, 1), row("b", 0, 3), { total: 9 }]).map((value) => value.total), [1, 2, 3]); +}); diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts new file mode 100644 index 0000000..8e1f44c --- /dev/null +++ b/src/runtime/grokInferenceProxy.test.ts @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import test from "node:test"; + +import { startGrokBrokerProxy, type GrokBrokerCredentialAuthority, type GrokBrokerUpstream } from "./grokBrokerProxy.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GrokInferenceGrants } from "./grokInferenceGrants.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +import type { InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +// The judge main request Grok 1.0.34 sends (live capture, `--json-schema` variant). +const judgeBody = (overrides: Record = {}): string => JSON.stringify({ + messages: [{ role: "system", content: "You are a strict judge." }, { role: "user", content: "..." }, { role: "user", content: "Rate the answer." }], + model: "grok-4.6", reasoning_effort: "low", + response_format: { type: "json_schema", json_schema: { name: "structured_output", schema: { type: "object", properties: { score: { type: "number" } }, required: ["score"], additionalProperties: false }, strict: true } }, + stream: true, stream_options: { include_usage: true }, ...overrides +}); +// The per-call session_title request the same CLI sends first (live capture). +const titleBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", temperature: 1, max_tokens: 100, messages: [{ role: "system", content: "title" }, { role: "user", content: "Rate the answer." }], tools: [{ type: "function", function: { name: "session_title", description: "", parameters: {} } }], tool_choice: { type: "function", function: { name: "session_title" } }, stream: true, stream_options: { include_usage: true } }); +const usageStream = (total: number) => Buffer.from(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "{\"score\":1}" } }] })}\n\ndata: ${JSON.stringify({ choices: [], usage: { prompt_tokens: total - 5, completion_tokens: 5, total_tokens: total } })}\n\ndata: [DONE]\n\n`); + +type Harness = Readonly<{ port: number; grants: GrokInferenceGrants; rows: InferenceUsageEntry[]; bodies: Record[]; proxy: Awaited> }>; +async function withProxy(run: (harness: Harness) => Promise, options: Readonly<{ authority?: GrokBrokerCredentialAuthority; upstream?: GrokBrokerUpstream }> = {}): Promise { + const rows: InferenceUsageEntry[] = [], bodies: Record[] = []; + const grants = new GrokInferenceGrants({ onSettled: (row) => rows.push(row) }); + const upstream: GrokBrokerUpstream = options.upstream ?? (async (request) => { bodies.push(JSON.parse(Buffer.from(request.body).toString("utf8")) as Record); return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(105) }; }); + const proxy = await startGrokBrokerProxy(options.authority ?? { accessToken: async () => "provider-token", markRejected: async () => undefined }, upstream, undefined, 0, grants); + try { await run({ port: proxy.port, grants, rows, bodies, proxy }); } finally { grants.close(); await proxy.close(); } +} + +function post(port: number, bearer: string, body: string, version = "1.0.34"): Promise> { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${bearer}`, "x-grok-client-version": version, "content-type": "application/json" } }, (response) => { + const chunks: Buffer[] = []; response.on("data", (chunk: Buffer) => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }); + req.on("error", reject); req.end(body); + }); +} + +test("a grant forwards the captured judge request re-serialized under the declared model and meters it into the inference rows only", async () => { + await withProxy(async ({ port, grants, rows, bodies }) => { + const issued = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const result = await post(port, issued.token, judgeBody().replace('"model":', '"model":"grok-4.5","model":')); + assert.equal(result.status, 200); assert.match(result.text, /score/u); + assert.equal(bodies.length, 1); assert.equal(bodies[0]!.model, "grok-4.6"); + assert.deepEqual(rows.map((row) => [row.grant, row.request, row.usage.total, row.usageSource, row.purpose]), [[issued.grantId, 0, 105, "upstream", "judge"]]); + assert.equal((await post(port, issued.token, judgeBody({ response_format: undefined }))).status, 200); + assert.equal(rows.length, 2); + }); +}); + +test("a grant refuses any tools member, the session_title request, and undeclared model or effort, before any upstream call or row", async () => { + await withProxy(async ({ port, grants, rows, bodies }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const refused = [ + judgeBody({ tools: [] }), judgeBody({ tools: [{ type: "function", function: { name: "read_file" } }] }), judgeBody({ tool_choice: "none" }), titleBody, + judgeBody({ model: "grok-4.5" }), judgeBody({ reasoning_effort: "high" }), judgeBody({ reasoning_effort: undefined }), + judgeBody({ stream: false }), judgeBody({ stream_options: undefined }), judgeBody({ temperature: 1 }), + judgeBody({ messages: [{ role: "tool", content: "x" }] }), judgeBody({ messages: [{ role: "assistant", content: null, tool_calls: [] }] }), + judgeBody({ response_format: { type: "json_object" } }) + ]; + // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. + for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); + assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); + // The title sink keeps its transient 503 shape on the grant path too. + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); + assert.equal(bodies.length, 0); assert.equal(rows.length, 0); + }); +}); + +test("an expired, released or unknown grant is refused", async () => { + await withProxy(async ({ port, grants, bodies }) => { + const released = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); grants.release(released.grantId); + assert.equal((await post(port, released.token, judgeBody())).status, 400); + assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 400); + assert.equal(bodies.length, 0); + }); + let now = 5_000; + const grants = new GrokInferenceGrants({ now: () => now }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(10) }), undefined, 0, grants); + try { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + assert.equal((await post(proxy.port, token, judgeBody())).status, 200); + now += 600_000; + assert.equal((await post(proxy.port, token, judgeBody())).status, 400); + } finally { grants.close(); await proxy.close(); } +}); + +test("a grant token never authorizes a subject turn and a turn capability never authorizes a grant request", async () => { + await withProxy(async ({ port, grants, proxy, bodies }) => { + const { token: grantToken } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const turnToken = proxy.capabilities.issue("agent-a", "turn-a"); + proxy.registerIsolationGuard("turn-a", async () => undefined); + proxy.registerTurn("turn-a", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }) }); + const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); + const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); + // Cross-use is a policy miss on both paths: refused 400, never a retryable 503. + assert.equal((await post(port, grantToken, leanBody)).status, 400); + assert.equal((await post(port, turnToken, judgeBody())).status, 400); + assert.equal(bodies.length, 0); + assert.equal((await post(port, turnToken, leanBody)).status, 200); + assert.equal((await post(port, grantToken, judgeBody())).status, 200); + }); +}); + +test("a stale realm answers a grant request with the distinct auth_stale failure, and a rejected refresh too", async () => { + let stale = true; + await withProxy(async ({ port, grants, bodies }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const result = await post(port, token, judgeBody()); + assert.equal(result.status, 401); assert.equal(result.text, GROK_INFERENCE_AUTH_STALE_BODY); assert.equal(bodies.length, 0); + }, { authority: { accessToken: async () => { if (stale) throw new Error("stale"); return "t"; }, markRejected: async () => undefined, isStale: () => stale } }); + stale = false; let rejected = 0; + await withProxy(async ({ port, grants, rows }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "optimizer" }); + const result = await post(port, token, judgeBody()); + assert.equal(result.status, 401); assert.equal(result.text, GROK_INFERENCE_AUTH_STALE_BODY); assert.equal(rejected, 1); + assert.deepEqual(rows.map((row) => row.usageSource), ["estimated"]); + }, { authority: { accessToken: async (force) => force ? "second" : "first", markRejected: async () => { rejected++; stale = true; throw new Error("stale"); }, isStale: () => stale }, upstream: async () => ({ status: 401, headers: { "content-type": "application/json" }, body: new Uint8Array() }) }); +}); + +test("a grant's request ceiling and one-in-flight rule hold on the wire", async () => { + let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); + let calls = 0; + await withProxy(async ({ port, grants, rows }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const first = post(port, token, judgeBody()); + while (calls === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const busy = await post(port, token, judgeBody()); assert.equal(busy.status, 429); assert.match(busy.text, /in flight/u); + release(); assert.equal((await first).status, 200); + const grant = grants.authorize(token)!; + for (let index = 1; index < 64; index++) { const admission = grant.meter.admit(); assert.ok("index" in admission); grants.settle(grant, admission.index, { input: 1, cacheRead: 0, cacheWrite: 0, output: 1, total: 2 }, 10); } + const over = await post(port, token, judgeBody()); assert.equal(over.status, 429); assert.match(over.text, /requests/u); + assert.equal(calls, 1); assert.equal(rows.length, 64); + }, { upstream: async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(50) }; } }); +}); + +test("a grant whose id equals a live turn id leaves that turn's capability, policy and meter untouched", async () => { + const shared = "0123456789abcdef0123456789abcdef"; + const grants = new GrokInferenceGrants({ grantId: () => shared }); + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(20) }; }, undefined, 0, grants); + try { + const turnToken = proxy.capabilities.issue("agent-a", shared); + const turnMeter = new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }); + proxy.registerIsolationGuard(shared, async () => undefined); + proxy.registerTurn(shared, { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: turnMeter }); + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "high", purpose: "judge" }); + assert.equal(issued.grantId, shared); + const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); + const leanBody = JSON.stringify({ model: "grok-4.6", reasoningEffort: undefined, reasoning_effort: "low", stream: true, messages: [], tools: lean }); + assert.equal((await post(proxy.port, turnToken, leanBody)).status, 200); + assert.equal((await post(proxy.port, issued.token, judgeBody({ model: "grok-4.5", reasoning_effort: "high" }))).status, 200); + assert.equal(turnMeter.snapshot().requests, 1); assert.equal(grants.authorize(issued.token)!.meter.snapshot().requests, 1); + grants.release(shared); + assert.equal((await post(proxy.port, turnToken, leanBody)).status, 200); + assert.equal(turnMeter.snapshot().requests, 2); assert.equal(calls, 3); + } finally { grants.close(); await proxy.close(); } +}); diff --git a/src/runtime/grokInferenceProxy.ts b/src/runtime/grokInferenceProxy.ts new file mode 100644 index 0000000..34acc99 --- /dev/null +++ b/src/runtime/grokInferenceProxy.ts @@ -0,0 +1,73 @@ +import { createHash } from "node:crypto"; +import type { ServerResponse } from "node:http"; + +import { GrokBrokerProxyRefusal } from "./grokBrokerProxy.js"; +import type { GrokBrokerCredentialAuthority, GrokBrokerUpstream } from "./grokBrokerProxy.js"; +import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; +import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; +import { authorizeGrokInferenceProxyRequest } from "./grokInferenceProxyRequest.js"; + +export type GrokInferenceProxyInput = Readonly<{ method: string; pathname: string; headers: Readonly>; body: Buffer; token: string }>; + +/** HTTP 401 with a fixed code: the broker realm is stale, distinct from a generic 503 broker failure. */ +export const GROK_INFERENCE_AUTH_STALE_BODY = '{"error":"auth_stale"}'; + +/** + * Serves one evaluator grant request; never throws. + * + * Order mirrors the subject path: the body is proven a grant-shaped request + * (declared model/effort, no tools) before the credential is read, the grant + * meter admits it before any upstream call, and exactly one ledger row is + * written once an admitted request settles. No worker isolation guard applies: + * grants are issued only to the organization uid. + * + * Stale realm: the grant shares the subject's credential authority, so a + * stale realm fails judges and subject turns alike (accepted shared fate). A + * grant request that finds the realm stale — before the upstream call or + * after a rejected refresh — is answered 401 `{"error":"auth_stale"}`, never + * the generic 503, so the evaluator can report it as a credential failure. + */ +export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, response: ServerResponse, grants: GrokInferenceGrants, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream): Promise { + let settle: ((usage: ReturnType) => void) | undefined; + try { + const grant = grants.authorize(input.token); + if (grant === undefined) throw new GrokBrokerProxyRefusal("unknown_or_expired_grant"); + let prepared: ReturnType; + try { prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } + let token = await authority.accessToken(false); const rejectedDigest = createHash("sha256").update(token).digest("hex"); + prepared = withBearer(prepared, token); token = ""; + const admission = grant.meter.admit(); + if ("refused" in admission) return json(response, 429, JSON.stringify({ error: "grant limit reached", limit: admission.refused })); + if ("busy" in admission) return json(response, 429, '{"error":"grant request in flight"}'); + settle = (usage) => { settle = undefined; grants.settle(grant, admission.index, usage, input.body.byteLength); }; + let result = await upstream(prepared, admission.signal); + if (result.status === 401) { + token = authority.refreshAfterRejection ? await authority.refreshAfterRejection(rejectedDigest) : await authority.accessToken(true); + const refreshedDigest = createHash("sha256").update(token).digest("hex"); prepared = withBearer(prepared, token); token = ""; + result = await upstream(prepared, admission.signal); + if (result.status === 401) await authority.markRejected(refreshedDigest); + } + settle?.(parseGrokUpstreamUsage(result.body, result.headers["content-type"])); + json(response, result.status, result.body, result.headers["content-type"]); + } catch (error) { + settle?.(undefined); + // Same rule as the subject path: a policy miss is non-retryable (400), because a + // retryable 503 makes the client re-send a request the broker will never accept, + // charging estimated usage for every attempt. 503 stays for transient faults only. + process.stderr.write(`[grok-proxy] inference refused: ${error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"}\n`); + if (authority.isStale?.() === true) json(response, 401, GROK_INFERENCE_AUTH_STALE_BODY); + else if (error instanceof GrokBrokerProxyRefusal) json(response, 400, JSON.stringify({ error: "broker refused this request", reason: error.reason })); + else json(response, 503, '{"error":"broker unavailable"}'); + } +} + +const withBearer = (prepared: ReturnType, token: string): ReturnType => { + if (!token || /[\r\n]/u.test(token)) throw new Error("broker credential authority unavailable"); + return { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; +}; + +function json(response: ServerResponse, status: number, body: string | Uint8Array, contentType = "application/json"): void { + if (response.headersSent) return; + response.writeHead(status, { "content-type": contentType, "cache-control": "no-store" }); response.end(body); +} diff --git a/src/runtime/grokInferenceProxyRequest.ts b/src/runtime/grokInferenceProxyRequest.ts new file mode 100644 index 0000000..51796cf --- /dev/null +++ b/src/runtime/grokInferenceProxyRequest.ts @@ -0,0 +1,54 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { GrokBrokerProxyInput, GrokBrokerUpstreamRequest } from "./grokBrokerProxyRequest.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; + +const MAX_BODY = 2 * 1024 * 1024; +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; +const BODY_MEMBERS: ReadonlySet = new Set(SPEC.bodyMembers); +const ROLES: ReadonlySet = new Set(SPEC.messageRoles); +type JsonRecord = Record; +const plain = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); +const rejected = (): Error => new Error("inference grant request rejected"); + +/** + * Authorizes one evaluator grant request body and rebuilds it for the provider. + * + * The accepted shape is exactly what Grok CLI 1.0.34 sends for a Paideia + * judge/optimizer call (`--tools read_file --disallowed-tools + * read_file,search_tool,use_tool --max-turns 1`, with and without + * `--system-prompt-override` and `--json-schema`; live stub capture): + * + * - `stream: true` with `stream_options: {include_usage: true}` — the CLI never + * sends a non-streaming request, so none is accepted; + * - `model` and `reasoning_effort` equal to the grant's declaration; + * - `messages` of plain `{role, content}` string turns, roles system/user/assistant; + * - optional `response_format` `{type: "json_schema", json_schema: {name, schema, strict}}`; + * - no `tools` and no `tool_choice` member at all, not even an empty one. The + * CLI's per-call `session_title` request carries both and is refused here. + * + * The client version must be the pinned CLI, and the forwarded body is the + * re-serialized parse, never the caller's bytes. + */ +export function authorizeGrokInferenceProxyRequest(input: Omit, bearer: string, policy: GrokBrokerModelPolicy): GrokBrokerUpstreamRequest { + const declared = parseGrokBrokerModelPolicy(policy); + if (input.method !== "POST" || input.pathname !== "/v1/chat/completions" || input.body.byteLength < 2 || input.body.byteLength > MAX_BODY) throw rejected(); + if (!bearer || /[\r\n]/u.test(bearer)) throw new Error("broker credential authority unavailable"); + const clientVersion = input.headers["x-grok-client-version"]; + if (clientVersion !== GROK_ENGINE_BROKER.grokCliVersion) throw rejected(); + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8"), (key, value: unknown) => { if (key === "__proto__") throw new Error(); return value; }); } catch { throw rejected(); } + if (!plain(parsed) || Object.keys(parsed).some((key) => !BODY_MEMBERS.has(key))) throw rejected(); + if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort || parsed.stream !== true) throw rejected(); + if (!plain(parsed.stream_options) || Object.keys(parsed.stream_options).length !== 1 || parsed.stream_options.include_usage !== true) throw rejected(); + if (!Array.isArray(parsed.messages) || parsed.messages.length === 0 || !parsed.messages.every(plainMessage)) throw rejected(); + if (parsed.response_format !== undefined && !jsonSchemaFormat(parsed.response_format)) throw rejected(); + return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: Buffer.from(JSON.stringify(parsed)) }; +} + +const plainMessage = (message: unknown): boolean => plain(message) && Object.keys(message).length === 2 && ROLES.has(message.role as string) && typeof message.content === "string"; + +const jsonSchemaFormat = (value: unknown): boolean => { + if (!plain(value) || Object.keys(value).length !== 2 || value.type !== "json_schema" || !plain(value.json_schema)) return false; + const schema = value.json_schema; + return Object.keys(schema).every((key) => key === "name" || key === "schema" || key === "strict") && typeof schema.name === "string" && plain(schema.schema) && (schema.strict === undefined || typeof schema.strict === "boolean"); +}; diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts new file mode 100644 index 0000000..2cc85d2 --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { resolveOrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; +import { parseGrokSlotPreflightReceipt, verifyGrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; + +const fresh = { expectedNonce: "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", minGeneration: 3 } as const; + +const fixture = async (name: string): Promise> => JSON.parse(await readFile(new URL(`./fixtures/grok-slot-preflight/${name}`, import.meta.url), "utf8")) as Record; +const projection = async () => { const input = await fixture("projection-input.json") as { config: unknown; agentId: string; options: Parameters[2] }; return resolveOrganizationGrokBrokerProjection(input.config, input.agentId, input.options); }; + +test("the committed valid receipt fixture proves the committed projection input", async () => { + const receipt = verifyGrokSlotPreflightReceipt(await fixture("receipt.valid.v2.json"), await projection(), fresh); + assert.equal(receipt.canaries.length, (await projection()).denyPaths.length); + assert.ok(receipt.canaries.every((canary) => canary.method === "sandboxed-read" && canary.result === "denied")); +}); + +test("the schema refuses a readable canary, an unknown member, duplicates and malformed digests or times", async () => { + const valid = await fixture("receipt.valid.v2.json"); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.readable-canary.json")), /invalid Grok slot preflight receipt/u); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.unknown-member.json")), /invalid Grok slot preflight receipt/u); + const canaries = valid.canaries as Record[]; + for (const bad of [ + { ...valid, canaries: [...canaries, canaries[0]] }, + { ...valid, canaries: [{ ...canaries[0], method: "stat" }] }, + { ...valid, canaries: [{ ...canaries[0], path: "/run/../etc" }] }, + { ...valid, canaries: [{ ...canaries[0], extra: true }] }, + { ...valid, canaries: [] }, + { ...valid, projection_sha256: "A".repeat(64) }, + { ...valid, worker_uid: 2_000 }, + { ...valid, created_at: "2026-09-17T12:00:00Z" }, + { ...valid, version: "noopolis.daimon.grok-slot-preflight.v3" } + ]) assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); +}); + +test("a receipt for a different projection, slot, profile or deny set is refused", async () => { + const projected = await projection(); + const valid = await fixture("receipt.valid.v2.json"); + // Mutation guard: dropping the digest comparison accepts this fixture. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected, fresh), /projection_sha256/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected, fresh), /canaries/u); + // Exact match, both halves: a canary for a path the projection does not deny is as wrong as a missing one. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.extra-canary.json"), projected, fresh), /canaries/u); + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }, fresh), /projection_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected, fresh), /sandbox_profile_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected, fresh), /slot/u); + // Mutation guard: never comparing the seccomp digest accepts a receipt taken under another profile. + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, seccomp_profile_sha256: "8".repeat(64) }, projected, fresh), /seccomp_profile_sha256/u); + assert.throws(() => parseGrokSlotPreflightReceipt({ ...valid, sandbox_runtime: "none" }), /invalid Grok slot preflight receipt/u); + assert.throws(() => parseGrokSlotPreflightReceipt((({ sandbox_runtime: _omit, ...rest }) => rest)(valid)), /invalid Grok slot preflight receipt/u); +}); + +test("a receipt from an earlier recycle is refused: another nonce, a lower generation, or a v1 receipt without freshness", async () => { + const projected = await projection(); + const valid = await fixture("receipt.valid.v2.json"); + assert.equal(verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, minGeneration: 1 }).generation, 3); + // Mutation guard: ignoring the nonce accepts a receipt written for another recycle request. + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, expectedNonce: "a".repeat(64) }), /stale: nonce/u); + // Mutation guard: ignoring the generation accepts a receipt older than the last one accepted. + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, minGeneration: 4 }), /stale: generation/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.legacy-v1.json"), projected, fresh), /invalid Grok slot preflight receipt/u); + for (const bad of [{ ...valid, nonce: "A".repeat(64) }, { ...valid, nonce: "ab" }, { ...valid, generation: 0 }, { ...valid, generation: 1.5 }, (({ nonce: _omit, ...rest }) => rest)(valid), (({ generation: _omit, ...rest }) => rest)(valid)]) { + assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); + } + for (const freshness of [{ expectedNonce: "short", minGeneration: 1 }, { expectedNonce: fresh.expectedNonce, minGeneration: 0 }, { expectedNonce: fresh.expectedNonce.toUpperCase(), minGeneration: 1 }]) { + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, freshness), /invalid Grok slot preflight freshness/u); + } +}); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts new file mode 100644 index 0000000..791b9b3 --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -0,0 +1,104 @@ +import path from "node:path"; +import { z } from "zod"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokBrokerProjectionSha256, type OrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; + +export const GROK_SLOT_PREFLIGHT_VERSION = GROK_ENGINE_BROKER.slotPreflightVersion; + +const sha256 = z.string().regex(/^[a-f0-9]{64}$/u); +const NONCE = /^[a-f0-9]{64}$/u; +const canonicalAbsolute = z.string().max(4_096).refine((value) => path.posix.isAbsolute(value) && path.posix.normalize(value) === value && value !== "/" && !value.endsWith("/") && !value.includes("\0"), "canonical absolute path"); + +/** + * One denied-path canary: the root supervisor ran a real sandboxed read of + * `path` as the slot's worker uid (a stub-model turn under the attested + * profile) and the read was denied. Only denials are representable — a + * supervisor that observed a readable path writes no receipt at all. + */ +export const grokSlotPreflightCanarySchema = z.strictObject({ + path: canonicalAbsolute, + method: z.literal("sandboxed-read"), + result: z.literal("denied") +}); + +/** + * `noopolis.daimon.grok-slot-preflight.v2`: what the root slot supervisor (P5) + * writes after provisioning or recycling one broker slot, and what an + * evaluator (Paideia, P4) must hold before it runs a Grok subject turn in that + * slot. It binds the slot to one exact projection by digest, so any change to + * the model, limits, deny list, profile, worker config, or pinned executable + * invalidates it. + * + * The projection digest is identical across recycles of the same slot, so v1 + * could not tell this recycle's receipt from an earlier one. v2 adds + * freshness: `generation` is the supervisor-owned per-slot counter, strictly + * increasing on every provision/recycle, and `nonce` echoes the 32 random + * bytes (hex) the evaluator passed in its recycle request. A v1 receipt is + * refused. + */ +export const grokSlotPreflightReceiptSchema = z.strictObject({ + version: z.literal(GROK_SLOT_PREFLIGHT_VERSION), + slot: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + worker_uid: z.number().int().min(GROK_ENGINE_BROKER.identities.firstWorkerUid).max(4_294_967_294), + /** Supervisor-owned, strictly increasing per slot across provisions and recycles; starts at 1. */ + generation: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER), + /** The caller's recycle nonce: 32 random bytes, lowercase hex. */ + nonce: z.string().regex(NONCE), + projection_sha256: sha256, + /** The bubblewrap/Landlock `daimon-strict` profile bytes' digest (the projection's `profileSha256`). */ + sandbox_profile_sha256: sha256, + /** The container seccomp profile the worker ran under. */ + seccomp_profile_sha256: sha256, + /** Grok 1.0.34 runs every profile inside bubblewrap; the supervisor observed it present and working. */ + sandbox_runtime: z.literal("bubblewrap"), + grok_executable_sha256: sha256, + canaries: z.array(grokSlotPreflightCanarySchema).min(1).max(256), + created_at: z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u).refine((value) => !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value, "exact RFC3339 timestamp") +}).superRefine((receipt, context) => { + const paths = receipt.canaries.map((canary) => canary.path); + if (new Set(paths).size !== paths.length) context.addIssue({ code: "custom", path: ["canaries"], message: "duplicate canary path" }); +}); + +export type GrokSlotPreflightReceipt = z.infer; + +/** Strict parse: unknown members, off-contract values, or duplicate canaries throw. */ +export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflightReceipt { + const result = grokSlotPreflightReceiptSchema.safeParse(value); + if (!result.success) throw new TypeError(`invalid Grok slot preflight receipt: ${result.error.issues.map((issue) => `${issue.path.join(".") || "receipt"}: ${issue.message}`).join("; ")}`); + return result.data; +} + +/** + * What the evaluator knows about *this* recycle: the nonce it sent, and the + * lowest generation it will accept — one above the last generation it + * accepted for the slot (1 for a slot it has never seen). + */ +export type GrokSlotPreflightFreshness = Readonly<{ expectedNonce: string; minGeneration: number }>; + +/** + * Parse a receipt and require that it proves *this* projection's slot: same + * digest, slot, worker uid, profile and executable, and a denied canary for + * exactly every projected deny path (no more, no fewer), under the projected + * seccomp profile and sandbox runtime — and that it is *this* recycle's + * receipt: the caller's nonce, at or above the caller's minimum generation. + * A receipt replayed from an earlier recycle fails on the nonce, and one + * whose generation went backwards fails on the generation. + */ +export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection, freshness: GrokSlotPreflightFreshness): GrokSlotPreflightReceipt { + if (freshness === null || typeof freshness !== "object" || typeof freshness.expectedNonce !== "string" || !NONCE.test(freshness.expectedNonce) || !Number.isSafeInteger(freshness.minGeneration) || freshness.minGeneration < 1) throw new TypeError("invalid Grok slot preflight freshness"); + const receipt = parseGrokSlotPreflightReceipt(value); + const mismatch = (member: string): never => { throw new Error(`Grok slot preflight receipt does not match the projection: ${member}`); }; + if (receipt.nonce !== freshness.expectedNonce) throw new Error("Grok slot preflight receipt is stale: nonce"); + if (receipt.generation < freshness.minGeneration) throw new Error("Grok slot preflight receipt is stale: generation"); + if (receipt.projection_sha256 !== grokBrokerProjectionSha256(projection)) mismatch("projection_sha256"); + if (receipt.slot !== projection.slot) mismatch("slot"); + if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); + if (receipt.sandbox_profile_sha256 !== projection.profileSha256) mismatch("sandbox_profile_sha256"); + if (receipt.grok_executable_sha256 !== projection.grokExecutableSha256) mismatch("grok_executable_sha256"); + if (receipt.seccomp_profile_sha256 !== projection.seccompProfileSha256) mismatch("seccomp_profile_sha256"); + if (receipt.sandbox_runtime !== projection.attestation.sandboxRuntime) mismatch("sandbox_runtime"); + const denied = receipt.canaries.map((canary) => canary.path).sort(); + if (denied.length !== projection.denyPaths.length || denied.some((entry, index) => entry !== projection.denyPaths[index])) mismatch("canaries"); + return receipt; +} diff --git a/src/runtime/grokWorkerAttestation.test.ts b/src/runtime/grokWorkerAttestation.test.ts index 7dc230d..92920e3 100644 --- a/src/runtime/grokWorkerAttestation.test.ts +++ b/src/runtime/grokWorkerAttestation.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { appendFile, chmod, mkdtemp, rm, stat, symlink, truncate, writeFile } from "node:fs/promises"; +import { appendFile, chmod, mkdir, mkdtemp, readFile, rm, stat, symlink, truncate, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -102,7 +102,7 @@ const eventsFile = async (dir: string, initial = ""): Promise => { }; const watermark = async (file: string, denyPaths: readonly string[] = []): Promise => { const info = await stat(file); - return { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), mtimeMs: Number(info.mtimeMs), denyPaths }; + return { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths }; }; const verify = (eventsPath: string, before: GrokWorkerAttestationSnapshot, brokerGid = self.gid): Promise => verifyGrokWorkerAttestation({ eventsPath, workerUid: self.uid, brokerGid, workspace }, before); @@ -220,8 +220,9 @@ test("refuses a sandbox profile that is not root-owned, and one reached through const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n`; await writeFile(profile, text); await chmod(profile, 0o444); - const events = await eventsFile(dir); - const input = { profilePath: profile, eventsPath: events, profileSha256: sha256(text), workerUid: self.uid, brokerGid: self.gid }; + await mkdir(path.join(dir, "sessions")); + const events = await eventsFile(path.join(dir, "sessions")); + const input = { profilePath: profile, eventsPath: events, profileSha256: sha256(text), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; // Owned by the test user rather than root: `secureOpen` must refuse it even // though its bytes hash correctly. await assert.rejects(prepareGrokWorkerAttestation(input), /attestation unavailable/u); @@ -244,3 +245,28 @@ test("holds the kernel's reported deny_paths to exactly what the pinned profile } assert.throws(() => parseGrokWorkerProfileApplied(bytes(["/a"]), workspace, []), /attestation unavailable/u); }); + +test("reads sandbox events only from the Grok 1.0.34 sessions log", async (t) => { + // Mutation-critical: 1.0.34 writes nothing to `$GROK_HOME/sandbox-events.jsonl`, + // so attesting that path would fail every turn as "not enforced" — or worse, + // accept a stale file. Restoring the old relation must turn this red. + const dir = await workspaceDir(); + t.after(() => rm(dir, { recursive: true, force: true })); + const input = { profilePath: path.join(dir, "sandbox.toml"), profileSha256: "0".repeat(64), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + await assert.rejects(prepareGrokWorkerAttestation({ ...input, eventsPath: path.join(dir, "sandbox-events.jsonl") }), /sessions\/sandbox-events\.jsonl/u); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, eventsPath: path.join(dir, "sessions", "sandbox-events.jsonl") }), /attestation unavailable/u); +}); + +test("attests real Grok 1.0.34 events: ProfileApplied with a non-empty deny list, then the denial it caused", async (t) => { + const fixture = await readFile(new URL("./fixtures/grok-1.0.34-sandbox-events.jsonl", import.meta.url)); + const liveWorkspace = "/var/lib/spawnfile/instance/workspace/agents/a1"; + assert.doesNotThrow(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, ["/run/paideia"])); + assert.throws(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, []), /attestation unavailable/u); + assert.throws(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, ["/run/paideia", "/run/training"]), /attestation unavailable/u); + const dir = await workspaceDir(); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = await eventsFile(dir); + const before = await watermark(file, ["/run/paideia"]); + await appendFile(file, fixture); + await verifyGrokWorkerAttestation({ eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace: liveWorkspace }, before); +}); diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index ce450f3..c441411 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -1,6 +1,12 @@ import { createHash } from "node:crypto"; import { constants } from "node:fs"; import { lstat,open } from "node:fs/promises"; +import path from "node:path"; + +import { assertGrokWorkerDenyPathsPlaceable } from "./grokWorkerDenyPlacement.js"; +import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; +import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** * The per-turn freshness watermark taken before the worker is launched: the @@ -8,22 +14,20 @@ import { lstat,open } from "node:fs/promises"; * `size` is pre-turn history and is never read back — a `ProfileApplied` down * there is a replay, not evidence about this turn. */ -export type GrokWorkerAttestationSnapshot=Readonly<{dev:number;ino:number;size:number;mtimeMs:number;denyPaths:readonly string[]}>; +export type GrokWorkerAttestationSnapshot=Readonly<{dev:number;ino:number;size:number;denyPaths:readonly string[]}>; type Snapshot=GrokWorkerAttestationSnapshot; export class GrokWorkerAttestationFailure extends Error { constructor(readonly failureClass:"profile_missing"|"profile_invalid"){super("Grok worker isolation attestation unavailable");} } /** * Reads the `deny` list out of a worker sandbox profile, but only after the * bytes match `profileSha256` exactly. * - * There is no minimum length. The floor used to be 3 (subscription realm, - * bootstrap credential, peer roots), then 1, and an empty list is now the - * *expected* shape: Grok 1.0.13 re-execs itself inside bubblewrap whenever - * `deny` is non-empty and then opens each deny-path placeholder — which it - * created at mode 000 — from a capability-stripped process, gets EACCES, and - * refuses to start ("possible __GROK_INSIDE_BWRAP spoof") without ever - * emitting a `ProfileApplied` event. Spawnfile therefore renders `deny = []` - * and confines the worker with unix permissions plus builtin-`strict` - * Landlock instead (`containerDaimonBrokerRender.ts`). + * There is no minimum length, and a populated list is supported. Grok 1.0.13 + * refused to start on any non-empty `deny` (its mode-000 placeholders failed + * an EACCES "__GROK_INSIDE_BWRAP spoof" check), so deployments rendered + * `deny = []`. Grok 1.0.34 runs every profile inside bubblewrap and enforces a + * non-empty list, and since its strict base reads all of `/run`, `/var` and + * `/tmp`, the deny list is what keeps evaluator and host-bind paths away from + * the worker. `grokWorkerSandboxProfile.ts` renders these bytes. * * The integrity guarantee is the hash pin, not the length. `profileSha256` * comes from `/etc/daimon-engine-broker/service.json`, which the root @@ -52,25 +56,105 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str if(!Array.isArray(parsed)||parsed.some((entry)=>typeof entry!=="string")||new Set(parsed).size!==parsed.length)throw new Error("Grok worker isolation attestation unavailable"); return [...parsed as string[]].sort(); } -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number}>):Promise{ - const profile=await secureOpen(input.profilePath,0,0,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} - const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),mtimeMs:Number(stat.mtimeMs),denyPaths};}finally{await events.close();} +/** + * Pre-launch half of the per-turn attestation. Besides the pinned profile and + * the events watermark it requires the 1.0.34 layout: events under + * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on + * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to + * the declared renderer output (`grokWorkerHomeAttestation.ts`). + * + * It also refuses a deny list bubblewrap could not materialize + * (`grokWorkerDenyPlacement.ts`), naming the entry and the ancestor that stops + * it. Without this the worker dies with a bare `bwrap: Can't create file at + * …: Permission denied` on *every* turn, because one unplaceable entry makes + * Grok refuse the whole profile. The broker cannot descend into a directory + * opened to the worker's group alone (`/tool-state` under a + * `2000: 0710` home), so an `EACCES` there is left undecided; root + * provisioning, which holds `CAP_DAC_READ_SEARCH`, is the authority that + * decides every entry. + */ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;workerGid?:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ + if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); + const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. + // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + await assertGrokWorkerDenyPathsPlaceable(denyPaths,{uid:input.workerUid,gid:input.workerGid??input.workerUid}); + const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); + if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); + await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); + const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} +} +/** The per-turn attestation input for one registration, carrying every registered worker so sibling temp is attested too. */ +export const grokBrokerAttestationInput=>(registration:T,registrations:readonly Readonly<{profilePath:string;workerUid:number}>[],configSha256:string)=>({...registration,brokerGid:2100,configSha256,registeredWorkers:registrations.map((entry)=>({profilePath:entry.profilePath,workerUid:entry.workerUid}))}); +/** + * The accepted `ProfileApplied` line of one turn, as an absolute byte range of + * the events file plus its digest. + */ +export type GrokWorkerAttestationLock={accepted?:Readonly<{offset:number;length:number;digest:string}>;refused?:GrokWorkerAttestationFailure["failureClass"]}; + +/** + * The per-turn isolation guard: every model request of a turn, and the + * post-turn check, go through the same lock. + * + * Why the first verification is trustworthy: the proxy awaits this guard before + * *every* upstream request, including the first, and the only worker-uid + * process that exists before request 1 is the Grok the launcher started under + * the pinned profile — tool children are created only after a model response. + * So the event accepted at request 1 was written before any tool child could + * write to the (worker-owned) events file. Later requests must find that exact + * line, byte for byte, at the same offset: a `ProfileApplied` appended later — + * which a tool child could forge — is never a substitute, and once a turn has + * been refused it stays refused. + */ +export function createGrokWorkerIsolationGuard(input:Parameters[0],before:Snapshot):()=>Promise{ + const lock:GrokWorkerAttestationLock={}; + return async()=>{ + if(lock.refused!==undefined)throw new GrokWorkerAttestationFailure(lock.refused); + try{await verifyGrokWorkerAttestation(input,before,lock);} + catch(error){const failure=error instanceof GrokWorkerAttestationFailure?error:new GrokWorkerAttestationFailure("profile_invalid");lock.refused=failure.failureClass;throw failure;} + }; } -export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:string;workerUid:number;brokerGid:number;workspace:string}>,before:Snapshot):Promise{ - let handle:Awaited>;try{handle=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);}catch{throw new GrokWorkerAttestationFailure("profile_invalid");}let bytes:Buffer|undefined;try{const stat=await handle.stat();if(Number(stat.dev)!==before.dev||Number(stat.ino)!==before.ino)throw new GrokWorkerAttestationFailure("profile_invalid");if(Number(stat.size)<=before.size)throw new GrokWorkerAttestationFailure("profile_missing");bytes=Buffer.alloc(Number(stat.size)-before.size);const read=await handle.read(bytes,0,bytes.length,before.size);if(read.bytesRead!==bytes.length)throw new GrokWorkerAttestationFailure("profile_invalid");const after=await handle.stat();if(Number(after.size)!==Number(stat.size)||Number(after.mtimeMs)!==Number(stat.mtimeMs))throw new GrokWorkerAttestationFailure("profile_invalid");parseGrokWorkerProfileApplied(bytes,input.workspace,before.denyPaths);}catch(error){if(error instanceof GrokWorkerAttestationFailure)throw error;throw new GrokWorkerAttestationFailure("profile_invalid");}finally{bytes?.fill(0);await handle.close();} +export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:string;workerUid:number;brokerGid:number;workspace:string}>,before:Snapshot,lock?:GrokWorkerAttestationLock):Promise{ + let handle:Awaited>;try{handle=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);}catch{throw new GrokWorkerAttestationFailure("profile_invalid");} + let bytes:Buffer|undefined; + try{ + const stat=await handle.stat(); + if(Number(stat.dev)!==before.dev||Number(stat.ino)!==before.ino)throw new GrokWorkerAttestationFailure("profile_invalid"); + if(Number(stat.size)<=before.size)throw new GrokWorkerAttestationFailure("profile_missing"); + const accepted=lock?.accepted; + const start=accepted===undefined?before.size:before.size+accepted.offset; + const length=accepted===undefined?Number(stat.size)-before.size:accepted.length; + if(start+length>Number(stat.size))throw new GrokWorkerAttestationFailure("profile_invalid"); + bytes=Buffer.alloc(length); + const read=await handle.read(bytes,0,bytes.length,start); + if(read.bytesRead!==bytes.length)throw new GrokWorkerAttestationFailure("profile_invalid"); + // The file must not change while it is read: a concurrent writer could + // otherwise show this check bytes that no single state of the file held. + const after=await handle.stat(); + if(Number(after.size)!==Number(stat.size)||Number(after.mtimeMs)!==Number(stat.mtimeMs))throw new GrokWorkerAttestationFailure("profile_invalid"); + if(accepted!==undefined){ + if(createHash("sha256").update(bytes).digest("hex")!==accepted.digest)throw new GrokWorkerAttestationFailure("profile_invalid"); + return; + } + const event=locateGrokWorkerProfileApplied(bytes,input.workspace,before.denyPaths); + if(lock!==undefined)lock.accepted={offset:event.offset,length:event.length,digest:createHash("sha256").update(bytes.subarray(event.offset,event.offset+event.length)).digest("hex")}; + }catch(error){if(error instanceof GrokWorkerAttestationFailure)throw error;throw new GrokWorkerAttestationFailure("profile_invalid");}finally{bytes?.fill(0);await handle.close();} } /** * Requires one fully-conforming `ProfileApplied` event *somewhere* in the * fresh region — not as its last line. * - * `sandbox-events.jsonl` is not a profile log. Grok 1.0.13 writes its whole + * `sandbox-events.jsonl` is not a profile log. Grok (1.0.13 and 1.0.34) writes its whole * sandbox event vocabulary there — verified by reading the shipped binary: * `ProfileApplied, ApplyFailed, FsViolation, NetViolation, BypassGranted, * BypassDenied` (one contiguous enum blob beside the record fields * `timestamp, event_type, read_only_paths, deny_paths, operation, target, * command, tool_call_id`, emitted from `xai_grok_sandbox::logging`), and its * own embedded documentation says so outright: "Sandbox events (profile - * applied, violations) are logged to `~/.grok/sandbox-events.jsonl`". + * applied, violations) are logged to `~/.grok/sandbox-events.jsonl`" (1.0.34 + * moved the file to `~/.grok/sessions/`; a 1.0.34 turn logs `ProfileApplied` + * followed by an `FsViolation` for every denied read). * * Requiring `ProfileApplied` to be the *last* line therefore failed on the * first denied access of any turn: the violation Grok logged next became the @@ -88,15 +172,19 @@ export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:str * guard exists for: a Grok that came up without kernel enforcement, which * emits no conforming `ProfileApplied` at all. */ -export function parseGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):void{ +export function parseGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):void{locateGrokWorkerProfileApplied(bytes,workspace,denyPaths);} +/** Byte range (within `bytes`) of the first fully-conforming `ProfileApplied` line. */ +export function locateGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):Readonly<{offset:number;length:number}>{ const expected=JSON.stringify([...denyPaths].sort()); - for(const line of Buffer.from(bytes).toString("utf8").split("\n")){ + const buffer=Buffer.from(bytes.buffer,bytes.byteOffset,bytes.byteLength); + for(let offset=0;offset; try{const parsed=JSON.parse(line) as unknown;if(parsed===null||typeof parsed!=="object"||Array.isArray(parsed))continue;event=parsed as Record;}catch{continue;} if(event.event_type!=="ProfileApplied")continue; const observed=Array.isArray(event.deny_paths)?event.deny_paths.filter((entry):entry is string=>typeof entry==="string").sort():[]; - if(event.profile==="daimon-strict"&&event.enforced===true&&event.restrict_network===true&&event.platform==="linux/landlock"&&event.workspace===workspace&&JSON.stringify(observed)===expected)return; + if(event.profile==="daimon-strict"&&event.enforced===true&&event.restrict_network===true&&event.platform==="linux/landlock"&&event.workspace===workspace&&JSON.stringify(observed)===expected)return{offset:lineOffset,length:end-lineOffset}; } throw new Error("Grok worker isolation attestation unavailable"); } diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts new file mode 100644 index 0000000..45480ed --- /dev/null +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, realpath, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { + GrokWorkerAttestationFailure, + grokBrokerAttestationInput, + prepareGrokWorkerAttestation, + verifyGrokWorkerAttestation, + type GrokWorkerAttestationSnapshot +} from "./grokWorkerAttestation.js"; + +const workspace = "/var/lib/daimon-workers/2200/workspace"; +const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; +const applied = (overrides: Record = {}): string => + `${JSON.stringify({ event_type: "ProfileApplied", profile: "daimon-strict", enforced: true, restrict_network: true, platform: "linux/landlock", workspace, deny_paths: [], ...overrides })}\n`; +const violation = `${JSON.stringify({ event_type: "FsViolation", profile: "daimon-strict", operation: "read", target: "/run/paideia/context.json" })}\n`; + +const fixture = async (t: { after(fn: () => Promise): void }, initial = "") => { + const dir = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = path.join(dir, "sandbox-events.jsonl"); + await writeFile(file, initial); + await chmod(file, 0o640); + const info = await stat(file); + const before: GrokWorkerAttestationSnapshot = { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths: [] }; + const input = { eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace }; + return { dir, file, before, input }; +}; +const refusal = async (run: Promise): Promise => { + try { await run; } catch (error) { + assert.ok(error instanceof GrokWorkerAttestationFailure, `expected a GrokWorkerAttestationFailure, got ${String(error)}`); + return error.failureClass; + } + throw new Error("expected the attestation to be refused"); +}; + +test("refuses an events file with a second hard link", async (t) => { + const { dir, file, before, input } = await fixture(t); + await appendFile(file, applied()); + await verifyGrokWorkerAttestation(input, before); + await link(file, path.join(dir, "second-name.jsonl")); + assert.equal(await refusal(verifyGrokWorkerAttestation(input, before)), "profile_invalid"); +}); + +test("refuses events that change while they are being read", async (t) => { + const { file, before, input } = await fixture(t); + await appendFile(file, applied()); + const probe = await open(file, "r"); + const prototype = Object.getPrototypeOf(probe) as { read: (...args: unknown[]) => Promise }; + await probe.close(); + const original = prototype.read; + const reading = mock.method(prototype, "read", async function (this: unknown, ...args: unknown[]) { + const result = await original.apply(this, args); + await appendFile(file, violation); + return result; + }); + try { assert.equal(await refusal(verifyGrokWorkerAttestation(input, before)), "profile_invalid"); } finally { reading.mock.restore(); } + assert.ok(reading.mock.callCount() >= 1); +}); + +test("prepare refuses a worker home that fails attestation even when profile, temp and events are valid", async (t) => { + // Run as a non-root owner so the profile, temp and events legs pass; the home + // leg (root-owned, read-only config) cannot, and must be what refuses. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = path.join(root, ".grok"); + await mkdir(path.join(home, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); + const profile = path.join(home, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + const events = path.join(home, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); +}); + +test("prepare refuses a worker without a private temp directory before the home check", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + const owner = { uid: self.uid, gid: Number((await stat(profile)).gid) }; + // No /tmp: the temp leg refuses (the home leg would refuse too, with a different message). + await assert.rejects(prepareGrokWorkerAttestation(input, owner), /temp isolation attestation unavailable/u); + // A profile outside /.grok cannot name the launcher's TMPDIR home. + await mkdir(path.join(root, "elsewhere", "sessions"), { recursive: true }); + const stray = { ...input, profilePath: path.join(root, "elsewhere", "sandbox.toml"), eventsPath: path.join(root, "elsewhere", "sessions", "sandbox-events.jsonl") }; + await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); + await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); +}); + +test("prepare refuses the current turn when a sibling registered worker's private temp is 0777", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-sibling-")); + t.after(() => rm(root, { recursive: true, force: true })); + const make = async (name: string) => { + const home = path.join(root, name), grok = path.join(home, ".grok"); + await mkdir(path.join(grok, "sessions"), { recursive: true }); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + return { home, profile: path.join(grok, "sandbox.toml"), events: path.join(grok, "sessions", "sandbox-events.jsonl") }; + }; + const own = await make("own"), sibling = await make("sibling"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(own.profile, text); await chmod(own.profile, 0o444); + await writeFile(own.events, ""); await chmod(own.events, 0o640); + const registration = { profilePath: own.profile, eventsPath: own.events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, workspace: "/w" }; + const input = grokBrokerAttestationInput(registration, [registration, { profilePath: sibling.profile, workerUid: self.uid }], "0".repeat(64)); + assert.deepEqual(input.registeredWorkers.map((entry) => entry.profilePath), [own.profile, sibling.profile]); + const seams = { uid: self.uid, gid: Number((await stat(own.profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }; + await chmod(root, 0o700); + // Sibling well provisioned: temp passes and the (root-only) home leg is what refuses. + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await chmod(path.join(sibling.home, "tmp"), 0o777); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); +}); + +test("prepare refuses a deny entry bubblewrap could not materialize, before any other leg", async (t) => { + // The production defect: the wake-acceptance store sits under a `0700` organization state directory, + // so bubblewrap — which materializes every deny target as the worker uid — could not create it and + // Grok refused the whole profile, failing every turn with `bwrap: Can't create file at …`. + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-guard-deny-"))); + t.after(async () => { await chmod(path.join(root, "state"), 0o700); await rm(root, { recursive: true, force: true }); }); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); + await mkdir(path.join(root, "state", "wake-acceptance"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const owner = { uid: self.uid, gid: Number((await stat(path.join(root, "tmp"))).gid) }; + const withDeny = async (denied: string) => { + const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["${denied}"]\n`; + await chmod(profile, 0o644).catch(() => undefined); + await writeFile(profile, text); await chmod(profile, 0o444); + return { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + }; + await chmod(path.join(root, "state"), 0o600); + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state", "wake-acceptance")), owner), + (error: Error) => /is not placeable/u.test(error.message) && error.message.includes(`cannot search ${path.join(root, "state")}`) + ); + // The lift: the private directory itself is placeable, so this leg passes and a later one refuses. + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state")), owner), + (error: Error) => !/is not placeable/u.test(error.message) + ); +}); diff --git a/src/runtime/grokWorkerDenyPlacement.test.ts b/src/runtime/grokWorkerDenyPlacement.test.ts new file mode 100644 index 0000000..f5013a0 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + assertGrokWorkerDenyPathPlacement, + assertGrokWorkerDenyPathShape, + assertGrokWorkerDenyPathsPlaceable, + GROK_WORKER_BASE_PROFILE_GRANTS, + grokWorkerCanSearch, + grokWorkerDenyPathChain, + GrokWorkerDenyPlacementError, + type GrokWorkerDenyPathEntry, + type GrokWorkerDenyPathStep +} from "./grokWorkerDenyPlacement.js"; +import { renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +const entry = (uid: number, gid: number, mode: number, kind: "dir" | "file" | "link" = "dir"): GrokWorkerDenyPathEntry => + ({ uid, gid, mode, isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" }); +const worker = { uid: 2200, gid: 2200 }; +const steps = (denyPath: string, entries: readonly (GrokWorkerDenyPathEntry | string)[]): GrokWorkerDenyPathStep[] => + grokWorkerDenyPathChain(denyPath).map((target, index) => { + const value = entries[index]; + return typeof value === "string" ? { path: target, code: value } : { path: target, entry: value }; + }); + +test("search permission follows owner, then group, then other — as the worker's cleared-group process does", () => { + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o700), worker), true); + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o677), worker), false, "owner bits win even when group and other would allow"); + assert.equal(grokWorkerCanSearch(entry(2000, 2200, 0o710), worker), true); + assert.equal(grokWorkerCanSearch(entry(2000, 2000, 0o700), worker), false); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o711), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o755), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 2000, 0o750), worker), false); +}); + +test("refuses a deny entry at or above any Grok 1.0.34 base-profile grant", () => { + for (const grant of GROK_WORKER_BASE_PROFILE_GRANTS) { + assert.throws(() => assertGrokWorkerDenyPathShape(grant), GrokWorkerDenyPlacementError, grant); + } + assert.throws(() => assertGrokWorkerDenyPathShape("/var"), /base profile grant \/var/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/run"]), /base profile grant \/run/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/var/lib/spawnfile/daimon/usage", "/tmp"]), /base profile grant \/tmp/u); + // Strictly below every grant is exactly what Grok accepts. + assertGrokWorkerDenyPathShape("/tmp/sub"); + assertGrokWorkerDenyPathShape("/var/lib/spawnfile/daimon/usage"); +}); + +test("refuses the wake-acceptance shape: a deny entry under a parent the worker cannot search", () => { + // `/state` is `2000:2000 0700`; the store beneath it is what production used to deny. + const denyPath = "/var/lib/spawnfile/instances/daimon/org/state/wake-acceptance"; + assert.throws( + () => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700), entry(2000, 2000, 0o700) + ]), worker), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && /cannot search \/var\/lib\/spawnfile\/instances\/daimon\/org\/state \(700 2000:2000\); deny that directory itself instead/u.test(error.message) + ); + // The lift production now emits: the private directory itself, whose own parent is traversable. + const lifted = "/var/lib/spawnfile/instances/daimon/org/state"; + assertGrokWorkerDenyPathPlacement(lifted, steps(lifted, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700) + ]), worker); +}); + +test("refuses a missing target, a symlink and a non-directory ancestor; leaves an undecidable EACCES alone", () => { + const denyPath = "/run/training/slot/state"; + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "ENOENT" + ]), worker), /does not exist; bubblewrap would have to create it as the worker uid/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o777, "link") + ]), worker), /is a symlink; bubblewrap refuses to bind over one/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o644, "file"), entry(0, 0, 0o755), entry(0, 0, 0o700) + ]), worker), /is not a directory/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "EPERM" + ]), worker), /could not be read \(EPERM\)/u); + // The broker (uid 2100) cannot descend into a `2000: 0710` runtime home the worker itself can + // search, so an EACCES *below a worker-searchable ancestor* is undecided here, never a refusal. + assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(2000, 2200, 0o710), "EACCES" + ]), worker); +}); + +test("walks the real filesystem and names the ancestor that stops a deny entry", async () => { + // realpath: macOS `/var` is a symlink, and a symlinked ancestor is refused on purpose — Daimon's + // registered paths are canonical, and bubblewrap must bind over the inode the deny entry names. + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "grok-deny-"))); + const state = path.join(root, "state"); + // This process stands in for root provisioning: it can stat every component, and judges searchability + // for the worker from the modes it reads. Here the worker is this uid, so only `state` blocks it. + const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; + try { + mkdirSync(path.join(state, "wake-acceptance"), { recursive: true }); + writeFileSync(path.join(state, "wake-acceptance", "store.jsonl"), "{}\n"); + chmodSync(state, 0o600); + await assert.rejects( + assertGrokWorkerDenyPathsPlaceable([path.join(state, "wake-acceptance")], self), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && error.message.includes(`cannot search ${state}`) && error.message.includes("deny that directory itself instead") + ); + // The lift: the unsearchable directory itself is placeable, and masks strictly more. + await assertGrokWorkerDenyPathsPlaceable([state], self); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "absent")], self), /does not exist/u); + symlinkSync(state, path.join(root, "link")); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "link")], self), /is a symlink/u); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable(["relative/path"], self), /not an absolute path/u); + } finally { + chmodSync(state, 0o700); + rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/src/runtime/grokWorkerDenyPlacement.ts b/src/runtime/grokWorkerDenyPlacement.ts new file mode 100644 index 0000000..f2995d2 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.ts @@ -0,0 +1,147 @@ +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +/** + * Paths Grok 1.0.34's strict base profile grants read or read-write. A `deny` + * entry that equals or contains one of them makes Grok refuse the profile + * outright (verified for `/tmp`, `/var/tmp`, `/run`, `/etc`, `/var` and + * `sessions`; `/tmp/sub` is accepted), so every entry must sit strictly below + * each grant it touches. + */ +export const GROK_WORKER_BASE_PROFILE_GRANTS = Object.freeze([ + "/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp" +] as const); + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +export class GrokWorkerDenyPlacementError extends Error { + constructor(readonly denyPath: string, readonly reason: string) { + super(`Grok worker sandbox deny path ${JSON.stringify(denyPath)} is not placeable: ${reason}`); + this.name = "GrokWorkerDenyPlacementError"; + } +} + +/** + * The shape half of the deny-placement policy: everything decidable without + * touching a filesystem, so the profile renderer can refuse a bad entry before + * its bytes are ever pinned or written. + * + * `renderGrokWorkerSandboxProfile` already refuses entries that are relative, + * non-canonical, `/`, trailing-slashed, or carrying a character TOML or Grok + * would reinterpret; this adds the base-profile grant rule. + */ +export function assertGrokWorkerDenyPathShape(denyPath: string): void { + const grant = GROK_WORKER_BASE_PROFILE_GRANTS.find((candidate) => within(candidate, denyPath)); + if (grant !== undefined) { + throw new GrokWorkerDenyPlacementError(denyPath, `it equals or contains the base profile grant ${grant}, which Grok refuses`); + } +} + +/** The subset of `lstat` the placement rules read; pure so every refusal is testable without root. */ +export type GrokWorkerDenyPathEntry = Readonly<{ + uid: number; + gid: number; + mode: number; + isDirectory: () => boolean; + isSymbolicLink: () => boolean; +}>; + +/** One resolved path component: the entry, or the errno that stopped the walk. */ +export type GrokWorkerDenyPathStep = Readonly<{ path: string; entry?: GrokWorkerDenyPathEntry; code?: string }>; + +export type GrokWorkerDenyPathWorker = Readonly<{ uid: number; gid: number }>; + +/** + * POSIX search permission: owner bits win, then group, then other. The worker + * runs with its supplementary groups cleared, so its primary gid is the only + * group that can apply. + */ +export const grokWorkerCanSearch = (entry: GrokWorkerDenyPathEntry, worker: GrokWorkerDenyPathWorker): boolean => + entry.uid === worker.uid ? (entry.mode & 0o100) !== 0 + : entry.gid === worker.gid ? (entry.mode & 0o010) !== 0 + : (entry.mode & 0o001) !== 0; + +/** The `/`-rooted ancestor chain of `denyPath`, deepest last, followed by the entry itself. */ +export const grokWorkerDenyPathChain = (denyPath: string): readonly string[] => { + const components = denyPath.split("/").slice(1); + return ["/", ...components.map((_, index) => `/${components.slice(0, index + 1).join("/")}`)]; +}; + +/** + * The placement half of the policy, as a pure function of an already-walked + * chain. + * + * Grok 1.0.34 materializes every `deny` entry inside bubblewrap **as the worker + * uid**, by bind-mounting `$GROK_HOME/sandbox-blocked-{file,dir}` over the + * target. So bwrap must be able to *resolve* the target as that uid: every + * ancestor directory needs the search bit for it, and the target must already + * exist — otherwise bwrap tries to create it and needs write on the parent, + * which a private parent never grants. A single unplaceable entry makes Grok + * refuse the whole profile, so every turn of that worker fails, not just that + * path. Verified matrix: `.runtime/grok-deny-placement/EVIDENCE.md`. + * + * The walk may legitimately stop early: a caller that is neither root nor the + * worker (the broker, uid 2100) cannot descend into a directory the worker's + * own group opens to it alone — `/tool-state` under a + * `2000: 0710` runtime home is exactly that. An `EACCES` below an + * ancestor the *worker* can search is therefore "not decidable from here", not + * a refusal; every decidable failure still refuses. + */ +export function assertGrokWorkerDenyPathPlacement( + denyPath: string, + steps: readonly GrokWorkerDenyPathStep[], + worker: GrokWorkerDenyPathWorker +): void { + assertGrokWorkerDenyPathShape(denyPath); + const chain = grokWorkerDenyPathChain(denyPath); + if (steps.length !== chain.length || steps.some((step, index) => step.path !== chain[index])) { + throw new GrokWorkerDenyPlacementError(denyPath, "its resolved path chain does not match the entry"); + } + for (const [index, step] of steps.entries()) { + const ancestor = index < steps.length - 1; + if (step.entry === undefined) { + if (step.code === "ENOENT") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} does not exist; bubblewrap would have to create it as the worker uid`); + if (step.code !== "EACCES") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} could not be read (${step.code ?? "unknown error"})`); + // Undecidable from here, and only after every shallower ancestor passed. + return; + } + if (step.entry.isSymbolicLink()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is a symlink; bubblewrap refuses to bind over one`); + if (!ancestor) return; + if (!step.entry.isDirectory()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is not a directory`); + if (!grokWorkerCanSearch(step.entry, worker)) { + throw new GrokWorkerDenyPlacementError(denyPath, `worker uid ${worker.uid} cannot search ${step.path} (${(step.entry.mode & 0o7777).toString(8)} ${step.entry.uid}:${step.entry.gid}); deny that directory itself instead`); + } + } +} + +/** Walks one deny path on the real filesystem, recording what stopped it rather than throwing. */ +export async function readGrokWorkerDenyPathChain(denyPath: string): Promise { + const steps: GrokWorkerDenyPathStep[] = []; + for (const target of grokWorkerDenyPathChain(denyPath)) { + try { + steps.push({ path: target, entry: await lstat(target) }); + } catch (error) { + steps.push({ path: target, code: (error as NodeJS.ErrnoException).code }); + break; + } + } + const chain = grokWorkerDenyPathChain(denyPath); + while (steps.length < chain.length) steps.push({ path: chain[steps.length]!, code: steps.at(-1)?.code ?? "EACCES" }); + return steps; +} + +/** + * Fails closed before a worker is ever launched with a profile Grok would + * refuse. Callers with the widest view run it: root provisioning at container + * start and on every slot recycle, and the direct (non-broker) path, which runs + * as the worker uid itself. + */ +export async function assertGrokWorkerDenyPathsPlaceable( + denyPaths: readonly string[], + worker: GrokWorkerDenyPathWorker +): Promise { + for (const denyPath of denyPaths) { + if (!path.posix.isAbsolute(denyPath)) throw new GrokWorkerDenyPlacementError(denyPath, "it is not an absolute path"); + assertGrokWorkerDenyPathPlacement(denyPath, await readGrokWorkerDenyPathChain(denyPath), worker); + } +} diff --git a/src/runtime/grokWorkerHomeAttestation.test.ts b/src/runtime/grokWorkerHomeAttestation.test.ts new file mode 100644 index 0000000..22f591e --- /dev/null +++ b/src/runtime/grokWorkerHomeAttestation.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, open, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +import { assertGrokWorkerConfigBytes, assertGrokWorkerHomeEntries, verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; + +type Kind = "dir" | "file" | "link"; +const entry = (kind: Kind, mode: number, uid = 0, nlink = 1) => ({ + uid, mode: (kind === "dir" ? 0o040000 : kind === "file" ? 0o100000 : 0o120000) | mode, nlink, + isFile: () => kind === "file", isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); +const files = ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"]; +const layout = (overrides: Record | undefined> = {}) => ({ + ".": entry("dir", 0o1771), sessions: entry("dir", 0o1771), ...Object.fromEntries(files.map((name) => [name, entry("file", 0o444)])), ...overrides +}); + +test("accepts the root-owned sticky worker home with root-owned read-only config and trust files", () => { + assert.doesNotThrow(() => assertGrokWorkerHomeEntries(layout())); + assert.doesNotThrow(() => assertGrokWorkerHomeEntries(layout({ ".": entry("dir", 0o711), sessions: entry("dir", 0o750) }))); +}); + +test("refuses a worker home where the worker could change its config or trust state", () => { + const refusals: Record> = { + "worker-owned home": layout({ ".": entry("dir", 0o1771, 2200) }), + "group-writable home without sticky bit": layout({ ".": entry("dir", 0o771) }), + "world-writable home": layout({ ".": entry("dir", 0o1777) }), + "worker-owned sessions": layout({ sessions: entry("dir", 0o700, 2200) }), + "group-writable sessions without sticky bit": layout({ sessions: entry("dir", 0o770) }) + }; + for (const name of files) { + refusals[`${name} missing`] = layout({ [name]: undefined }); + refusals[`${name} worker-owned`] = layout({ [name]: entry("file", 0o444, 2200) }); + refusals[`${name} owner-writable`] = layout({ [name]: entry("file", 0o644) }); + refusals[`${name} group-writable`] = layout({ [name]: entry("file", 0o464) }); + refusals[`${name} symlink`] = layout({ [name]: entry("link", 0o444) }); + refusals[`${name} hard-linked`] = layout({ [name]: entry("file", 0o444, 0, 2) }); + } + for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerHomeEntries(entries), /attestation unavailable/u, label); +}); + +test("refuses a real home that is not root-owned even when every file exists", async (t) => { + const home = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-")); + t.after(() => rm(home, { recursive: true, force: true })); + await mkdir(path.join(home, "sessions")); + for (const name of files) await writeFile(path.join(home, name), "", { mode: 0o444 }); + await assert.rejects(verifyGrokWorkerHome(home, "0".repeat(64)), /attestation unavailable/u); +}); + +test("accepts only the renderer's exact config bytes for the declared model policy", () => { + const declared = { model: "grok-4.6", reasoningEffort: "low" } as const; + const bytes = Buffer.from(renderGrokBrokerWorkerConfig(declared)); + assert.doesNotThrow(() => assertGrokWorkerConfigBytes(bytes, grokBrokerWorkerConfigSha256(declared))); + for (const tampered of [ + renderGrokBrokerWorkerConfig({ model: "grok-4.6", reasoningEffort: "high" }), + renderGrokBrokerWorkerConfig(declared).replace(/\[skills\]\ndisabled = \[[^\]]*\]\n/u, ""), + renderGrokBrokerWorkerConfig(declared).replace('session_summary = "daimon-session-title-disabled"', 'session_summary = "grok-4.6"'), + `${renderGrokBrokerWorkerConfig(declared)}\n[mcp_servers.extra]\nurl = "http://127.0.0.1:1/mcp"\n` + ]) assert.throws(() => assertGrokWorkerConfigBytes(Buffer.from(tampered), grokBrokerWorkerConfigSha256(declared)), /attestation unavailable/u); +}); + +const ownHome = async (t: { after(fn: () => Promise): void }) => { + const home = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-owned-")); + t.after(async () => { await chmod(home, 0o700); await rm(home, { recursive: true, force: true }); }); + await mkdir(path.join(home, "sessions")); + const declared = { model: "grok-4.6", reasoningEffort: "low" } as const; + for (const name of files) await writeFile(path.join(home, name), name === "config.toml" ? renderGrokBrokerWorkerConfig(declared) : "", { mode: 0o444 }); + await chmod(path.join(home, "sessions"), 0o1771); await chmod(home, 0o1771); + return { home, sha: grokBrokerWorkerConfigSha256(declared), uid: process.getuid?.() ?? 0 }; +}; + +test("attests a correctly laid out home whose config is the declared renderer output", async (t) => { + const { home, sha, uid } = await ownHome(t); + await verifyGrokWorkerHome(home, sha, uid); + await assert.rejects(verifyGrokWorkerHome(home, grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "high" }), uid), /attestation unavailable/u); +}); + +test("refuses a config.toml whose opened inode is not the one lstat saw", async (t) => { + const { home, sha, uid } = await ownHome(t); + const probe = await open(path.join(home, "config.toml"), "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + return Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }); + }); + try { await assert.rejects(verifyGrokWorkerHome(home, sha, uid), /attestation unavailable/u); } finally { swapped.mock.restore(); } + await verifyGrokWorkerHome(home, sha, uid); +}); diff --git a/src/runtime/grokWorkerHomeAttestation.ts b/src/runtime/grokWorkerHomeAttestation.ts new file mode 100644 index 0000000..47b15ba --- /dev/null +++ b/src/runtime/grokWorkerHomeAttestation.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import { constants, type Stats } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Partial> & Readonly<{ isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; + +/** + * The worker must not be able to change what Grok reads before its next turn. + * + * Grok writes its own state (sessions, hooks, locks, docs) into `$GROK_HOME`, + * so the directory is worker-group writable — but root-owned and sticky, so a + * worker can neither rename nor unlink a root-owned file in it. Every file + * that decides a turn's behaviour is root-owned with no write bit: `config.toml` + * (model, effort, MCP, skills), `sandbox.toml`, `trusted_folders.toml` (trust + * would re-enable cwd `AGENTS.md` and project skills), and the managed and + * requirements layers that can override user config. A missing file fails + * too: the worker could create it. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerHomeEntries(entries: Readonly>, rootUid: number = HOME.directory.uid): void { + const directory = (entry: Entry | undefined): boolean => + entry !== undefined && entry.isDirectory() && !entry.isSymbolicLink() && entry.uid === rootUid + && (Number(entry.mode) & 0o002) === 0 && ((Number(entry.mode) & 0o020) === 0 || (Number(entry.mode) & 0o1000) !== 0); + if (!directory(entries["."]) || !directory(entries[HOME.sessionsDirectory.relativePath])) throw unavailable(); + for (const name of HOME.readOnlyFiles.names) { + const entry = entries[name]; + if (entry === undefined || !entry.isFile() || entry.isSymbolicLink() || entry.uid !== rootUid || entry.nlink !== 1 || (Number(entry.mode) & 0o7222) !== 0) throw unavailable(); + } +} + +/** The worker's `config.toml` must be exactly the renderer's bytes for the declared policy. */ +export function assertGrokWorkerConfigBytes(bytes: Uint8Array, configSha256: string): void { + if (!/^[0-9a-f]{64}$/u.test(configSha256) || createHash("sha256").update(bytes).digest("hex") !== configSha256) throw unavailable(); +} + +/** + * Attests the worker home layout and that `config.toml` is exactly the + * renderer's bytes for the declared model policy (`configSha256`). + */ +export async function verifyGrokWorkerHome(grokHome: string, configSha256: string, rootUid: number = HOME.directory.uid): Promise { + const entries: Record = {}; + for (const name of [".", HOME.sessionsDirectory.relativePath, ...HOME.readOnlyFiles.names]) { + try { entries[name] = await lstat(path.join(grokHome, name)); } catch { entries[name] = undefined; } + } + assertGrokWorkerHomeEntries(entries, rootUid); + let handle: Awaited> | undefined; + try { + handle = await open(path.join(grokHome, "config.toml"), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const opened = await handle.stat(); + const before = entries["config.toml"]!; + // Same inode as the lstat above, as in `secureOpen`: a rename between the two cannot swap in other bytes. + if (!opened.isFile() || opened.size > 65_536 || opened.dev !== before.dev || opened.ino !== before.ino || opened.uid !== before.uid || opened.mode !== before.mode || opened.nlink !== 1) throw unavailable(); + assertGrokWorkerConfigBytes(await handle.readFile(), configSha256); + } catch { throw unavailable(); } finally { await handle?.close().catch(() => undefined); } +} + +const unavailable = (): Error => new Error("Grok worker isolation attestation unavailable"); diff --git a/src/runtime/grokWorkerIsolationGuard.test.ts b/src/runtime/grokWorkerIsolationGuard.test.ts new file mode 100644 index 0000000..cae9d81 --- /dev/null +++ b/src/runtime/grokWorkerIsolationGuard.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { appendFile, chmod, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + createGrokWorkerIsolationGuard, + GrokWorkerAttestationFailure, + verifyGrokWorkerAttestation, + type GrokWorkerAttestationSnapshot +} from "./grokWorkerAttestation.js"; + +const workspace = "/var/lib/daimon-workers/2200/workspace"; +const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; +const applied = (overrides: Record = {}): string => + `${JSON.stringify({ event_type: "ProfileApplied", profile: "daimon-strict", enforced: true, restrict_network: true, platform: "linux/landlock", workspace, deny_paths: [], ...overrides })}\n`; +const violation = `${JSON.stringify({ event_type: "FsViolation", profile: "daimon-strict", operation: "read", target: "/run/paideia/context.json" })}\n`; + +const fixture = async (t: { after(fn: () => Promise): void }, initial = "") => { + const dir = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = path.join(dir, "sandbox-events.jsonl"); + await writeFile(file, initial); + await chmod(file, 0o640); + const info = await stat(file); + const before: GrokWorkerAttestationSnapshot = { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths: [] }; + const input = { eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace }; + return { dir, file, before, input }; +}; +const refusal = async (run: Promise): Promise => { + try { await run; } catch (error) { + assert.ok(error instanceof GrokWorkerAttestationFailure, `expected a GrokWorkerAttestationFailure, got ${String(error)}`); + return error.failureClass; + } + throw new Error("expected the attestation to be refused"); +}; + +test("a turn refused at its first request stays refused after a conforming ProfileApplied is appended", async (t) => { + // Before request 1 only the launcher-started Grok can have written events; a + // line appended after it can come from a tool child and must never repair the turn. + const { file, before, input } = await fixture(t); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, applied({ enforced: false })); + assert.equal(await refusal(guard()), "profile_invalid"); + await appendFile(file, applied()); + assert.equal(await refusal(guard()), "profile_invalid"); + // Without the lock the same bytes would be accepted, so the refusal above is the lock's doing. + await verifyGrokWorkerAttestation(input, before); +}); + +test("later requests need the accepted event at the same offset, not any newly appended one", async (t) => { + const { file, before, input } = await fixture(t, "{\"event_type\":\"stale\"}\n"); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, `${violation}${applied()}`); + await guard(); + await appendFile(file, `${violation}${applied()}`); + await guard(); + + // Rewrite the accepted line in place (same length, different bytes) and append a fresh conforming one. + const bytes = await readFile(file, "utf8"); + const acceptedAt = bytes.indexOf(applied()); + const handle = await open(file, "r+"); + try { await handle.write(Buffer.from(applied({ workspace: workspace.replace("workspace", "workspacX") })), 0, undefined, acceptedAt); } finally { await handle.close(); } + await appendFile(file, applied()); + assert.equal(await refusal(guard()), "profile_invalid"); +}); + +test("a truncated accepted event region is refused on the next request", async (t) => { + const { file, before, input } = await fixture(t); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, applied()); + await guard(); + await truncate(file, 10); + await appendFile(file, `\n${applied()}`); + assert.equal(await refusal(guard()), "profile_invalid"); +}); diff --git a/src/runtime/grokWorkerSandboxProfile.test.ts b/src/runtime/grokWorkerSandboxProfile.test.ts new file mode 100644 index 0000000..7c24264 --- /dev/null +++ b/src/runtime/grokWorkerSandboxProfile.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; + +import { parseGrokWorkerSandboxProfile } from "./grokWorkerAttestation.js"; +import { grokWorkerEventsPathFor, grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +test("renders a non-empty deny list deterministically and round-trips through the attestation parser", () => { + const profile = renderGrokWorkerSandboxProfile(["/run/training/inputs", "/run/paideia", "/run/paideia"]); + assert.equal(profile, '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["/run/paideia", "/run/training/inputs"]\n'); + assert.equal(renderGrokWorkerSandboxProfile(["/run/paideia", "/run/training/inputs"]), profile); + const digest = createHash("sha256").update(profile).digest("hex"); + assert.equal(grokWorkerSandboxProfileSha256(["/run/training/inputs", "/run/paideia"]), digest); + assert.deepEqual(parseGrokWorkerSandboxProfile(Buffer.from(profile), digest), ["/run/paideia", "/run/training/inputs"]); + assert.equal(renderGrokWorkerSandboxProfile(), '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'); +}); + +test("refuses deny paths that are relative, non-canonical, root, globbed, or TOML-breaking", () => { + for (const entry of ["run/paideia", "/run/../etc", "/run/paideia/", "/", "/run/*", '/run/"x', "/run/a\nb", "/run/a\\b", ""]) { + assert.throws(() => renderGrokWorkerSandboxProfile([entry]), /deny path/u, JSON.stringify(entry)); + } +}); + +test("derives the Grok 1.0.34 sandbox events log from the profile location", () => { + assert.equal(grokWorkerEventsPathFor("/var/lib/daimon-workers/2200/.grok/sandbox.toml"), "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl"); +}); diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts new file mode 100644 index 0000000..881ffb1 --- /dev/null +++ b/src/runtime/grokWorkerSandboxProfile.ts @@ -0,0 +1,51 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { assertGrokWorkerDenyPathShape } from "./grokWorkerDenyPlacement.js"; + +export const GROK_WORKER_SANDBOX_PROFILE = "daimon-strict" as const; +export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events.jsonl" as const; + +/** + * The only source of `daimon-strict` sandbox profile bytes. + * + * Grok 1.0.34 runs every Landlock profile inside bubblewrap, and there a + * non-empty `deny` list works: each entry is bind-masked for both the shell + * tool and in-process `read_file` (P0: `/run/paideia` denied, controls intact). + * The strict base still reads all of `/run`, `/var`, `/tmp` and `/etc`, so the + * deny list — not Landlock's allowlist — is what keeps evaluator and host-bind + * paths away from the worker. Which paths to deny is a registration input + * supplied by the deployment; Daimon only renders, pins, and attests them. + * + * Entries are sorted and deduplicated so equal sets render equal bytes (and the + * same `profileSha256`). A path that is not absolute and canonical, or that + * carries a character TOML or Grok would reinterpret, is refused — and so is + * one that equals or contains a base-profile grant, the first half of the + * deny-placement policy (`grokWorkerDenyPlacement.ts`). The other half — + * the entry exists and every ancestor is searchable by the worker uid — needs + * a filesystem, so it is asserted by whoever provisions the paths and, on the + * direct path, before every turn. + */ +export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = []): string { + const denied = [...new Set(denyPaths)].sort(); + for (const entry of denied) { + if (!path.posix.isAbsolute(entry) || path.posix.normalize(entry) !== entry || entry === "/" || entry.endsWith("/") || /["\\\u0000-\u001f\u007f*?[\]]/u.test(entry)) { + throw new TypeError("invalid Grok worker sandbox deny path"); + } + assertGrokWorkerDenyPathShape(entry); + } + return [ + `[profiles.${GROK_WORKER_SANDBOX_PROFILE}]`, + 'extends = "strict"', + "restrict_network = true", + `deny = [${denied.map((entry) => JSON.stringify(entry)).join(", ")}]`, + "" + ].join("\n"); +} + +export const grokWorkerSandboxProfileSha256 = (denyPaths: readonly string[] = []): string => + createHash("sha256").update(renderGrokWorkerSandboxProfile(denyPaths)).digest("hex"); + +/** `$GROK_HOME` is the profile's directory; 1.0.34 logs sandbox events under `sessions/`. */ +export const grokWorkerEventsPathFor = (profilePath: string): string => + path.posix.join(path.posix.dirname(profilePath), GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH); diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts new file mode 100644 index 0000000..a5a728c --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp, type GrokWorkerTmpOptions } from "./grokWorkerTmpAttestation.js"; + +const worker = 2200, sibling = 2201; +type Entry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>; +const dir = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); +const file = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o100000 | mode, isDirectory: () => false }); +type Entries = { privateTmps: { uid: number; entry: Entry | undefined }[]; shared: (Entry | undefined)[] }; +const good = (): Entries => ({ privateTmps: [{ uid: worker, entry: dir(0o700, worker, worker) }, { uid: sibling, entry: dir(0o700, sibling, sibling) }], shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); +const withShared = (shared: Entry | undefined): Entries => ({ ...good(), shared: [shared, dir(0o1774, 0, 2000)] }); +const withOwn = (entry: Entry | undefined, uid = worker): Entries => ({ ...good(), privateTmps: [{ uid, entry }, good().privateTmps[1]!] }); +const withSibling = (entry: Entry | undefined): Entries => ({ ...good(), privateTmps: [good().privateTmps[0]!, { uid: sibling, entry }] }); +const refused = /temp isolation attestation unavailable/u; + +test("accepts private worker temps and shared temp roots the workers cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good())); + // Per contract the shared group only has to be below the worker range: the broker group 2100 is fine. + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1774, 0, 2100)))); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1770, 0, 2000)))); +}); + +test("refuses shared temp roots that are missing, not directories, not root-owned, worker-grouped, or open to others", () => { + const cases: Record = { + "missing": withShared(undefined), + "regular file": withShared(file(0o1774, 0, 2000)), + "owned by the org user": withShared(dir(0o1774, 2000, 2000)), + "group 2200 (a worker group)": withShared(dir(0o1774, 0, 2200)), + "other search (1775)": withShared(dir(0o1775, 0, 2000)), + "other write (1776)": withShared(dir(0o1776, 0, 2000)), + "default /tmp (1777)": withShared(dir(0o1777, 0, 0)), + "no shared roots at all": { ...good(), shared: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("refuses a private temp that is missing, not a directory, owned by someone else, or has any group or other bit", () => { + const cases: Record = { + "missing": withOwn(undefined), + "regular file": withOwn(file(0o600, worker, worker)), + "owned by another worker": withOwn(dir(0o700, sibling, sibling)), + "group read (0740)": withOwn(dir(0o740, worker, worker)), + "other execute (0701)": withOwn(dir(0o701, worker, worker)), + "other write (0702)": withOwn(dir(0o702, worker, worker)), + "other read (0704)": withOwn(dir(0o704, worker, worker)), + "worker uid below the worker range": withOwn(dir(0o700, 2100, 2100), 2100), + "no workers at all": { ...good(), privateTmps: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("a misprovisioned sibling worker's temp refuses the current worker's turn", () => { + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o777, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o770, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(undefined)), refused); +}); + +test("on a real filesystem: sibling 0777 temp, symlinked temps and symlinked shared roots are refused", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const uid = process.getuid?.() ?? 0; + const own = path.join(root, "own"), other = path.join(root, "other"), shared = path.join(root, "shared"), elsewhere = path.join(root, "elsewhere"); + for (const directory of [path.join(own, "tmp"), path.join(other, "tmp"), shared, elsewhere]) { await mkdir(directory, { recursive: true }); await chmod(directory, 0o700); } + // Seams: this test runs unprivileged, so the owner and worker-range floor are the test user. + const options: GrokWorkerTmpOptions = { sharedRoots: [shared], sharedOwnerUid: uid, firstWorkerUid: uid }; + const workers = [{ home: own, uid }, { home: other, uid }]; + await verifyGrokWorkerTmp(workers, options); + + await chmod(path.join(other, "tmp"), 0o777); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "sibling 0777"); + await chmod(path.join(other, "tmp"), 0o700); + + await rm(path.join(own, "tmp"), { recursive: true }); await symlink(elsewhere, path.join(own, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "private temp symlink to a valid directory"); + await rm(path.join(own, "tmp")); await mkdir(path.join(own, "tmp"), { mode: 0o700 }); + + const linkedShared = path.join(root, "linked-shared"); await symlink(elsewhere, linkedShared); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [linkedShared] }), refused, "shared root symlink to a valid directory"); + const regular = path.join(root, "regular"); await writeFile(regular, ""); await chmod(regular, 0o600); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [regular] }), refused, "shared root regular file"); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [] }), refused, "empty shared roots"); + await verifyGrokWorkerTmp(workers, options); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts new file mode 100644 index 0000000..863a9ab --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -0,0 +1,68 @@ +import type { Stats } from "node:fs"; +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Readonly<{ isDirectory(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; +const FIRST_WORKER_UID = GROK_ENGINE_BROKER.identities.firstWorkerUid; + +export type GrokWorkerTmpWorker = Readonly<{ home: string; uid: number }>; +/** Test seams only; production uses the manifest defaults. */ +export type GrokWorkerTmpOptions = Readonly<{ sharedRoots?: readonly string[]; sharedOwnerUid?: number; firstWorkerUid?: number }>; + +/** + * Temp-directory isolation, checked before every turn for *every* registered + * worker, not only the one about to run: a misprovisioned sibling temp + * directory (group- or world-writable) would be a place this worker could + * write into and that sibling would read from. + * + * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` + * read-write, and it refuses to start when either (or any ancestor of a + * granted path) is in `deny` — verified live: `deny = ["/tmp"]`, + * `["/var/tmp"]`, `["/run"]`, `["/etc"]` all fail with "could not apply the + * sandbox profile", while `["/tmp/sub"]` works. So the kernel profile cannot + * keep evaluator temp files away from the worker. Unix modes can, because the + * launcher drops the worker to its own uid/gid with no supplementary groups: + * + * - shared temp roots are root-owned, owned by a group below the worker range, + * and give "other" at most read (Grok opens the directory; without search + * the worker can list names but cannot open, stat, or create anything); a + * deployment that lets workers traverse or write them is refused; + * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which + * strict grants read-write) is a real directory owned by the worker with no + * group or other access, and every registered worker's is checked. + * + * + * Entries come from `lstat`, so a symlink is never a directory here: a + * symlinked temp root or private temp is refused by the directory check. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmps: readonly Readonly<{ uid: number; entry: Entry | undefined }>[]; shared: readonly (Entry | undefined)[] }>, options: GrokWorkerTmpOptions = {}): void { + const shared = HOME.sharedTmp; + const firstWorkerUid = options.firstWorkerUid ?? FIRST_WORKER_UID; + if (entries.shared.length === 0 || entries.privateTmps.length === 0) throw unavailable(); + for (const entry of entries.shared) { + if (entry === undefined || !entry.isDirectory() || entry.uid !== (options.sharedOwnerUid ?? shared.uid) || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + for (const { uid, entry } of entries.privateTmps) { + if (!Number.isSafeInteger(uid) || uid < firstWorkerUid || entry === undefined || !entry.isDirectory() || entry.uid !== uid || (Number(entry.mode) & 0o077) !== 0) throw unavailable(); + } +} + +/** `workers` must list every registered worker (the running one included). */ +export async function verifyGrokWorkerTmp(workers: readonly GrokWorkerTmpWorker[], options: GrokWorkerTmpOptions = {}): Promise { + const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; + assertGrokWorkerTmpEntries({ + privateTmps: await Promise.all(workers.map(async (worker) => ({ uid: worker.uid, entry: await inspect(path.join(worker.home, HOME.privateTmp.relativeToWorkerHome)) }))), + shared: await Promise.all((options.sharedRoots ?? HOME.sharedTmp.paths).map(inspect)) + }, options); +} + +/** A registration's worker home is the parent of its `/.grok/sandbox.toml`. */ +export const grokWorkerHomeForProfile = (profilePath: string): string | undefined => + path.basename(path.dirname(profilePath)) === ".grok" ? path.dirname(path.dirname(profilePath)) : undefined; + +const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 3111163..e16c59c 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -3,6 +3,25 @@ export * from "./contractManifest.js"; export * from "./agySubscriptionRealm.js"; export { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; export { createOrganizationRuntimeControlHost } from "./organizationRuntimeControl.js"; +export { CODEX_SANDBOX_PROJECTION_VERSION, resolveOrganizationCodexSandboxProjection, + type OrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; +export { GROK_BROKER_PROJECTION_VERSION, grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, + verifyGrokBrokerRegistrationMatchesProjection, type OrganizationGrokBrokerProjection, type OrganizationGrokBrokerProjectionOptions } from "./grokBrokerProjection.js"; +export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, type GrokSlotPreflightFreshness, + verifyGrokSlotPreflightReceipt, type GrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; +export { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInferenceClientConfigSha256, renderGrokInferenceClientConfig, renderProductionGrokInferenceClientConfig, type GrokInferenceClientConfigInput } from "./grokInferenceClientConfig.js"; +export { GROK_INFERENCE_PROXY_BASE_URL, ENGINE_BROKER_INFERENCE_FAILURE_CODES, type EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; +export { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION, GROK_INFERENCE_PURPOSES, type GrokInferencePurpose } from "./inferenceUsageLedger.js"; +export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; +export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; +export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; +export { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { TURN_SEAL_LEDGER, TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; +export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, + type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; export { WakeTransitionLockBlockedError } from "./wakeAcceptanceStore.js"; export { OFFLINE_RECONCILIATION_BLOCKED_CODE, @@ -38,3 +57,4 @@ export { type WakeReceiptState } from "./wakeAcceptanceTypes.js"; export type { OrganizationRuntimeControlHost, OrganizationRuntimeControlOptions } from "./organizationRuntimeControl.js"; +export { EngineBrokerControlClient, EngineBrokerInferenceGrantRefused, type EngineBrokerInferenceGrant } from "./engineBrokerControlClient.js"; diff --git a/src/runtime/inferenceUsageLedger.ts b/src/runtime/inferenceUsageLedger.ts new file mode 100644 index 0000000..7a389c3 --- /dev/null +++ b/src/runtime/inferenceUsageLedger.ts @@ -0,0 +1,63 @@ +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; +import { recordLedgerLines } from "./turnRequestLedger.js"; + +/** + * Evaluator inference spend, one row per upstream model request of one grant. + * + * This is a separate stream at a separate path (`service.json` + * `inferenceLedgerPath`), never the subject usage ledger: the org wake fuse and + * Spawnfile's subject accounting sum that ledger, and a judge's tokens are not + * a subject wake's. Rows carry `kind: "inference"` so a reader that is pointed + * at the wrong file can still tell them apart (`wakeFuse.ts` skips them). + * + * Grants are not sealed in a durable registry: each request is appended when + * it settles. `(grant, request)` identifies a row; readers dedupe on it + * ({@link dedupeInferenceUsageRows}). + */ +export const INFERENCE_USAGE_LEDGER_VERSION = GROK_ENGINE_BROKER.inferenceGrants.ledgerVersion; +export const GROK_INFERENCE_PURPOSES = GROK_ENGINE_BROKER.inferenceGrants.purposes; +export type GrokInferencePurpose = (typeof GROK_INFERENCE_PURPOSES)[number]; + +export type InferenceUsageEntry = Readonly<{ + grant: string; + purpose: GrokInferencePurpose; + model: GrokBrokerModel; + request: number; + usage: EngineBrokerTurnUsage; + usageSource: "upstream" | "estimated"; + startedAt: string; + endedAt: string; + at?: string; +}>; + +export const renderInferenceUsageLine = (entry: InferenceUsageEntry): string => { + if (!/^[a-f0-9]{32}$/u.test(entry.grant) || !(GROK_INFERENCE_PURPOSES as readonly string[]).includes(entry.purpose) || !(GROK_BROKER_MODELS as readonly string[]).includes(entry.model) || !Number.isSafeInteger(entry.request) || entry.request < 0) throw new TypeError("invalid inference usage entry"); + const { usage } = entry; + return `${JSON.stringify({ + v: INFERENCE_USAGE_LEDGER_VERSION, kind: "inference", purpose: entry.purpose, grant: entry.grant, request: entry.request, + at: entry.at ?? new Date().toISOString(), started_at: entry.startedAt, ended_at: entry.endedAt, model: entry.model, + input: usage.input, cache_read: usage.cacheRead, cache_write: usage.cacheWrite, output: usage.output, total: usage.total, + usage_source: entry.usageSource + })}\n`; +}; + +/** Advisory, never rejects: an evaluator request that already spent tokens must not fail on its ledger. */ +export const recordInferenceUsage = async (file: string, entry: InferenceUsageEntry): Promise => { + let line: string; + try { line = renderInferenceUsageLine(entry); } catch { return false; } + return recordLedgerLines(file, line); +}; + +/** Keeps the first row of each `(grant, request)`; rows without both keys are not inference rows and are dropped. */ +export const dedupeInferenceUsageRows = >(rows: readonly T[]): T[] => { + const seen = new Set(); + return rows.filter((row) => { + if (typeof row.grant !== "string" || typeof row.request !== "number") return false; + const key = `${row.grant}\0${row.request}`; + if (seen.has(key)) return false; + seen.add(key); return true; + }); +}; diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index f8e468b..be65878 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -11,3 +11,159 @@ forbidden. Prompt and scoped capability bytes cross as inherited sealed file descriptors, not protocol strings. Workers must start in a private process group with no-new-privileges, no capabilities, no core dump, and a parent-death signal. Unsupported platforms fail closed. + +The worker argv is one compiled constant: the lean Grok 1.0.34 flags, the +`DBL_GROK_SYSTEM_PROMPT` operating contract, the `--tools` allowlist and the +`--max-turns` backstop live in `engineBrokerLauncher.h`, mirrored byte-for-byte +by `src/contracts/grokWorkerContract.ts` and checked by `launcherArgv.test.ts`. +Model and reasoning effort are per-deployment and belong to the worker +`config.toml`, never to the argv. + +The launcher sets exactly two turn-scoped capabilities in the worker +environment: `DAIMON_MCP_CAPABILITY` and `DAIMON_PROVIDER_CAPABILITY` (Grok +1.0.34 ignores `[auth_provider.*]` helpers for custom models, so the worker +config reads the proxy capability through `env_key`). `--auth-provider` mode +remains for callers of the older contract. + +It also exports `TMPDIR=/tmp`, the worker's private temp +directory, derived only from the root-owned registration. + +Received descriptors carry `MSG_CMSG_CLOEXEC` and can already occupy fds 3-5, +so `launch()` lifts prompt, capability, output, executable and status fds above +16 before `dup2`-ing them into place; a `dup2` onto itself keeps close-on-exec +and the prompt used to vanish at exec. `fixtureWorker.c` reads +`/proc/self/fd/3` so the integration suite fails if that regresses. + +The executable is re-hashed on every spawn (~58 ms for the 136 MB Grok binary). +Holding one verified descriptor and `execveat`-ing it would not make a replaced +binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path +(`/usr/local/bin/grok`), so the image path's root ownership, not the launcher +descriptor, is what protects the sandboxed process. + +**The worker's end of that pipe is a blocking pipe.** `O_NONBLOCK` is a +property of the open file description, not of a descriptor, so creating the +merged stdout/stderr pipe with `pipe2(..., O_NONBLOCK)` handed non-blocking +writes to the worker along with `pipes[1]`: Grok 1.0.34 makes the first EAGAIN +from a headless stdout write fatal (`stdout write failed: Resource temporarily +unavailable (os error 11)`) and exits 1 before it issues a single model +request, so the turn burns a wake and buys nothing. It stayed invisible until +the MCP tools became reachable and the init frame that enumerates them grew to +roughly 9.5 KB — past a pipe buffer, which is not always the 64 KiB default +(8 KiB inside the Docker Desktop VM this suite runs in). So the pipe is created +`O_CLOEXEC` only and `O_NONBLOCK` is set afterwards on `pipes[0]` alone, the +read end this process polls; that one is load bearing, because the post-exit +drain loop has no `poll` and would otherwise park on a write end some surviving +grandchild still holds. + +A blocking child cannot wedge the launcher. `serve()` runs in its own forked +handler per connection, so one worker's backpressure never reaches another +turn; `supervise` drains the pipe on every pass of a 250 ms `poll`, and both +bounds act on a child that is asleep in `write()`: crossing `DBL_MAX_OUTPUT` +stops reading (`p[1].events = 0`) and `kill(-pid, SIGKILL)`s the worker's whole +process group in the same iteration, and a client disconnect does the same — +neither is refusable by a process sleeping on a pipe. `worker_spill_case` is +the cover: the fixture shrinks its own stdout pipe to the kernel minimum, +reports the capacity it actually got, and writes four times that in one +`write`, so it straddles the buffer on any host without assuming 64 KiB while +staying under `DBL_MAX_OUTPUT`. + +**Crossing `DBL_MAX_OUTPUT` is a reported status, not a lost turn.** This is +worth stating because it has been guessed at twice: a trip sets +`output_limited`, stops reading, `SIGKILL`s the worker's process group, reaps +it, and then — `disconnected` is still 0, so the branch at the end of +`supervise` runs — writes the complete 128-byte result frame with +`DBL_STATUS_OUTPUT_FAILED`, `DBL_STAGE_OUTPUT`, `DBL_FAILURE_OUTPUT_LIMIT` and +`output_length = 0`. `closed_result` admits exactly that shape, the client +relays it, and `decodeNativeBrokerResult` raises a named +`NativeBrokerTurnFailure`. So a trip costs the turn its *text* and nothing +else: the broker still seals the turn and still meters the spend the proxy +measured. A lost terminal frame, an unnamed transport failure or an unmetered +turn therefore cannot be explained by this bound, and the only branch that +sends nothing at all is a client that already disconnected. + +**Both readers of that buffer trip the same bound**, through +`output_limit_crossed`. The poll loop always did; the post-exit drain did not, +so a worker that exited with more than `DBL_MAX_OUTPUT` still in the pipe left +`used` at the buffer's last byte with `output_limited` clear, and the turn was +published `DBL_STATUS_OK` with `output_length = DBL_MAX_OUTPUT + 1` — which +`closed_result` refuses, so the client replaced it with a fabricated +`prelaunch_failed`/`protocol` frame carrying no pid and no start ticks. That +frame says the worker never ran, about a turn that ran and whose work may have +succeeded, which is the one class of lie this boundary must never tell. +The window is real but narrow: `poll` is level-triggered, so the loop sees any +buffered byte, and the drain can only inherit data written in the gap between +`poll()` returning and `waitpid()` reaping. It is therefore **not +reproducible on demand in this suite** — the fix is by construction, and the +adversarial cases that do cross the bound (`output_boundary_case`, +`worker_flood_case`, `worker_spill_case`) only prove it did not regress. Do not +add a test that claims to cover it by feeding the bound through the poll loop: +that routes around the defect. + +The bound is the whole turn's stdout, not one frame, and it is now 256 KiB. +**This supersedes the "known limit, deliberately not raised yet" this file +carried while the pipe's blocking mode was the variable under test.** The +measurement that decided it: a live brokered turn emitted 26,482 bytes for +four tool calls, 23,320 of them one tool-result frame carrying all four +(`.runtime/grok-p1b/worker-a2-output.jsonl`); JSON framing and escaping +inflated those payloads by 1.007x, so a turn's stdout is close to the sum of +its tool results. The nine-tool-call turn this was raised for is about 210 KB +of the same shape, against a 64 KiB bound — so 64 KiB was reachable by an +ordinary working turn, and crossing it costs that turn its whole text. The new +number is not headroom-by-guess: it is the control protocol's own `text` bound +(`engineBrokerProtocol.ts`, 262144), the next boundary this output has to +cross, so a larger launcher bound would only move the refusal one layer up. +`worker_turn_case` writes exactly that measured shape — a 9,728-byte init +frame and nine 23,320-byte frames, 219,608 bytes — and asserts it is published +whole; restoring 65536 turns it red. + +**A bound that hangs would be worse than no bound, and this one does not.** +The hypothesis that a worker parks forever in `write()` once the bound is +crossed — plausible after the pipe became blocking, because a write that +cannot complete now blocks instead of erroring — was tested, not reasoned +about, and it is false. `worker_stream_case` writes eight times the bound in +frame-sized writes and then sleeps far longer than this suite, so it is asleep +inside `write()` with its pipe full when the trip fires and nothing but the +launcher can end it; the launcher answers `output_limit` with `term_signal` +SIGKILL in seconds. Its socket carries a 30-second deadline so a park fails +red instead of parking the runner. Deleting the `output_limited` half of +`if (disconnected || output_limited) kill(-pid, SIGKILL)` is the mutation that +proves it: the case then times out on that deadline, and +`output_boundary_case` does not notice, because its worker has already exited +by the time the trip fires. That is the boundary the two cases straddle — +a worker gone at the trip against a worker alive and blocked at it. + +The result frame's last word is `diagnostic_length`, not padding: on +`DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` +bytes of the worker's merged stdout/stderr and sends them after the fixed +frame, while `output_length` stays 0 as before. Every other failure sends none, +and `closed_result` refuses a frame that mixes the two. The bytes are the +worker's own, so the broker redacts them before they cross any boundary. + +**The window keeps both ends.** A worker that dies early prints its error +first and then echoes its own input, so a pure tail kept the echo: the one live +capture this had ever produced was 512 bytes of the agent's own prompt read +back, with the error already off the front and erased here. `diagnostic_window` +keeps the first `DBL_MAX_DIAGNOSTIC / 2`, then `DBL_DIAGNOSTIC_ELISION` naming +the bytes dropped, then the last `DBL_MAX_DIAGNOSTIC / 2`, all inside the same +bound — the marker is sized against `used`, the largest count it can carry, so +the budget holds for every input, and a `snprintf` that will not fit falls back +to the tail. Output that already fits is left in place, byte-identical, with no +marker. The marker text is byte-identical to the TypeScript window's +(`boundedDiagnosticWindow`), so one grep finds an elision on either side of the +boundary. + +That elision is a *cut*, and a cut can split a turn capability in half, leaving +a fragment exact redaction can never match. The answer used where Daimon owns +both ends — retain one whole secret more than is reported — cannot work here, +because what this window keeps is exactly what it sends: a margin reserved here +would be sent too. So the fragment is scrubbed where the capabilities are +known, in `engineBrokerNativeClient.ts` (`scrubCutFragments`), on both sides of +every marker and at the window's outer ends. + +Changing any of the six pinned launcher sources means rebuilding: `node +--import tsx src/runtime/native/build.ts`, then re-pin +`artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256` in the contract manifest and +re-emit it. `artifactsManifest.test.ts` fails by design until that is done. The +adversarial suite is `docker build -f Dockerfile.integration -t .` in this +folder and `docker run --rm --privileged `; `worker_flood_case` is the +head-and-tail cover and fails first if the window regresses to a tail. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index f028a3c..7d34af4 100755 Binary files a/src/runtime/native/artifacts/daimon-engine-broker-arm64 and b/src/runtime/native/artifacts/daimon-engine-broker-arm64 differ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json index 85dafe3..a694add 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58","binary_sha256":"sha256:ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64 b/src/runtime/native/artifacts/daimon-engine-broker-x64 index 3b0b5d1..f7c022d 100755 Binary files a/src/runtime/native/artifacts/daimon-engine-broker-x64 and b/src/runtime/native/artifacts/daimon-engine-broker-x64 differ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index 5514ae9..4dba2b9 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58","binary_sha256":"sha256:e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/artifactsManifest.test.ts b/src/runtime/native/artifactsManifest.test.ts new file mode 100644 index 0000000..259bcdc --- /dev/null +++ b/src/runtime/native/artifactsManifest.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../../contracts/runtimeContractManifest.js"; + +const sources = ["engineBrokerLauncher.c", "engineBrokerLauncher.h", "engineBrokerLauncherCore.inc", "engineBrokerLauncherServer.inc", "engineBrokerLauncherModes.inc", "engineBrokerLauncherMain.inc"]; +const read = (name: string): Buffer => readFileSync(new URL(`./${name}`, import.meta.url)); + +test("the manifest pins the committed native artifacts, their provenance, and the current launcher source", () => { + const source = createHash("sha256"); + for (const name of sources) source.update(read(name)); + assert.equal(GROK_ENGINE_BROKER.artifacts.sourceSha256, source.digest("hex"), "launcher source changed without rebuilding the native artifacts"); + for (const [architecture, pinned] of [["x64", GROK_ENGINE_BROKER.artifacts.x64Sha256], ["arm64", GROK_ENGINE_BROKER.artifacts.arm64Sha256]] as const) { + const binary = read(`artifacts/daimon-engine-broker-${architecture}`); + const provenance = JSON.parse(read(`artifacts/daimon-engine-broker-${architecture}.provenance.json`).toString("utf8")) as Record; + const digest = createHash("sha256").update(binary).digest("hex"); + assert.equal(digest, pinned, architecture); + assert.equal(provenance.binary_sha256, `sha256:${digest}`, architecture); + assert.equal(provenance.source_sha256, `sha256:${GROK_ENGINE_BROKER.artifacts.sourceSha256}`, architecture); + assert.equal(provenance.install_path, GROK_ENGINE_BROKER.nativeExecutablePath, architecture); + } +}); diff --git a/src/runtime/native/copyArtifact.ts b/src/runtime/native/copyArtifact.ts index 1166aa6..be3c30d 100644 --- a/src/runtime/native/copyArtifact.ts +++ b/src/runtime/native/copyArtifact.ts @@ -4,8 +4,14 @@ import { fileURLToPath } from 'node:url'; await import('./verifyArtifacts.ts'); const root = path.dirname(fileURLToPath(import.meta.url)); -const architecture = process.arch; -if (!['x64', 'arm64'].includes(architecture) || process.platform !== 'linux') { +// The broker artifacts are prebuilt, provenance-verified Linux executables checked into +// this repository, so staging one is a packaging step and not a host capability: the +// published tarball must carry `dist/runtime/native/daimon-engine-broker` on every packing +// host, because the runtime image installs that tarball with no lifecycle script that could +// stage it later. `DAIMON_ENGINE_BROKER_ARCH` selects the packaged Linux target when it is +// not the host's own architecture. +const architecture = process.env.DAIMON_ENGINE_BROKER_ARCH?.trim() || process.arch; +if (!['x64', 'arm64'].includes(architecture)) { if (process.env.DAIMON_REQUIRE_ENGINE_BROKER === '1') throw new Error('native engine broker is Linux x64/arm64 only'); process.exit(0); } diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 76e7ce9..7542b51 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -8,7 +8,28 @@ #define DBL_MAX_PROMPT 65536u #define DBL_MAX_TOKEN 4096u #define DBL_MAX_CAPABILITY_BUNDLE (DBL_MAX_TOKEN * 2u + 4u) -#define DBL_MAX_OUTPUT 65536u +/* The WHOLE turn's stdout, not one frame, and sized against real turns rather + than headroom-by-guess. A live four-tool-call brokered turn emitted 26,482 + bytes, 23,320 of them one tool-result frame carrying four results + (`.runtime/grok-p1b/worker-a2-output.jsonl`), so 64 KiB was reachable by an + ordinary working turn: the nine-tool-call turn this was raised for lands + around 210 KB of the same shape, and a trip costs the turn its whole text. + The number is the control protocol's own `text` bound + (`engineBrokerProtocol.ts`, 262144), because that is the next boundary the + output must cross: a larger launcher bound would only move the refusal one + layer up. A runaway worker is still stopped here — crossing it stops + reading, SIGKILLs the worker's process group and reports `output_limit`. */ +#define DBL_MAX_OUTPUT 262144u +/* Bounded tail of the worker's own merged stdout/stderr, kept only for a + worker that exited on its own account (`DBL_STATUS_WORKER_FAILED`), so the + reason reaches the host instead of `exit=1`. It is a diagnostic, never the + turn's output: `output_length` stays 0 on every failure. */ +#define DBL_MAX_DIAGNOSTIC 512u +/* The marker that joins the two ends of an elided diagnostic, byte-identical + to the TypeScript window's (`boundedDiagnosticWindow` in + `src/pi/cliChildOutput.ts`), so one grep finds every elision on either side + of the boundary. Its own bytes are paid for out of DBL_MAX_DIAGNOSTIC. */ +#define DBL_DIAGNOSTIC_ELISION "[\xe2\x80\xa6 %llu bytes elided \xe2\x80\xa6]" #ifndef DBL_REGISTRY #define DBL_REGISTRY "/etc/daimon-engine-broker/registrations.bin" #endif @@ -21,6 +42,12 @@ #ifndef DBL_EXECUTABLE #define DBL_EXECUTABLE "/usr/local/bin/grok" #endif +/* Lean Grok worker contract; mirrored byte-for-byte by src/contracts/grokWorkerContract.ts + and checked by launcherArgv.test.ts. */ +#define DBL_GROK_SYSTEM_PROMPT \ + "You are a headless Daimon agent; no human is present. Your identity, instructions and wake event are in the user prompt. Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know. If a tool result says output was saved to a file, read that path with read_file. If a tool fails, do not retry it in a loop: stop and report the failure. Your final answer is a private note to the runtime: one line, or empty." +#define DBL_GROK_TOOLS "run_terminal_cmd,read_file,grep,list_dir,search_tool,use_tool" +#define DBL_GROK_MAX_TURNS "48" struct dbl_request { uint32_t version, slot; @@ -67,13 +94,13 @@ struct dbl_result { int32_t worker_pid, exit_code, term_signal; uint64_t start_ticks; char turn_id[65]; - uint32_t stage, failure_class, profile_applied, reserved; + uint32_t stage, failure_class, profile_applied, diagnostic_length; }; #define DBL_RESULT_SIZE 128u #define DBL_RESULT_STAGE_OFFSET 108u #define DBL_RESULT_FAILURE_CLASS_OFFSET 112u #define DBL_RESULT_PROFILE_APPLIED_OFFSET 116u -#define DBL_RESULT_RESERVED_OFFSET 120u +#define DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET 120u _Static_assert(sizeof(struct dbl_result) == DBL_RESULT_SIZE, "dbl_result ABI size"); _Static_assert(__builtin_offsetof(struct dbl_result, stage) == @@ -85,8 +112,8 @@ _Static_assert(__builtin_offsetof(struct dbl_result, failure_class) == _Static_assert(__builtin_offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET, "dbl_result profile offset"); -_Static_assert(__builtin_offsetof(struct dbl_result, reserved) == - DBL_RESULT_RESERVED_OFFSET, - "dbl_result reserved offset"); +_Static_assert(__builtin_offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, + "dbl_result diagnostic length offset"); #endif diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index 8329892..332fb0a 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -25,7 +25,6 @@ #include static __attribute__((noreturn)) void die(void) { _exit(111); } -static int bounded(const char *s, size_t n) { return memchr(s, 0, n) != NULL; } static int safe_component(const char *s, size_t n) { size_t i, l = strnlen(s, n); if (!l || l == n) @@ -157,12 +156,27 @@ static int load_registration(uint32_t slot, struct dbl_registration *out) { close(fd); return -1; } +/* Absolute, NUL-terminated within n, no empty, "." or ".." component and no + trailing slash: the launcher derives HOME, GROK_HOME and TMPDIR from it. */ +static int canonical_path(const char *s, size_t n) { + size_t l = strnlen(s, n), i = 0; + if (l < 2 || l == n || s[0] != '/' || s[l - 1] == '/') + return 0; + while (i < l) { + size_t start = ++i; + while (i < l && s[i] != '/') + i++; + if (i == start || (i - start == 1 && s[start] == '.') || + (i - start == 2 && s[start] == '.' && s[start + 1] == '.')) + return 0; + } + return 1; +} static int valid_registration(const struct dbl_registration *r, const struct dbl_request *q) { return r->uid >= 2200 && r->gid >= 2200 && - bounded(r->workspace, sizeof(r->workspace)) && - bounded(r->home, sizeof(r->home)) && r->workspace[0] == '/' && - r->home[0] == '/' && + canonical_path(r->workspace, sizeof(r->workspace)) && + canonical_path(r->home, sizeof(r->home)) && safe_component(r->agent_id, sizeof(r->agent_id)) && strcmp(r->agent_id, q->agent_id) == 0; } @@ -248,101 +262,3 @@ static uint64_t start_ticks(pid_t pid) { } return 0; } - -static __attribute__((noreturn)) void launch_fail(int status_fd, - uint32_t code) { - (void)full_write(status_fd, &code, sizeof(code)); - _exit(111); -} -static pid_t launch(const struct dbl_registration *r, int executable, - int prompt, int capability, int output, uint32_t *failure, - uint64_t *observed_start_ticks) { - unsigned char mcp[DBL_MAX_TOKEN + 1] = {0}; - int status_pipe[2]; - if (capability_bundle(capability, NULL, mcp) || pipe2(status_pipe, O_CLOEXEC)) - return -1; - pid_t p = fork(); - if (p < 0) { - close(status_pipe[0]); - close(status_pipe[1]); - erase(mcp, sizeof(mcp)); - return -1; - } - if (p != 0) { - uint32_t code = 0; - close(status_pipe[1]); - if (full_read(status_pipe[0], observed_start_ticks, - sizeof(*observed_start_ticks)) || - !*observed_start_ticks) { - close(status_pipe[0]); - erase(mcp, sizeof(mcp)); - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - ssize_t got = read(status_pipe[0], &code, sizeof(code)); - close(status_pipe[0]); - erase(mcp, sizeof(mcp)); - if (got == 0) - return p; - if (got == (ssize_t)sizeof(code)) - *failure = code; - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - close(status_pipe[0]); - if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) - launch_fail(status_pipe[1], 1); - uint64_t identity = start_ticks(getpid()); - if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) - launch_fail(status_pipe[1], 1); - struct rlimit z = {0, 0}; - if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || - prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || - setresuid(r->uid, r->uid, r->uid)) - launch_fail(status_pipe[1], 2); - zero_caps(); - if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || - prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) - launch_fail(status_pipe[1], 3); - if (dup2(prompt, 3) < 0 || dup2(capability, 4) < 0 || - dup2(output, STDOUT_FILENO) < 0 || dup2(output, STDERR_FILENO) < 0) - launch_fail(status_pipe[1], 4); - if (executable != 5 && dup2(executable, 5) < 0) - launch_fail(status_pipe[1], 5); - close_other_fds(status_pipe[1]); - char *const argv[] = {"grok", - "--sandbox", - "daimon-strict", - "--always-approve", - "--no-subagents", - "--prompt-file", - "/proc/self/fd/3", - "--no-memory", - "--disable-web-search", - "--cwd", - (char *)r->workspace, - "--output-format", - "streaming-messages-json", - "--model", - "daimon-broker-grok", - NULL}; - char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24]; - snprintf(home, sizeof(home), "HOME=%s", r->home); - snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); - snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); - erase(mcp, sizeof(mcp)); - char *const envp[] = {home, - grok, - mcp_env, - "DAIMON_CAPABILITY_FD=4", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LANG=C.UTF-8", - "TZ=UTC", - NULL}; - syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); - erase(mcp_env, sizeof(mcp_env)); - launch_fail(status_pipe[1], 6); -} - diff --git a/src/runtime/native/engineBrokerLauncherIntegrationCore.inc b/src/runtime/native/engineBrokerLauncherIntegrationCore.inc index 25ff703..0194abf 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationCore.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationCore.inc @@ -215,7 +215,7 @@ static void client_case(void) { result_bytes == sizeof(result) && result_padding_zero(&result) && result.status == DBL_STATUS_OK && result.stage == DBL_STAGE_OUTPUT && result.failure_class == DBL_FAILURE_NONE && !result.profile_applied && - !result.reserved && result.output_length < DBL_MAX_OUTPUT, + !result.diagnostic_length && result.output_length < DBL_MAX_OUTPUT, "client result"); char *out = calloc(1, result.output_length + 1); check(read(output[0], out, result.output_length) == @@ -369,7 +369,7 @@ static void malformed_response_case(int truncated) { if (truncated) write(peer, &result, sizeof(result) / 2); else { - result.reserved = 1; + result.diagnostic_length = DBL_MAX_DIAGNOSTIC + 1; write(peer, &result, sizeof(result)); } close(peer); @@ -380,11 +380,11 @@ static void malformed_response_case(int truncated) { observed.status == DBL_STATUS_PRELAUNCH_FAILED && observed.stage == DBL_STAGE_REQUEST && observed.failure_class == DBL_FAILURE_PROTOCOL && - !observed.reserved, + !observed.diagnostic_length, "fixed protocol diagnostic"); close(output[0]); int status; waitpid(child, &status, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, - truncated ? "truncated result" : "reserved result"); + truncated ? "truncated result" : "over-bound diagnostic result"); } diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 8458226..7b4bdc8 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -83,6 +83,198 @@ static void output_boundary_case(const char *provider, int overflow) { close(p); close(c); } +/* A worker that exits nonzero publishes no output, but its own last words — + stderr and stdout share one pipe — must survive as the bounded diagnostic + tail, because the host otherwise sees nothing but `exit=1`. */ +static void worker_failure_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-failure", "mcp.Failure-2"); + struct dbl_request q = request(); + struct dbl_result r; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_WORKER_FAILED && r.stage == DBL_STAGE_WAIT && + r.failure_class == DBL_FAILURE_EXEC && r.exit_code == 1 && + r.term_signal == 0 && r.worker_pid > 0 && r.worker_uid == 2200 && + r.start_ticks > 0 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker failure diagnostic length"); + char *reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length) && + strstr(reason, "grok: session store unwritable"), + "worker failure reason"); + char extra; + check(read(s, &extra, 1) == 0, "worker failure EOF"); + free(reason); + close(s); + close(p); + close(c); +} +/* A worker that dies after printing far more than the window: the error is at + the START of what it printed and the echo at the end, which is the shape a + pure tail got wrong. The distinctive head run STRADDLES the head's own cut — + its first bytes are inside the retained head and its later bytes are elided — + so a tail-only window loses it entirely and this case fails. */ +static void worker_flood_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-flood", "mcp.Flood-2"); + struct dbl_request q = request(); + struct dbl_result r; + size_t printed = strlen("HEAD-OF-ERROR grok: profile refused ") + 4096u + + strlen(" TAIL-OF-ECHO"); + unsigned long long elided = 0; + char *reason, *marker, *shown; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && r.status == DBL_STATUS_WORKER_FAILED && + r.exit_code == 1 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker flood diagnostic length"); + reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length), "worker flood diagnostic"); + check(!strncmp(reason, "HEAD-OF-ERROR grok: profile refused", + strlen("HEAD-OF-ERROR grok: profile refused")), + "worker flood head"); + shown = strstr(reason, " TAIL-OF-ECHO"); + check(shown && shown[strlen(" TAIL-OF-ECHO")] == 0, "worker flood tail"); + marker = strstr(reason, "[\xe2\x80\xa6 "); + check(marker && sscanf(marker, "[\xe2\x80\xa6 %llu bytes elided", &elided) == 1, + "worker flood marker"); + /* The count is exactly what was dropped: everything printed, less the two + ends that survived (the whole window less the marker itself). */ + size_t marker_length = (size_t)(strchr(marker, ']') - marker) + 1u; + check((size_t)elided + (size_t)r.diagnostic_length - marker_length == printed, + "worker flood elided count"); + char extra; + check(read(s, &extra, 1) == 0, "worker flood EOF"); + free(reason); + close(s); + close(p); + close(c); +} +/* A worker's single write larger than the pipe buffer must complete, not kill + it. The launcher created its stdout/stderr pipe O_NONBLOCK as a whole, and + O_NONBLOCK is a property of the open file description, so the worker + inherited a non-blocking stdout: Grok 1.0.34 turns the first EAGAIN from a + headless stdout write into `stdout write failed: Resource temporarily + unavailable` and exits 1 before it ever reaches the model. Once its init + frame outgrew a pipe buffer that was every turn, at $0 apiece. + The fixture shrinks its own stdout pipe to the kernel minimum, reports the + capacity it actually got, and writes four times that in ONE write, so the + case straddles the buffer on any host instead of assuming 64 KiB — and + stays under DBL_MAX_OUTPUT, so it is the write that is bounded here, never + the turn's output. */ +static void worker_spill_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("spill-output", "mcp.Spill-2"); + struct dbl_request q = request(); + struct dbl_result r; + unsigned capacity = 0, count = 0; + size_t head, filler = 0; + char *out, *line, extra; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && + r.output_length > 0 && r.output_length <= DBL_MAX_OUTPUT, + "worker spill survived its own oversized write"); + out = calloc(1, (size_t)r.output_length + 1u); + check(out && read_all(s, out, r.output_length), "worker spill output"); + line = strchr(out, '\n'); + check(line && sscanf(out, "SPILL cap=%u count=%u", &capacity, &count) == 2 && + capacity > 0 && count == r.output_length && + count >= capacity * 4u, + "worker spill straddles the pipe buffer"); + head = (size_t)(line - out) + 1u; + while (head + filler + 10u < (size_t)r.output_length && + out[head + filler] == 's') + filler++; + check(head + filler + 10u == (size_t)r.output_length && + !memcmp(out + r.output_length - 10, "SPILL-TAIL", 10), + "worker spill bytes arrived intact"); + check(read(s, &extra, 1) == 0, "worker spill EOF"); + free(out); + close(s); + close(p); + close(c); +} +/* An ordinary working turn must not be able to hit the bound, and its output + must arrive whole. The fixture writes the shape one live turn measured — a + 9,728-byte init frame and nine 23,320-byte tool-result frames, 219,608 + bytes in all — in frame-sized writes. That is more than three times the + 64 KiB this bound used to be, so at the old value this same turn was + published as `output_limit` with its whole text discarded; it is the case + that straddles the raise, and restoring 65536 turns it red. */ +static void worker_turn_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("turn-output", "mcp.Turn-2"); + struct dbl_request q = request(); + struct dbl_result r; + const uint32_t expected = 9728u + 23320u * 9u; + char *out, extra; + size_t index; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + expected > 65536u && + r.output_length == expected, + "ordinary turn published whole"); + out = calloc(1, (size_t)expected + 1u); + check(out && read_all(s, out, expected), "ordinary turn output"); + check(!memcmp(out, "TURN-HEAD", 9) && + !memcmp(out + expected - 9, "TURN-TAIL", 9), + "ordinary turn ends intact"); + for (index = 9; index < (size_t)expected - 9u; index++) + if (out[index] != 'u') + break; + check(index == (size_t)expected - 9u, "ordinary turn bytes intact"); + check(read(s, &extra, 1) == 0, "ordinary turn EOF"); + free(out); + close(s); + close(p); + close(c); +} +/* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is + still working and still writing. `output_boundary_case` writes one byte past + the bound and then exits on its own account, so it proves the arithmetic and + not the termination: the worker was already gone when the trip fired. This + one straddles that boundary from the other side. The fixture writes eight + times the bound in ordinary frame-sized writes and then sleeps far longer + than this suite, so it fills its own pipe repeatedly and is asleep inside + `write()` when the trip happens, and NOTHING but the launcher can end it. + A launcher that stopped reading without killing — or killed without + answering — parks the worker and this client forever, which is why the + socket carries a deadline: a park has to fail red here, not hang the runner. + The deadline also bounds the answer: it must arrive while the fixture is + still sleeping, so a pass cannot be the fixture exiting by itself. */ +static void worker_stream_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stream-output", "mcp.Stream-2"); + struct dbl_request q = request(); + struct dbl_result r; + struct timeval deadline = {.tv_sec = 30, .tv_usec = 0}; + char extra; + check(!setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &deadline, sizeof(deadline)), + "worker stream deadline"); + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r), + "worker stream answered before the deadline"); + check(r.status == DBL_STATUS_OUTPUT_FAILED && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_OUTPUT_LIMIT && + r.output_length == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && r.start_ticks > 0 && + r.term_signal == SIGKILL, + "worker stream stopped at the bound"); + check(read(s, &extra, 1) == 0, "worker stream EOF"); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -90,6 +282,11 @@ static void org_cases(void) { client_input_reject(5, 1); output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); + worker_failure_case(); + worker_flood_case(); + worker_spill_case(); + worker_turn_case(); + worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); @@ -110,8 +307,22 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && + strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null tmpdir=/tmp/worker-home/tmp\n") && !strstr(out, "EVIL"), "fixed worker boundary"); + check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && + strstr(out, "=--no-subagents\n") && strstr(out, "=--no-memory\n") && + strstr(out, "=--disable-web-search\n") && + strstr(out, "=--tools\n") && strstr(out, "=" DBL_GROK_TOOLS "\n") && + strstr(out, "=--max-turns\n") && + strstr(out, "=" DBL_GROK_MAX_TURNS "\n") && + strstr(out, "=--system-prompt-override\n") && + strstr(out, "=" DBL_GROK_SYSTEM_PROMPT "\n") && + strstr(out, "=--prompt-file\n") && + strstr(out, "=/proc/self/fd/3\n") && + !strstr(out, "--reasoning-effort") && + strstr(out, " argc=23 "), + "lean worker argv"); free(out); close(s); q = request(); diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index 0b816dd..ef59692 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -1,3 +1,110 @@ +/* Non-canonical or unterminated homes/workspaces are refused at registration: + HOME, GROK_HOME and TMPDIR are derived from them. */ +static void noncanonical_registration_cases(struct dbl_registration r) { + const char *homes[] = {"/tmp/worker-home/../other", "/tmp//worker-home", + "/tmp/worker-home/", "/tmp/./worker-home", + "/tmp/worker-home/..", "relative/home", NULL}; + struct dbl_registration bad[8]; + size_t count = 0; + for (; homes[count]; count++) { + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 0, sizeof(bad[count].home)); + strcpy(bad[count].home, homes[count]); + } + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 'a', sizeof(bad[count].home)); /* over-long: no NUL */ + bad[count].home[0] = '/'; + count++; + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + strcpy(bad[count].workspace, "/tmp/workspace/../etc"); + count++; + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r) && + write(f, bad, sizeof(bad[0]) * count) == + (ssize_t)(sizeof(bad[0]) * count), + "noncanonical registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + for (size_t i = 0; i < count; i++) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Path-1", "mcp.Path-2"); + struct dbl_request q = request(); + struct dbl_result result; + q.slot = 100 + (uint32_t)i; + send_request(s, &q, p, c); + if (!read_all(s, &result, sizeof(result)) || + result.status != DBL_STATUS_PRELAUNCH_FAILED || + result.stage != DBL_STAGE_REGISTRATION || result.worker_pid != 0) + _exit(10 + (int)i); + close(s); + close(p); + close(c); + } + _exit(0); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "noncanonical registration refused"); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} +/* A worker that fails before or at exec must not run anything: the launcher + child exits instead. The registered "executable" is a #! script; fd 5 is + close-on-exec, so execveat(AT_EMPTY_PATH) cannot run it (a script needs + /dev/fd/5 after exec) and the turn fails at the exec stage. */ +static void exec_failure_case(struct dbl_registration r) { + const char *marker = "/tmp/exec-failure-script-ran"; + unlink(marker); + check(rename("/usr/local/bin/grok", "/usr/local/bin/grok.fixture") == 0, + "exec failure fixture move"); + int script = open("/usr/local/bin/grok", O_CREAT | O_EXCL | O_WRONLY, 0755); + const char body[] = "#!/bin/sh\ntouch /tmp/exec-failure-script-ran\n"; + check(script >= 0 && write(script, body, sizeof(body) - 1) == + (ssize_t)(sizeof(body) - 1), + "exec failure script"); + close(script); + digest("/usr/local/bin/grok", r.executable_sha256); + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "exec failure registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Exec-1", "mcp.Exec-2"); + struct dbl_request q = request(); + struct dbl_result result; + send_request(s, &q, p, c); + _exit(read_all(s, &result, sizeof(result)) && + result.status == DBL_STATUS_PRELAUNCH_FAILED && + result.stage == DBL_STAGE_EXEC && + result.failure_class == DBL_FAILURE_EXEC && + result.worker_pid == 0 + ? 0 + : 1); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "exec failure result"); + sleep(1); + check(access(marker, F_OK) != 0, "exec failure ran nothing"); + unlink("/usr/local/bin/grok"); + check(rename("/usr/local/bin/grok.fixture", "/usr/local/bin/grok") == 0, + "exec failure fixture restore"); + digest("/usr/local/bin/grok", r.executable_sha256); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} int main(void) { setbuf(stdout, NULL); alarm(40); @@ -7,7 +114,8 @@ int main(void) { DBL_RESULT_FAILURE_CLASS_OFFSET && offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET && - offsetof(struct dbl_result, reserved) == DBL_RESULT_RESERVED_OFFSET, + offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, "result ABI"); mkdir("/run/daimon-engine-broker", 0755); mkdir("/etc/daimon-engine-broker", 0755); @@ -43,8 +151,13 @@ int main(void) { auth_adversarial_cases(); puts("native-stage auth complete"); pid_t broker = fork(); - if (!broker) + if (!broker) { + /* A real launcher stdin, so a worker inheriting it is visible. */ + int launcher_stdin = open("/etc/hostname", O_RDONLY); + if (launcher_stdin < 0 || dup2(launcher_stdin, STDIN_FILENO) < 0) + _exit(1); execl("/opt/daimon/bin/daimon-engine-broker", "daimon-engine-broker", NULL); + } check(broker > 0, "broker fork"); wait_launcher_ready(broker); root_peer_rejects(); @@ -56,6 +169,11 @@ int main(void) { } int status; waitpid(org, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); + exec_failure_case(r); + puts("native-stage exec failure complete"); + noncanonical_registration_cases(r); + puts("native-stage noncanonical registration complete"); kill(broker, SIGKILL); waitpid(broker, 0, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); diff --git a/src/runtime/native/engineBrokerLauncherModes.inc b/src/runtime/native/engineBrokerLauncherModes.inc index ac78544..012da6b 100644 --- a/src/runtime/native/engineBrokerLauncherModes.inc +++ b/src/runtime/native/engineBrokerLauncherModes.inc @@ -2,7 +2,7 @@ static int client_mode(void) { struct dbl_request request; uint32_t prompt_length = 0, capability_length = 0; unsigned char *prompt = NULL, capability[DBL_MAX_CAPABILITY_BUNDLE] = {0}, - output[DBL_MAX_OUTPUT] = {0}, extra; + output[DBL_MAX_OUTPUT + DBL_MAX_DIAGNOSTIC] = {0}, extra; struct dbl_result result; struct sockaddr_un a = {.sun_family = AF_UNIX}; int s = -1, p = -1, c = -1, ok = -1; @@ -37,7 +37,8 @@ static int client_mode(void) { 1) || client_send(s, &request, p, c) || full_read(s, &result, sizeof(result)) || !closed_result(&result, request.turn_id) || - full_read(s, output, result.output_length) || read(s, &extra, 1) != 0) { + full_read(s, output, result.output_length + result.diagnostic_length) || + read(s, &extra, 1) != 0) { struct dbl_result failure; memset(&failure, 0, sizeof(failure)); failure.version = DBL_VERSION; @@ -50,8 +51,11 @@ static int client_mode(void) { ok = 0; goto done; } + /* The trailer is the turn's output on success and the worker's bounded + diagnostic tail on failure; `closed_result` keeps the two exclusive. */ if (full_write(STDOUT_FILENO, &result, sizeof(result)) || - full_write(STDOUT_FILENO, output, result.output_length)) + full_write(STDOUT_FILENO, output, + result.output_length + result.diagnostic_length)) goto done; ok = 0; done: diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index a6cb347..bd3dca2 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -1,3 +1,210 @@ +static __attribute__((noreturn)) void launch_fail(int status_fd, + uint32_t code) { + (void)full_write(status_fd, &code, sizeof(code)); + _exit(111); +} +static pid_t launch(const struct dbl_registration *r, int executable, + int prompt, int capability, int output, uint32_t *failure, + uint64_t *observed_start_ticks) { + unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; + int status_pipe[2]; + if (capability_bundle(capability, provider, mcp) || + pipe2(status_pipe, O_CLOEXEC)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + pid_t p = fork(); + if (p < 0) { + close(status_pipe[0]); + close(status_pipe[1]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + if (p != 0) { + uint32_t code = 0; + close(status_pipe[1]); + if (full_read(status_pipe[0], observed_start_ticks, + sizeof(*observed_start_ticks)) || + !*observed_start_ticks) { + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + ssize_t got = read(status_pipe[0], &code, sizeof(code)); + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + if (got == 0) + return p; + if (got == (ssize_t)sizeof(code)) + *failure = code; + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + close(status_pipe[0]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) + launch_fail(status_pipe[1], 1); + uint64_t identity = start_ticks(getpid()); + if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) + launch_fail(status_pipe[1], 1); + struct rlimit z = {0, 0}; + if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || + prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || + setresuid(r->uid, r->uid, r->uid)) + launch_fail(status_pipe[1], 2); + zero_caps(); + if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + launch_fail(status_pipe[1], 3); + /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto + itself keeps close-on-exec (the prompt vanished at exec) and an + overlapping order clobbers a source. Lift all of them above the targets + first so every dup2 below changes the fd number and clears CLOEXEC. */ + int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); + if (status_fd < 0) + launch_fail(status_pipe[1], 4); + int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); + int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), + high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), + high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), + high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); + if (null_input < 0 || high_prompt < 0 || high_capability < 0 || + high_output < 0 || high_executable < 0 || + dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || + dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || + dup2(high_output, STDERR_FILENO) < 0) + launch_fail(status_fd, 4); + /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok + image descriptor out of the worker and every tool child (execveat with + AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ + if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) + launch_fail(status_fd, 5); + close_other_fds(status_fd); + char *const argv[] = {"grok", + "--sandbox", + "daimon-strict", + "--always-approve", + "--no-subagents", + "--prompt-file", + "/proc/self/fd/3", + "--no-memory", + "--disable-web-search", + "--no-plan", + "--verbatim", + "--system-prompt-override", + DBL_GROK_SYSTEM_PROMPT, + "--tools", + DBL_GROK_TOOLS, + "--max-turns", + DBL_GROK_MAX_TURNS, + "--cwd", + (char *)r->workspace, + "--output-format", + "streaming-messages-json", + "--model", + "daimon-broker-grok", + NULL}; + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], + provider_env[DBL_MAX_TOKEN + 32]; + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). + A truncated path must never name a different directory: fail instead. */ + if ((size_t)snprintf(home, sizeof(home), "HOME=%s", r->home) >= sizeof(home) || + (size_t)snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home) >= + sizeof(grok) || + (size_t)snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home) >= + sizeof(tmp)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + launch_fail(status_fd, 6); + } + snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); + /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so + the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped + proxy capability from here. It is as exposed as the MCP capability. */ + snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", + provider); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + char *const envp[] = {home, + grok, + tmp, + mcp_env, + provider_env, + "DAIMON_CAPABILITY_FD=4", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LANG=C.UTF-8", + "TZ=UTC", + NULL}; + syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); + erase(provider_env, sizeof(provider_env)); + erase(mcp_env, sizeof(mcp_env)); + launch_fail(status_fd, 6); +} + +/* Both ends of a failed worker's own output, inside DBL_MAX_DIAGNOSTIC. + A pure tail was the wrong end for the process this exists for: a worker that + dies early prints its error first and then echoes its input, so the tail is + the echo. One live turn reported 512 bytes of the agent's own prompt read + back, with the error already off the front and erased here — the head is + only recoverable where it still exists, which is here. + The marker is paid for out of the same budget and sized against `used`, the + largest count it can ever carry, so the result never exceeds the bound for + any input; output that already fits is left exactly as it is, in place and + byte-identical, with no marker at all. A snprintf that will not fit falls + back to the tail this replaced. */ +static size_t diagnostic_window(unsigned char *bytes, size_t used) { + char marker[64]; + int width, final; + size_t budget, head, tail; + if (used <= DBL_MAX_DIAGNOSTIC) + return used; + width = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)used); + budget = (width > 0 && (size_t)width + 2u <= DBL_MAX_DIAGNOSTIC) + ? DBL_MAX_DIAGNOSTIC - (size_t)width + : 0u; + if (!budget) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + head = budget / 2u; + tail = budget - head; + final = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)(used - head - tail)); + if (final <= 0 || (size_t)final > (size_t)width) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + /* The head stays where it is; the tail moves up behind the marker. */ + memmove(bytes + head + (size_t)final, bytes + (used - tail), tail); + memcpy(bytes + head, marker, (size_t)final); + return head + (size_t)final + tail; +} +/* The one place the total-output bound is decided, because both readers fill + the same buffer and both must trip it. The post-exit drain had no check: a + worker that exited with more than DBL_MAX_OUTPUT still in the pipe left + `used` at the buffer's last byte with `output_limited` clear, so the turn + was published OK with `output_length = DBL_MAX_OUTPUT + 1` — which + `closed_result` refuses, so the client replaced it with a fabricated + `prelaunch_failed`/`protocol` frame. That frame says the worker never ran, + about a turn that ran and whose work may well have succeeded, and it carries + neither the worker's pid nor its start ticks. An honest `output_failed` is + the whole of what changes here. */ +static int output_limit_crossed(size_t used, struct dbl_result *out) { + if (used <= DBL_MAX_OUTPUT) + return 0; + out->status = DBL_STATUS_OUTPUT_FAILED; + out->stage = DBL_STAGE_OUTPUT; + out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; + return 1; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -23,12 +230,9 @@ static void supervise(int client, pid_t pid, int output, ssize_t got = read(output, bytes + used, sizeof(bytes) - used); if (got > 0) used += (size_t)got; - if (used > DBL_MAX_OUTPUT) { + if (output_limit_crossed(used, out)) { output_limited = 1; p[1].events = 0; - out->status = DBL_STATUS_OUTPUT_FAILED; - out->stage = DBL_STAGE_OUTPUT; - out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; } } if (disconnected || output_limited) @@ -47,6 +251,8 @@ static void supervise(int client, pid_t pid, int output, if (got <= 0) break; used += (size_t)got; + if (output_limit_crossed(used, out)) + output_limited = 1; } if (WIFEXITED(status)) out->exit_code = WEXITSTATUS(status); @@ -75,12 +281,25 @@ static void supervise(int client, pid_t pid, int output, out->output_length = (uint32_t)used; } if (out->status != DBL_STATUS_OK) { + /* A failed turn publishes no output, but a worker that exited on its own + account said why on the pipe it shares with stdout, and that window is + the only reason the host can ever see: without it a failure reads + `exit=1`. Keep a bounded window of it and erase the rest here; the + broker redacts it before it crosses any boundary. The other failures get + none: an output-limit window is the very payload the bound refused to + publish, a cancelled turn has no reader left, and a prelaunch failure ran + nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED + ? 0 + : diagnostic_window(bytes, used); + erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; - erase(bytes, sizeof(bytes)); + out->diagnostic_length = (uint32_t)keep; + used = keep; } if (!disconnected) { full_write(client, out, sizeof(*out)); - if (out->status == DBL_STATUS_OK) + if (used) full_write(client, bytes, used); } erase(bytes, sizeof(bytes)); @@ -126,7 +345,14 @@ static void serve(int client) { goto done; out.stage = DBL_STAGE_EXEC; out.failure_class = DBL_FAILURE_EXEC; - if (pipe2(pipes, O_CLOEXEC | O_NONBLOCK)) + /* O_NONBLOCK lives on the open file description, so setting it on the pipe + as a whole handed it to the worker with `pipes[1]`: Grok 1.0.34 makes one + EAGAIN from a headless stdout write fatal and exits 1 before any model + request. Only the read end this process polls is non-blocking, so the + post-exit drain cannot park; a blocking child cannot wedge the launcher, + because `supervise` drains every pass and SIGKILLs the worker's process + group the moment the bound is crossed (`native/AGENTS.md`). */ + if (pipe2(pipes, O_CLOEXEC) || fcntl(pipes[0], F_SETFL, O_NONBLOCK)) goto done; pid = launch(&r, exe, fds[0], fds[1], pipes[1], &launch_failure, &launch_start_ticks); @@ -184,7 +410,8 @@ static int client_send(int socket_fd, const struct dbl_request *r, int prompt, return sendmsg(socket_fd, &m, MSG_NOSIGNAL) == (ssize_t)sizeof(*r) ? 0 : -1; } static int closed_result(const struct dbl_result *r, const char turn_id[65]) { - if (r->version != DBL_VERSION || r->reserved || r->profile_applied > 1 || + if (r->version != DBL_VERSION || r->diagnostic_length > DBL_MAX_DIAGNOSTIC || + r->profile_applied > 1 || r->stage > DBL_STAGE_ATTESTATION || r->failure_class > DBL_FAILURE_ATTESTATION_PROFILE_INVALID || r->output_length > DBL_MAX_OUTPUT || memcmp(r->turn_id, turn_id, 65)) @@ -193,12 +420,14 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_NONE && r->profile_applied == 0 && r->worker_pid > 0 && r->worker_uid >= 2200 && r->start_ticks && - r->exit_code == 0 && r->term_signal == 0; + r->exit_code == 0 && r->term_signal == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_PRELAUNCH_FAILED) return r->stage >= DBL_STAGE_PEER && r->stage <= DBL_STAGE_EXEC && r->failure_class >= DBL_FAILURE_PEER && r->failure_class <= DBL_FAILURE_EXEC && r->worker_pid == 0 && - r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0; + r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_WORKER_FAILED) return r->stage == DBL_STAGE_WAIT && (r->failure_class == DBL_FAILURE_EXEC || @@ -208,10 +437,12 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { if (r->status == DBL_STATUS_OUTPUT_FAILED) return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_OUTPUT_LIMIT && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_CANCELLED) return r->stage == DBL_STAGE_WAIT && r->failure_class == DBL_FAILURE_CANCELLED && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; return 0; } diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 6baaad7..14fd55a 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -3,6 +3,21 @@ #include #include #include +#include +#include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");int turn=!strcmp(provider,"turn-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(turn){char*blob=malloc(TURN_BYTES);if(!blob)return 39;memset(blob,'u',TURN_BYTES);memcpy(blob,"TURN-HEAD",9);memcpy(blob+TURN_BYTES-9,"TURN-TAIL",9);size_t sent=0;while(sentd_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i readFileSync(new URL(`./${name}`, import.meta.url), "utf8"); +const unquote = (literal: string): string => { + assert.match(literal, /^"[^"\\]*"$/u, "launcher literals must not need escaping"); + return literal.slice(1, -1); +}; + +/** `#define NAME "value"` (optionally continued onto the next line) from the launcher header. */ +const defines = (): ReadonlyMap => { + const header = read("engineBrokerLauncher.h").replace(/\\\n\s*/gu, ""); + return new Map([...header.matchAll(/^#define (DBL_GROK_[A-Z_]+)\s+("[^"\n]*")$/gmu)].map((match) => [match[1]!, unquote(match[2]!)])); +}; + +/** The compiled worker argv, token by token, exactly as `launch()` passes it to `execveat`. */ +const compiledArgv = (): readonly string[] => { + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + const block = source.match(/char \*const argv\[\] = \{([\s\S]*?)NULL\};/u); + assert.ok(block, "launcher argv array not found"); + const values = defines(); + return block[1]!.split(",").map((token) => token.trim()).filter(Boolean).map((token) => { + if (token.startsWith('"')) return unquote(token); + if (token === "(char *)r->workspace") return "/registered/workspace"; + const value = values.get(token); + assert.ok(value !== undefined, `unexpected launcher argv token ${token}`); + return value; + }); +}; + +test("the native launcher compiles exactly the lean Grok worker argv Daimon renders", () => { + const argv = compiledArgv(); + assert.equal(argv[0], "grok"); + assert.deepEqual(argv.slice(1), renderGrokBrokerWorkerArgs("/proc/self/fd/3", "/registered/workspace")); + for (const flag of ["--verbatim", "--no-plan", "--no-subagents", "--no-memory", "--disable-web-search", "--always-approve"]) assert.ok(argv.includes(flag), flag); + assert.equal(argv[argv.indexOf("--tools") + 1], GROK_WORKER_TOOL_IDS.join(",")); + assert.equal(argv[argv.indexOf("--max-turns") + 1], String(GROK_WORKER_MAX_TURNS)); + assert.equal(argv.includes("--reasoning-effort"), false, "effort is declared per deployment in config.toml, never compiled"); + assert.equal(argv.includes("--disallowed-tools"), false, "--disallowed-tools is ignored under --tools"); +}); + +test("the compiled system prompt is byte-identical to the contract prompt pinned in the manifest", () => { + const compiled = defines().get("DBL_GROK_SYSTEM_PROMPT"); + assert.equal(compiled, DAIMON_GROK_SYSTEM_PROMPT); + assert.equal(createHash("sha256").update(DAIMON_GROK_SYSTEM_PROMPT).digest("hex"), GROK_ENGINE_BROKER.worker.systemPromptSha256); + assert.match(DAIMON_GROK_SYSTEM_PROMPT, /^[\x20-\x7e]+$/u); + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.length >= 320 && DAIMON_GROK_SYSTEM_PROMPT.length <= 700, "roughly 80-150 tokens"); + for (const tool of ["daimon__moltnet_read", "daimon__moltnet_send", "use_tool", "search_tool", "read_file"]) assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(tool), tool); +}); + +test("the launcher exports the turn provider capability under the env_key the worker config reads", () => { + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); + assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); +}); + +test("the launcher gives every worker a private TMPDIR under its registered home", () => { + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\) >=\s*sizeof\(tmp\)/u); + assert.match(source, /canonical_path\(r->home, sizeof\(r->home\)\)/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); + assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); +}); diff --git a/src/runtime/organizationRuntime.test.ts b/src/runtime/organizationRuntime.test.ts index f5f1c7f..60f98f8 100644 --- a/src/runtime/organizationRuntime.test.ts +++ b/src/runtime/organizationRuntime.test.ts @@ -287,16 +287,14 @@ test("accepts only the narrow optional Codex workspace policy", () => { } }); -test("rejects model and reasoningEffort on a non-codex engine", () => { +test("rejects model and reasoningEffort on agy, and codexSandbox on every non-codex engine", () => { + const withModel = valid(); + withModel.agents[0]!.engine = { kind: "agy", model: "gpt-5-codex" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); + const withEffort = valid(); + withEffort.agents[0]!.engine = { kind: "agy", reasoningEffort: "high" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); for (const kind of ["grok", "agy"] as const) { - const withModel = valid(); - withModel.agents[0]!.engine = { kind, model: "gpt-5-codex" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); - - const withEffort = valid(); - withEffort.agents[0]!.engine = { kind, reasoningEffort: "high" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); - const withPolicy = valid(); withPolicy.agents[0]!.engine = { kind, codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; assert.throws(() => parseOrganizationRuntimeConfig(withPolicy), /codex-only/); diff --git a/src/runtime/organizationRuntime.ts b/src/runtime/organizationRuntime.ts index 76684fe..f201306 100644 --- a/src/runtime/organizationRuntime.ts +++ b/src/runtime/organizationRuntime.ts @@ -35,10 +35,10 @@ export { export type OrganizationRuntimeEngineKind = "codex" | "grok" | "agy"; /** - * `model`/`reasoningEffort` are codex-only: grok and agy own their own model - * selection (their subscription auth and model selection are Daimon-owned), - * and `organizationRuntimeParsing.ts` rejects either field on a non-codex - * engine at parse time rather than silently ignoring it. The type stays flat + * `model`/`reasoningEffort` are accepted for codex (open model name) and for + * grok (closed broker lists, declared together); agy owns its own model + * selection and `organizationRuntimeParsing.ts` rejects either field there at + * parse time rather than silently ignoring it. The type stays flat * — not a `kind`-discriminated union — because every parsed value already * satisfies the invariant; callers that need it narrow on `kind === "codex"`. */ diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts new file mode 100644 index 0000000..8ec4178 --- /dev/null +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeHost, type OrganizationRuntimeWakeRequest } from "./organizationRuntime.js"; +import { createOrganizationRuntimeControlHostWithCoreForTest } from "./organizationRuntimeControl.js"; +import { ACTIVITY_V2_VERSION } from "./wakeAcceptanceTypes.js"; + +const token = "control-secret"; +const config = { + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_CLOSURE_TOKEN" }, + agents: [{ id: "alpha", name: "Alpha", instructions: "Act.", workspacePath: "/runtime/workspace", runtimeHomePath: "/runtime/home", engine: { kind: "codex" as const } }] +}; +const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; +const delivery = (deliveryId = "delivery-1") => ({ token, agent_id: "alpha", delivery_id: deliveryId, event: { version: "noopolis.daimon.wake.v2", kind: "manual" as const, text: "hello", occurred_at: "2026-09-18T00:00:00.000Z" } }); + +const core = { + async start(): Promise {}, + async wake(request: OrganizationRuntimeWakeRequest) { return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; }, + async health() { return { version: "noopolis.daimon.organization-runtime-health.v1" as const, state: "running" as const, agents: [{ agentId: "alpha", engine: "codex" as const, state: "idle" as const }] }; }, + async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; }, + async stop() { return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } +} as unknown as OrganizationRuntimeHost; + +/** + * A caller proving that a native execution closed has exactly one authority to + * read — the v2 activity projection — and the runtime that owns it is stopped by + * the time the proof is taken: the worker stops its own host as soon as a delivery + * closes its execution and stays deferred. Before this, `activityV2` answered + * `undefined` there (HTTP 503 `native_host_unavailable` through the caller's + * worker route), so a trial whose subject really ran, spent its budget and simply + * did not do the work reported as an unscorable infrastructure failure. + */ +test("a stopped control host still answers the closure query it alone can settle", async () => { + const root = await privateRoot(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + // Nothing has started: there is no projection to seal and none is invented. + assert.equal(await control.activityV2(token), undefined); + await control.start(); + const accepted = await control.accept(delivery()); + assert.equal(accepted.state, "accepted"); + const live = await control.activityV2(token); + assert.equal(live?.state, "running"); + assert.equal(live?.items.length, 1); + + assert.equal((await control.stop()).state, "stopped"); + const sealed = await control.activityV2(token); + assert.equal(sealed?.version, ACTIVITY_V2_VERSION); + // The same projection, said to be final: nothing can be admitted after it, so + // an empty execution list is a stronger quiescence statement than a live poll. + assert.equal(sealed?.state, "stopped"); + assert.deepEqual(sealed?.executions, []); + assert.equal(sealed?.items.length, 1); + assert.equal(sealed?.items[0]?.delivery_id, "delivery-1"); + assert.equal(sealed?.items[0]?.active, false); + // Repeating the query repeats the seal rather than draining it. + assert.deepEqual(await control.activityV2(token), sealed); + // The seal is not a bypass of authentication, and the store-backed routes that + // have no post-stop answer still report absence instead of an empty runtime. + assert.equal(await control.activityV2("wrong-token"), undefined); + assert.equal(await control.availability(token), undefined); + assert.equal(await control.wakeReceipt(token, accepted.state === "accepted" ? accepted.acceptance_id : ""), undefined); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +/** A host that was never started cannot attest anything, and a second stop keeps the seal. */ +test("an unstarted host seals nothing and a repeated stop does not erase the seal", async () => { + const root = await privateRoot(); + const unstarted = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + assert.equal((await unstarted.stop()).state, "stopped"); + assert.equal(await unstarted.activityV2(token), undefined); + + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + await control.start(); + await control.accept(delivery("delivery-2")); + await control.stop(); + const sealed = await control.activityV2(token); + assert.equal(sealed?.state, "stopped"); + await control.stop(); + assert.deepEqual(await control.activityV2(token), sealed); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +async function privateRoot(): Promise { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-closure-")); await chmod(root, 0o700); return root; } diff --git a/src/runtime/organizationRuntimeControl.ts b/src/runtime/organizationRuntimeControl.ts index e4d9c5c..386b6c5 100644 --- a/src/runtime/organizationRuntimeControl.ts +++ b/src/runtime/organizationRuntimeControl.ts @@ -47,6 +47,19 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt let fusePoll: ReturnType | undefined; let started = false; let stopping = false; + let sealedActivity: OrganizationRuntimeActivityV2 | undefined; + + /** + * One projection, read the same way live and at shutdown. `active` is decided by + * the dispatcher's own execution authority rather than the record's flag alone, + * so a stopped host — whose dispatcher has already awaited every in-flight turn + * — reports exactly the executions that were still admitted when it stopped. + */ + const projectActivity = async (current: WakeAcceptanceStore, state: "running" | "stopped"): Promise => { + const executions = dispatcher?.activeExecutions() ?? []; + const items = (await current.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); + return { version: ACTIVITY_V2_VERSION, state, items, executions }; + }; const hardReason = (): BlockReason | undefined => { if (!started || stopping) return stopping ? "host_stopping" : "host_stopped"; @@ -132,10 +145,15 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt return await store.status(acceptanceId); }, async activityV2(token) { - if (!tokensEqual(expectedToken, token) || store === undefined) return undefined; - const executions = dispatcher?.activeExecutions() ?? []; - const items = (await store.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); - return { version: ACTIVITY_V2_VERSION, items, executions }; + if (!tokensEqual(expectedToken, token)) return undefined; + // A stopped host is not an unanswerable one. Its sealed projection is a + // *stronger* statement about quiescence than a live poll, because nothing + // can be admitted after it, and a caller proving that an execution closed + // has no other authority to read. A host that never started, or one whose + // seal could not be taken, still answers nothing: absence stays absence + // rather than becoming a fabricated idle runtime. + if (store === undefined) return sealedActivity; + return await projectActivity(store, "running"); }, async availability(token) { if (!tokensEqual(expectedToken, token) || !store || !fuse) return undefined; @@ -161,6 +179,11 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt await Promise.allSettled(persistence); const result = await host.stop(); await dispatcher?.stop(); + // The last moment the store can be read, and the only one at which the + // dispatcher has finished every admitted turn. Seal the projection here so + // the closure query keeps an accurate answer once the store is closed; a + // fault leaves it absent instead of inventing one. + if (store) { try { sealedActivity = await projectActivity(store, "stopped"); } catch { /* an unreadable final state stays absent */ } } await store?.close(); await fuse?.close(); store = undefined; fuse = undefined; schedules = undefined; started = false; return result; diff --git a/src/runtime/organizationRuntimeGrokEngine.test.ts b/src/runtime/organizationRuntimeGrokEngine.test.ts new file mode 100644 index 0000000..93ea7ab --- /dev/null +++ b/src/runtime/organizationRuntimeGrokEngine.test.ts @@ -0,0 +1,58 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; + +import { startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { + ORGANIZATION_RUNTIME_CONFIG_SCHEMA, + ORGANIZATION_RUNTIME_VERSION, + validateOrganizationRuntimeConfig, + parseOrganizationRuntimeConfig, + type OrganizationRuntimeAgentConfig, + type OrganizationRuntimeEngineIntent +} from "./organizationRuntime.js"; + +const valid = () => ({ + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_TOKEN" }, + agents: [{ id: "editor", name: "Editor", instructions: "Write a concise report.", workspacePath: "/runtime/workspaces/editor", runtimeHomePath: "/runtime/homes/editor", engine: { kind: "codex" } as OrganizationRuntimeEngineIntent }] +}); + +test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { + const declared = valid(); + declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); + const bare = valid(); + bare.agents[0]!.engine = { kind: "grok" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); + for (const engine of [ + { kind: "grok", model: "grok-4.6" }, + { kind: "grok", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, + { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } + ]) { + const invalid = valid(); + invalid.agents[0]!.engine = engine as never; + assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); + } +}); + +test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { + const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; + const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); + for (const engine of [ + { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, + { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } + ]) { + const config = valid(); + config.agents[0]!.engine = engine as never; + assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); + } +}); + +test("a declared Grok model is refused on the direct path that cannot enforce it", async () => { + const config = { ...valid().agents[0]!, engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; + await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); +}); diff --git a/src/runtime/organizationRuntimeParsing.ts b/src/runtime/organizationRuntimeParsing.ts index a6f1d39..cc65b39 100644 --- a/src/runtime/organizationRuntimeParsing.ts +++ b/src/runtime/organizationRuntimeParsing.ts @@ -1,3 +1,4 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; import { parseAttention } from "./attention.js"; import { ORGANIZATION_RUNTIME_CODEX_REASONING_EFFORTS, @@ -187,18 +188,19 @@ function cronValues(field: string, [minimum, maximum]: readonly [number, number] function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent { const input = object(value, label); const kind = string(input.kind, `${label}.kind`); if (!ENGINE_KINDS.has(kind)) throw new TypeError(`${label}.kind is not a supported engine`); - // `model`/`reasoningEffort` are codex-only: grok and agy own their own model - // selection, so either field on a non-codex engine is rejected explicitly - // here (a clear, named error) rather than falling through to the generic - // "must contain exactly" rejection every other unexpected key gets below. - const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : ["kind"]; + // `codexSandbox` is codex-only; `model`/`reasoningEffort` are accepted for + // codex (open model name, Codex effort list) and for grok (closed broker + // lists, declared together or not at all). agy owns its own model selection. + const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : kind === "grok" ? ["kind", "model", "reasoningEffort"] : ["kind"]; const extras = Object.keys(input).filter((key) => !allowed.includes(key)); if (extras.length > 0) { - if (kind !== "codex" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { - throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only; their subscription auth, model selection, and sandbox policy are Daimon-owned`); + if (kind === "agy" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { + throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only for agy; its subscription auth, model selection, and sandbox policy are Daimon-owned`); } + if (kind === "grok" && extras.includes("codexSandbox")) throw new TypeError(`${label}.codexSandbox is codex-only; the Grok worker sandbox is Daimon-owned`); throw new TypeError(`${label} must contain exactly ${allowed.join(", ")}`); } + if (kind === "grok") return grokEngine(input, label); return { kind: kind as OrganizationRuntimeEngineKind, ...(input.model === undefined ? {} : { model: nonEmpty(input.model, `${label}.model`) }), @@ -207,6 +209,22 @@ function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent }; } +/** + * A Grok model is declared explicitly or not at all: both members from the + * closed broker lists, never one of them with the other inherited. Omitting + * both keeps a config parseable by pre-declaration producers; the brokered + * paths that need a model (`grokBrokerProjection.ts`, `service.json` v2) + * require it there instead of defaulting it here. + */ +function grokEngine(input: RecordValue, label: string): OrganizationRuntimeEngineIntent { + if ((input.model === undefined) !== (input.reasoningEffort === undefined)) throw new TypeError(`${label}.model and ${label}.reasoningEffort must be declared together for grok`); + if (input.model === undefined) return { kind: "grok" }; + const model = string(input.model, `${label}.model`), effort = string(input.reasoningEffort, `${label}.reasoningEffort`); + if (!(GROK_BROKER_MODELS as readonly string[]).includes(model)) throw new TypeError(`${label}.model must be one of ${GROK_BROKER_MODELS.join(", ")}`); + if (!(GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(effort)) throw new TypeError(`${label}.reasoningEffort must be one of ${GROK_BROKER_REASONING_EFFORTS.join(", ")}`); + return { kind: "grok", model, reasoningEffort: effort }; +} + function codexSandbox(value: unknown, label: string): OrganizationRuntimeEngineIntent["codexSandbox"] { const input = object(value, label); exact(input, ["mode", "networkAccess", "webSearch"], label); diff --git a/src/runtime/physicalReadiness.test.ts b/src/runtime/physicalReadiness.test.ts index bdbb248..b645cce 100644 --- a/src/runtime/physicalReadiness.test.ts +++ b/src/runtime/physicalReadiness.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; +import { assertRuntimeDirectory, prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; const agent = (workspacePath: string, runtimeHomePath: string): OrganizationRuntimeAgentConfig => ({ @@ -46,3 +46,68 @@ test("preflight requires safe workspace and private runtime roots, and proves ph await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, workspace)]), /overlap/); } finally { await rm(root, { recursive: true, force: true }); } }); + +const withRoots = async (body: (root: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-physical-")); + try { await body(root); } finally { await rm(root, { force: true, recursive: true }); } +}; + +const runtime = { uid: 2000, gid: 2000, firstWorkerUid: 2200 }; +const entry = (mode: number, uid = 2000, gid = 2000, kind: "dir" | "link" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : 0o120000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); + +test("a brokered Grok runtime home is accepted at exactly 2000: 0710 and nothing wider", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2200), "runtimeHomePath", "worker-traversable", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2201), "runtimeHomePath", "worker-traversable", runtime)); + const refusals: Record> = { + "0700 (no worker traversal, pre-P1b layout)": entry(0o700, 2000, 2200), + "0711 (world traverse)": entry(0o711, 2000, 2200), + "0712": entry(0o712, 2000, 2200), + "0714": entry(0o714, 2000, 2200), + "0730 (group write)": entry(0o730, 2000, 2200), + "0750 (group read)": entry(0o750, 2000, 2200), + "0770": entry(0o770, 2000, 2200), + "0777": entry(0o777, 2000, 2200), + "2710 (setgid)": entry(0o2710, 2000, 2200), + "owned by a worker": entry(0o710, 2200, 2200), + "owned by root": entry(0o710, 0, 2200), + "group is the runtime's own": entry(0o710, 2000, 2000), + "group below the worker range": entry(0o710, 2000, 2100), + "a symlink": entry(0o710, 2000, 2200, "link") + }; + for (const [label, candidate] of Object.entries(refusals)) { + assert.throws(() => assertRuntimeDirectory(candidate, "runtimeHomePath", "worker-traversable", runtime), /runtimeHomePath/u, label); + } +}); + +test("every other engine's runtime home stays exactly 0700, and a workspace stays group/other-write free", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "runtimeHomePath", "private", runtime)); + for (const mode of [0o710, 0o701, 0o750, 0o770, 0o711, 0o755, 0o2700]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "runtimeHomePath", "private", runtime), /must have mode 0700/u, mode.toString(8)); + } + // The brokered Grok workspace contract (2000: 0750) passes the workspace shape. + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o750, 2000, 2200), "workspacePath", "safe", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "workspacePath", "safe", runtime)); + for (const mode of [0o770, 0o720, 0o702, 0o777]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "workspacePath", "safe", runtime), /must not grant group or other write/u, mode.toString(8)); + } +}); + +test("the engine kind decides the runtime home shape on a real filesystem", async () => { + await withRoots(async (root) => { + const workspace = path.join(root, "workspace"), home = path.join(root, "home"); + await mkdir(workspace, { mode: 0o700 }); + await mkdir(home, { mode: 0o710 }); + const grok = { ...agent(workspace, home), engine: { kind: "grok" as const, model: "grok-4.6" as const, reasoningEffort: "low" as const } }; + // 0710 reaches the Grok branch: only the worker-group requirement is left to refuse it here. + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /group-owned by the agent's Grok worker group/u); + // The same home refuses a Codex agent for being wider than 0700. + await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, home)]), /must have mode 0700/u); + await chmod(home, 0o700); + const authority = await prepareOrganizationRuntimePaths([agent(workspace, home)]); + await authority.close(); + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /must have mode 0710 for a brokered Grok agent/u); + }); +}); diff --git a/src/runtime/physicalReadiness.ts b/src/runtime/physicalReadiness.ts index 075ea47..50ce64a 100644 --- a/src/runtime/physicalReadiness.ts +++ b/src/runtime/physicalReadiness.ts @@ -2,9 +2,26 @@ import { constants, type Stats } from "node:fs"; import { lstat, open, realpath } from "node:fs/promises"; import path from "node:path"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; type Identity = Readonly<{ dev: number; ino: number; uid: number; mode: number }>; +/** + * `private` is every engine's runtime home: 0700, nothing but the runtime user. + * + * `worker-traversable` is the brokered Grok shape, and only that shape: the + * agent's own sandboxed worker runs as another uid and must be able to *walk + * into* this home to read the setgid `tool-output/` spill directory the + * truncation notice sends it to (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`). + * Traverse-only means `0710`: no group read (the worker cannot list the home or + * see the acceptance store, telemetry, memory or credential names) and no group + * write. Anything wider — `0711`, `0750`, `0770`, any world bit — is refused, + * as is a group that is not a worker group. Daimon cannot tell *which* worker + * gid belongs to this agent; the per-slot mapping is the deployment's + * provisioning contract, re-checked by the slot preflight receipt's worker-uid + * canaries. + */ +type DirectoryShape = "safe" | "private" | "worker-traversable"; type Directory = { readonly configured: string; readonly real: string; readonly fd: Awaited>; readonly identity: Identity; closed: boolean }; /** @@ -20,6 +37,11 @@ export type OrganizationRuntimePathAuthority = Readonly<{ close(): Promise; }>; +/** Only a brokered Grok agent's home is worker-traversable; every other engine keeps 0700. */ +function runtimeHomeShape(agent: OrganizationRuntimeAgentConfig): DirectoryShape { + return agent.engine.kind === "grok" ? "worker-traversable" : "private"; +} + export async function prepareOrganizationRuntimePaths( agents: readonly OrganizationRuntimeAgentConfig[] ): Promise { @@ -28,7 +50,7 @@ export async function prepareOrganizationRuntimePaths( try { for (const agent of agents) { workspaces.set(agent.id, await verifyDirectory(agent.workspacePath, "workspacePath", "safe")); - homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", "private")); + homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", runtimeHomeShape(agent))); } const roots = [...workspaces.values(), ...homes.values()]; for (let left = 0; left < roots.length; left += 1) for (let right = left + 1; right < roots.length; right += 1) { @@ -51,7 +73,7 @@ export async function prepareOrganizationRuntimePaths( if (workspace === undefined || home === undefined) throw new Error(`no runtime path authority for ${agent.id}`); await Promise.all([ verifyIdentity(workspace, "workspacePath", "safe"), - verifyIdentity(home, "runtimeHomePath", "private") + verifyIdentity(home, "runtimeHomePath", runtimeHomeShape(agent)) ]); }; return { @@ -70,10 +92,10 @@ export async function prepareOrganizationRuntimePaths( }; } -async function verifyDirectory(configured: string, label: string, mode: "safe" | "private"): Promise { +async function verifyDirectory(configured: string, label: string, shape: DirectoryShape): Promise { await assertNoSymlinkComponents(configured); const before = await lstat(configured); - assertDirectory(before, label, mode); + assertDirectory(before, label, shape); const fd = await open(configured, constants.O_RDONLY | directoryFlag() | noFollow()); try { const opened = await fd.stat(); @@ -89,7 +111,7 @@ async function verifyDirectory(configured: string, label: string, mode: "safe" | } } -async function verifyIdentity(directory: Directory, label: string, mode: "safe" | "private"): Promise { +async function verifyIdentity(directory: Directory, label: string, shape: DirectoryShape): Promise { if (directory.closed) throw new Error(`${label} authority is closed`); await assertNoSymlinkComponents(directory.configured); const entry = await lstat(directory.configured); @@ -97,7 +119,7 @@ async function verifyIdentity(directory: Directory, label: string, mode: "safe" if (!sameIdentity(identity(entry), directory.identity) || !sameIdentity(identity(opened), directory.identity)) { throw new Error(`${label} changed after readiness validation`); } - assertDirectory(entry, label, mode); + assertDirectory(entry, label, shape); if (await realpath(directory.configured) !== directory.real) throw new Error(`${label} changed after readiness validation`); } @@ -112,12 +134,28 @@ async function assertNoSymlinkComponents(target: string): Promise { } } -function assertDirectory(entry: Stats, label: string, mode: "safe" | "private"): void { +/** Identity of the process Daimon runs as; a seam so every refusal is testable unprivileged. */ +export type RuntimeIdentity = Readonly<{ uid: number; gid: number; firstWorkerUid?: number }>; +type DirectoryEntry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean; isSymbolicLink(): boolean }>; + +/** Pure shape check for a caller-prepared runtime root. */ +export function assertRuntimeDirectory(entry: DirectoryEntry, label: string, shape: DirectoryShape, runtime: RuntimeIdentity): void { if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error(`${label} must be an existing real directory`); - if (entry.uid !== process.getuid?.()) throw new Error(`${label} must be owned by the runtime user`); - const permissions = entry.mode & 0o777; - if (mode === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); - if (mode === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); + if (entry.uid !== runtime.uid) throw new Error(`${label} must be owned by the runtime user`); + const permissions = Number(entry.mode) & 0o7777; + if (shape === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); + if (shape === "worker-traversable") { + const home = GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome; + if (permissions !== home.mode) throw new Error(`${label} must have mode 0710 for a brokered Grok agent`); + if (entry.gid < (runtime.firstWorkerUid ?? GROK_ENGINE_BROKER.identities.firstWorkerUid) || entry.gid === runtime.gid) { + throw new Error(`${label} must be group-owned by the agent's Grok worker group`); + } + } + if (shape === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); +} + +function assertDirectory(entry: Stats, label: string, shape: DirectoryShape): void { + assertRuntimeDirectory(entry, label, shape, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); } function identity(entry: Stats): Identity { return { dev: entry.dev, ino: entry.ino, uid: entry.uid, mode: entry.mode & 0o7777 }; } diff --git a/src/runtime/portableCredentialBoundary.test.ts b/src/runtime/portableCredentialBoundary.test.ts new file mode 100644 index 0000000..8d00742 --- /dev/null +++ b/src/runtime/portableCredentialBoundary.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { chmod, link, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const execute = promisify(execFile); +const reader = fileURLToPath(new URL("./portableCredentialMaterial.ts", import.meta.url)); +const child = `const {materializePortableCredential}=await import(process.argv[1]);const home=process.argv[2];try{await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:home+'/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home);process.stdout.write('UNEXPECTED_IMPORT');}catch{process.stdout.write('MATERIALIZATION_REFUSED');}`; + +for (const kind of ["mode", "hardlink", "directory", "empty", "oversize", "symlink"] as const) { + test(`real filesystem rejects unsafe credential ${kind}`, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-credential-boundary-")); + const source = `${root}/.daimon-inbound/codex-auth`; + const agent: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: "codex" } }; + try { + await mkdir(path.dirname(source), { mode: 0o700 }); + if (kind === "directory") await mkdir(source, { mode: 0o600 }); + else if (kind === "symlink") { await writeFile(`${root}/target`, "dummy", { mode: 0o600 }); await symlink(`${root}/target`, source); } + else { + await writeFile(source, kind === "empty" ? "" : kind === "oversize" ? "x".repeat(65537) : "dummy", { mode: 0o600 }); + if (kind === "mode") await chmod(source, 0o644); + if (kind === "hardlink") await link(source, `${root}/second-link`); + } + await assert.rejects(materializePortableCredential(agent, root), /credential materialization failed/u); + await assert.rejects(readFile(`${root}/.codex/auth.json`), { code: "ENOENT" }); + } finally { await rm(root, { recursive: true, force: true }); } + }); +} + +test("a real FIFO is rejected promptly; deleting nonblocking open makes the subprocess hang", { skip: process.platform === "win32", timeout: 15000 }, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-credential-fifo-")); + try { + await mkdir(`${root}/.daimon-inbound`, { mode: 0o700 }); + await execute("mkfifo", ["-m", "600", `${root}/.daimon-inbound/codex-auth`]); + const run = (modulePath: string) => execute(process.execPath, ["--import", "tsx", "--input-type=module", "-e", child, modulePath, root], { timeout: 3000, killSignal: "SIGKILL" }); + assert.equal((await run(reader)).stdout, "MATERIALIZATION_REFUSED"); + const original = await readFile(reader, "utf8"); + const mutated = original.replace(" | constants.O_NONBLOCK", "").replace('"./contractManifest.js"', JSON.stringify(fileURLToPath(new URL("./contractManifest.ts", import.meta.url)))); + assert.notEqual(mutated, original); + await writeFile(`${root}/package.json`, '{"type":"module"}'); + const mutant = `${root}/mutant.ts`; await writeFile(mutant, mutated); + await assert.rejects(run(mutant), (error: NodeJS.ErrnoException & { killed?: boolean; signal?: string }) => error.killed === true && error.signal === "SIGKILL"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialFailures.test.ts b/src/runtime/portableCredentialFailures.test.ts new file mode 100644 index 0000000..475e801 --- /dev/null +++ b/src/runtime/portableCredentialFailures.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +test("keeps directory and existing-credential protections and refuses undeclared engines", async () => { + for (const mode of ["engine", "directory-mode", "directory-link", "destination-mode", "destination-directory", "repair-destination"] as const) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-guards-")); + const config: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: mode === "engine" ? "grok" : "codex" } }; + try { + await fs.mkdir(`${root}/.daimon-inbound`, { mode: 0o700 }); + await fs.writeFile(`${root}/.daimon-inbound/codex-auth`, "dummy", { mode: 0o600 }); + if (mode === "directory-mode") await fs.chmod(`${root}/.daimon-inbound`, 0o755); + if (mode === "directory-link") { await fs.rename(`${root}/.daimon-inbound`, `${root}/source`); await fs.symlink(`${root}/source`, `${root}/.daimon-inbound`); } + if (mode.startsWith("destination") || mode === "repair-destination") await fs.mkdir(`${root}/.codex`, { mode: mode === "repair-destination" ? 0o755 : 0o700 }); + if (mode === "destination-mode") await fs.writeFile(`${root}/.codex/auth.json`, "existing", { mode: 0o644 }); + if (mode === "destination-directory") await fs.mkdir(`${root}/.codex/auth.json`, { mode: 0o600 }); + if (mode === "repair-destination") { + assert.equal(await materializePortableCredential(config, root), "created"); + assert.equal((await fs.stat(`${root}/.codex`)).mode & 0o777, 0o700); + } else await assert.rejects(materializePortableCredential(config, root), /credential material/u); + } finally { await fs.rm(root, { recursive: true, force: true }); } + } +}); + +for (const failure of ["mkdir", "rename", "source-lstat", "existing-lstat", "temporary-close"] as const) test(`does not install credentials after ${failure} failure`, async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-failure-")); + const config: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: "codex" } }; + const source = `${root}/.daimon-inbound/codex-auth`, destination = `${root}/.codex/auth.json`; + const fail = (): never => { throw Object.assign(new Error("private-path-and-contents-must-not-leak"), { code: "EIO" }); }; + try { + await fs.mkdir(path.dirname(source), { mode: 0o700 }); await fs.writeFile(source, "dummy", { mode: 0o600 }); + if (failure === "mkdir") mock.method(fs, "mkdir", fail); + if (failure === "rename") mock.method(fs, "rename", fail); + if (failure.endsWith("lstat")) { + const original = fs.lstat; + mock.method(fs, "lstat", (...args: Parameters) => args[0] === (failure === "source-lstat" ? source : destination) ? fail() : original(...args)); + } + if (failure === "temporary-close") { + const original = fs.open; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await original(...args); + if (String(args[0]).endsWith(".tmp")) { const close = handle.close.bind(handle); mock.method(handle, "close", async () => { await close(); fail(); }); } + return handle; + }); + } + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(config, root), (error: Error) => error.message === "agent agent:codex codex credential materialization failed"); + await assert.rejects(fs.readFile(destination), { code: "ENOENT" }); + const files = await fs.readdir(`${root}/.codex`).catch(() => []); + assert.equal(files.some(file => file.endsWith(".tmp")), false); + } finally { mock.restoreAll(); syncBuiltinESMExports(); await fs.rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialMaterial.linux.test.ts b/src/runtime/portableCredentialMaterial.linux.test.ts new file mode 100644 index 0000000..70d1ed3 --- /dev/null +++ b/src/runtime/portableCredentialMaterial.linux.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const image = process.env.DAIMON_CREDENTIAL_BIND_TEST_IMAGE; +const script = `import fs from 'node:fs/promises'; +const home='/native-home',source=home+'/.daimon-inbound/codex-auth'; +const before=await fs.lstat(source); +const {materializePortableCredential}=await import('/opt/paideia/credential-test/reader.ts'); +const result=await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:'/tmp/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home); +if(result!=='created'||await fs.readFile(home+'/.codex/auth.json','utf8')!=='non-secret-dummy-credential')throw Error('dummy credential was not materialized'); +const after=await fs.lstat(source);process.stdout.write(JSON.stringify({result,uid:process.getuid(),beforeUid:before.uid,afterUid:after.uid,descriptorRefreshObserved:before.uid!==after.uid,verified:true}));`; + +test("actual read-only Docker bind materializes a fresh dummy credential without metadata prewarming", { skip: !image, timeout: 30000 }, async t => { + const parent = fileURLToPath(new URL("../../.runtime/", import.meta.url)); + await mkdir(parent, { recursive: true, mode: 0o700 }); + const root = await mkdtemp(path.join(parent, "daimon-dummy-credential-bind-")); + try { + const home = `${root}/home`, code = `${root}/code`, source = `${root}/dummy-credential`; + await mkdir(`${home}/.daimon-inbound`, { recursive: true, mode: 0o700 }); await mkdir(code); + await writeFile(source, "non-secret-dummy-credential", { mode: 0o600 }); + // Execute the edited reader without building dist; its unchanged contract + // constant comes from the image's public runtime export, never private code. + const bytes = (await readFile(fileURLToPath(new URL("./portableCredentialMaterial.ts", import.meta.url)), "utf8")) + .replace('"./contractManifest.js"', '"@noopolis/daimon/runtime"'); + await writeFile(`${code}/reader.ts`, bytes); + const result = await promisify(execFile)("docker", ["run", "--rm", "--network", "none", "--read-only", + "--user", `${process.getuid?.() ?? 501}:${process.getgid?.() ?? 20}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "--tmpfs", "/tmp:rw,nosuid,nodev", "--mount", `type=bind,source=${home},target=/native-home`, + "--mount", `type=bind,source=${source},target=/native-home/.daimon-inbound/codex-auth,readonly`, + "--mount", `type=bind,source=${code},target=/opt/paideia/credential-test,readonly`, + "--entrypoint", "/usr/local/bin/node", image!, "--experimental-strip-types", "--input-type=module", "-e", script + ], { timeout: 20000, maxBuffer: 16384 }); + const proof = JSON.parse(result.stdout) as { result: string; uid: number; beforeUid: number; afterUid: number; verified: boolean; descriptorRefreshObserved: boolean }; + assert.equal(proof.result, "created"); assert.equal(proof.verified, true); assert.equal(proof.afterUid, proof.uid); + t.diagnostic(JSON.stringify(proof)); + // Root can read the dummy file via DAC_OVERRIDE; the reader must still + // reject its genuinely different owner rather than relying on open EACCES. + const unsafeOwner = `import fs from 'node:fs/promises'; +const home='/native-home',source=home+'/.daimon-inbound/codex-auth';await fs.mkdir(home+'/.daimon-inbound',{mode:448});await fs.writeFile(source,'dummy',{mode:384});await fs.chown(source,501,20); +const opened=await fs.open(source,'r');if((await opened.stat()).uid===process.getuid())throw Error('owner fixture invalid');await opened.close(); +const {materializePortableCredential}=await import('/opt/paideia/credential-test/reader.ts');let refused=false;try{await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:'/tmp/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home);}catch{refused=true;} +if(!refused)throw Error('unsafe owner imported');try{await fs.access(home+'/.codex/auth.json');throw Error('destination exists');}catch(e){if(e.code!=='ENOENT')throw e;}process.stdout.write('UNSAFE_OWNER_REFUSED');`; + const negative = await promisify(execFile)("docker", ["run", "--rm", "--network", "none", "--read-only", "--user", "0:0", + "--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--security-opt", "no-new-privileges", + "--tmpfs", "/tmp:rw,nosuid,nodev", "--tmpfs", "/native-home:rw,mode=0700", + "--mount", `type=bind,source=${code},target=/opt/paideia/credential-test,readonly`, "--entrypoint", "/usr/local/bin/node", + image!, "--experimental-strip-types", "--input-type=module", "-e", unsafeOwner], { timeout: 10000, maxBuffer: 16384 }); + assert.equal(negative.stdout, "UNSAFE_OWNER_REFUSED"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialMaterial.ts b/src/runtime/portableCredentialMaterial.ts index 0dc0a71..3d729fd 100644 --- a/src/runtime/portableCredentialMaterial.ts +++ b/src/runtime/portableCredentialMaterial.ts @@ -1,4 +1,4 @@ -import { constants } from "node:fs"; +import { constants, type Stats } from "node:fs"; import { chmod, lstat, mkdir, open, rename, unlink } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import path from "node:path"; @@ -15,6 +15,7 @@ type FileIdentity = Readonly<{ mtimeMs: number; size: number; uid: number; + nlink: number; }>; /** @@ -72,13 +73,25 @@ async function readCredential( ): Promise<{ bytes: Buffer; identity: FileIdentity }> { let handle: Awaited> | undefined; try { - const before = await assertCredential(filePath, mode, agent); - handle = await open(filePath, constants.O_RDONLY | noFollow()); - const opened = identity(await handle.stat()); - if (!sameIdentity(before, opened)) throw new Error("credential changed during import"); - const bytes = await handle.readFile(); - const after = identity(await handle.stat()); - if (!sameIdentity(before, after) || bytes.length !== before.size) { + // Opening a read-only bind can refresh its metadata. Validate the actual + // descriptor before consulting the refreshed path or reading any bytes. + // Nonblocking open lets us reject a FIFO by type instead of hanging here. + handle = await open(filePath, constants.O_RDONLY | noFollow() | constants.O_NONBLOCK); + const before = assertCredentialEntry(await handle.stat(), mode, agent); + if (!sameIdentity(before, await assertCredential(filePath, mode, agent))) throw new Error("credential changed during import"); + // One extra byte detects growth without allowing a changing file to drive + // an unbounded read/allocation. Partial reads advance within this buffer. + const buffer = Buffer.alloc(before.size + 1); + let length = 0; + while (length < buffer.length) { + const result = await handle.read(buffer, length, buffer.length - length, length); + if (result.bytesRead === 0) break; + length += result.bytesRead; + } + const bytes = buffer.subarray(0, length); + const after = assertCredentialEntry(await handle.stat(), mode, agent); + const afterPath = await assertCredential(filePath, mode, agent); + if (!sameIdentity(before, after) || !sameIdentity(before, afterPath) || bytes.length !== before.size) { throw new Error("credential changed during import"); } return { bytes, identity: before }; @@ -147,6 +160,10 @@ async function assertCredential( if ((error as NodeJS.ErrnoException).code === "ENOENT") throw error; throw unavailable(agent, error); } + return assertCredentialEntry(entry, mode, agent); +} + +function assertCredentialEntry(entry: Stats, mode: number, agent: PortableAgent): FileIdentity { if (!entry.isFile() || entry.isSymbolicLink() || entry.uid !== process.getuid?.() || entry.nlink !== 1 || (entry.mode & 0o777) !== mode || entry.size === 0 || entry.size > MAX_CREDENTIAL_BYTES) { @@ -172,6 +189,7 @@ function identity(value: Awaited>): FileIdentity { mtimeMs: number; size: number; uid: number; + nlink: number; }; return { dev: numeric.dev, @@ -179,12 +197,13 @@ function identity(value: Awaited>): FileIdentity { mode: numeric.mode & 0o7777, mtimeMs: numeric.mtimeMs, size: numeric.size, - uid: numeric.uid + uid: numeric.uid, + nlink: numeric.nlink }; } function sameIdentity(left: FileIdentity, right: FileIdentity): boolean { return left.dev === right.dev && left.ino === right.ino && left.mode === right.mode - && left.mtimeMs === right.mtimeMs && left.size === right.size && left.uid === right.uid; + && left.mtimeMs === right.mtimeMs && left.size === right.size && left.uid === right.uid && left.nlink === right.nlink; } function noFollow(): number { return (constants as typeof constants & { O_NOFOLLOW?: number }).O_NOFOLLOW ?? 0; diff --git a/src/runtime/portableCredentialRead.test.ts b/src/runtime/portableCredentialRead.test.ts new file mode 100644 index 0000000..9245de0 --- /dev/null +++ b/src/runtime/portableCredentialRead.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { constants, type Stats } from "node:fs"; +import fs from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const altered = (entry: Stats, patch: Partial): Stats => Object.assign(Object.create(Object.getPrototypeOf(entry)), entry, patch); +async function fixture(run: (agent: OrganizationRuntimeAgentConfig, source: string, destination: string) => Promise): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-descriptor-")); + const agent: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused.", workspacePath: `${root}/workspace`, runtimeHomePath: `${root}/home`, engine: { kind: "codex" } }; + const source = `${agent.runtimeHomePath}/.daimon-inbound/codex-auth`, destination = `${agent.runtimeHomePath}/.codex/auth.json`; + await fs.mkdir(path.dirname(source), { recursive: true, mode: 0o700 }); + await fs.writeFile(source, "dummy-credential", { mode: 0o600 }); + try { await run(agent, source, destination); } + finally { mock.restoreAll(); syncBuiltinESMExports(); await fs.rm(root, { recursive: true, force: true }); } +} + +test("opens before validation when a bind's pathname metadata refreshes on open", async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, originalLstat = fs.lstat; + const order: string[] = []; + let opened = false; + mock.method(fs, "lstat", async (...args: Parameters) => { + const entry = await originalLstat(...args); + if (args[0] !== source) return entry; + order.push("path"); + return opened ? entry : altered(entry as Stats, { uid: (process.getuid?.() ?? 0) + 1 }); + }); + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + order.push("open"); opened = true; + assert.ok((Number(args[1]) & constants.O_NOFOLLOW) !== 0); + assert.ok((Number(args[1]) & constants.O_NONBLOCK) !== 0); + const stat = handle.stat.bind(handle), read = handle.read.bind(handle); + mock.method(handle, "stat", async () => { order.push("descriptor"); return stat(); }); + mock.method(handle, "read", (buffer: Buffer, offset: number, length: number, position: number) => { order.push("read"); return read(buffer, offset, length, position); }); + } + return handle; + }); + syncBuiltinESMExports(); + assert.equal(await materializePortableCredential(agent, agent.runtimeHomePath), "created"); + assert.deepEqual(order, ["open", "descriptor", "path", "read", "read", "descriptor", "path"]); + assert.equal(await fs.readFile(destination, "utf8"), "dummy-credential"); + }); +}); + +for (const [name, patch] of [ + ["owner", { uid: (process.getuid?.() ?? 0) + 1 }], ["mode", { mode: 0o100644 }], + ["link count", { nlink: 2 }], ["empty", { size: 0 }], ["oversize", { size: 65537 }], + ["directory", { mode: 0o040600 }] +] as const) test(`rejects unsafe descriptor ${name} before reading`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; + let reads = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const entry = await handle.stat(), read = handle.read.bind(handle); + mock.method(handle, "stat", async () => altered(entry, patch)); + mock.method(handle, "read", (buffer: Buffer, offset: number, length: number, position: number) => { reads++; return read(buffer, offset, length, position); }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(reads, 0); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +for (const phase of ["before", "after"] as const) test(`rejects a pathname replaced ${phase} the read`, async () => { + await fixture(async (agent, source, destination) => { + const originalLstat = fs.lstat; + let paths = 0; + mock.method(fs, "lstat", async (...args: Parameters) => { + const entry = await originalLstat(...args); + if (args[0] !== source) return entry; + paths++; + return paths >= (phase === "before" ? 1 : 2) ? altered(entry as Stats, { ino: Number(entry.ino) + 1 }) : entry; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +test("rejects descriptor metadata and byte-length changes during the read", async () => { + for (const changed of ["mtime", "bytes"] as const) await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const entry = await handle.stat(); let stats = 0; + mock.method(handle, "stat", async () => ++stats === 1 || changed === "bytes" ? entry : altered(entry, { mtimeMs: entry.mtimeMs + 1 })); + if (changed === "bytes") mock.method(handle, "read", async (buffer: Buffer) => ({ bytesRead: 0, buffer })); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +test("assembles partial descriptor reads within one expected-size-plus-one buffer", async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, expected = await fs.readFile(source); + const buffers = new Set(); let calls = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "readFile", () => { throw Error("unbounded read forbidden"); }); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + calls++; buffers.add(buffer); + assert.equal(buffer.length, expected.length + 1); + assert.equal(position, offset); assert.ok(offset + length <= buffer.length); + return read(buffer, offset, Math.min(length, 3), position); + }); + } + return handle; + }); + syncBuiltinESMExports(); + assert.equal(await materializePortableCredential(agent, agent.runtimeHomePath), "created"); + assert.deepEqual(await fs.readFile(destination), expected); + assert.equal(buffers.size, 1); assert.equal(calls, Math.ceil(expected.length / 3) + 1); + }); +}); + +for (const partial of [false, true]) test(`growth after safe metadata remains byte-bounded (${partial ? "partial" : "full"} reads)`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, expectedSize = (await fs.stat(source)).size; + const buffers = new Set(); let calls = 0, bytes = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "readFile", () => { throw Error("unbounded read forbidden"); }); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + if (calls++ === 0) await fs.truncate(source, 16 * 1024 * 1024); + buffers.add(buffer); assert.equal(buffer.length, expectedSize + 1); + assert.ok(length > 0 && offset + length <= buffer.length); + const result = await read(buffer, offset, partial ? 1 : length, position); bytes += result.bytesRead; return result; + }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(bytes, expectedSize + 1); assert.equal(buffers.size, 1); + assert.equal(calls, partial ? expectedSize + 1 : 1); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +for (const size of [0, 5]) test(`rejects truncation to ${size} bytes after safe metadata`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; let calls = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + if (calls++ === 0) await fs.truncate(source, size); + return read(buffer, offset, length, position); + }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(calls, size === 0 ? 1 : 2); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); diff --git a/src/runtime/productionAgentTools.ts b/src/runtime/productionAgentTools.ts index a23f304..1b7c702 100644 --- a/src/runtime/productionAgentTools.ts +++ b/src/runtime/productionAgentTools.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, mkdir, open, readdir, rename, unlink } from "node:fs/promises"; +import { lstat, open, readdir, rename, unlink } from "node:fs/promises"; import path from "node:path"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; @@ -13,6 +13,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/ import type { OrganizationRuntimeAgentConfig, OrganizationRuntimeMcpServer } from "./organizationRuntime.js"; import { moltnetOperationResult, readMoltnetPages } from "./moltnetMachineRead.js"; import { McpToolCallError, MCP_TOOL_RESULT_MAX_BYTES, renderMcpToolResult, replayMcpReceipt, type McpUpstreamResult } from "./mcpToolResult.js"; +import { ensureRuntimeHomeDirectory } from "./runtimeHomeLayout.js"; import { capToolResult, resolveExemptToolNames, resolveToolResultMaxBytes, TOOL_OUTPUT_DIRECTORY_NAME } from "./toolResultSpill.js"; import { cliChildEnvironment } from "../pi/cliEnvironment.js"; import type { PiWakeEnvironmentContextRef } from "../pi/piAgentWakeSupport.js"; @@ -31,7 +32,7 @@ const MAX_RESULT = 65_536; const TIMEOUT = 10_000; const DAIMON_ACTION_ID_PREFIX = "daimon-"; export async function createProductionAgentTools(agent: OrganizationRuntimeAgentConfig, wakeContext: PiWakeEnvironmentContextRef = {}): Promise { - await mkdir(path.join(agent.runtimeHomePath, "tool-state"), { recursive: true, mode: 0o700 }); + await ensureRuntimeHomeDirectory(agent.runtimeHomePath, "tool-state"); // Resolved once, at agent start: a malformed bound is a configuration error // that should refuse the agent, not a surprise thrown from the middle of a // tool call the model is waiting on. diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts new file mode 100644 index 0000000..8b1a867 --- /dev/null +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -0,0 +1,218 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; +import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory, RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; + +/** + * A brokered Grok runtime home is traversable by its worker uid (0710), so + * anything Daimon creates inside it must stay 0700 — a default `mkdir` would + * make telemetry (prompts, replies, trajectories) readable by the model's own + * sandboxed worker. + */ +const withTraversableHome = async (body: (home: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-home-layout-")); + const home = path.join(root, "runtime-home"); + await mkdir(home, { mode: 0o710 }); + try { await body(home); } finally { await rm(root, { force: true, recursive: true }); } +}; +const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; + +const traceRecord = { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } +} as unknown as Parameters[1]; + +test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { + assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); +}); + +test("telemetry directories Daimon creates in a traversable runtime home are private", async () => { + await withTraversableHome(async (home) => { + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + await rm(path.join(home, "telemetry"), { recursive: true }); + + await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +/** + * The half the mode argument never covered. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and `assertRuntimeDirectory` checks the home and not what Daimon creates + * inside it. So a `telemetry/` left at 0755 by a pre-branch Daimon — or + * pre-created by a deployment — stayed 0755 under a Grok agent's deliberately + * traversable 0710 home, where it is the sandboxed worker reading its own + * agent's prompts, replies and causal history. + * + * The boundary these assertions straddle: a fresh install against an existing + * one. Every writer below is reached through its real entry point, because the + * hole was never in the mode constant — it was in what the call sites did with + * it. + * + * Mutation: restore `mkdir(directory, { recursive: true, mode })` in + * `ensureRuntimeHomeDirectory` and every assertion here goes red while the + * fresh-install test above stays green, which is exactly how this shipped. + */ +test("a runtime-home subdirectory that already exists is made private, not left as it was found", async () => { + await withTraversableHome(async (home) => { + // Pre-created by a deployment, or by a Daimon that predates the mode. + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // An existing ancestor is the same hole one level up: `telemetry/turns` can + // be created privately under a `telemetry/` that stays world-readable. + await chmod(path.join(home, "telemetry"), 0o755); + await mkdir(path.join(home, "telemetry", "turns"), { mode: 0o755 }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE, "the ancestor is corrected too"); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // And the home itself is never touched: which mode it should carry is + // `physicalReadiness.ts`'s judgement, and for a Grok agent it is 0710. + assert.equal(await mode(home), 0o710); + }); +}); + +/** + * Refuse rather than widen, and never follow a link to do it. + * + * A directory the runtime does not own cannot be made private by it, and + * writing an agent's telemetry into it anyway is the failure the correction + * exists to prevent. A symlink planted where a directory belongs is the same + * fault with an attacker attached, so the correction goes through an + * `O_DIRECTORY|O_NOFOLLOW` handle and the `fchmod` lands on the directory that + * was stat'd. + * + * Mutation: drop the owner check and the first case silently proceeds; drop + * `O_NOFOLLOW` and the second chmods the link's target instead of refusing. + */ +test("a runtime-home subdirectory owned by another user, or replaced by a symlink, is refused", async () => { + await withTraversableHome(async (home) => { + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + const foreign = (process.getuid?.() ?? 0) + 4_242; + await assert.rejects(ensureRuntimeHomeDirectory(home, "telemetry", foreign), /owned by the runtime user/u); + assert.equal(await mode(path.join(home, "telemetry")), 0o755, "a refusal corrects nothing and widens nothing"); + + const elsewhere = path.join(home, "elsewhere"); + await mkdir(elsewhere, { mode: 0o755 }); + await symlink(elsewhere, path.join(home, "tool-state")); + await assert.rejects(ensureRuntimeHomeDirectory(home, "tool-state")); + assert.equal(await mode(elsewhere), 0o755, "the link's target must not be chmod'ed through it"); + }); +}); + +test("the home itself is created when absent and never re-moded when present", async () => { + await withTraversableHome(async (home) => { + const fresh = path.join(home, "fresh-home"); + assert.equal(await ensureRuntimeHome(fresh), fresh); + assert.equal(await mode(fresh), RUNTIME_HOME_SUBDIRECTORY_MODE); + await chmod(fresh, 0o710); + await ensureRuntimeHome(fresh); + assert.equal(await mode(fresh), 0o710, "a Grok agent's traversable home is not this helper's judgement"); + }); +}); + +/** + * The rule that keeps the correction from being reintroducible. + * + * `mkdir(..., { mode })` reads like a guarantee and is one only for a + * directory that does not exist yet, so the mode constant stays private to + * this module: a call site that wants a private directory under a runtime home + * asks `ensureRuntimeHomeDirectory` for one and gets the assertion with it. + * + * Mutation: import the constant into any writer and pass it to `mkdir` again, + * and this goes red — which is the shape the 0755 hole had. + */ +test("the private mode is used only where it is also asserted", async () => { + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || target === "src/runtime/runtimeHomeLayout.ts") continue; + if ((await readFile(target, "utf8")).includes("RUNTIME_HOME_SUBDIRECTORY_MODE")) offenders.push(target); + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); + +// Creations that are not inside an agent's runtime home. +const OUTSIDE_RUNTIME_HOME = [ + "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" +]; + +/** + * Files that may name a runtime home in a `mkdir` of their own. + * + * `runtimeHomeLayout.ts` is the correction itself. `wakeAcceptanceFs.ts` + * creates the home and its store directory and then *asserts* each one — + * refusing a directory it finds wider rather than correcting it — which closes + * the same hole the other way and is its own documented contract. + */ +const MAY_MKDIR_A_RUNTIME_HOME = ["src/runtime/runtimeHomeLayout.ts", "src/pi/wakeAcceptanceFs.ts"]; + +const mkdirCalls = (source: string): string[] => { + const calls: string[] = []; + for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { + let depth = 0; + for (let cursor = index + "mkdir".length; cursor < source.length; cursor += 1) { + if (source[cursor] === "(") depth += 1; + else if (source[cursor] === ")") { depth -= 1; if (depth === 0) { calls.push(source.slice(index, cursor + 1)); break; } } + } + } + return calls; +}; + +test("no runtime-home directory is created without an explicit private mode, and none is created outside the layout", async () => { + // Source policy: a default `mkdir` under an agent's runtime home would be 0755, + // the home of a brokered Grok agent is traversable by its worker uid, and a + // `mode:` argument only covers the install where the directory is new. + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; + const source = await readFile(target, "utf8"); + for (const call of mkdirCalls(source)) { + // A `mode:` argument is a create-only mode: it decides nothing for a + // directory that already exists, so naming a runtime home in a `mkdir` + // is the defect whether or not a mode is passed. + if (/runtimeHome/iu.test(call) && !MAY_MKDIR_A_RUNTIME_HOME.includes(target)) { + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + continue; + } + // The workspace is a caller-prepared root with its own contract (group-readable for Grok). + if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + } + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts new file mode 100644 index 0000000..d1b5c1f --- /dev/null +++ b/src/runtime/runtimeHomeLayout.ts @@ -0,0 +1,84 @@ +import { constants } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; + +/** + * Mode for every directory Daimon creates inside an agent's runtime home. + * + * A brokered Grok agent's runtime home is `0710` + * (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`) so its sandboxed + * worker can traverse into the setgid `tool-output/` spill directory. Traverse + * is all it may have: anything Daimon creates in that home — telemetry traces + * (prompts, replies, world trajectories), tool state, receipts, the engine's + * XDG directories and the private `.tmp` — stays `0700`, so a default + * `mkdir` (0755 under the usual umask) never turns a traversable home into a + * readable one. + */ +export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; + +/** + * The home itself, created if it is absent and otherwise left exactly as it is. + * + * Create-only is the whole contract here. A brokered Grok agent's home is + * deliberately `0710` and an organization's may be `0700`; which of the two is + * correct is `physicalReadiness.ts`'s judgement, made against the agent's + * declared engine, and a layout helper that "corrected" a traversable home to + * `0700` would break the worker's only route to its own spills. + */ +export const ensureRuntimeHome = async (runtimeHomePath: string): Promise => { + await mkdir(runtimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + return runtimeHomePath; +}; + +/** + * One directory Daimon owns *below* a runtime home, private on every install. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and that is the common case rather than the exotic one: a `telemetry/` left + * at `0755` by a pre-branch Daimon, or pre-created by a deployment, stayed + * `0755` forever. Under a Grok agent's traversable `0710` home that is the + * worker reading its own agent's prompts, replies and causal history — the + * home is traverse-only precisely so that nothing but `tool-output/` is + * readable. `assertRuntimeDirectory` checks the home, and nothing checked what + * Daimon created inside it. + * + * So every level below the home is asserted and corrected on the way down, + * through a handle rather than a path: `O_DIRECTORY|O_NOFOLLOW` refuses a + * symlink planted where a directory belongs, and the `fchmod` that follows + * lands on the directory that was stat'd. A directory owned by anyone but the + * runtime user is **refused**, never widened and never silently accepted — the + * runtime cannot make someone else's directory private, and proceeding would + * write an agent's telemetry into it anyway. + * + * `owner` is a seam so both refusals are testable unprivileged, exactly as + * `physicalReadiness.ts`'s `RuntimeIdentity` is. + */ +export async function ensureRuntimeHomeDirectory(runtimeHomePath: string, relative: string, owner: number = process.getuid?.() ?? -1): Promise { + const segments = relative.split("/").filter((segment) => segment.length > 0); + if (segments.length === 0 || segments.some((segment) => segment === "." || segment === "..")) { + throw new Error(`runtime home subdirectory must name a path below the home: ${relative}`); + } + let current = await ensureRuntimeHome(runtimeHomePath); + for (const segment of segments) { + current = path.join(current, segment); + await mkdir(current, { mode: RUNTIME_HOME_SUBDIRECTORY_MODE }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + await assertPrivateDirectory(current, owner); + } + return current; +} + +const flag = (name: "O_DIRECTORY" | "O_NOFOLLOW"): number => (constants as typeof constants & Partial>)[name] ?? 0; + +async function assertPrivateDirectory(directory: string, owner: number): Promise { + const handle = await open(directory, constants.O_RDONLY | flag("O_DIRECTORY") | flag("O_NOFOLLOW")); + try { + const entry = await handle.stat(); + if (!entry.isDirectory()) throw new Error(`runtime home path is not a directory: ${directory}`); + if (entry.uid !== owner) throw new Error(`runtime home subdirectory must be owned by the runtime user: ${directory}`); + if ((entry.mode & 0o7777) !== RUNTIME_HOME_SUBDIRECTORY_MODE) await handle.chmod(RUNTIME_HOME_SUBDIRECTORY_MODE); + } finally { + await handle.close(); + } +} diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index b83a587..a4b94c6 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { chmod, chown, lstat, mkdir, mkdtemp, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { mock } from "node:test"; import { renderMcpToolResult, MCP_TOOL_RESULT_MAX_BYTES, type McpUpstreamResult } from "./mcpToolResult.js"; import { + assertSpillDirectoryStat, capToolResult, DEFAULT_TOOL_RESULT_MAX_BYTES, MIN_TOOL_RESULT_MAX_BYTES, @@ -150,3 +151,91 @@ test("the bound and the exemption list come from the environment, and a nonsense assert.deepEqual([...resolveExemptToolNames({})], []); assert.deepEqual([...resolveExemptToolNames({ [TOOL_RESULT_EXEMPT_ENV]: " mcp_a_b , mcp_c_d ," })], ["mcp_a_b", "mcp_c_d"]); }); + +test("spilled files are readable by the directory's (worker) group and never by other users", async () => { + await withDirectory(async (directory) => { + // What a deployment provisions for a brokered worker: setgid tool-output in a group that is not the runtime's own. + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { await chown(directory, process.getuid?.() ?? -1, workerGroup); await chmod(directory, 0o2750); } else await chmod(directory, 0o700); + const previous = process.umask(0o077); + let capped; + try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } + assert.ok(capped.spillPath, "the spill was written"); + const file = await stat(capped.spillPath); + assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); + assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); + }); +}); + +const big = { content: [{ type: "text" as const, text: `HEAD${"y".repeat(100_000)}TAIL` }] }; +const spillName = "daimon-abc123.mcp_desk_archive_dump.log"; + +test("a symlinked, world-open, or group-open-without-setgid spill directory is refused and nothing is written", async () => { + await withDirectory(async (root) => { + const target = path.join(root, "target"); await mkdir(target, { mode: 0o700 }); + const linked = path.join(root, "linked"); await symlink(target, linked); + const capped = await cap(big, { spillDirectory: linked }); + assert.equal(capped.spillPath, undefined); + assert.equal(capped.details.full_output_saved, false); + assert.deepEqual(await readdir(target), [], "nothing written through the symlink"); + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { + // A foreign group without setgid: files would not inherit it, so it is refused. + const noSetgid = path.join(root, "foreign-group-no-setgid"); await mkdir(noSetgid); await chown(noSetgid, process.getuid?.() ?? -1, workerGroup); await chmod(noSetgid, 0o750); + assert.equal((await cap(big, { spillDirectory: noSetgid })).spillPath, undefined, "0750 foreign group without setgid"); + } + // 2750 in the runtime's own group is not a worker grant; 0701/0704 exceed 2750. + for (const mode of [0o777, 0o755, 0o750, 0o2770, 0o2757, 0o2750, 0o701, 0o704]) { + const directory = path.join(root, `mode-${mode.toString(8)}`); await mkdir(directory); await chmod(directory, mode); + const refused = await cap(big, { spillDirectory: directory }); + assert.equal(refused.spillPath, undefined, mode.toString(8)); + assert.deepEqual((await readdir(directory)).filter((name) => !name.startsWith(".")), [], mode.toString(8)); + } + }); +}); + +test("a destination symlink or a pre-existing 0666 file is replaced by a 0640 regular file without touching the target", async () => { + await withDirectory(async (root) => { + const directory = path.join(root, "tool-output"); await mkdir(directory, { mode: 0o700 }); + const victim = path.join(root, "victim.txt"); await writeFile(victim, "VICTIM", { mode: 0o644 }); + await symlink(victim, path.join(directory, spillName)); + const first = await cap(big, { spillDirectory: directory }); + assert.equal(first.spillPath, path.join(directory, spillName)); + const replaced = await lstat(first.spillPath!); + assert.ok(replaced.isFile() && !replaced.isSymbolicLink()); + assert.equal(replaced.mode & 0o777, 0o640); + assert.equal(await readFile(victim, "utf8"), "VICTIM", "the symlink target is untouched"); + + await rm(first.spillPath!); await writeFile(first.spillPath!, "stale", { mode: 0o666 }); await chmod(first.spillPath!, 0o666); + const second = await cap(big, { spillDirectory: directory }); + const rewritten = await lstat(second.spillPath!); + assert.equal(rewritten.mode & 0o777, 0o640); + assert.equal(await readFile(second.spillPath!, "utf8"), big.content[0].text); + }); +}); + +test("the spill directory must be owned by the runtime itself", () => { + const runtime = { uid: 2000, gid: 2000 }; + const entry = (uid: number, gid: number, mode: number) => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2000, 0o700), runtime)); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2200, 0o2750), runtime)); + for (const [label, candidate] of [["owned by a worker", entry(2200, 2200, 0o700)], ["owned by root", entry(0, 2200, 0o2750)], ["not a directory", { ...entry(2000, 2000, 0o700), isDirectory: () => false }]] as const) { + assert.throws(() => assertSpillDirectoryStat(candidate, runtime), /spill directory/u, label); + } +}); + +test("a spill directory whose opened inode differs from the named one is refused", async () => { + await withDirectory(async (directory) => { + const probe = await open(directory, "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; let calls = 0; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + calls += 1; + return calls === 1 ? Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }) : real; + }); + try { assert.equal((await cap(big, { spillDirectory: directory })).spillPath, undefined); } finally { swapped.mock.restore(); } + assert.deepEqual(await readdir(directory), []); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index 6e993a2..5955577 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -116,7 +116,8 @@ const measure = (content: readonly unknown[], details: Record): Buffer.byteLength(safeStringify({ content, structuredContent: details }), "utf8"); /** Cut on a code-point boundary, from the front. */ -const headUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const headUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let end = Math.max(0, maxBytes); @@ -125,7 +126,8 @@ const headUtf8 = (value: string, maxBytes: number): string => { }; /** Cut on a code-point boundary, from the back. */ -const tailUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const tailUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let start = Math.max(0, bytes.byteLength - maxBytes); @@ -181,6 +183,20 @@ const notice = (input: Readonly<{ + ` or \`grep -n "" ${input.spillPath}\` — and do NOT repeat this tool call to see it: an identical call returns this same truncation.]`; }; +/** + * Spilled files are group-readable and never other-readable. + * + * A brokered Grok worker runs as its own uid and reads a spill with + * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to + * the very agent the notice sends there. The group grant reaches exactly that + * agent's worker only when the deployment provisions `tool-output` as + * `: 2750` (setgid, so each file inherits the worker + * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon + * creates itself stays 0700, so for every other engine nothing new is exposed. + */ +export const SPILL_FILE_MODE = 0o640; +export const SPILL_DIRECTORY_MAX_MODE = 0o2750; + /** * Write the full payload where the agent can read it, atomically. * @@ -190,12 +206,66 @@ const notice = (input: Readonly<{ */ const writeSpill = async (directory: string, name: string, text: string): Promise => { await mkdir(directory, { recursive: true, mode: 0o700 }); + const pinned = await pinSpillDirectory(directory); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } - return file; + try { + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + let written: Awaited>; + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); written = await handle.stat(); } finally { await handle.close(); } + if (!written.isFile() || written.nlink !== 1 || written.uid !== process.getuid?.()) throw new Error("spill file is not a private regular file"); + // Node has no openat: re-check that the directory entry still names the pinned inode before publishing into it. + await assertSameDirectory(directory, pinned); + // rename replaces a pre-existing destination entry (a symlink included) without following it. + await rename(temporary, file); + const published = await lstat(file); + if (!published.isFile() || published.dev !== written.dev || published.ino !== written.ino) throw new Error("spill file was replaced"); + return file; + } catch (error) { + await unlink(temporary).catch(() => undefined); + throw error; + } finally { + await pinned.handle.close(); + } +}; + +type PinnedDirectory = Readonly<{ handle: Awaited>; dev: number; ino: number }>; + +/** + * The spill directory must be a real directory owned by this runtime and no + * wider than `2750`: either Daimon's own `0700`, or a deployment-provisioned + * setgid directory whose group is not the runtime's own (a worker group). A + * symlinked, foreign-owned, world-accessible, or group-open-without-setgid + * directory is refused and nothing is written. Daimon cannot know *which* + * worker gid belongs to this agent; that mapping is the deployment's + * provisioning contract (`GROK_ENGINE_BROKER.worker.home.spillDirectory`). + */ +const pinSpillDirectory = async (directory: string): Promise => { + const before = await lstat(directory); + if (before.isSymbolicLink() || !before.isDirectory()) throw new Error("spill directory is not a real directory"); + const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("spill directory was replaced"); + assertSpillDirectoryStat(opened, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); + return { handle, dev: Number(opened.dev), ino: Number(opened.ino) }; + } catch (error) { await handle.close(); throw error; } +}; + +/** Pure so a foreign owner is testable without root. */ +export const assertSpillDirectoryStat = (entry: Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>, runtime: Readonly<{ uid: number; gid: number }>): void => { + const mode = Number(entry.mode) & 0o7777; + const groupOpen = (mode & 0o070) !== 0; + if (!entry.isDirectory() || entry.uid !== runtime.uid || (mode & ~SPILL_DIRECTORY_MAX_MODE) !== 0 + || (groupOpen && ((mode & 0o2000) === 0 || entry.gid === runtime.gid))) { + throw new Error("spill directory is not a private or provisioned worker-group directory"); + } +}; + +const assertSameDirectory = async (directory: string, pinned: PinnedDirectory): Promise => { + const [now, held] = await Promise.all([lstat(directory), pinned.handle.stat()]); + if (now.isSymbolicLink() || Number(now.dev) !== pinned.dev || Number(now.ino) !== pinned.ino || Number(held.ino) !== pinned.ino) throw new Error("spill directory was replaced"); }; /** Newest-first retention, so a busy agent cannot fill its own runtime home. */ diff --git a/src/runtime/turnRequestLedger.test.ts b/src/runtime/turnRequestLedger.test.ts index 954c069..5944103 100644 --- a/src/runtime/turnRequestLedger.test.ts +++ b/src/runtime/turnRequestLedger.test.ts @@ -96,9 +96,23 @@ test("a real rollout reaches the stream end to end, one line per request", async assert.equal(rows.length, 4); assert.equal(rows.every((row) => row.thread === FIXTURE_THREAD && row.wake === "wake-1" && row.requests === 4), true); assert.deepEqual(rows.map((row) => row.fresh_input), [15_742, 248, 4_276, 10_068]); + // Each request carries its own rollout-frame interval, not the wake's append time. + // Mutation guard: stamping every row with `at` collapses these to one value. + assert.deepEqual(rows.map((row) => [row.started_at, row.ended_at]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); }); +test("a request without measured timestamps carries none rather than the wake end", () => { + const [row] = renderTurnRequestLines({ agent: "a", wake: "w", thread: FIXTURE_THREAD, at: "2026-09-05T02:00:00.000Z", requests: [request({ startedAt: "not-a-time" })] }).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line)); + assert.equal("started_at" in row, false); + assert.equal("ended_at" in row, false); +}); + test("a missing or malformed rollout writes nothing and still resolves", async () => { await withDirectory(async (directory) => { const home = path.join(directory, "codex-home"); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index 21d5fbb..d0d2153 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -2,6 +2,7 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; import { readCodexRolloutRequests, type CodexRequestUsage } from "../pi/codexRolloutUsage.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES } from "./turnUsageLedger.js"; /** @@ -91,10 +92,102 @@ export const renderTurnRequestLines = (entry: TurnRequestEntry): string => { cache_write: request.cacheWrite, output: request.output, reasoning: request.reasoning, - total: request.total + total: request.total, + ...requestClockFields(request) })}\n`).join(""); }; +const TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u; +/** Per-request `started_at`/`ended_at`, each only when it was measured; `at` stays the append time. */ +const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: string }>): Record => ({ + ...(request.startedAt !== undefined && TIMESTAMP.test(request.startedAt) ? { started_at: request.startedAt } : {}), + ...(request.endedAt !== undefined && TIMESTAMP.test(request.endedAt) ? { ended_at: request.endedAt } : {}) +}); + +/** One Grok broker model request: usage from the worker stream, timing from the proxy. */ +/** + * `usageSource`: `stream` (the worker's own per-request frame), `upstream` + * (the provider response the proxy saw), or `estimated` (no valid usage; the + * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). + */ +/** + * `toolCalls` are the tool-call names that request's response carried, as the + * proxy read them (`grokBrokerTurnMeter.ts`): names only, bounded, `[]` for a + * response that called nothing, and absent when no response could be decoded. + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated"; toolCalls?: readonly string[] }>; +/** + * `requestCount` is the turn's admitted request count when it exceeds the rows: + * a killed turn's in-flight request was sent upstream but never reported usage, + * so it has no row yet still counts in every row's `requests`. + */ +export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; requestCount?: number; at?: string }>; + +/** + * Grok rows share the Codex row's field meaning: `input` is the whole prompt + * side the request replayed (`input_tokens + cache_read`), `cached_input` the + * cache read, `fresh_input` the uncached remainder. Grok does not separate + * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the + * broker idempotency key and `thread` the Grok session id when the stream + * named one. + * + * `tool_calls` is what a turn's rows could not say before: whether the model + * ever *tried* to call anything. Two live turns ended with correctly mounted + * tools and no visible attempt, and the rows recorded timings and tokens only, + * so the question could not be answered after the fact. It is names only — + * never arguments, never message content, never a bearer — bounded, `[]` for a + * response that called nothing, and absent for a response that could not be + * decoded, because a fabricated empty list is byte-identical to a measured one. + * + * It is an additive field inside the unchanged + * `noopolis.daimon.turn-requests.v1` row, deliberately without a version bump: + * Spawnfile's usage reader (`spawnfile/src/runtime/usageLedger.ts`) drops every + * line whose `v` it does not recognise while ignoring fields it does not know, + * and Paideia only relocates this stream's path + * (`DAIMON_TURN_REQUESTS_LEDGER_PATH`). A bump is what would blind them; a new + * field is not. + */ +export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { + const at = entry.at ?? new Date().toISOString(); + return entry.requests.map((request) => `${JSON.stringify({ + v: TURN_REQUEST_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at, + turn: entry.turn, + ...(entry.session === undefined ? {} : { thread: bounded(entry.session) }), + model: entry.model, + request: request.index, + requests: Math.max(entry.requests.length, entry.requestCount ?? 0), + input: request.input + request.cacheRead, + cached_input: request.cacheRead, + fresh_input: request.input, + cache_write: request.cacheWrite, + output: request.output, + total: request.total, + ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), + ...(request.toolCalls === undefined ? {} : { tool_calls: request.toolCalls }), + ...requestClockFields(request) + })}\n`).join(""); +}; + +/** + * Append already-rendered, newline-terminated ledger lines in one write, with the + * same rotation and file mode as both ledgers. Advisory: never rejects. The + * broker uses it to append the exact bytes it sealed into a turn record. + */ +export const recordLedgerLines = async (file: string, lines: string): Promise => { + if (lines.length === 0) return false; + try { await rotate(file); await appendLines(file, lines); return true; } catch { return false; } +}; + +/** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */ +export const recordGrokTurnRequests = async (file: string, entry: GrokTurnRequestEntry): Promise => { + if (entry.requests.length === 0) return false; + try { await rotate(file); await appendLines(file, renderGrokTurnRequestLines(entry)); return true; } catch { return false; } +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/turnUsageLedger.test.ts b/src/runtime/turnUsageLedger.test.ts index e6be905..7ddefad 100644 --- a/src/runtime/turnUsageLedger.test.ts +++ b/src/runtime/turnUsageLedger.test.ts @@ -15,6 +15,7 @@ import { TURN_USAGE_LEDGER_VERSION, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES, + dedupeTurnUsageRows, type TurnUsageEntry } from "./turnUsageLedger.js"; @@ -148,7 +149,7 @@ test("a failed wake's row carries the outcome and a reason from the closed vocab assert.equal(failed.reason, "token_ceiling"); assert.equal(failed.total, measurement.total, "the numbers are the ones the engine reported, not the outcome's"); - assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); + assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "request_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); for (const reason of TURN_USAGE_FAILURE_REASONS) { assert.equal(JSON.parse(renderTurnUsageLine(entry({ outcome: { status: "failed", reason } }))).reason, reason); } @@ -183,3 +184,13 @@ test("the outcome survives an append and a read back of the ledger file", async for (const record of written) assert.equal(record.v, TURN_USAGE_LEDGER_VERSION, "an added field, not a version bump"); }); }); + +test("broker rows carry the turn key, closed limit reason and closed model, and readers dedupe on the turn", () => { + const turn = "c".repeat(64); + const row = JSON.parse(renderTurnUsageLine(entry({ turn, limitReason: "requests", model: "grok-4.6", outcome: { status: "failed", reason: "request_ceiling" } }))); + assert.deepEqual([row.turn, row.limit_reason, row.model, row.reason], [turn, "requests", "grok-4.6", "request_ceiling"]); + assert.equal(row.total, row.input + row.cache_read + row.cache_write + row.output); + const forged = JSON.parse(renderTurnUsageLine(entry({ turn: "not-a-turn", limitReason: "budget" as never, model: "gpt-5" as never }))); + assert.deepEqual([forged.turn, forged.limit_reason, forged.model], [undefined, undefined, undefined]); + assert.deepEqual(dedupeTurnUsageRows([{ turn, total: 1 }, { total: 2 }, { turn, total: 3 }, { total: 4 }]), [{ turn, total: 1 }, { total: 2 }, { total: 4 }]); +}); diff --git a/src/runtime/turnUsageLedger.ts b/src/runtime/turnUsageLedger.ts index 5ca205a..6256caf 100644 --- a/src/runtime/turnUsageLedger.ts +++ b/src/runtime/turnUsageLedger.ts @@ -1,6 +1,10 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + /** * Append-only per-turn token accounting for one container. * @@ -81,6 +85,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; /** * - `token_ceiling` — the turn's own reported usage crossed the per-wake ceiling. + * - `request_ceiling` — the broker refused a model request past the turn's request limit. * - `wake_timeout` — the wall-clock bound fired before the child finished. * - `output_limit` — the retained-output bound was exceeded. * - `engine_exit` — the child exited non-zero (or died) after reporting usage. @@ -89,6 +94,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; */ export const TURN_USAGE_FAILURE_REASONS = [ "token_ceiling", + "request_ceiling", "wake_timeout", "output_limit", "engine_exit", @@ -110,6 +116,17 @@ export type TurnUsageEntry = Readonly<{ usage: TurnUsageMeasurement; at?: string; outcome?: TurnUsageOutcome; + /** + * Broker rows only. `turn` is the idempotency key readers dedupe on (the + * broker turn id, a sha256 hex); `limitReason` and `model` come from the + * closed broker vocabularies and are dropped rather than written through + * when they are not members. + */ + turn?: string; + limitReason?: EngineBrokerLimitReason; + model?: GrokBrokerModel; + /** Broker rows only: how many of the turn's requests were charged an estimate because their response carried no valid usage. */ + estimatedRequests?: number; }>; /** @@ -166,9 +183,27 @@ export const renderTurnUsageLine = (entry: TurnUsageEntry): string => `${JSON.st calls: entry.usage.calls, notional_usd: entry.usage.notionalUsd, complete: entry.usage.complete, - ...outcomeFields(entry.outcome) + ...outcomeFields(entry.outcome), + ...brokerFields(entry) })}\n`; +const brokerFields = (entry: TurnUsageEntry): Record => ({ + ...(entry.turn !== undefined && /^[a-f0-9]{64}$/u.test(entry.turn) ? { turn: entry.turn } : {}), + ...(entry.limitReason !== undefined && ENGINE_BROKER_LIMIT_REASONS.includes(entry.limitReason) ? { limit_reason: entry.limitReason } : {}), + ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}), + ...(entry.estimatedRequests !== undefined && Number.isSafeInteger(entry.estimatedRequests) && entry.estimatedRequests > 0 ? { estimated_requests: entry.estimatedRequests } : {}) +}); + +/** + * Collapse rows that share a `turn` key to the first one, keeping rows without + * a key untouched. Every reader that sums the ledger applies this, because a + * turn key exists precisely so a re-appended turn can never be counted twice. + */ +export const dedupeTurnUsageRows = >(rows: readonly T[]): T[] => { + const seen = new Set(); + return rows.filter((row) => { if (typeof row.turn !== "string") return true; if (seen.has(row.turn)) return false; seen.add(row.turn); return true; }); +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/wakeAcceptanceTypes.ts b/src/runtime/wakeAcceptanceTypes.ts index 1bc8e40..e21de59 100644 --- a/src/runtime/wakeAcceptanceTypes.ts +++ b/src/runtime/wakeAcceptanceTypes.ts @@ -71,6 +71,12 @@ export type OrganizationRuntimeActivityV2Item = OrganizationRuntimeWakeReceiptSt }>; export type OrganizationRuntimeActivityV2 = Readonly<{ version: typeof ACTIVITY_V2_VERSION; + /** + * Whether this projection was read from a live runtime or sealed as the host + * stopped. Optional on the wire because a projection published before the seal + * existed must still parse; its absence means "not stated", never "running". + */ + state?: "running" | "stopped"; items: readonly OrganizationRuntimeActivityV2Item[]; executions?: readonly Readonly<{ agent_id: string; execution_id: string; state: "running"; delivery_ids: readonly string[] }>[]; }>; diff --git a/src/runtime/wakeFuse.test.ts b/src/runtime/wakeFuse.test.ts index 9c6bf50..a526063 100644 --- a/src/runtime/wakeFuse.test.ts +++ b/src/runtime/wakeFuse.test.ts @@ -229,3 +229,15 @@ test("DAIMON_WAKE_FUSE=off never touches the usage ledger, missing or not", asyn const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory, { DAIMON_WAKE_FUSE: "off" }) }); assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); })); + +test("usage rows sharing a broker turn key count once toward the token ceiling", async () => await withDirectory(async (directory) => { + const turn = "b".repeat(64); + // 600 + 600 would trip a 1000-token ceiling; the duplicate turn row must not. + await writeFile(path.join(directory, "usage.jsonl"), [ + JSON.stringify({ at: "2026-08-30T00:00:00.000Z", total: 600, turn }), + JSON.stringify({ at: "2026-08-30T00:00:00.001Z", total: 600, turn }) + ].join("\n") + "\n"); + const now = () => new Date("2026-08-30T00:00:00.000Z"); + const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory), now }); + assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); +})); diff --git a/src/runtime/wakeFuse.ts b/src/runtime/wakeFuse.ts index 9786dff..04c8038 100644 --- a/src/runtime/wakeFuse.ts +++ b/src/runtime/wakeFuse.ts @@ -201,10 +201,15 @@ async function readFuseRecords(directory: string): Promise { let total = 0; + // Rows carrying a `turn` idempotency key count once per turn, whichever file holds them. + const turns = new Set(); for (const file of [`${ledgerPath}.1`, ledgerPath]) { for (const line of await lines(file)) { try { - const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown }; + const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown; turn?: unknown; kind?: unknown }; + // Evaluator inference rows belong to their own ledger; one here (a misconfigured path) is never subject spend. + if (value.kind === "inference") continue; + if (typeof value.turn === "string") { if (turns.has(value.turn)) continue; turns.add(value.turn); } if ((agentId === undefined || value.agent === agentId) && typeof value.at === "string" && !Number.isNaN(Date.parse(value.at)) && value.at >= since && typeof value.total === "number" && Number.isFinite(value.total) && value.total >= 0) total += value.total; } catch { /* usage accounting is advisory input; malformed lines are skipped */ } }