From 7e01937a9431ae8f76ebd1f19065dcc36eb4936e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 13 Sep 2026 21:26:51 +0200 Subject: [PATCH 01/84] feat: delegate canonical agent training to paideia --- README.md | 1 + specs/AGENTS.md | 1 + specs/INDEX.md | 1 + specs/SPEC.md | 3 + specs/TRAINING.md | 356 ++++++++++++++++++++++++++ src/cli/AGENTS.md | 3 + src/cli/paideiaDelegation.test.ts | 191 ++++++++++++++ src/cli/paideiaDelegation.ts | 139 ++++++++++ src/cli/paideiaSupervisor.test.ts | 49 ++++ src/cli/paideiaSupervisor.ts | 49 ++++ src/cli/runCli.ts | 18 +- src/cli/trainCommand.test.ts | 84 ++++++ src/cli/trainCommand.ts | 54 ++++ src/compiler/AGENTS.md | 1 + src/compiler/index.ts | 1 + src/compiler/training/AGENTS.md | 10 + src/compiler/training/CLAUDE.md | 1 + src/compiler/training/context.test.ts | 154 +++++++++++ src/compiler/training/context.ts | 101 ++++++++ src/compiler/training/contract.ts | 42 +++ src/compiler/training/index.ts | 2 + 21 files changed, 1257 insertions(+), 4 deletions(-) create mode 100644 specs/TRAINING.md create mode 100644 src/cli/paideiaDelegation.test.ts create mode 100644 src/cli/paideiaDelegation.ts create mode 100644 src/cli/paideiaSupervisor.test.ts create mode 100644 src/cli/paideiaSupervisor.ts create mode 100644 src/cli/trainCommand.test.ts create mode 100644 src/cli/trainCommand.ts create mode 100644 src/compiler/training/AGENTS.md create mode 120000 src/compiler/training/CLAUDE.md create mode 100644 src/compiler/training/context.test.ts create mode 100644 src/compiler/training/context.ts create mode 100644 src/compiler/training/contract.ts create mode 100644 src/compiler/training/index.ts diff --git a/README.md b/README.md index 11eccbd2..0f6e5975 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ Portability is capability-specific: the compiler reports each feature as **suppo | Explore complete projects | [Examples](examples/) | | Add an adapter or contribute | [Contributing](CONTRIBUTING.md) | | Integrate with deployment tooling | [Target and lifecycle contracts](specs/TARGETS.md) | +| Plan isolated agent training | [Paideia training handoff](specs/TRAINING.md) | | Find a detailed contract | [Specification index](specs/INDEX.md) | Spawnfile is part of [Noopolis](https://github.com/noopolis). [Moltnet](https://moltnet.dev) supplies messaging; [Daimon](https://github.com/noopolis/daimon) runs individual agents; [Simfile](https://simfile.org) builds simulation worlds around organizations. You can use Spawnfile on its own. diff --git a/specs/AGENTS.md b/specs/AGENTS.md index df2162bd..3afb14a2 100644 --- a/specs/AGENTS.md +++ b/specs/AGENTS.md @@ -9,6 +9,7 @@ specs/ ├── COMPILER.md # Compiler architecture and internal contracts ├── CONTAINERS.md # Container compilation spec ├── RUNTIMES.md # Runtime registry, version pinning, adapter lifecycle +├── TRAINING.md # Canonical source handoff and Paideia CLI delegation ├── CAUSAL.md # Shared causal wire and Stele read/verify contract ├── ECOSYSTEM_RUNTIME_BOUNDARIES.md # Cross-project runtime authority and enforcement gates ├── USAGE_ACCOUNTING_DESIGN.md # Daimon turn-usage envelope and Spawnfile aggregation design diff --git a/specs/INDEX.md b/specs/INDEX.md index 3254bbc2..36c7288c 100644 --- a/specs/INDEX.md +++ b/specs/INDEX.md @@ -16,6 +16,7 @@ These are the source of truth. Implementation in `src/` must stay aligned with t | [SURFACES.md](SURFACES.md) | evolving | Communication surfaces — platform messaging, HTTP, webhook, runtime support matrix, and lowering notes | | [RUNTIMES.md](RUNTIMES.md) | evolving | Runtime registry model — version pinning, status tracking, adapter lifecycle | | [STATUS.md](STATUS.md) | evolving | Operational status — static and live status, deployment records, Docker targets, runtime probes, and Moltnet metadata-only diagnostics | +| [TRAINING.md](TRAINING.md) | implemented handoff; native preparation integration required | Canonical agent selection and versioned Paideia delegation, dry-run and source provenance | | [DISTRIBUTION.md](DISTRIBUTION.md) | evolving | Image distribution — self-describing images, sourceless run/status, deployment record v2, publish, registry drift, and the network binding contract | | [CAUSAL.md](CAUSAL.md) | evolving | Causal event envelope — producer wire rules plus the shared Stele read/verify and reconciliation contract | | [TARGETS.md](TARGETS.md) | evolving | Project-neutral target-resource public contracts and staged target-adapter boundary | diff --git a/specs/SPEC.md b/specs/SPEC.md index 98f54507..e664a10f 100644 --- a/specs/SPEC.md +++ b/specs/SPEC.md @@ -1749,6 +1749,7 @@ spawnfile model clear-fallbacks [path] spawnfile validate [path] spawnfile view [path] spawnfile compile [path] [--out ] +spawnfile train [path] [--agent ] --train --test --out [--dry-run] spawnfile status [path | ] [--out ] [--live] [--deployment ] [--image] [--pull] [--pull-check] spawnfile up [path | ] [--out ] [--auth-profile ] [--env-file ] [--detach] [--deployment ] [--context ] [--image] [--pull] spawnfile dev up [path] [--out ] [--auth-profile ] [--env-file ] [--deployment ] [--context ] @@ -1762,6 +1763,7 @@ spawnfile publish [path] --tag [--out ] ``` See `DISTRIBUTION.md` for `publish`, image-reference `up`/`status`, and the `--image`/`--pull`/`--pull-check` flags. +See `TRAINING.md` for canonical source handoff, Paideia requirements and delegated training outcomes. ### Exit Codes @@ -1772,6 +1774,7 @@ All commands share one convention: - `1` — runtime failure: a compile, build, Docker, or other operation that failed after input validation passed. Per-command notes below reference this convention rather than restating exit numbers. +`train` preserves Paideia's completed failed-check exit 1 and cancellation exits 130/143; empty success receipts fail. #### `spawnfile init` diff --git a/specs/TRAINING.md b/specs/TRAINING.md new file mode 100644 index 00000000..01a92469 --- /dev/null +++ b/specs/TRAINING.md @@ -0,0 +1,356 @@ +# Canonical agent training + +`spawnfile train` resolves one agent from its full project and delegates to an installed +Paideia CLI. Spawnfile owns canonical source resolution and native compilation; +Paideia owns datasets, evaluation, cost planning, optimization and isolated trials. + +```sh +spawnfile train ./Spawnfile --agent agent:writer \ + --train evals/train.paideia.yaml --test evals/test.paideia.yaml \ + --editable agents/writer/AGENTS.md --cost-config local-costs.yaml --dry-run +``` + +Paideia supplies the cost-config format and training options. Dataset roles are explicit. +`--agent` is an exact resolved node ID; omission is allowed only for one-agent projects. +`--paideia-command` selects an installed executable, default `paideia`; no shell, +automatic installation, model-provider fallback or production launch is involved. + +## Public handoff + +The child invocation is `paideia train --spawnfile-context FILE` followed by the +explicit Paideia options. `FILE` is private evaluator-only JSON, removed after exit. +Dry-run requires neither `--out` nor an optimizer bridge; actual training requires `--out`. +The receiver must save any provenance needed later in its own protected experiment. +The strict `spawnfile.training-context.v1` schema appears below and is generated from +`src/compiler/training/contract.ts`; it is a wire contract, not an internal import API. + +Sources cover the full graph's manifests, resolved documents and skill entry files. +Every pin has an absolute `sourcePath`, project-relative POSIX `destinationPath`, and +SHA-256 of the actual file bytes. `destinationPath` preserves source editing locations; +it is **not** a compiled runtime destination. Source files outside the project root +are unsupported in v1. Effective documents retain canonical role order and inheritance. + +`project.sourceDigest` is SHA-256 of UTF-8 `JSON.stringify` over the `sources` array +projected to `{destinationPath,sha256}` in that key order, sorted by destinationPath +using code-point lexical order. All digests use the `sha256:` prefix. Absolute roots +do not affect this digest. The receiver must revalidate files and mappings before use. + +Resources disclose declaration digests and pins, not mounted or verified archives. +The resource definition digest uses the compiler's recursively key-sorted JSON. +`pin` is the declared bundle SHA or Git `ref`; branch/tag-only Git and volumes use null. +This receipt is not a complete packaged-resource closure. No environment values, +transport configuration, resource URLs or credentials are serialized. + +`agent.engine` is an explicit runtime engine option or null. Model identity/auth method +use canonical model resolution; absent native model defaults remain null. Runtime-added +instructions, tool schemas, skills loaded at runtime, and native model defaults must be +established from actual compilation/runtime receipts, never invented from this context. + +## Execution and outcomes + +Dry-run resolves local sources and lets Paideia validate datasets and estimate costs. +It does not compile, use Docker/auth, call models, or start an optimizer. Its final JSON +receipt must have `schema: paideia.training-cost-plan.v1` and `modelCallsMade: 0`. +Unimplemented native preparation is reported as unsupported, not ready to execute. + +Actual execution requires a supported preparation integration. Each candidate must +reach native files through Spawnfile compilation; no generic Pi fallback, second agent +declaration or replacement flattened prompt is authorized by this entrypoint. Packaging +exclusion, state isolation and single-agent preparation are not supplied by this handoff. + +Repeated options: `--editable`, `--resource`, `--judge`, `--validation-group`. Other +forwarded options: `--train`, `--test`, `--optimizer-model`, `--bridge-command`, `--out`, +`--max-trials`, `--max-proposals`, `--seed`, `--timeout-ms`, `--view`, `--cost-config`. +The canonical runtime/model/instruction selection cannot be replaced by generic CLI flags. + +Exit 0 requires the mode's final receipt. Completed actual runs also require +`status: completed` and a nonempty `index` path; exit 1 preserves completed failed checks. +Receiver error exits are propagated; empty success is a runtime failure. A supervisor +retains the owned POSIX process-group identity after the native child exits. Cancellation +forwards SIGTERM and escalates after one second; completion also removes group stragglers. +The parent verifies group/output quiescence before returning 0 or cancellation 130/143; +unknown cleanup is a runtime error. No signal uses an identity after its supervisor is reaped. +This delegation requires POSIX process groups; escaped processes and remote effects are not observed. +The child deadline is Paideia's declared deadline plus five seconds for cleanup. + +## JSON Schema + + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "spawnfile.training-context.v1" + }, + "producer": { + "type": "object", + "properties": { + "package": { + "type": "string", + "const": "spawnfile" + }, + "version": { + "type": "string", + "minLength": 1 + } + }, + "required": ["package", "version"], + "additionalProperties": false + }, + "project": { + "type": "object", + "properties": { + "root": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "manifest": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "sourceDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "required": ["root", "manifest", "sourceDigest"], + "additionalProperties": false + }, + "agent": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "runtime": { + "type": "string", + "minLength": 1 + }, + "engine": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "authMethod": { + "type": "string", + "minLength": 1 + } + }, + "required": ["provider", "name", "authMethod"], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": ["id", "name", "source", "runtime", "engine", "model"], + "additionalProperties": false + }, + "sources": { + "minItems": 1, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sourcePath": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "destinationPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!\\/)(?!.*(?:^|\\/)\\.\\.(?:\\/|$))[^\\\\]+$" + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "required": ["sourcePath", "destinationPath", "sha256"], + "additionalProperties": false + } + }, + "documents": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "sourcePath": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "destinationPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!\\/)(?!.*(?:^|\\/)\\.\\.(?:\\/|$))[^\\\\]+$" + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "minLength": 1 + } + }, + "required": ["sourcePath", "destinationPath", "sha256", "role"], + "additionalProperties": false + } + }, + "skills": { + "maxItems": 1000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sourcePath": { + "type": "string", + "minLength": 1, + "pattern": "^(?:\\/|[A-Za-z]:[\\\\/])" + }, + "destinationPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!\\/)(?!.*(?:^|\\/)\\.\\.(?:\\/|$))[^\\\\]+$" + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "requiresMcp": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["sourcePath", "destinationPath", "sha256", "name", "ref", "requiresMcp"], + "additionalProperties": false + } + }, + "resources": { + "maxItems": 1000, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "enum": [ + "bundle", + "git", + "volume" + ] + }, + "mount": { + "type": "string", + "minLength": 1 + }, + "mode": { + "type": "string", + "enum": [ + "mutable", + "readonly" + ] + }, + "sharing": { + "type": "string", + "enum": [ + "per_agent", + "team" + ] + }, + "definitionDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "pin": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + } + }, + "required": ["id", "kind", "mount", "mode", "sharing", "definitionDigest", "pin"], + "additionalProperties": false + } + }, + "requirements": { + "type": "object", + "properties": { + "nativeCompilation": { + "type": "boolean", + "const": true + }, + "isolatedPreparation": { + "type": "boolean", + "const": true + } + }, + "required": ["nativeCompilation", "isolatedPreparation"], + "additionalProperties": false + } + }, + "required": ["version", "producer", "project", "agent", "sources", "documents", "skills", "resources", "requirements"], + "additionalProperties": false +} +``` + diff --git a/src/cli/AGENTS.md b/src/cli/AGENTS.md index fa3d7937..e145d123 100644 --- a/src/cli/AGENTS.md +++ b/src/cli/AGENTS.md @@ -13,6 +13,9 @@ src/cli/ ├── composedLifecycleContractSet.ts # Closed machine command/contract inventory ├── evidenceExportHelperCommand.ts # Local helper construction command ├── compileBuildCommands.ts # `compile` and `build` command registration +├── trainCommand.ts # Canonical agent selection and Paideia CLI option forwarding +├── paideiaDelegation.ts # Private versioned context handoff, child lifecycle and completion receipts +├── paideiaSupervisor.ts # Packaged group leader retaining identity until native child/group cleanup ├── lifecycleCommands.ts # Thin lifecycle/compile/build/run/publish/up/down registration composition ├── lifecyclePlanningCommands.ts # Durable lifecycle plan and lookup command registration ├── runPublishCommands.ts # `run` and `publish` command registration diff --git a/src/cli/paideiaDelegation.test.ts b/src/cli/paideiaDelegation.test.ts new file mode 100644 index 00000000..f25b2889 --- /dev/null +++ b/src/cli/paideiaDelegation.test.ts @@ -0,0 +1,191 @@ +import { access, chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { TrainingContext } from "../compiler/training/index.js"; +import { delegatePaideiaTraining } from "./paideiaDelegation.js"; + +const directories: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +const digest = `sha256:${"1".repeat(64)}`; +const context: TrainingContext = { + version: "spawnfile.training-context.v1", producer: { package: "spawnfile", version: "0.1.17" }, + project: { root: "/isolated/project", manifest: "/isolated/project/Spawnfile", sourceDigest: digest }, + agent: { id: "agent:writer", name: "writer", source: "/isolated/project/Spawnfile", runtime: "daimon", engine: null, model: null }, + sources: [{ sourcePath: "/isolated/project/Spawnfile", destinationPath: "Spawnfile", sha256: digest }], + documents: [], skills: [], resources: [], requirements: { nativeCompilation: true, isolatedPreparation: true } +}; +const dryReceipt = 'console.log(JSON.stringify({schema:"paideia.training-cost-plan.v1",modelCallsMade:0}));'; +const completed = 'console.log(JSON.stringify({status:"completed",index:"/isolated/invocation.json"}));'; + +async function command(body: string): Promise { + const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-paideia-child-")); + directories.push(directory); + const executable = path.join(directory, "paideia"); + // Generated process fixture, not an alternate maintained implementation. + await writeFile(executable, `#!${process.execPath}\n${body}\n`); + await chmod(executable, 0o700); + return executable; +} + +async function invoke(body: string, overrides: Partial[0]> = {}) { + const stdout: string[] = [], stderr: string[] = []; + const result = delegatePaideiaTraining({ context, command: await command(body), args: ["--dry-run"], dryRun: true, + timeoutMs: 5000, streams: { stdout: (line) => stdout.push(line), stderr: (line) => stderr.push(line) }, ...overrides }); + return { result, stdout, stderr }; +} + +describe("Paideia public CLI delegation", () => { + it("passes literal argv and a private context, forwards streams and removes the temporary handoff", async () => { + const injected = "`touch /tmp/spawnfile-must-not-execute` $(false) with spaces"; + const { result, stdout, stderr } = await invoke(` +const fs = require("node:fs"); +const args = process.argv.slice(2), file = args[2]; +console.log(JSON.stringify({args,context:JSON.parse(fs.readFileSync(file,"utf8")),mode:fs.statSync(file).mode & 511,file})); +process.stderr.write("diagnostic without newline"); +${dryReceipt}`, { args: ["--train", injected, "--editable", "a.md", "--editable", "b.md", "--dry-run"] }); + expect(await result).toBe(0); + const observed = JSON.parse(stdout[0]!); + expect(observed.args.slice(0, 2)).toEqual(["train", "--spawnfile-context"]); + expect(observed.args.slice(3)).toEqual(["--train", injected, "--editable", "a.md", "--editable", "b.md", "--dry-run"]); + expect(observed.context).toEqual(context); + expect(observed.mode).toBe(0o600); + expect(stderr).toEqual(["diagnostic without newline"]); + await expect(access(observed.file)).rejects.toThrow(); + }); + + it.each([ + "", 'console.log(" ");', 'console.log("not json");', 'console.log("null");', + 'console.log(JSON.stringify({schema:"paideia.training-cost-plan.v1",modelCallsMade:1}));', + `${dryReceipt} console.log("done");` + ])("rejects empty, malformed or nonfinal success receipts: %s", async (body) => { + const run = await invoke(body); + await expect(run.result).rejects.toThrow("required final training receipt"); + }); + + it("handles a receipt without a newline and ordinary blank lines", async () => { + const run = await invoke('process.stdout.write("\\n\\r\\n" + JSON.stringify({schema:"paideia.training-cost-plan.v1",modelCallsMade:0}));'); + expect(await run.result).toBe(0); + }); + + it.each([0, 1])("requires actual completion and preserves the completed outcome %s", async (exitCode) => { + const run = await invoke(`${completed} process.exitCode=${exitCode};`, { dryRun: false, args: [] }); + expect(await run.result).toBe(exitCode); + }); + + it("rejects a dry-run receipt as an actual training completion", async () => { + const run = await invoke(dryReceipt, { dryRun: false, args: [] }); + await expect(run.result).rejects.toThrow("required final training receipt"); + }); + + it("rejects an incomplete actual-completion receipt", async () => { + const run = await invoke('console.log(JSON.stringify({status:"completed"}));', { dryRun: false }); + await expect(run.result).rejects.toThrow("required final training receipt"); + }); + + it("propagates receiver errors without requiring a success receipt", async () => { + const run = await invoke('console.error("unsupported native preparation"); process.exitCode=2;'); + expect(await run.result).toBe(2); + expect(run.stderr).toEqual(["unsupported native preparation"]); + }); + + it("reports missing executables without fallback", async () => { + const run = await invoke(dryReceipt, { command: "/does-not-exist/paideia" }); + await expect(run.result).rejects.toThrow("Could not start Paideia"); + }); + + it("cancels a running child and cleans up the context", async () => { + const controller = new AbortController(); + let contextPath = ""; + const run = await invoke('console.log(process.argv[4]); setInterval(()=>{},100);', { + signal: controller.signal, streams: { stdout: (line) => { contextPath = line; controller.abort(); }, stderr: () => undefined } + }); + expect(await run.result).toBe(130); + await expect(access(contextPath)).rejects.toThrow(); + }); + + it("does not spawn when already cancelled", async () => { + const controller = new AbortController(); controller.abort(); + const run = await invoke("throw Error('must not run');", { signal: controller.signal }); + expect(await run.result).toBe(130); + expect(run.stdout).toEqual([]); + }); + + it.each([["SIGINT", 130], ["SIGTERM", 143]] as const)("forwards parent %s and removes its signal handler", async (signal, expected) => { + const registered = vi.spyOn(process, "once"); + let listener: (() => void) | undefined; + const run = await invoke('console.log("ready"); setInterval(()=>{},100);', { + streams: { stdout: () => { + listener = registered.mock.calls.findLast(([name]) => name === signal)?.[1] as (() => void) | undefined; + expect(listener).toBeTypeOf("function"); listener!(); + }, stderr: () => undefined } + }); + expect(await run.result).toBe(expected); + expect(process.listeners(signal)).not.toContain(listener); + }); + + it("force-stops a child that ignores graceful termination", async () => { + const controller = new AbortController(); + const run = await invoke('process.on("SIGTERM",()=>{}); console.log("ready"); setInterval(()=>{},100);', { + signal: controller.signal, streams: { stdout: () => controller.abort(), stderr: () => undefined } + }); + expect(await run.result).toBe(130); + }); + + it.each([true, false])("terminates descendants with inherited output=%s after the native leader exits on cancellation", async (inherited) => { + await descendantTrial(inherited, true); + }); + + it.each([true, false])("confirms supervisor and descendant quiescence before success with inherited output=%s", async (inherited) => { + await descendantTrial(inherited, false); + }); + + it("reports unknown group cleanup as an error and never signals after the supervisor is reaped", async () => { + const original = process.kill.bind(process); + const kill = vi.spyOn(process, "kill").mockImplementation((pid, signal) => signal === 0 ? true : original(pid, signal)); + const run = await invoke(dryReceipt); + await expect(run.result).rejects.toThrow("quiescence is unknown"); + const firstProbe = kill.mock.calls.findIndex(([, signal]) => signal === 0); + expect(firstProbe).toBeGreaterThan(-1); + expect(kill.mock.calls.slice(firstProbe).every(([, signal]) => signal === 0)).toBe(true); + }); + + it("bounds hung children and oversized output", async () => { + const hung = await invoke('setInterval(()=>{},100);', { timeoutMs: 100 }); + await expect(hung.result).rejects.toThrow("command deadline"); + const noisy = await invoke('process.stdout.write("x".repeat(1024*1024+1)); setInterval(()=>{},100);'); + await expect(noisy.result).rejects.toThrow("bounded JSON-line contract"); + }); + + it("preserves a child signal outcome", async () => { + const run = await invoke('process.kill(process.pid,"SIGTERM");'); + expect(await run.result).toBe(143); + }); + + it("rejects an oversized context before spawning", async () => { + const run = await invoke("throw Error('must not run');", { context: { ...context, + producer: { package: "spawnfile", version: "v".repeat(1024 * 1024) } } }); + await expect(run.result).rejects.toThrow("context exceeds 1 MiB"); + }); +}); + +async function descendantTrial(inherited: boolean, cancel: boolean): Promise { + const folder = await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-descendant-")); directories.push(folder); + const ready = path.join(folder, "ready"); + const descendant = 'process.on("SIGTERM",()=>{}); require("node:fs").writeFileSync(' + JSON.stringify(ready) + ',"ready");setInterval(()=>{},100);'; + const leader = 'const {spawn}=require("node:child_process"),fs=require("node:fs");process.on("SIGTERM",()=>process.exit(0));' + + `const descendant=spawn(process.execPath,["-e",${JSON.stringify(descendant)}],{stdio:${JSON.stringify(inherited ? ["ignore", "inherit", "inherit"] : "ignore")}});` + + `const timer=setInterval(()=>{if(!fs.existsSync(${JSON.stringify(ready)}))return;clearInterval(timer);` + + 'console.log(JSON.stringify({leader:process.pid,supervisor:process.ppid,descendant:descendant.pid}));' + + (cancel ? 'setInterval(()=>{},100);' : `${dryReceipt} process.exit(0);`) + '},10);'; + const controller = new AbortController(); + let pids: { leader: number; supervisor: number; descendant: number } | undefined; + const run = await invoke(leader, { signal: controller.signal, streams: { stdout: (line) => { + const value = JSON.parse(line); + if (typeof value.descendant === "number") { pids = value; if (cancel) controller.abort(); } + }, stderr: () => undefined } }); + expect(await run.result).toBe(cancel ? 130 : 0); + expect(pids).toBeDefined(); + for (const pid of Object.values(pids!)) expect(() => process.kill(pid, 0)).toThrow(/ESRCH/u); +} diff --git a/src/cli/paideiaDelegation.ts b/src/cli/paideiaDelegation.ts new file mode 100644 index 00000000..b82bf8c0 --- /dev/null +++ b/src/cli/paideiaDelegation.ts @@ -0,0 +1,139 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; + +import { trainingContextSchema, type TrainingContext } from "../compiler/training/index.js"; +import { SpawnfileError } from "../shared/index.js"; +import type { PaideiaProcessOutcome } from "./paideiaSupervisor.js"; +import type { CliStreams } from "./runCli.js"; + +export interface DelegatePaideiaTrainingOptions { + context: TrainingContext; + command: string; + args: readonly string[]; + dryRun: boolean; + timeoutMs: number; + streams: CliStreams; + signal?: AbortSignal; +} + +const MAX_LINE_BYTES = 1024 * 1024; +const validReceipt = (line: string, dryRun: boolean): boolean => { + try { + const value: unknown = JSON.parse(line); + if (typeof value !== "object" || value === null) return false; + const receipt = value as Record; + return dryRun + ? receipt.schema === "paideia.training-cost-plan.v1" && receipt.modelCallsMade === 0 + : receipt.status === "completed" && typeof receipt.index === "string" && receipt.index.length > 0; + } catch { return false; } +}; +const failure = (message: string): SpawnfileError => new SpawnfileError("runtime_error", message); + +const runChild = (options: DelegatePaideiaTrainingOptions, contextPath: string): Promise => new Promise((resolve, reject) => { + if (options.signal?.aborted) { resolve(130); return; } + // Node strips types in a source checkout; the packaged build selects its emitted JS. + const extension = path.extname(fileURLToPath(import.meta.url)); + const supervisor = fileURLToPath(new URL(`./paideiaSupervisor${extension}`, import.meta.url)); + const child = spawn(process.execPath, ["--experimental-strip-types", supervisor, options.command, + "train", "--spawnfile-context", contextPath, ...options.args], { + shell: false, detached: true, stdio: ["ignore", "pipe", "pipe", "ipc"] + }); + const pid = child.pid; + let stdout = "", stderr = "", lastLine = ""; + let stopCode: number | undefined, error: Error | undefined, outcome: PaideiaProcessOutcome | undefined; + let reaped = false, closed = false, finished = false; + let killTimer: ReturnType | undefined; + const kill = (signal: NodeJS.Signals): void => { + if (pid === undefined || reaped) return; + try { process.kill(-pid, signal); } + catch (caught) { if ((caught as NodeJS.ErrnoException).code !== "ESRCH") error = failure("Could not signal the owned Paideia process group"); } + }; + const groupExists = (): boolean => { + if (pid === undefined) return false; + try { process.kill(-pid, 0); return true; } + catch (caught) { return (caught as NodeJS.ErrnoException).code !== "ESRCH"; } + }; + const stop = (code: number, cause?: Error): void => { + if (stopCode !== undefined) return; + stopCode = code; error = cause; + kill("SIGTERM"); + killTimer = setTimeout(() => kill("SIGKILL"), 1000); + killTimer.unref(); + }; + const interrupt = (): void => stop(130), terminate = (): void => stop(143), abort = (): void => stop(130); + const timer = setTimeout(() => stop(1, failure("Paideia training exceeded its command deadline")), options.timeoutMs); + timer.unref(); + process.once("SIGINT", interrupt); process.once("SIGTERM", terminate); + options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) abort(); + + const consume = (chunk: string, channel: "stdout" | "stderr"): void => { + const lines = ((channel === "stdout" ? stdout : stderr) + chunk).split("\n"); + const remainder = lines.pop()!; + if ([remainder, ...lines].some((line) => Buffer.byteLength(line, "utf8") > MAX_LINE_BYTES)) { + stop(1, failure("Paideia output exceeded the bounded JSON-line contract")); return; + } + for (const raw of lines) { + const line = raw.replace(/\r$/u, ""); + if (!line.trim()) continue; + if (channel === "stdout") lastLine = line; + options.streams[channel](line); + } + if (channel === "stdout") stdout = remainder; else stderr = remainder; + }; + child.stdout!.setEncoding("utf8").on("data", (chunk: string) => consume(chunk, "stdout")); + child.stderr!.setEncoding("utf8").on("data", (chunk: string) => consume(chunk, "stderr")); + child.once("message", (message: PaideiaProcessOutcome) => { + outcome = message; + if (message.type === "paideia.process.launch-error") error = failure(`Could not start Paideia: ${message.message}`); + // The native child exited, but our supervisor still holds the group identity. + // Stop stragglers before permitting its leader to disappear or reporting completion. + kill("SIGKILL"); + }); + const finish = async (): Promise => { + if (finished) return; + finished = true; + clearTimeout(timer); clearTimeout(killTimer); + process.removeListener("SIGINT", interrupt); process.removeListener("SIGTERM", terminate); + options.signal?.removeEventListener("abort", abort); + const deadline = Date.now() + 1500; + // Check only: never signal a numeric group identity after its supervisor was reaped. + while ((groupExists() || !closed) && Date.now() < deadline) await delay(20); + if (groupExists() || !closed) { + error = failure("Paideia process cleanup is incomplete; group or output quiescence is unknown"); + child.stdout!.destroy(); child.stderr!.destroy(); + } + if (stdout.trim()) { lastLine = stdout; options.streams.stdout(stdout); } + if (stderr.trim()) options.streams.stderr(stderr); + if (error) { reject(error); return; } + if (stopCode !== undefined) { resolve(stopCode); return; } + if (outcome?.type !== "paideia.process.exited") { reject(failure("Paideia supervisor exited without a native outcome")); return; } + if (outcome.signal) { resolve(outcome.signal === "SIGINT" ? 130 : outcome.signal === "SIGTERM" ? 143 : 1); return; } + const exitCode = outcome.code ?? 1; + if ((exitCode === 0 || exitCode === 1) && !validReceipt(lastLine, options.dryRun)) { + reject(failure("Paideia exited without the required final training receipt")); return; + } + resolve(exitCode); + }; + child.once("error", (caught) => { error = failure(`Could not start Paideia supervisor: ${caught.message}`); }); + child.once("exit", () => { reaped = true; void finish(); }); + child.once("close", () => { closed = true; void finish(); }); +}); + +/** Delegates through the public CLI, never importing Paideia or selecting a fallback adapter. */ +export const delegatePaideiaTraining = async (options: DelegatePaideiaTrainingOptions): Promise => { + if (options.signal?.aborted) return 130; + if (process.platform === "win32") throw failure("Paideia delegation requires POSIX process-group supervision"); + const bytes = JSON.stringify(trainingContextSchema.parse(options.context)); + if (Buffer.byteLength(bytes, "utf8") > MAX_LINE_BYTES) throw new SpawnfileError("validation_error", "Training context exceeds 1 MiB"); + const temporary = await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-")); + const contextPath = path.join(temporary, "context.json"); + try { + await writeFile(contextPath, bytes, { mode: 0o600, flag: "wx" }); + return await runChild(options, contextPath); + } finally { await rm(temporary, { recursive: true, force: true }); } +}; diff --git a/src/cli/paideiaSupervisor.test.ts b/src/cli/paideiaSupervisor.test.ts new file mode 100644 index 00000000..e8a1478e --- /dev/null +++ b/src/cli/paideiaSupervisor.test.ts @@ -0,0 +1,49 @@ +import { EventEmitter } from "node:events"; +import type { ChildProcess } from "node:child_process"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { supervisePaideia, type PaideiaSupervisorHost } from "./paideiaSupervisor.js"; + +const cleanups: Array<() => void> = []; +afterEach(() => { cleanups.splice(0).forEach((cleanup) => cleanup()); }); +function fixture() { + const process = Object.assign(new EventEmitter(), { pid: 1234, send: vi.fn(), kill: vi.fn(), exit: vi.fn() }); + const child = new EventEmitter(); + const launch = vi.fn(() => child as ChildProcess); + const host = { process, launch } as unknown as PaideiaSupervisorHost; + return { process, child, launch, host }; +} + +describe("Paideia group supervisor", () => { + it("passes argv unchanged and reports the real child outcome once while retaining group ownership", () => { + const run = fixture(); + cleanups.push(supervisePaideia(["paideia", "train", "literal $argument"], run.host)); + expect(run.launch).toHaveBeenCalledWith("paideia", ["train", "literal $argument"]); + run.child.emit("exit", 1, null); run.child.emit("error", new Error("later")); + expect(run.process.send).toHaveBeenCalledExactlyOnceWith({ type: "paideia.process.exited", code: 1, signal: null }); + expect(run.process.exit).not.toHaveBeenCalled(); + run.process.emit("SIGTERM"); run.process.emit("SIGINT"); + expect(run.process.exit).not.toHaveBeenCalled(); + }); + + it("reports launch failures without inventing completion", () => { + const run = fixture(); cleanups.push(supervisePaideia(["missing"], run.host)); + run.child.emit("error", new Error("ENOENT")); + expect(run.process.send).toHaveBeenCalledWith({ type: "paideia.process.launch-error", message: "ENOENT" }); + }); + + it("cleans its own still-live group if its parent disappears", () => { + const run = fixture(); cleanups.push(supervisePaideia(["paideia"], run.host)); + run.process.emit("disconnect"); + expect(run.process.kill).toHaveBeenCalledWith(-1234, "SIGKILL"); + expect(run.process.exit).toHaveBeenCalledWith(1); + }); + + it("requires a private parent channel and removes owned listeners on disposal", () => { + const run = fixture(); + expect(() => supervisePaideia([], run.host)).toThrow("private IPC"); + expect(() => supervisePaideia(["paideia"], { ...run.host, process: { ...run.host.process, send: undefined } })).toThrow("private IPC"); + const cleanup = supervisePaideia(["paideia"], run.host); cleanup(); + expect(run.process.listenerCount("disconnect")).toBe(0); + }); +}); diff --git a/src/cli/paideiaSupervisor.ts b/src/cli/paideiaSupervisor.ts new file mode 100644 index 00000000..9b5fb933 --- /dev/null +++ b/src/cli/paideiaSupervisor.ts @@ -0,0 +1,49 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export type PaideiaProcessOutcome = + | { type: "paideia.process.exited"; code: number | null; signal: NodeJS.Signals | null } + | { type: "paideia.process.launch-error"; message: string }; + +type SupervisorProcess = Pick; +export interface PaideiaSupervisorHost { + process: SupervisorProcess; + launch(command: string, args: string[]): ChildProcess; +} +const defaultHost: PaideiaSupervisorHost = { + process, + launch: (command, args) => spawn(command, args, { shell: false, stdio: ["ignore", "inherit", "inherit"] }) +}; + +/** Remains the owned group leader until the parent closes the entire group. */ +export const supervisePaideia = (argv: readonly string[], host: PaideiaSupervisorHost = defaultHost): (() => void) => { + const [command, ...args] = argv; + if (!command || !host.process.send) throw new Error("Paideia supervisor requires an executable and private IPC"); + const hold = setInterval(() => undefined, 1000); + const ignore = (): void => undefined; + const disconnect = (): void => { + // This process is still alive: its own group identity cannot have been recycled. + try { host.process.kill(-host.process.pid, "SIGKILL"); } + finally { host.process.exit(1); } + }; + host.process.on("SIGINT", ignore).on("SIGTERM", ignore).on("disconnect", disconnect); + let reported = false; + const report = (outcome: PaideiaProcessOutcome): void => { + if (reported) return; + reported = true; + host.process.send!(outcome); + }; + const child = host.launch(command, args); + child.once("error", (error) => report({ type: "paideia.process.launch-error", message: error.message })); + child.once("exit", (code, signal) => report({ type: "paideia.process.exited", code, signal })); + return () => { + clearInterval(hold); + host.process.removeListener("SIGINT", ignore).removeListener("SIGTERM", ignore).removeListener("disconnect", disconnect); + }; +}; + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { supervisePaideia(process.argv.slice(2)); } + catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; } +} diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index 82dcdc9a..9da2c183 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -22,6 +22,7 @@ import { buildUpReceipt, clearProjectModelFallbacks, compileProject, + createTrainingContext, initProject, listInitTemplates, publishProject, @@ -65,6 +66,8 @@ import { registerStatusCommand } from "./statusCommand.js"; import { registerUsageCommand } from "./usageCommand.js"; import { registerProductionTargetCommands } from "./targetProductionCommands.js"; import { registerViewCommand } from "./viewCommand.js"; +import { registerTrainCommand } from "./trainCommand.js"; +import { delegatePaideiaTraining } from "./paideiaDelegation.js"; const packageJsonPath = new URL("../../package.json", import.meta.url); @@ -95,6 +98,8 @@ const createDefaultRenderEnvironment = (): CliRenderEnvironment => ({ }); export interface CliHandlers { + createTrainingContext: typeof createTrainingContext; + delegatePaideiaTraining: typeof delegatePaideiaTraining; buildCompilePlan: typeof buildCompilePlan; buildOrganizationView: typeof buildOrganizationView; buildProject: typeof buildProject; compileProject: typeof compileProject; publishProject: typeof publishProject; @@ -127,6 +132,7 @@ export interface CliHandlers { } const createDefaultHandlers = (): CliHandlers => ({ + createTrainingContext, delegatePaideiaTraining, buildCompilePlan, buildOrganizationView, buildProject, compileProject, publishProject, addAgentProject, addProjectModelFallback, addProjectSurface, addSubagentProject, addTeamProject, clearProjectModelFallbacks, @@ -141,7 +147,7 @@ const createDefaultHandlers = (): CliHandlers => ({ }); export interface RunCliOptions { - handlers?: Partial; renderEnvironment?: CliRenderEnvironment; stdin?: AsyncIterable; streams?: CliStreams; + handlers?: Partial; renderEnvironment?: CliRenderEnvironment; stdin?: AsyncIterable; streams?: CliStreams; signal?: AbortSignal; } const isCliStreams = (value: CliStreams | RunCliOptions | undefined): value is CliStreams => { @@ -152,7 +158,7 @@ const isCliStreams = (value: CliStreams | RunCliOptions | undefined): value is C const normalizeRunCliOptions = ( optionsOrStreams?: CliStreams | RunCliOptions, handlerOverrides: Partial = {} -): Required => isCliStreams(optionsOrStreams) +): Required> & Pick => isCliStreams(optionsOrStreams) ? { handlers: handlerOverrides, renderEnvironment: createDefaultRenderEnvironment(), @@ -163,7 +169,8 @@ const normalizeRunCliOptions = ( handlers: optionsOrStreams?.handlers ?? handlerOverrides, renderEnvironment: optionsOrStreams?.renderEnvironment ?? createDefaultRenderEnvironment(), stdin: optionsOrStreams?.stdin ?? process.stdin, - streams: optionsOrStreams?.streams ?? createDefaultStreams() + streams: optionsOrStreams?.streams ?? createDefaultStreams(), + signal: optionsOrStreams?.signal }; const writeCommanderOutput = ( @@ -229,7 +236,7 @@ export const runCli: RunCli = async ( const streams = cliOptions.streams; const handlers = { ...createDefaultHandlers(), ...cliOptions.handlers }; const isTargetInvocation = argv[0] === "target"; - let commandExitCode: 0 | 1 | 2 = 0; + let commandExitCode = 0; const program = new Command(); program.name("spawnfile").description("Spawnfile v0.1 compiler").version(readPackageVersion()); program.exitOverride(); @@ -334,6 +341,9 @@ export const runCli: RunCli = async ( commandExitCode = exitCode; }, handlers); registerViewCommand(program, handlers, streams, cliOptions.renderEnvironment); + registerTrainCommand(program, handlers, streams, readPackageVersion(), (code) => { + commandExitCode = code; + }, cliOptions.signal); registerProductionTargetCommands(program, streams, cliOptions.stdin, (exitCode) => { commandExitCode = exitCode; }); diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts new file mode 100644 index 00000000..8f490e62 --- /dev/null +++ b/src/cli/trainCommand.test.ts @@ -0,0 +1,84 @@ +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { runCli } from "./runCli.js"; +import type { DelegatePaideiaTrainingOptions } from "./paideiaDelegation.js"; + +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); +async function project(): Promise { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-train-command-"))); + directories.push(root); + await writeFile(path.join(root, "Spawnfile"), 'spawnfile_version: "0.1"\nkind: agent\nname: author\nruntime: daimon\n'); + return root; +} +const base = ["--train", "train.paideia.yaml", "--test", "test.paideia.yaml", "--out", "local output"]; + +describe("spawnfile train", () => { + it("resolves the real canonical project and forwards only explicit Paideia options", async () => { + const root = await project(), stdout: string[] = [], stderr: string[] = []; + const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 0); + const forbidden = vi.fn(async () => { throw new Error("must not compile, build or authenticate"); }); + const code = await runCli(["train", root, ...base, "--agent", "agent:author", "--dry-run", + "--paideia-command", "/opt/paideia with space", "--editable", "a.md", "--editable", "b.md", "--resource", "archive=/private/source", + "--judge", "editor=fable", "--validation-group", "previous", "--cost-config", "prices.yaml", "--max-trials", "3", "--timeout-ms", "5000"], { + streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) }, + handlers: { delegatePaideiaTraining: delegate, compileProject: forbidden, buildProject: forbidden, importCodexAuth: forbidden } + }); + expect(code).toBe(0); expect(forbidden).not.toHaveBeenCalled(); + expect(delegate).toHaveBeenCalledOnce(); + const options = delegate.mock.calls[0]![0]; + expect(options.context.agent.id).toBe("agent:author"); + expect(options.command).toBe("/opt/paideia with space"); + expect(options.dryRun).toBe(true); expect(options.timeoutMs).toBe(10_000); + expect(options.args).toEqual(["--train", "train.paideia.yaml", "--test", "test.paideia.yaml", "--editable", "a.md", "--editable", "b.md", + "--resource", "archive=/private/source", "--judge", "editor=fable", "--validation-group", "previous", "--out", "local output", "--max-trials", "3", "--timeout-ms", "5000", "--cost-config", "prices.yaml", "--dry-run"]); + expect(stderr).toEqual([]); + }); + + it.each([1, 2, 130, 143])("propagates the delegated exit %s", async (exitCode) => { + const code = await runCli(["train", await project(), ...base], { handlers: { delegatePaideiaTraining: async () => exitCode }, + streams: { stdout: () => undefined, stderr: () => undefined } }); + expect(code).toBe(exitCode); + }); + + it("forwards cancellation and default executable/timeout without forcing dry-run", async () => { + const controller = new AbortController(); + let captured: DelegatePaideiaTrainingOptions | undefined; + expect(await runCli(["train", await project(), ...base, "--optimizer-model", "fable", "--bridge-command", "bridge", "--max-proposals", "1", "--seed", "0", "--view", "0"], { + signal: controller.signal, handlers: { delegatePaideiaTraining: async (options) => { captured = options; return 2; } }, + streams: { stdout: () => undefined, stderr: () => undefined } + })).toBe(2); + expect(captured).toMatchObject({ command: "paideia", timeoutMs: 185_000, signal: controller.signal, dryRun: false }); + expect(captured?.args).toContain("--bridge-command"); + expect(captured?.args).not.toContain("--dry-run"); + }); + + it("allows dry-run without output or optimizer bridge, but rejects an actual run without output", async () => { + const root = await project(), delegated = vi.fn(async () => 0); + const args = ["train", root, "--train", "train.paideia.yaml", "--test", "test.paideia.yaml"]; + const options = { handlers: { delegatePaideiaTraining: delegated }, streams: { stdout: () => undefined, stderr: () => undefined } }; + expect(await runCli([...args, "--dry-run"], options)).toBe(0); + expect(delegated).toHaveBeenCalledOnce(); + expect(await runCli(args, options)).toBe(2); + expect(delegated).toHaveBeenCalledOnce(); + }); + + it.each(["0", "1.5", "-1", "NaN", "3600001"])("rejects timeout %s before extracting or delegating", async (timeout) => { + const forbidden = vi.fn(async () => { throw Error("must not start"); }); + const code = await runCli(["train", "/missing", ...base, "--timeout-ms", timeout], { + handlers: { createTrainingContext: forbidden, delegatePaideiaTraining: forbidden }, streams: { stdout: () => undefined, stderr: () => undefined } + }); + expect(code).toBe(2); expect(forbidden).not.toHaveBeenCalled(); + }); + + it("rejects generic runtime/instruction overrides and missing required datasets", async () => { + const forbidden = vi.fn(async () => { throw Error("must not start"); }); + for (const args of [["train", ...base, "--runtime", "pi"], ["train", ...base, "--instructions", "prompt.md"], ["train", "--dry-run"]]) { + expect(await runCli(args, { handlers: { delegatePaideiaTraining: forbidden }, streams: { stdout: () => undefined, stderr: () => undefined } })).toBe(2); + } + expect(forbidden).not.toHaveBeenCalled(); + }); +}); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts new file mode 100644 index 00000000..2ab6edca --- /dev/null +++ b/src/cli/trainCommand.ts @@ -0,0 +1,54 @@ +import type { Command } from "commander"; + +import { SpawnfileError } from "../shared/index.js"; +import type { CliHandlers, CliStreams } from "./runCli.js"; + +const forwarded = ["train", "test", "editable", "resource", "judge", "validation-group", "optimizer-model", + "bridge-command", "out", "max-trials", "max-proposals", "seed", "timeout-ms", "view", "cost-config"] as const; +const repeated = new Set(["editable", "resource", "judge", "validation-group"]); +const key = (name: string): string => name.replace(/-([a-z])/gu, (_, letter: string) => letter.toUpperCase()); + +export const registerTrainCommand = ( + program: Command, + handlers: CliHandlers, + streams: CliStreams, + packageVersion: string, + setExitCode: (code: number) => void, + signal?: AbortSignal +): void => { + const command = program.command("train") + .description("Train one canonical agent through Paideia's isolated native integration") + .argument("[path]", "Canonical project directory or Spawnfile path", process.cwd()) + .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") + .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") + .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models"); + for (const name of forwarded) { + const flag = `--${name} `; + if (repeated.has(name)) command.option(flag, `Paideia ${name}; repeatable`, (value: string, previous: string[]) => [...previous, value], []); + else if (["train", "test"].includes(name)) command.requiredOption(flag, `Paideia ${name}`); + else command.option(flag, `Paideia ${name}`); + } + command.action(async (inputPath: string, options: Record) => { + if (options.dryRun !== true && typeof options.out !== "string") { + throw new SpawnfileError("validation_error", "Actual training requires --out; dry-run does not write an output directory"); + } + const timeout = options.timeoutMs === undefined ? 180_000 : Number(options.timeoutMs); + if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 3_600_000 || + (options.timeoutMs !== undefined && !/^\d+$/u.test(String(options.timeoutMs)))) { + throw new SpawnfileError("validation_error", "--timeout-ms must be an integer from 1 to 3600000"); + } + const context = await handlers.createTrainingContext(inputPath, { + agent: options.agent as string | undefined, packageVersion + }); + const args: string[] = []; + for (const name of forwarded) { + const value = options[key(name)]; + if (typeof value === "string") args.push(`--${name}`, value); + else if (Array.isArray(value)) for (const item of value) args.push(`--${name}`, item); + } + if (options.dryRun === true) args.push("--dry-run"); + setExitCode(await handlers.delegatePaideiaTraining({ context, args, + command: options.paideiaCommand as string, dryRun: options.dryRun === true, + timeoutMs: timeout + 5000, streams, signal })); + }); +}; diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 273b82b8..b2a7bda0 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -6,6 +6,7 @@ This folder owns graph resolution, effective configuration, compile planning, an ```text src/compiler/ +├── training/ # Versioned canonical source context for Paideia; no evaluation or launch ├── index.ts # Barrel for compiler-facing exports ├── types.ts # Internal compiler plan and resolved-node types ├── helpers.ts # Deterministic helper utilities diff --git a/src/compiler/index.ts b/src/compiler/index.ts index ca46f96f..4568f74c 100644 --- a/src/compiler/index.ts +++ b/src/compiler/index.ts @@ -19,3 +19,4 @@ export * from "./updateProjectSurfaces.js"; export * from "./upReceipt.js"; export * from "./worldBindings.js"; export * from "./view/index.js"; +export * from "./training/index.js"; diff --git a/src/compiler/training/AGENTS.md b/src/compiler/training/AGENTS.md new file mode 100644 index 00000000..9e9641f7 --- /dev/null +++ b/src/compiler/training/AGENTS.md @@ -0,0 +1,10 @@ +# Canonical Training Context + +- `contract.ts` owns the strict, versioned public JSON handoff to Paideia. +- `context.ts` resolves the full compiler graph and pins source files without compiling or launching it. +- Preserve resolved inheritance and exact agent IDs. Never create a second agent declaration. +- Source mappings are project-relative editable files, not runtime-native destinations or flattened prompts. +- Do not serialize secret values, arbitrary environments or transport credentials. +- Paideia owns datasets, evaluation, budgets and optimization. Its native integration must consume Spawnfile compilation. +- Dry-run extraction performs local reads only; no Docker, auth, deployment or model calls. +- Keep files below 400 lines and tests adjacent. Test real graph resolution and negative source/selection cases. diff --git a/src/compiler/training/CLAUDE.md b/src/compiler/training/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/context.test.ts b/src/compiler/training/context.test.ts new file mode 100644 index 00000000..fcdbbec3 --- /dev/null +++ b/src/compiler/training/context.test.ts @@ -0,0 +1,154 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { stringify } from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createTrainingContext } from "./context.js"; +import { trainingContextJsonSchema, trainingContextSchema } from "./contract.js"; + +vi.mock("node:fs/promises", async () => { + const actual = await vi.importActual("node:fs/promises"); + return { ...actual, readFile: vi.fn(actual.readFile) }; +}); +const directories: string[] = []; +const actual = await vi.importActual("node:fs/promises"); +afterEach(async () => { + vi.mocked(readFile).mockImplementation(actual.readFile); + await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); +}); +const hash = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; + +async function project(team = false): Promise { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-context-"))); + directories.push(root); + await mkdir(path.join(root, "agents/author"), { recursive: true }); + await mkdir(path.join(root, "skills/reporting"), { recursive: true }); + await writeFile(path.join(root, "AGENTS.md"), "Root system\n"); + await writeFile(path.join(root, "SOUL.md"), "Inherited soul\n"); + await writeFile(path.join(root, "agents/author/AGENTS.md"), "Local author system\n"); + await writeFile(path.join(root, "skills/reporting/SKILL.md"), "---\nname: reporting\ndescription: Report evidence.\n---\nRead carefully.\n"); + const workspace = { docs: { system: "AGENTS.md", soul: "SOUL.md" }, skills: [{ ref: "./skills/reporting" }] }; + const declaration = team ? { + spawnfile_version: "0.1", kind: "team", name: "publication", mode: "hierarchical", lead: "author", + shared: { workspace, environment: { env: { SECRET_LIKE_VALUE: "must-not-be-in-receipt" } } }, + members: [{ id: "author", ref: "./agents/author" }, { id: "reviewer", runtime: "daimon", workspace: { docs: { system: "AGENTS.md" } } }] + } : { spawnfile_version: "0.1", kind: "agent", name: "author", runtime: "daimon", workspace }; + await writeFile(path.join(root, "Spawnfile"), stringify(declaration)); + if (team) await writeFile(path.join(root, "agents/author/Spawnfile"), stringify({ + spawnfile_version: "0.1", kind: "agent", name: "author", + runtime: { name: "daimon", options: { engine: "codex" } }, + execution: { model: { primary: { provider: "openai", name: "gpt-5.5" }, auth: { method: "codex" } } }, + workspace: { docs: { system: "AGENTS.md" }, resources: [ + { id: "source", kind: "git", url: "https://user:private-token@example.invalid/code.git", ref: "a".repeat(40), mount: "./source", mode: "readonly" }, + { id: "moving", kind: "git", url: "https://example.invalid/other.git", branch: "main", mount: "./moving", mode: "readonly" }, + { id: "tools", kind: "bundle", source: "missing-but-uncompiled.tar", sha256: hash("tools"), mount: "./tools", mode: "readonly" }, + { id: "state", kind: "volume", name: "never-attach", mount: "./state", mode: "mutable", sharing: "team" } + ] } + })); + return root; +} + +describe("canonical training context", () => { + it("pins the full graph and preserves effective inherited docs, skills and model auth", async () => { + const root = await project(true); + const context = await createTrainingContext(root, { agent: "agent:author", packageVersion: "0.1.17" }); + expect(context.agent).toEqual({ id: "agent:author", name: "author", source: path.join(root, "agents/author/Spawnfile"), + runtime: "daimon", engine: "codex", model: { provider: "openai", name: "gpt-5.5", authMethod: "codex" } }); + expect(context.documents.map((document) => [document.role, document.destinationPath])).toEqual([ + ["system", "agents/author/AGENTS.md"], ["soul", "SOUL.md"] + ]); + expect(context.skills[0]).toMatchObject({ name: "reporting", destinationPath: "skills/reporting/SKILL.md" }); + expect(context.sources.map((source) => source.destinationPath)).toEqual([ + "AGENTS.md", "SOUL.md", "Spawnfile", "agents/author/AGENTS.md", "agents/author/Spawnfile", "skills/reporting/SKILL.md" + ]); + expect(context.project.sourceDigest).toBe(hash(JSON.stringify(context.sources.map(({ destinationPath, sha256 }) => ({ destinationPath, sha256 }))))); + expect(context.resources.map((resource) => [resource.id, resource.pin])).toEqual([ + ["moving", null], ["source", "a".repeat(40)], ["state", null], ["tools", hash("tools")] + ]); + expect(JSON.stringify(context)).not.toMatch(/must-not-be-in-receipt|private-token|never-attach|example\.invalid/u); + expect(context.requirements).toEqual({ nativeCompilation: true, isolatedPreparation: true }); + expect(await actual.readdir(root)).not.toContain(".spawn"); + }); + + it("infers only a single agent and retains unknown runtime defaults", async () => { + const context = await createTrainingContext(await project(), { packageVersion: "0.1.17" }); + expect(context.agent).toMatchObject({ id: "agent:author", engine: null, model: null }); + expect(trainingContextSchema.parse(context)).toEqual(context); + expect(trainingContextJsonSchema).toMatchObject({ type: "object", additionalProperties: false }); + expect(trainingContextSchema.safeParse({ ...context, environment: { secret: "no" } }).success).toBe(false); + expect(trainingContextSchema.safeParse({ ...context, sources: [{ ...context.sources[0], destinationPath: "../escape" }] }).success).toBe(false); + }); + + it("selects an inline member through its actual parent manifest", async () => { + const root = await project(true); + const context = await createTrainingContext(root, { agent: "agent:reviewer", packageVersion: "0.1.17" }); + expect(context.agent.source).toBe(path.join(root, "Spawnfile")); + expect(context.documents.find((document) => document.role === "system")?.destinationPath).toBe("AGENTS.md"); + expect(context.resources).toEqual([]); + }); + + it("rejects omitted, fuzzy, team and unknown selections in a multi-agent graph", async () => { + const root = await project(true); + for (const agent of [undefined, "author", "team:publication", "agent:missing"]) { + await expect(createTrainingContext(root, { agent, packageVersion: "0.1.17" })).rejects.toThrow(/--agent|No canonical agent/u); + } + }); + + it("makes changed source bytes visible in the fingerprint without absolute-root dependence", async () => { + const root = await project(); + const initial = await createTrainingContext(root, { packageVersion: "0.1.17" }); + const clone = await project(); + expect((await createTrainingContext(clone, { packageVersion: "0.1.17" })).project.sourceDigest).toBe(initial.project.sourceDigest); + await writeFile(path.join(root, "AGENTS.md"), "Updated system\n"); + const updated = await createTrainingContext(root, { packageVersion: "0.1.17" }); + expect(updated.project.sourceDigest).not.toBe(initial.project.sourceDigest); + expect(updated.documents.find((document) => document.role === "system")?.sha256).toBe(hash("Updated system\n")); + }); + + it("rejects a symlink escaping the project", async () => { + const root = await project(), outside = await project(); + await rm(path.join(root, "AGENTS.md")); + await symlink(path.join(outside, "AGENTS.md"), path.join(root, "AGENTS.md")); + await expect(createTrainingContext(root, { packageVersion: "0.1.17" })).rejects.toThrow(/inside the canonical project root|Symlinks are not allowed/u); + }); + + it("rejects a referenced agent outside the canonical project root", async () => { + const root = await project(), outside = await project(); + await writeFile(path.join(root, "Spawnfile"), stringify({ spawnfile_version: "0.1", kind: "team", name: "external", mode: "swarm", + members: [{ id: "author", ref: path.relative(root, outside) }] })); + await expect(createTrainingContext(root, { packageVersion: "0.1.17" })).rejects.toThrow(/inside the canonical project root|escapes/u); + }); + + it.each(["AGENTS.md", "Spawnfile"])("rejects %s changing during capture", async (name) => { + const root = await project(); + let reads = 0; + const target = path.join(root, name); + vi.mocked(readFile).mockImplementation((async (...args: Parameters) => { + if (String(args[0]) === target && ++reads === (name === "Spawnfile" ? 3 : 2)) { + await writeFile(target, name === "Spawnfile" ? `${await actual.readFile(target, "utf8")}# changed\n` : "Changed during resolution\n"); + } + return actual.readFile(...args); + }) as typeof readFile); + await expect(createTrainingContext(root, { packageVersion: "0.1.17" })).rejects.toThrow(/changed/u); + }); + + it("rejects a manifest changed before its first pin instead of pairing old resolution with new bytes", async () => { + const root = await project(), target = path.join(root, "Spawnfile"); + let reads = 0; + vi.mocked(readFile).mockImplementation((async (...args: Parameters) => { + if (String(args[0]) === target && ++reads === 2) { + await writeFile(target, (await actual.readFile(target, "utf8")).replace("name: author", "name: changed")); + } + return actual.readFile(...args); + }) as typeof readFile); + await expect(createTrainingContext(root, { packageVersion: "0.1.17" })).rejects.toThrow("between canonical resolution"); + }); + + it("keeps the documented JSON schema identical to the exported contract", async () => { + const doc = await actual.readFile(new URL("../../../specs/TRAINING.md", import.meta.url), "utf8"); + const schema = doc.match(/\n```json\n([\s\S]*?)\n```/u)?.[1]; + expect(JSON.parse(schema!)).toEqual(trainingContextJsonSchema); + }); +}); diff --git a/src/compiler/training/context.ts b/src/compiler/training/context.ts new file mode 100644 index 00000000..999838f2 --- /dev/null +++ b/src/compiler/training/context.ts @@ -0,0 +1,101 @@ +import { createHash } from "node:crypto"; +import { readFile, realpath, stat } from "node:fs/promises"; +import path from "node:path"; + +import { SpawnfileError } from "../../shared/index.js"; +import { getManifestPath } from "../../filesystem/index.js"; +import { buildCompilePlan } from "../buildCompilePlan.js"; +import { stableStringify } from "../helpers.js"; +import { resolveEffectiveModelTarget } from "../modelEnv.js"; +import type { CompilePlanNode, ResolvedAgentNode } from "../types.js"; +import { TRAINING_CONTEXT_VERSION, trainingContextSchema, type TrainingContext, type TrainingSource } from "./contract.js"; + +const sha256 = (value: string | Buffer): string => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const invalid = (message: string): never => { throw new SpawnfileError("validation_error", message); }; +const inside = (root: string, file: string): boolean => { + const relative = path.relative(root, file); + return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +}; + +export interface CreateTrainingContextOptions { + agent?: string; + packageVersion: string; +} + +/** Resolves the complete project, preserving inherited selection and source provenance. */ +export const createTrainingContext = async ( + inputPath: string, + options: CreateTrainingContextOptions +): Promise => { + const plan = await buildCompilePlan(await realpath(getManifestPath(inputPath))); + const agents = plan.nodes.filter((node): node is CompilePlanNode & { value: ResolvedAgentNode } => node.kind === "agent"); + const selected = options.agent === undefined + ? agents.length === 1 ? agents[0] : undefined + : agents.find((node) => node.id === options.agent); + if (!selected) invalid(options.agent === undefined + ? "Training requires --agent with an exact node id when the project does not contain exactly one agent" + : `No canonical agent matches ${options.agent}`); + const agent = selected!.value; + const manifest = await realpath(plan.root); + const root = path.dirname(manifest); + const pins = new Map(); + + const pin = async (source: string, expectedContent?: string): Promise => { + const sourcePath = path.resolve(source); + if (!inside(root, sourcePath) || !inside(root, await realpath(sourcePath))) { + invalid("Training context v1 requires source files inside the canonical project root"); + } + const info = await stat(sourcePath); + if (!info.isFile() || info.size > 16 * 1024 * 1024) invalid("Training source must be a regular file no larger than 16 MiB"); + const bytes = await readFile(sourcePath); + if (expectedContent !== undefined && bytes.toString("utf8") !== expectedContent) { + invalid("Training source changed during canonical graph resolution"); + } + const next = { sourcePath, destinationPath: path.relative(root, sourcePath).split(path.sep).join("/"), sha256: sha256(bytes) }; + const previous = pins.get(sourcePath); + if (previous && previous.sha256 !== next.sha256) invalid("Training source changed while its context was captured"); + pins.set(sourcePath, next); + return next; + }; + + await pin(manifest); + // Manifests for inline nodes live at sourcePath; their synthetic node source is not a file. + for (const node of plan.nodes) { + await pin(node.value.kind === "agent" ? node.value.sourcePath ?? node.value.source : node.value.source); + for (const document of node.value.docs) await pin(document.sourcePath, document.content); + const skills = node.value.kind === "agent" ? node.value.skills : node.value.shared.skills; + for (const skill of skills) await pin(skill.sourcePath, skill.content); + } + const documents = await Promise.all(agent.docs.map(async (document) => ({ + ...await pin(document.sourcePath, document.content), role: document.role + }))); + const skills = await Promise.all(agent.skills.map(async (skill) => ({ + ...await pin(skill.sourcePath, skill.content), name: skill.name, ref: skill.ref, requiresMcp: skill.requiresMcp + }))); + // A manifest can change after graph resolution but before its first pin. Re-resolve + // through the compiler owner, then seal all captured bytes against that graph. + if (stableStringify(plan) !== stableStringify(await buildCompilePlan(manifest))) { + invalid("Training source changed between canonical resolution and provenance capture"); + } + for (const source of [...pins.keys()]) await pin(source); + const sources = [...pins.values()].sort((left, right) => left.destinationPath < right.destinationPath ? -1 : left.destinationPath > right.destinationPath ? 1 : 0); + const declaredPrimary = agent.execution?.model?.primary; + const primary = declaredPrimary ? resolveEffectiveModelTarget(declaredPrimary, agent.execution) : undefined; + return trainingContextSchema.parse({ + version: TRAINING_CONTEXT_VERSION, + producer: { package: "spawnfile", version: options.packageVersion }, + project: { root, manifest, sourceDigest: sha256(JSON.stringify(sources.map(({ destinationPath, sha256: hash }) => ({ destinationPath, sha256: hash })))) }, + agent: { + id: selected!.id, name: agent.name, source: path.resolve(agent.sourcePath ?? agent.source), runtime: agent.runtime.name, + engine: typeof agent.runtime.options.engine === "string" ? agent.runtime.options.engine : null, + model: primary ? { provider: primary.provider, name: primary.name, authMethod: primary.auth.method } : null + }, + sources, documents, skills, + resources: (agent.workspaceResources ?? []).map((resource) => ({ + id: resource.id, kind: resource.kind, mount: resource.mount, mode: resource.mode, sharing: resource.sharing, + definitionDigest: sha256(stableStringify(resource)), + pin: resource.kind === "bundle" ? resource.sha256 : resource.kind === "git" ? resource.ref ?? null : null + })), + requirements: { nativeCompilation: true, isolatedPreparation: true } + }); +}; diff --git a/src/compiler/training/contract.ts b/src/compiler/training/contract.ts new file mode 100644 index 00000000..e2a47da3 --- /dev/null +++ b/src/compiler/training/contract.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; + +export const TRAINING_CONTEXT_VERSION = "spawnfile.training-context.v1" as const; + +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +const absolutePath = z.string().min(1).regex(/^(?:\/|[A-Za-z]:[\\/])/u); +const relativePath = z.string().min(1).regex(/^(?!\/)(?!.*(?:^|\/)\.\.(?:\/|$))[^\\]+$/u); +const text = z.string().min(1); + +export const trainingSourceSchema = z.object({ + sourcePath: absolutePath, + destinationPath: relativePath, + sha256: digest +}).strict(); + +/** Trusted compiler metadata only; this is neither a prompt nor permission to launch. */ +export const trainingContextSchema = z.object({ + version: z.literal(TRAINING_CONTEXT_VERSION), + producer: z.object({ package: z.literal("spawnfile"), version: text }).strict(), + project: z.object({ root: absolutePath, manifest: absolutePath, sourceDigest: digest }).strict(), + agent: z.object({ + id: text, name: text, source: absolutePath, runtime: text, + engine: text.nullable(), + model: z.object({ provider: text, name: text, authMethod: text }).strict().nullable() + }).strict(), + sources: z.array(trainingSourceSchema).min(1).max(10_000), + documents: z.array(trainingSourceSchema.extend({ role: text }).strict()).max(128), + skills: z.array(trainingSourceSchema.extend({ + name: text, ref: text, requiresMcp: z.array(text) + }).strict()).max(1_000), + resources: z.array(z.object({ + id: text, kind: z.enum(["bundle", "git", "volume"]), mount: text, + mode: z.enum(["mutable", "readonly"]), sharing: z.enum(["per_agent", "team"]), + definitionDigest: digest, pin: text.nullable() + }).strict()).max(1_000), + requirements: z.object({ nativeCompilation: z.literal(true), isolatedPreparation: z.literal(true) }).strict() +}).strict(); + +export type TrainingContext = z.infer; +export type TrainingSource = z.infer; + +export const trainingContextJsonSchema = z.toJSONSchema(trainingContextSchema); diff --git a/src/compiler/training/index.ts b/src/compiler/training/index.ts new file mode 100644 index 00000000..e904fe1a --- /dev/null +++ b/src/compiler/training/index.ts @@ -0,0 +1,2 @@ +export * from "./contract.js"; +export * from "./context.js"; From 275f20a86fecec1b957f2af23747c415beade99b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 13 Sep 2026 23:59:07 +0200 Subject: [PATCH 02/84] feat: forward bounded judge citation repairs --- specs/TRAINING.md | 9 ++++++++- src/cli/paideiaDelegation.test.ts | 6 ++++-- src/cli/trainCommand.test.ts | 18 ++++++++++++++++-- src/cli/trainCommand.ts | 4 ++-- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 01a92469..627aa8c9 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -58,11 +58,18 @@ reach native files through Spawnfile compilation; no generic Pi fallback, second declaration or replacement flattened prompt is authorized by this entrypoint. Packaging exclusion, state isolation and single-agent preparation are not supplied by this handoff. -Repeated options: `--editable`, `--resource`, `--judge`, `--validation-group`. Other +Repeated options: `--editable`, `--resource`, `--judge`, `--judge-citation-repairs`, `--validation-group`. Other forwarded options: `--train`, `--test`, `--optimizer-model`, `--bridge-command`, `--out`, `--max-trials`, `--max-proposals`, `--seed`, `--timeout-ms`, `--view`, `--cost-config`. The canonical runtime/model/instruction selection cannot be replaced by generic CLI flags. +`--judge-citation-repairs NAME=0|1` is forwarded literally to Paideia. The receiver +requires a matching named judge, unique selections and an exact `0` or `1` before +starting models. Default `0` preserves one judge call per check; `1` reserves one +additional bounded citation repair. It never retries valid quality failures, +authentication/quota failures or malformed JSON. Dry-run records the route policy +and reserves both judge attempts without adding subject trials or optimizer proposals. + Exit 0 requires the mode's final receipt. Completed actual runs also require `status: completed` and a nonempty `index` path; exit 1 preserves completed failed checks. Receiver error exits are propagated; empty success is a runtime failure. A supervisor diff --git a/src/cli/paideiaDelegation.test.ts b/src/cli/paideiaDelegation.test.ts index f25b2889..ab0f54f4 100644 --- a/src/cli/paideiaDelegation.test.ts +++ b/src/cli/paideiaDelegation.test.ts @@ -44,11 +44,13 @@ const fs = require("node:fs"); const args = process.argv.slice(2), file = args[2]; console.log(JSON.stringify({args,context:JSON.parse(fs.readFileSync(file,"utf8")),mode:fs.statSync(file).mode & 511,file})); process.stderr.write("diagnostic without newline"); -${dryReceipt}`, { args: ["--train", injected, "--editable", "a.md", "--editable", "b.md", "--dry-run"] }); +${dryReceipt}`, { args: ["--train", injected, "--editable", "a.md", "--editable", "b.md", + "--judge-citation-repairs", "editor=1", "--judge-citation-repairs", injected, "--dry-run"] }); expect(await result).toBe(0); const observed = JSON.parse(stdout[0]!); expect(observed.args.slice(0, 2)).toEqual(["train", "--spawnfile-context"]); - expect(observed.args.slice(3)).toEqual(["--train", injected, "--editable", "a.md", "--editable", "b.md", "--dry-run"]); + expect(observed.args.slice(3)).toEqual(["--train", injected, "--editable", "a.md", "--editable", "b.md", + "--judge-citation-repairs", "editor=1", "--judge-citation-repairs", injected, "--dry-run"]); expect(observed.context).toEqual(context); expect(observed.mode).toBe(0o600); expect(stderr).toEqual(["diagnostic without newline"]); diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index 8f490e62..f9ff0052 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -23,7 +23,8 @@ describe("spawnfile train", () => { const forbidden = vi.fn(async () => { throw new Error("must not compile, build or authenticate"); }); const code = await runCli(["train", root, ...base, "--agent", "agent:author", "--dry-run", "--paideia-command", "/opt/paideia with space", "--editable", "a.md", "--editable", "b.md", "--resource", "archive=/private/source", - "--judge", "editor=fable", "--validation-group", "previous", "--cost-config", "prices.yaml", "--max-trials", "3", "--timeout-ms", "5000"], { + "--judge", "editor=fable", "--judge", "grounding=other-model", "--judge-citation-repairs", "editor=1", + "--judge-citation-repairs", "grounding=0", "--validation-group", "previous", "--cost-config", "prices.yaml", "--max-trials", "3", "--timeout-ms", "5000"], { streams: { stdout: (value) => stdout.push(value), stderr: (value) => stderr.push(value) }, handlers: { delegatePaideiaTraining: delegate, compileProject: forbidden, buildProject: forbidden, importCodexAuth: forbidden } }); @@ -34,7 +35,9 @@ describe("spawnfile train", () => { expect(options.command).toBe("/opt/paideia with space"); expect(options.dryRun).toBe(true); expect(options.timeoutMs).toBe(10_000); expect(options.args).toEqual(["--train", "train.paideia.yaml", "--test", "test.paideia.yaml", "--editable", "a.md", "--editable", "b.md", - "--resource", "archive=/private/source", "--judge", "editor=fable", "--validation-group", "previous", "--out", "local output", "--max-trials", "3", "--timeout-ms", "5000", "--cost-config", "prices.yaml", "--dry-run"]); + "--resource", "archive=/private/source", "--judge", "editor=fable", "--judge", "grounding=other-model", + "--judge-citation-repairs", "editor=1", "--judge-citation-repairs", "grounding=0", "--validation-group", "previous", + "--out", "local output", "--max-trials", "3", "--timeout-ms", "5000", "--cost-config", "prices.yaml", "--dry-run"]); expect(stderr).toEqual([]); }); @@ -81,4 +84,15 @@ describe("spawnfile train", () => { } expect(forbidden).not.toHaveBeenCalled(); }); + it("forwards literal invalid repair selections for Paideia to validate and preserves receiver failure", async () => { + const root = await project(); + const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 2); + const literal = "unknown=$(false) with spaces"; + expect(await runCli(["train", root, ...base, "--judge-citation-repairs", literal, + "--judge-citation-repairs", literal, "--dry-run"], { + handlers: { delegatePaideiaTraining: delegate }, streams: { stdout: () => undefined, stderr: () => undefined } + })).toBe(2); + expect(delegate.mock.calls[0]![0].args).toEqual(["--train", "train.paideia.yaml", "--test", "test.paideia.yaml", + "--judge-citation-repairs", literal, "--judge-citation-repairs", literal, "--out", "local output", "--dry-run"]); + }); }); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 2ab6edca..5998d361 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -3,9 +3,9 @@ import type { Command } from "commander"; import { SpawnfileError } from "../shared/index.js"; import type { CliHandlers, CliStreams } from "./runCli.js"; -const forwarded = ["train", "test", "editable", "resource", "judge", "validation-group", "optimizer-model", +const forwarded = ["train", "test", "editable", "resource", "judge", "judge-citation-repairs", "validation-group", "optimizer-model", "bridge-command", "out", "max-trials", "max-proposals", "seed", "timeout-ms", "view", "cost-config"] as const; -const repeated = new Set(["editable", "resource", "judge", "validation-group"]); +const repeated = new Set(["editable", "resource", "judge", "judge-citation-repairs", "validation-group"]); const key = (name: string): string => name.replace(/-([a-z])/gu, (_, letter: string) => letter.toUpperCase()); export const registerTrainCommand = ( From 5d729550c5f0d0bdfd91a03375dec4f52d8feea2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Mon, 14 Sep 2026 02:21:22 +0200 Subject: [PATCH 03/84] feat: delegate YAML training controls to Paideia --- src/cli/trainCommand.test.ts | 10 +++++++++- src/cli/trainCommand.ts | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index f9ff0052..9face9b3 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -41,6 +41,14 @@ describe("spawnfile train", () => { expect(stderr).toEqual([]); }); + it("delegates YAML-owned test selection and permits the complete YAML time budget", async () => { + const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 0); + expect(await runCli(["train", await project(), "--train", "train.paideia.yaml", "--dry-run"], { + handlers: { delegatePaideiaTraining: delegate }, streams: { stdout: () => undefined, stderr: () => undefined } + })).toBe(0); + expect(delegate.mock.calls[0]![0]).toMatchObject({ timeoutMs: 3_605_000, args: ["--train", "train.paideia.yaml", "--dry-run"] }); + }); + it.each([1, 2, 130, 143])("propagates the delegated exit %s", async (exitCode) => { const code = await runCli(["train", await project(), ...base], { handlers: { delegatePaideiaTraining: async () => exitCode }, streams: { stdout: () => undefined, stderr: () => undefined } }); @@ -54,7 +62,7 @@ describe("spawnfile train", () => { signal: controller.signal, handlers: { delegatePaideiaTraining: async (options) => { captured = options; return 2; } }, streams: { stdout: () => undefined, stderr: () => undefined } })).toBe(2); - expect(captured).toMatchObject({ command: "paideia", timeoutMs: 185_000, signal: controller.signal, dryRun: false }); + expect(captured).toMatchObject({ command: "paideia", timeoutMs: 3_605_000, signal: controller.signal, dryRun: false }); expect(captured?.args).toContain("--bridge-command"); expect(captured?.args).not.toContain("--dry-run"); }); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 5998d361..02fdfb87 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -25,14 +25,14 @@ export const registerTrainCommand = ( for (const name of forwarded) { const flag = `--${name} `; if (repeated.has(name)) command.option(flag, `Paideia ${name}; repeatable`, (value: string, previous: string[]) => [...previous, value], []); - else if (["train", "test"].includes(name)) command.requiredOption(flag, `Paideia ${name}`); + else if (name === "train") command.requiredOption(flag, `Paideia ${name}`); else command.option(flag, `Paideia ${name}`); } command.action(async (inputPath: string, options: Record) => { if (options.dryRun !== true && typeof options.out !== "string") { throw new SpawnfileError("validation_error", "Actual training requires --out; dry-run does not write an output directory"); } - const timeout = options.timeoutMs === undefined ? 180_000 : Number(options.timeoutMs); + const timeout = options.timeoutMs === undefined ? 3_600_000 : Number(options.timeoutMs); if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 3_600_000 || (options.timeoutMs !== undefined && !/^\d+$/u.test(String(options.timeoutMs)))) { throw new SpawnfileError("validation_error", "--timeout-ms must be an integer from 1 to 3600000"); From 938a77193714116645545e934acae3bb9498cdc9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Mon, 14 Sep 2026 03:50:44 +0200 Subject: [PATCH 04/84] feat: forward explicit training resume to Paideia --- specs/TRAINING.md | 5 +++++ src/cli/trainCommand.test.ts | 10 ++++++++++ src/cli/trainCommand.ts | 4 +++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 627aa8c9..da1ddb76 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -11,6 +11,11 @@ spawnfile train ./Spawnfile --agent agent:writer \ ``` Paideia supplies the cost-config format and training options. Dataset roles are explicit. + +`--resume` forwards to Paideia for the same output directory. Paideia validates +unchanged canonical inputs, the isolated integration's execution identity and +cumulative budgets before restoring its optimizer and native evidence. Spawnfile +does not interpret checkpoints, repeat trials or deploy the optimized candidate. `--agent` is an exact resolved node ID; omission is allowed only for one-agent projects. `--paideia-command` selects an installed executable, default `paideia`; no shell, automatic installation, model-provider fallback or production launch is involved. diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index 9face9b3..f2960a9c 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -49,6 +49,16 @@ describe("spawnfile train", () => { expect(delegate.mock.calls[0]![0]).toMatchObject({ timeoutMs: 3_605_000, args: ["--train", "train.paideia.yaml", "--dry-run"] }); }); + it("forwards explicit resume to Paideia without changing canonical agent context", async () => { + const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 0); + expect(await runCli(["train", await project(), ...base, "--resume"], { + handlers: { delegatePaideiaTraining: delegate }, streams: { stdout: () => undefined, stderr: () => undefined } + })).toBe(0); + expect(delegate.mock.calls[0]![0].args).toEqual([...base, "--resume"]); + expect(delegate.mock.calls[0]![0].context.agent.id).toBe("agent:author"); + expect(delegate.mock.calls[0]![0].dryRun).toBe(false); + }); + it.each([1, 2, 130, 143])("propagates the delegated exit %s", async (exitCode) => { const code = await runCli(["train", await project(), ...base], { handlers: { delegatePaideiaTraining: async () => exitCode }, streams: { stdout: () => undefined, stderr: () => undefined } }); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 02fdfb87..cd770bf0 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -21,7 +21,8 @@ export const registerTrainCommand = ( .argument("[path]", "Canonical project directory or Spawnfile path", process.cwd()) .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") - .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models"); + .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") + .option("--resume", "Resume the exact persisted training experiment in --out"); for (const name of forwarded) { const flag = `--${name} `; if (repeated.has(name)) command.option(flag, `Paideia ${name}; repeatable`, (value: string, previous: string[]) => [...previous, value], []); @@ -47,6 +48,7 @@ export const registerTrainCommand = ( else if (Array.isArray(value)) for (const item of value) args.push(`--${name}`, item); } if (options.dryRun === true) args.push("--dry-run"); + if (options.resume === true) args.push("--resume"); setExitCode(await handlers.delegatePaideiaTraining({ context, args, command: options.paideiaCommand as string, dryRun: options.dryRun === true, timeoutMs: timeout + 5000, streams, signal })); From a2046153bb61a87c06584b720b976f19e3ba0665 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Mon, 14 Sep 2026 15:01:31 +0200 Subject: [PATCH 05/84] feat: isolate training in one managed container --- package.json | 11 +- runtime-images/training/AGENTS.md | 11 ++ runtime-images/training/CLAUDE.md | 1 + runtime-images/training/Dockerfile | 49 ++++++ specs/AGENTS.md | 1 + specs/INDEX.md | 1 + specs/TRAINING.md | 4 + specs/TRAINING_CONTAINERS.md | 102 ++++++++++++ src/auth/AGENTS.md | 1 + src/auth/index.ts | 2 + src/auth/trainingAuth.failure.test.ts | 38 +++++ src/auth/trainingAuth.fifo.test.ts | 18 ++ src/auth/trainingAuth.test.ts | 22 +++ src/auth/trainingAuth.ts | 66 ++++++++ src/cli/paideiaDelegation.test.ts | 19 +-- src/cli/paideiaDelegation.ts | 8 + src/cli/trainCommand.test.ts | 7 +- src/cli/trainCommand.ts | 6 + src/compiler/training/AGENTS.md | 1 + src/compiler/training/container/AGENTS.md | 9 + src/compiler/training/container/CLAUDE.md | 1 + src/compiler/training/container/contract.ts | 20 +++ .../container/fixtures.test-helper.ts | 48 ++++++ src/compiler/training/container/index.ts | 4 + .../training/container/launch.test.ts | 156 ++++++++++++++++++ src/compiler/training/container/launch.ts | 118 +++++++++++++ .../training/container/prepare.test.ts | 86 ++++++++++ src/compiler/training/container/prepare.ts | 66 ++++++++ .../training/container/process.test.ts | 40 +++++ src/compiler/training/container/process.ts | 50 ++++++ 30 files changed, 948 insertions(+), 18 deletions(-) create mode 100644 runtime-images/training/AGENTS.md create mode 120000 runtime-images/training/CLAUDE.md create mode 100644 runtime-images/training/Dockerfile create mode 100644 specs/TRAINING_CONTAINERS.md create mode 100644 src/auth/trainingAuth.failure.test.ts create mode 100644 src/auth/trainingAuth.fifo.test.ts create mode 100644 src/auth/trainingAuth.test.ts create mode 100644 src/auth/trainingAuth.ts create mode 100644 src/compiler/training/container/AGENTS.md create mode 120000 src/compiler/training/container/CLAUDE.md create mode 100644 src/compiler/training/container/contract.ts create mode 100644 src/compiler/training/container/fixtures.test-helper.ts create mode 100644 src/compiler/training/container/index.ts create mode 100644 src/compiler/training/container/launch.test.ts create mode 100644 src/compiler/training/container/launch.ts create mode 100644 src/compiler/training/container/prepare.test.ts create mode 100644 src/compiler/training/container/prepare.ts create mode 100644 src/compiler/training/container/process.test.ts create mode 100644 src/compiler/training/container/process.ts diff --git a/package.json b/package.json index ea2122f4..688356f0 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "test:verdict": "node -e \"const fs=require('fs'); const file='coverage/vitest-results.json'; if (!fs.existsSync(file)) { console.error('VERDICT: results file missing'); process.exit(1); } let result; try { result=JSON.parse(fs.readFileSync(file, 'utf8')); } catch { console.error('VERDICT: results file unparseable'); process.exit(1); } const total=result.numTotalTests; const totalSuites=result.numTotalTestSuites; const passedTests=result.numPassedTests; const failedTests=result.numFailedTests; const pendingTests=result.numPendingTests; const todoTests=result.numTodoTests; const failedSuites=result.numFailedTestSuites; if (typeof result.success !== 'boolean' || !Number.isInteger(passedTests) || passedTests < 0 || !Number.isInteger(failedTests) || failedTests < 0 || !Number.isInteger(pendingTests) || pendingTests < 0 || !Number.isInteger(todoTests) || todoTests < 0 || !Number.isInteger(failedSuites) || failedSuites < 0 || !Number.isInteger(totalSuites) || totalSuites < 0) { console.error('VERDICT: results schema unrecognised'); process.exit(1); } if (!Number.isInteger(total) || total <= 0) { console.error('VERDICT: no tests found'); process.exit(1); } if (passedTests + failedTests + pendingTests + todoTests !== total) { console.error('VERDICT: results counters do not reconcile (tests='+total+', passed='+passedTests+', failed='+failedTests+', skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } const failed=result.success === false || failedTests > 0 || failedSuites > 0; if (failed) { console.error('VERDICT: test failures found (tests='+failedTests+', suites='+failedSuites+', success='+result.success+')'); process.exit(2); } if (passedTests === 0) { console.error('VERDICT: no tests actually ran (skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } console.log('VERDICT: tests passed (tests='+total+', suites='+totalSuites+', passed='+passedTests+', skipped='+pendingTests+', todo='+todoTests+')');\"", "test:coverage-verdict": "node --import tsx -e \"import fs from 'node:fs'; const file='coverage/coverage-summary.json'; const fail=(message)=>{ console.error('COVERAGE VERDICT: '+message); process.exit(1); }; let summary; try { summary=JSON.parse(fs.readFileSync(file, 'utf8')); } catch (error) { fail(error && error.code === 'ENOENT' ? 'summary file missing' : 'summary file unparseable'); } import('./vitest.config.ts').then(({default:config})=>{ const thresholds=config.test && config.test.coverage && config.test.coverage.thresholds; if (!thresholds || typeof thresholds !== 'object' || !summary || typeof summary !== 'object' || !summary.total || typeof summary.total !== 'object') fail('summary or threshold schema unrecognised'); const metricNames=['branches','functions','lines','statements']; const metrics=metricNames.filter((metric)=>Object.prototype.hasOwnProperty.call(thresholds, metric)); if (metrics.length === 0) fail('no threshold metrics configured'); let shortfall=false; for (const metric of metrics) { const threshold=thresholds[metric]; const pct=summary.total[metric] && summary.total[metric].pct; if (typeof threshold !== 'number' || !Number.isFinite(threshold) || typeof pct !== 'number' || !Number.isFinite(pct)) fail('missing or non-numeric value for '+metric); console.log('COVERAGE: '+metric+' '+pct+'% (threshold '+threshold+'%)'); if (pct < threshold) shortfall=true; } if (shortfall) { console.error('COVERAGE VERDICT: thresholds not met'); process.exit(2); } console.log('COVERAGE VERDICT: thresholds met'); }).catch((error)=>fail('could not load vitest config: '+error.message));\"", "test:node": "mkdir -p coverage || exit 1; rm -f coverage/node-test.tap || exit 1; node --import tsx --test --test-reporter=tap src/runtime/pi/appControlDeliveryMetadata.test.ts > coverage/node-test.tap 2>&1; node_status=$?; cat coverage/node-test.tap; cat_status=$?; npm run test:node-verdict; verdict_status=$?; if [ \"$verdict_status\" -eq 2 ]; then exit 2; fi; if [ \"$verdict_status\" -ne 0 ]; then exit 1; fi; if [ \"$cat_status\" -ne 0 ]; then echo 'NODE VERDICT: could not read TAP output' >&2; exit 1; fi; if [ \"$node_status\" -ne 0 ]; then echo \"NODE VERDICT: node:test runner exited $node_status despite a passing TAP summary\" >&2; exit 1; fi; exit 0", - "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' — cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", + "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' \u2014 cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", "test:product-state-volume": "node --experimental-strip-types scripts/product-state-volume-integration.test.ts", "test:boundaries": "vitest run --coverage.enabled=false src/ownership/rootOwnershipBoundary.test.ts src/deployment/providerRuntimeBoundary.test.ts src/ownership/mnemePublicImportBoundary.test.ts src/target/containerBundleArchive.test.ts", "test:causal-conformance": "tsx src/ledger/causalConformanceCli.ts", @@ -81,5 +81,12 @@ "typescript": "^5.9.3", "vitest": "^3.2.4" }, - "packageManager": "pnpm@10.32.1+sha512.a706938f0e89ac1456b6563eab4edf1d1faf3368d1191fc5c59790e96dc918e4456ab2e67d613de1043d2e8c81f87303e6b40d4ffeca9df15ef1ad567348f2be" + "packageManager": "pnpm@10.32.1+sha512.a706938f0e89ac1456b6563eab4edf1d1faf3368d1191fc5c59790e96dc918e4456ab2e67d613de1043d2e8c81f87303e6b40d4ffeca9df15ef1ad567348f2be", + "exports": { + "./auth": { + "types": "./dist/auth/index.d.ts", + "import": "./dist/auth/index.js" + }, + "./*": "./*" + } } diff --git a/runtime-images/training/AGENTS.md b/runtime-images/training/AGENTS.md new file mode 100644 index 00000000..749d10fe --- /dev/null +++ b/runtime-images/training/AGENTS.md @@ -0,0 +1,11 @@ +# Training image + +Owns the single-container training dependency recipe. Spawnfile launches and +stops the outer container; Paideia supervises experiments and native trial child +processes inside it. No Docker socket or host home is mounted. + +Build inputs are explicit package distributions, locked dependency manifests, +verified native executables and an installed integration entrypoint. Credentials, +cases and results are runtime mounts, never image contents. Native runtime and +Python base images must be immutable. Do not download unpinned CLI installers. +The recipe is opt-in during incubation; no published runtime is implied. diff --git a/runtime-images/training/CLAUDE.md b/runtime-images/training/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/runtime-images/training/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile new file mode 100644 index 00000000..1a1582a7 --- /dev/null +++ b/runtime-images/training/Dockerfile @@ -0,0 +1,49 @@ +ARG NATIVE_IMAGE +ARG PYTHON_IMAGE +FROM ${PYTHON_IMAGE} AS python +FROM ${NATIVE_IMAGE} AS training + +# Both parents must use the same architecture and Debian release. +COPY --from=python /usr/local /opt/python +COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/ +ENV LD_LIBRARY_PATH=/opt/python/lib + +WORKDIR /opt/training/paideia +COPY paideia/package.json paideia/package-lock.json ./ +RUN npm ci --omit=dev --omit=peer --ignore-scripts \ + && mkdir -p node_modules/@noopolis \ + && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon node_modules/@noopolis/daimon + +WORKDIR /opt/training/spawnfile +COPY spawnfile/package.json spawnfile/package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts + +COPY bridge /opt/training/paideia/bridges/dspy +RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \ + && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir -r /opt/training/paideia/bridges/dspy/requirements.lock \ + && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir setuptools==80.9.0 \ + && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy + +COPY claude/package.json claude/package-lock.json /opt/training/claude/ +RUN cd /opt/training/claude && npm ci --omit=dev +COPY grok /opt/training/bin/grok + +COPY paideia/dist /opt/training/paideia/dist +COPY spawnfile/dist /opt/training/spawnfile/dist +COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/ +COPY integration /opt/training/integration +COPY bootstrap /opt/training/bootstrap +COPY train /opt/training/bin/train +RUN chmod 0555 /opt/training/bin/train /opt/training/bin/grok \ + && chmod -R a+rX /opt/training/bootstrap /opt/training/integration \ + && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \ + && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \ + && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \ + && mkdir -p /opt/training/integration/node_modules/@noopolis \ + && ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia \ + && ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile \ + && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon +ENV PATH=/opt/training/bin:/opt/training/paideia/bridges/dspy/.venv/bin:/opt/spawnfile/runtime-installs/daimon/bin:/usr/local/bin:/usr/bin:/bin +ENV HOME=/home/training +WORKDIR /work +ENTRYPOINT ["/opt/training/bin/train"] diff --git a/specs/AGENTS.md b/specs/AGENTS.md index 3afb14a2..c9691c97 100644 --- a/specs/AGENTS.md +++ b/specs/AGENTS.md @@ -10,6 +10,7 @@ specs/ ├── CONTAINERS.md # Container compilation spec ├── RUNTIMES.md # Runtime registry, version pinning, adapter lifecycle ├── TRAINING.md # Canonical source handoff and Paideia CLI delegation +├── TRAINING_CONTAINERS.md # Single-container training launch boundary ├── CAUSAL.md # Shared causal wire and Stele read/verify contract ├── ECOSYSTEM_RUNTIME_BOUNDARIES.md # Cross-project runtime authority and enforcement gates ├── USAGE_ACCOUNTING_DESIGN.md # Daimon turn-usage envelope and Spawnfile aggregation design diff --git a/specs/INDEX.md b/specs/INDEX.md index 36c7288c..437cecae 100644 --- a/specs/INDEX.md +++ b/specs/INDEX.md @@ -17,6 +17,7 @@ These are the source of truth. Implementation in `src/` must stay aligned with t | [RUNTIMES.md](RUNTIMES.md) | evolving | Runtime registry model — version pinning, status tracking, adapter lifecycle | | [STATUS.md](STATUS.md) | evolving | Operational status — static and live status, deployment records, Docker targets, runtime probes, and Moltnet metadata-only diagnostics | | [TRAINING.md](TRAINING.md) | implemented handoff; native preparation integration required | Canonical agent selection and versioned Paideia delegation, dry-run and source provenance | +| [TRAINING_CONTAINERS.md](TRAINING_CONTAINERS.md) | single-container launcher; end-to-end validation pending | Whole-experiment image, declared mounts, native auth staging and verified lifecycle | | [DISTRIBUTION.md](DISTRIBUTION.md) | evolving | Image distribution — self-describing images, sourceless run/status, deployment record v2, publish, registry drift, and the network binding contract | | [CAUSAL.md](CAUSAL.md) | evolving | Causal event envelope — producer wire rules plus the shared Stele read/verify and reconciliation contract | | [TARGETS.md](TARGETS.md) | evolving | Project-neutral target-resource public contracts and staged target-adapter boundary | diff --git a/specs/TRAINING.md b/specs/TRAINING.md index da1ddb76..9abe17df 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -4,6 +4,10 @@ Paideia CLI. Spawnfile owns canonical source resolution and native compilation; Paideia owns datasets, evaluation, cost planning, optimization and isolated trials. +The [container boundary](TRAINING_CONTAINERS.md) runs the complete experiment +inside one immutable image. Actual training requires explicit image and mount +configuration; dry-run remains a host-only estimate. + ```sh spawnfile train ./Spawnfile --agent agent:writer \ --train evals/train.paideia.yaml --test evals/test.paideia.yaml \ diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md new file mode 100644 index 00000000..9338270e --- /dev/null +++ b/specs/TRAINING_CONTAINERS.md @@ -0,0 +1,102 @@ +# Training in one container + +`spawnfile train` runs the complete model-bearing experiment in one immutable +Docker image: Paideia, DSPy, the integration, Daimon and inference CLIs. The host +validates declarations, mounts explicit inputs, forwards output and supervises +container termination. No Docker socket enters the container. Dry-run remains a +host-only estimate and does not use Docker or model authentication. + +## Launch contract + +Actual training requires both `--training-image` (a digest reference or immutable +image ID already installed locally) and `--training-config` (JSON below). +`--paideia-command` applies only to dry-run; actual execution always starts the +image-owned `/opt/training/bin/train`. No shell or executable from the host is +mounted or selected. The image must contain all dependencies and its integration. + +```json +{ + "version": "spawnfile.training-container.v1", + "dockerContext": "desktop-linux", + "inputs": [ + { "source": "/absolute/project", "destination": "/run/training/inputs/project" }, + { "source": "/absolute/evaluation", "destination": "/run/training/inputs/integration" } + ], + "output": { "source": "/absolute/generated/run", "destination": "/run/training/output" }, + "auth": [ + { "source": "/absolute/credential-leaf", "provider": "codex" } + ] +} +``` + +All sources must already exist at exactly canonical paths (no `..`, redundant +separators or symlink aliases). Host input roots must not overlap. Inputs are read-only and +cannot overlap the writable output root. Auth declarations accept only regular +leaf files, mounted read-only at `/run/paideia-auth/`; no whole CLI home +or configuration directory is accepted. This checks declared paths, not arbitrary +secret contents inside user-selected input bytes. Input roots must contain only +the experiment's intended source and evidence. + +The selected Docker context must resolve to a local Unix socket. Remote daemon +bind staging is unsupported. The launcher resolves the pinned image before +creating a uniquely labelled container, then uses the immutable image ID. + +The image entrypoint receives: + +```text +/opt/training/bin/train train --spawnfile-context /run/paideia/context.json ... +``` + +Canonical context source paths and CLI dataset/resource/output paths are mapped +through the declared bindings. YAML-relative data paths continue to resolve in +that mapped dataset tree. Absolute paths embedded inside integration settings or +YAML must already use container paths; the launcher does not rewrite arbitrary +file contents. Bridge executables must already exist under `/opt/training`. +Runtime `HOME=/home/training`, `/tmp` and `/work` are fresh writable tmpfs mounts; +the image root is read-only. Launch uses the non-root host uid/gid, drops all +capabilities and keeps no-new-privileges. The existing Codex native namespace +compatibility options disable the outer seccomp/AppArmor profiles; native sandbox +preflight must still verify the agent boundary before cognition. Image startup +owns its runtime configuration. + +`--view ` accepts one explicit port from 1 to 65535 and publishes it only +on host `127.0.0.1` at that same port. Port zero is unsupported in container mode. +The trusted image integration binds its Paideia viewer to the container interface; +public URLs still use host loopback. Removing the owned container removes that +port mapping. Persisted events also remain available for later local replay. + +## Native subscription bootstrap + +The public `spawnfile/auth` module exports `stageTrainingAuth({home,provider,source?})`. +It copies opaque bytes from `/run/paideia-auth/` by default into an +existing canonical runtime home. An explicit provisioned source leaf is accepted. +It creates only the fixed private directory and native auth leaf, never overwrites +an existing or refreshed credential, and returns a non-secret versioned receipt. + +| Provider | Destination relative to runtime home | +| --- | --- | +| codex | `.daimon-inbound/codex-auth` | +| grok | `.grok/auth.json` | +| claude | `.claude/.credentials.json` | + +The image startup can stage Grok/Claude into its clean shared home. A native trial +preparation callback stages Codex into that trial's fresh home before Daimon +starts. Credentials remain writable only inside the runtime home; renewed state +is not silently copied back to the host bootstrap file. + +## Completion and cancellation + +The host forwards container stdout/stderr and requires the final Paideia training +receipt, a matching stopped container exit status, and a real completion artifact +within the declared output root. A successful `docker create` or client exit alone +never means the experiment completed. + +Cancellation and timeout stop the Docker client and force-remove only a container +whose exact ID, unique ownership label, name and image match. Absence is checked +through a successful Docker listing. Unknown cleanup is an error and preserves +its private mounted context for diagnosis; it is never reported as quiescent. + +This is a single-container boundary. Native Daimon sandbox policy still controls +individual agent access inside it. The image build, usable private integration, +authenticated model run and live terminal receipt require end-to-end verification +before calling this deployment ready. diff --git a/src/auth/AGENTS.md b/src/auth/AGENTS.md index de826ddd..15c2046d 100644 --- a/src/auth/AGENTS.md +++ b/src/auth/AGENTS.md @@ -7,6 +7,7 @@ This folder owns Spawnfile-managed auth profiles and auth import flows. ```text src/auth/ ├── index.ts # Barrel exports +├── trainingAuth.ts # Public opaque training ingress staging to fixed native leaves ├── types.ts # Auth profile types ├── paths.ts # Spawnfile auth home and profile path helpers ├── profileStore.ts # Read/write auth profiles and imported auth material diff --git a/src/auth/index.ts b/src/auth/index.ts index 646efa87..826a4e93 100644 --- a/src/auth/index.ts +++ b/src/auth/index.ts @@ -10,3 +10,5 @@ export * from "./credentialProvisioningRequest.js"; export * from "./credentialWorldBindings.js"; export * from "./targetSecretSourceLifecycle.js"; export * from "./targetSecretSourceResolver.js"; +export { stageTrainingAuth } from "./trainingAuth.js"; +export type { TrainingAuthStageOptions, TrainingAuthProvider } from "./trainingAuth.js"; diff --git a/src/auth/trainingAuth.failure.test.ts b/src/auth/trainingAuth.failure.test.ts new file mode 100644 index 00000000..4bcd4be1 --- /dev/null +++ b/src/auth/trainingAuth.failure.test.ts @@ -0,0 +1,38 @@ +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +const control = vi.hoisted(()=>({ failWrite:false, mutateSource:false, growSource:false })); +vi.mock("node:fs/promises",async(importOriginal)=>{ + const native=await importOriginal(); + return {...native,realpath:async(...args:Parameters)=>{ + if(String(args[0]).startsWith("/run/paideia-auth/")) throw Error("fixed ingress intercepted for test: "+args[0]); + return native.realpath(...args); + },open:async(...args:Parameters)=>{ + const file=await native.open(...args); + if(String(args[0]).includes(".stage-") && control.failWrite) file.writeFile=async()=>{throw Error("injected disk full");}; + if(!String(args[0]).includes(".stage-") && control.growSource) { + const original=file.read.bind(file);file.read=(async(...readArgs:unknown[])=>{const buffer=readArgs[0] as Buffer;buffer.fill(120);return {bytesRead:buffer.length,buffer};}) as typeof original; + } + if(!String(args[0]).includes(".stage-") && control.mutateSource){const original=file.stat.bind(file);let calls=0;file.stat=(async(options:unknown)=>{const value=await original(options as {bigint:true});if(++calls===2)value.ctimeNs+=1n;return value;}) as typeof file.stat;} + return file; + }}; +}); +import {stageTrainingAuth} from "./trainingAuth.js"; +const roots:string[]=[]; +afterEach(async()=>{control.failWrite=false;control.mutateSource=false;control.growSource=false;for(const root of roots.splice(0))await rm(root,{recursive:true,force:true});}); +const setup=async()=>{const root=await realpath(await mkdtemp(path.join(os.tmpdir(),"training-auth-failure-")));roots.push(root);const home=path.join(root,"home"),source=path.join(root,"source");await mkdir(home,{mode:0o700});await writeFile(source,"complete-fake-credential");return {home,source,provider:"codex" as const};}; +it("never publishes a partial credential and allows a clean retry after write failure",async()=>{ + const f=await setup();control.failWrite=true;await expect(stageTrainingAuth(f)).rejects.toThrow("disk full");expect(await readdir(path.join(f.home,".daimon-inbound"))).toEqual([]); + control.failWrite=false;const receipt=await stageTrainingAuth(f);expect(await readFile(receipt.destination,"utf8")).toBe("complete-fake-credential"); +}); +it("rejects even a nanosecond source identity change before publication",async()=>{ + const f=await setup();control.mutateSource=true;await expect(stageTrainingAuth(f)).rejects.toThrow("changed during staging");expect(await readdir(path.join(f.home,".daimon-inbound"))).toEqual([]); +}); + +it("bounds an unexpectedly growing source to the preallocated limit",async()=>{ + const f=await setup();control.growSource=true;await expect(stageTrainingAuth(f)).rejects.toThrow("changed during staging");expect(await readdir(path.join(f.home,".daimon-inbound"))).toEqual([]); +}); +it("selects the fixed default ingress without opening real credentials",async()=>{ + const f=await setup();await expect(stageTrainingAuth({home:f.home,provider:"codex"})).rejects.toThrow("fixed ingress intercepted for test: /run/paideia-auth/codex"); +}); diff --git a/src/auth/trainingAuth.fifo.test.ts b/src/auth/trainingAuth.fifo.test.ts new file mode 100644 index 00000000..bf83580f --- /dev/null +++ b/src/auth/trainingAuth.fifo.test.ts @@ -0,0 +1,18 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, realpath, rm } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +const execute = promisify(execFile); +it("rejects a FIFO before any blocking read in a bounded child", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-auth-fifo-"))); + try { + const source = path.join(root, "fifo"), home = path.join(root, "home"); + await mkdir(home, { mode: 0o700 }); await execute("mkfifo", [source]); + const module = new URL("./trainingAuth.ts", import.meta.url).href; + const script = `const {stageTrainingAuth}=await import(${JSON.stringify(module)});try{await stageTrainingAuth(${JSON.stringify({home,source,provider:"codex"})});throw Error("unexpected stage")}catch(error){if(!String(error).includes("bounded nonempty regular leaf"))throw error;console.log("FIFO_REJECTED_WITHOUT_BLOCKING")}`; + const result = await execute(process.execPath, ["--experimental-strip-types", "--input-type=module", "-e", script], { timeout: 2500 }); + expect(result.stdout.trim()).toBe("FIFO_REJECTED_WITHOUT_BLOCKING"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/auth/trainingAuth.test.ts b/src/auth/trainingAuth.test.ts new file mode 100644 index 00000000..c72893c3 --- /dev/null +++ b/src/auth/trainingAuth.test.ts @@ -0,0 +1,22 @@ +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { stageTrainingAuth, type TrainingAuthProvider } from "./trainingAuth.js"; +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root,{recursive:true,force:true}); }); +const setup = async () => { const root = await realpath(await mkdtemp(path.join(os.tmpdir(),"spawnfile-training-auth-")));roots.push(root);const home=path.join(root,"home"),source=path.join(root,"auth");await mkdir(home,{mode:0o700});await writeFile(source,"opaque-fixture-only");return {root,home,source}; }; +it.each(["codex","grok","claude"] as const)("stages only %s auth to its native private leaf without overwriting renewal",async(provider)=>{ + const f=await setup();const receipt=await stageTrainingAuth({...f,provider}); + expect(receipt.version).toBe("spawnfile.training-auth-stage.v1");expect(await readFile(receipt.destination,"utf8")).toBe("opaque-fixture-only");expect((await stat(receipt.destination)).mode&0o777).toBe(0o600); + await writeFile(receipt.destination,"renewed");await expect(stageTrainingAuth({...f,provider})).rejects.toThrow("renewed credential preserved");expect(await readFile(receipt.destination,"utf8")).toBe("renewed"); +}); +it("rejects source and destination symlinks, nonprivate ingress, directories and empty/oversized leaves",async()=>{ + const f=await setup(),alias=path.join(f.root,"alias");await symlink(f.source,alias); + for(const source of [alias,f.home]) await expect(stageTrainingAuth({...f,source,provider:"codex"})).rejects.toThrow(); + for(const bytes of ["","x".repeat(1024*1024+1)]){await writeFile(f.source,bytes);await expect(stageTrainingAuth({...f,provider:"codex"})).rejects.toThrow();} + await writeFile(f.source,"fixture");await symlink(f.home,path.join(f.home,".daimon-inbound"));await expect(stageTrainingAuth({...f,provider:"codex"})).rejects.toThrow("private and canonical"); + await rm(path.join(f.home,".daimon-inbound"));await mkdir(path.join(f.home,".daimon-inbound"));await chmod(path.join(f.home,".daimon-inbound"),0o755);await expect(stageTrainingAuth({...f,provider:"codex"})).rejects.toThrow("private"); + await expect(stageTrainingAuth({...f,provider:"other" as TrainingAuthProvider})).rejects.toThrow("Unsupported"); + await expect(stageTrainingAuth({...f,home:"relative",provider:"codex"})).rejects.toThrow("canonical"); +}); diff --git a/src/auth/trainingAuth.ts b/src/auth/trainingAuth.ts new file mode 100644 index 00000000..cef43962 --- /dev/null +++ b/src/auth/trainingAuth.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, mkdir, open, realpath, unlink } from "node:fs/promises"; +import path from "node:path"; + +export type TrainingAuthProvider = "codex" | "grok" | "claude"; +export interface TrainingAuthStageOptions { + /** Caller-provisioned, existing private runtime home; never a host home mount. */ + home: string; + provider: TrainingAuthProvider; + /** Explicit provisioned leaf; default is the fixed training ingress. */ + source?: string; +} +const targets: Record = { + codex: [".daimon-inbound", "codex-auth"], + grok: [".grok", "auth.json"], + claude: [".claude", ".credentials.json"] +}; +/** Stages opaque credential bytes once. Never imports configuration, logs bytes, or overwrites renewed auth. */ +export const stageTrainingAuth = async (options: TrainingAuthStageOptions): Promise<{ version: "spawnfile.training-auth-stage.v1"; provider: TrainingAuthProvider; destination: string }> => { + if (!Object.hasOwn(targets, options.provider)) throw Error("Unsupported training auth provider"); + const home = path.resolve(options.home), source = options.source ?? `/run/paideia-auth/${options.provider}`; + if (!path.isAbsolute(options.home) || await realpath(home) !== home || !(await lstat(home)).isDirectory()) throw Error("Training home must be a canonical existing directory"); + if (!path.isAbsolute(source) || await realpath(source) !== path.resolve(source)) throw Error("Training auth source must be a canonical regular leaf"); + const input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await input.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > 1_048_576n) throw Error("Training auth source must be a bounded nonempty regular leaf"); + const [folder, leaf] = targets[options.provider]; + const directory = path.join(home, folder); + try { await mkdir(directory, { mode: 0o700 }); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } + const existing = await lstat(directory); + if (!existing.isDirectory() || existing.isSymbolicLink() || existing.mode % 512 !== 0o700 || await realpath(directory) !== directory) throw Error("Training auth directory must be private and canonical"); + const bytes = Buffer.alloc(Number(before.size) + 1); + let primary: unknown; + const destination = path.join(directory, leaf), temporary = path.join(directory, `.stage-${randomUUID()}`); + try { + let length = 0; + while (length < bytes.length) { + const read = await input.read(bytes, length, bytes.length - length, null); + if (read.bytesRead === 0) break; + length += read.bytesRead; + } + const after = await input.stat({ bigint: true }); + if (BigInt(length) !== before.size || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs || after.ino !== before.ino || after.dev !== before.dev || after.birthtimeNs !== before.birthtimeNs) throw Error("Training auth source changed during staging"); + const output = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await output.writeFile(bytes.subarray(0, length)); await output.sync(); } + finally { await output.close(); } + // Exclusive publication preserves renewed credentials and never exposes a partial leaf. + try { await link(temporary, destination); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") throw Error("Training auth already present; renewed credential preserved"); + throw error; + } + } catch (error) { primary = error; throw error; } finally { + bytes.fill(0); + await unlink(temporary).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return; + if (primary) throw new AggregateError([primary, error], "Training auth staging failed and temporary cleanup is incomplete"); + throw error; + }); + } + return { version: "spawnfile.training-auth-stage.v1", provider: options.provider, destination }; + } finally { await input.close(); } +}; diff --git a/src/cli/paideiaDelegation.test.ts b/src/cli/paideiaDelegation.test.ts index ab0f54f4..891fadd5 100644 --- a/src/cli/paideiaDelegation.test.ts +++ b/src/cli/paideiaDelegation.test.ts @@ -71,19 +71,12 @@ ${dryReceipt}`, { args: ["--train", injected, "--editable", "a.md", "--editable" expect(await run.result).toBe(0); }); - it.each([0, 1])("requires actual completion and preserves the completed outcome %s", async (exitCode) => { - const run = await invoke(`${completed} process.exitCode=${exitCode};`, { dryRun: false, args: [] }); - expect(await run.result).toBe(exitCode); - }); - - it("rejects a dry-run receipt as an actual training completion", async () => { - const run = await invoke(dryReceipt, { dryRun: false, args: [] }); - await expect(run.result).rejects.toThrow("required final training receipt"); - }); - - it("rejects an incomplete actual-completion receipt", async () => { - const run = await invoke('console.log(JSON.stringify({status:"completed"}));', { dryRun: false }); - await expect(run.result).rejects.toThrow("required final training receipt"); + it("rejects every host actual-training path before invoking a model executable", async () => { + for (const body of [completed, dryReceipt, 'console.log("unexpected");']) { + const run = await invoke(body, { dryRun: false, args: [] }); + await expect(run.result).rejects.toThrow("host execution is disabled"); + expect(run.stdout).toEqual([]); + } }); it("propagates receiver errors without requiring a success receipt", async () => { diff --git a/src/cli/paideiaDelegation.ts b/src/cli/paideiaDelegation.ts index b82bf8c0..82439fe4 100644 --- a/src/cli/paideiaDelegation.ts +++ b/src/cli/paideiaDelegation.ts @@ -6,6 +6,7 @@ import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { trainingContextSchema, type TrainingContext } from "../compiler/training/index.js"; +import { launchTrainingContainer } from "../compiler/training/container/index.js"; import { SpawnfileError } from "../shared/index.js"; import type { PaideiaProcessOutcome } from "./paideiaSupervisor.js"; import type { CliStreams } from "./runCli.js"; @@ -15,6 +16,8 @@ export interface DelegatePaideiaTrainingOptions { command: string; args: readonly string[]; dryRun: boolean; + trainingImage?: string; + trainingConfig?: string; timeoutMs: number; streams: CliStreams; signal?: AbortSignal; @@ -127,6 +130,11 @@ const runChild = (options: DelegatePaideiaTrainingOptions, contextPath: string): /** Delegates through the public CLI, never importing Paideia or selecting a fallback adapter. */ export const delegatePaideiaTraining = async (options: DelegatePaideiaTrainingOptions): Promise => { if (options.signal?.aborted) return 130; + if (!options.dryRun) { + if (!options.trainingImage || !options.trainingConfig) throw failure("Actual training requires --training-image and --training-config; host execution is disabled"); + return launchTrainingContainer({ image: options.trainingImage, configPath: options.trainingConfig, + context: options.context, args: options.args, timeoutMs: options.timeoutMs, streams: options.streams, signal: options.signal }); + } if (process.platform === "win32") throw failure("Paideia delegation requires POSIX process-group supervision"); const bytes = JSON.stringify(trainingContextSchema.parse(options.context)); if (Buffer.byteLength(bytes, "utf8") > MAX_LINE_BYTES) throw new SpawnfileError("validation_error", "Training context exceeds 1 MiB"); diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index f2960a9c..6f54b9fc 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -14,6 +14,7 @@ async function project(): Promise { await writeFile(path.join(root, "Spawnfile"), 'spawnfile_version: "0.1"\nkind: agent\nname: author\nruntime: daimon\n'); return root; } +const container = ["--training-image", `sha256:${"a".repeat(64)}`, "--training-config", "/training.json"]; const base = ["--train", "train.paideia.yaml", "--test", "test.paideia.yaml", "--out", "local output"]; describe("spawnfile train", () => { @@ -51,7 +52,7 @@ describe("spawnfile train", () => { it("forwards explicit resume to Paideia without changing canonical agent context", async () => { const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 0); - expect(await runCli(["train", await project(), ...base, "--resume"], { + expect(await runCli(["train", await project(), ...base, ...container, "--resume"], { handlers: { delegatePaideiaTraining: delegate }, streams: { stdout: () => undefined, stderr: () => undefined } })).toBe(0); expect(delegate.mock.calls[0]![0].args).toEqual([...base, "--resume"]); @@ -60,7 +61,7 @@ describe("spawnfile train", () => { }); it.each([1, 2, 130, 143])("propagates the delegated exit %s", async (exitCode) => { - const code = await runCli(["train", await project(), ...base], { handlers: { delegatePaideiaTraining: async () => exitCode }, + const code = await runCli(["train", await project(), ...base, ...container], { handlers: { delegatePaideiaTraining: async () => exitCode }, streams: { stdout: () => undefined, stderr: () => undefined } }); expect(code).toBe(exitCode); }); @@ -68,7 +69,7 @@ describe("spawnfile train", () => { it("forwards cancellation and default executable/timeout without forcing dry-run", async () => { const controller = new AbortController(); let captured: DelegatePaideiaTrainingOptions | undefined; - expect(await runCli(["train", await project(), ...base, "--optimizer-model", "fable", "--bridge-command", "bridge", "--max-proposals", "1", "--seed", "0", "--view", "0"], { + expect(await runCli(["train", await project(), ...base, ...container, "--optimizer-model", "fable", "--bridge-command", "bridge", "--max-proposals", "1", "--seed", "0", "--view", "0"], { signal: controller.signal, handlers: { delegatePaideiaTraining: async (options) => { captured = options; return 2; } }, streams: { stdout: () => undefined, stderr: () => undefined } })).toBe(2); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index cd770bf0..bee119da 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -21,6 +21,8 @@ export const registerTrainCommand = ( .argument("[path]", "Canonical project directory or Spawnfile path", process.cwd()) .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") + .option("--training-image ", "Pinned image containing the complete training environment") + .option("--training-config ", "Explicit local Docker inputs, output and auth leaf bindings") .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") .option("--resume", "Resume the exact persisted training experiment in --out"); for (const name of forwarded) { @@ -33,6 +35,9 @@ export const registerTrainCommand = ( if (options.dryRun !== true && typeof options.out !== "string") { throw new SpawnfileError("validation_error", "Actual training requires --out; dry-run does not write an output directory"); } + if (options.dryRun !== true && (typeof options.trainingImage !== "string" || typeof options.trainingConfig !== "string")) { + throw new SpawnfileError("validation_error", "Actual training requires --training-image and --training-config; host execution is disabled"); + } const timeout = options.timeoutMs === undefined ? 3_600_000 : Number(options.timeoutMs); if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 3_600_000 || (options.timeoutMs !== undefined && !/^\d+$/u.test(String(options.timeoutMs)))) { @@ -50,6 +55,7 @@ export const registerTrainCommand = ( if (options.dryRun === true) args.push("--dry-run"); if (options.resume === true) args.push("--resume"); setExitCode(await handlers.delegatePaideiaTraining({ context, args, + trainingImage: options.trainingImage as string | undefined, trainingConfig: options.trainingConfig as string | undefined, command: options.paideiaCommand as string, dryRun: options.dryRun === true, timeoutMs: timeout + 5000, streams, signal })); }); diff --git a/src/compiler/training/AGENTS.md b/src/compiler/training/AGENTS.md index 9e9641f7..825728fa 100644 --- a/src/compiler/training/AGENTS.md +++ b/src/compiler/training/AGENTS.md @@ -1,5 +1,6 @@ # Canonical Training Context +- `container/` owns the single-container actual-training launch; no model process runs on the host. - `contract.ts` owns the strict, versioned public JSON handoff to Paideia. - `context.ts` resolves the full compiler graph and pins source files without compiling or launching it. - Preserve resolved inheritance and exact agent IDs. Never create a second agent declaration. diff --git a/src/compiler/training/container/AGENTS.md b/src/compiler/training/container/AGENTS.md new file mode 100644 index 00000000..d26458e1 --- /dev/null +++ b/src/compiler/training/container/AGENTS.md @@ -0,0 +1,9 @@ +# Training Container Launcher + +- One immutable image runs the whole experiment; the host only prepares declared mounts and supervises Docker. +- `contract.ts` validates explicit launch configuration. `prepare.ts` maps canonical context and CLI paths into declared mounts. +- `process.ts` owns bounded Docker subprocess transport. `launch.ts` owns exact container identity, streaming, receipt verification and cleanup. +- Never mount a Docker socket, host home, arbitrary environment, or host executable. Auth bindings are explicit read-only leaf files. +- Dry-run stays in the existing host estimator. Actual training must never fall back to host execution. +- Docker bind mounts currently require an explicitly selected local Unix-socket context. Remote daemon staging is unsupported. +- Keep files under 400 lines; adjacent negative tests must prove ownership, cancellation and final receipt checks. diff --git a/src/compiler/training/container/CLAUDE.md b/src/compiler/training/container/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/container/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/container/contract.ts b/src/compiler/training/container/contract.ts new file mode 100644 index 00000000..fdebbf68 --- /dev/null +++ b/src/compiler/training/container/contract.ts @@ -0,0 +1,20 @@ +import path from "node:path"; +import { z } from "zod"; + +const hostPath = z.string().min(1).refine((value) => path.isAbsolute(value) && !/[,\r\n\0]/u.test(value), "Expected an absolute bind path"); +const inputPath = z.string().regex(/^\/run\/training\/inputs\/[A-Za-z0-9._/-]+$/u).refine((value) => path.posix.normalize(value) === value && !value.endsWith("/")); +export const trainingImageSchema = z.string().regex(/^(?:sha256:[a-f0-9]{64}|[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[a-f0-9]{64})$/u); +export const trainingContainerConfigSchema = z.object({ + version: z.literal("spawnfile.training-container.v1"), + dockerContext: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), + inputs: z.array(z.object({ source: hostPath, destination: inputPath }).strict()).min(1).max(64), + output: z.object({ source: hostPath, destination: z.literal("/run/training/output") }).strict(), + auth: z.array(z.object({ source: hostPath, provider: z.enum(["codex", "grok", "claude"]) }).strict()).max(3) +}).strict().superRefine((value, context) => { + const destinations = value.inputs.map((entry) => entry.destination); + if (destinations.some((entry, index) => destinations.some((other, otherIndex) => otherIndex !== index && (entry === other || entry.startsWith(`${other}/`))))) { + context.addIssue({ code: "custom", message: "Input destinations must not overlap" }); + } + if (new Set(value.auth.map((entry) => entry.provider)).size !== value.auth.length) context.addIssue({ code: "custom", message: "Duplicate auth provider" }); +}); +export type TrainingContainerConfig = z.infer; diff --git a/src/compiler/training/container/fixtures.test-helper.ts b/src/compiler/training/container/fixtures.test-helper.ts new file mode 100644 index 00000000..6a990382 --- /dev/null +++ b/src/compiler/training/container/fixtures.test-helper.ts @@ -0,0 +1,48 @@ +import { mkdtemp, mkdir, realpath, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { TrainingContext } from "../contract.js"; +import type { TrainingDockerProcess } from "./process.js"; + +export const image = `sha256:${"a".repeat(64)}`; +export const id = "b".repeat(64); +export const fixture = async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-container-test-"))); + const project = path.join(root, "project"), output = path.join(root, "output"); + await mkdir(project); await mkdir(output); + await writeFile(path.join(project, "Spawnfile"), "fixture"); + await writeFile(path.join(project, "train.yaml"), "fixture"); + await writeFile(path.join(output, "index.json"), "{}"); + await writeFile(path.join(root, "auth-leaf"), "fake-fixture-token"); + const source = { sourcePath: path.join(project, "Spawnfile"), destinationPath: "Spawnfile", sha256: image }; + const context: TrainingContext = { version: "spawnfile.training-context.v1", producer: { package: "spawnfile", version: "test" }, + project: { root: project, manifest: source.sourcePath, sourceDigest: image }, + agent: { id: "agent:a", name: "a", source: source.sourcePath, runtime: "daimon", engine: null, model: null }, + sources: [source], documents: [{ ...source, role: "system" }], skills: [{ ...source, name: "skill", ref: "skill", requiresMcp: [] }], resources: [], + requirements: { nativeCompilation: true, isolatedPreparation: true } }; + const config = { version: "spawnfile.training-container.v1", dockerContext: "desktop-linux", inputs: [{ source: project, destination: "/run/training/inputs/project" }], + output: { source: output, destination: "/run/training/output" }, auth: [{ source: path.join(root, "auth-leaf"), provider: "grok" }] }; + const configPath = path.join(root, "launch.json"); await writeFile(configPath, JSON.stringify(config)); + const args = ["--train", path.join(project, "train.yaml"), "--out", output]; + return { root, project, output, context, config, configPath, args }; +}; +export const dockerFixture = (override?: (args: readonly string[], options: Parameters[1]) => Promise<{code:number;stdout:string;stderr:string} | undefined>) => { + const calls: string[][] = []; let name = ""; + const process: TrainingDockerProcess = async (args, options) => { + calls.push([...args]); + const custom = await override?.(args, options); if (custom) return custom; + const result = (stdout: string, code = 0) => ({ stdout, code, stderr: "" }); + if (args[0] === "context") return result(JSON.stringify("unix:///var/run/docker.sock")); + const command = args[2]; + if (command === "image") return result(image); + if (command === "create") { name = args[args.indexOf("--name") + 1]!; return result(id); } + if (command === "inspect") { + if (args[4] === "{{json .State}}") return result(JSON.stringify({ Running: false, ExitCode: 0 })); + return result([id, `/${name}`, image, { "com.spawnfile.training.owner": name }].map((part) => JSON.stringify(part)).join("\n")); + } + if (command === "start") { options.stdout?.('measuring'); options.stdout?.('{"status":"completed","index":"/run/training/output/index.json"}'); return result(""); } + if (command === "rm" || command === "container") return result(""); + throw Error(`Unexpected fake Docker command ${args}`); + }; + return { process, calls }; +}; diff --git a/src/compiler/training/container/index.ts b/src/compiler/training/container/index.ts new file mode 100644 index 00000000..3c50bf28 --- /dev/null +++ b/src/compiler/training/container/index.ts @@ -0,0 +1,4 @@ +export { trainingContainerConfigSchema, trainingImageSchema } from "./contract.js"; +export type { TrainingContainerConfig } from "./contract.js"; +export { launchTrainingContainer } from "./launch.js"; +export type { LaunchTrainingContainerOptions } from "./launch.js"; diff --git a/src/compiler/training/container/launch.test.ts b/src/compiler/training/container/launch.test.ts new file mode 100644 index 00000000..13e5dde6 --- /dev/null +++ b/src/compiler/training/container/launch.test.ts @@ -0,0 +1,156 @@ +import { readFile, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import { fixture, dockerFixture, image, id } from "./fixtures.test-helper.js"; +import { launchTrainingContainer } from "./launch.js"; +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const setup = async () => { const value = await fixture(); roots.push(value.root); return value; }; +const streams = () => ({ stdout: vi.fn(), stderr: vi.fn() }); +it("launches one immutable container, streams a verified persisted completion and removes only its verified identity", async () => { + const f = await setup(); let observedContext: unknown; + const docker = dockerFixture(async (args) => { + if (args[2] === "create") { + const mount = args.find((entry) => entry.includes("dst=/run/paideia/context.json"))!; + observedContext = JSON.parse(await readFile(mount.split("src=")[1]!.split(",")[0]!, "utf8")); + } + return undefined; + }); const output = streams(); + expect(await launchTrainingContainer({ ...f, image, timeoutMs: 1000, streams: output, process: docker.process })).toBe(0); + expect(observedContext).toMatchObject({ project: { root: "/run/training/inputs/project" } }); + const create = docker.calls.find((args) => args[2] === "create")!; + expect(create).toContain("/opt/training/bin/train"); expect(create).toContain("HOME=/home/training"); + expect(create).toContain("--user"); expect(create).toContain(`${process.getuid!()}:${process.getgid!()}`); + expect(create).toContain("--security-opt=seccomp=unconfined"); expect(create).toContain("--security-opt=apparmor=unconfined"); + expect(create.some((value) => value.startsWith("/work:") && value.includes(`uid=${process.getuid!()}`))).toBe(true); + expect(create).toContain(image); expect(create).not.toContain("--privileged"); + expect(create.join(" ")).not.toContain("docker.sock"); + expect(create.join(" ")).toContain("dst=/run/paideia-auth/grok,readonly"); + expect(output.stdout).toHaveBeenLastCalledWith('{"status":"completed","index":"/run/training/output/index.json"}'); + expect(docker.calls.some((args) => args[2] === "rm" && args.at(-1) === id)).toBe(true); +}); +it("rejects remote contexts and missing images before container creation", async () => { + const f = await setup(); + for (const remote of [true, false]) { + const docker = dockerFixture(async (args) => args[0] === "context" && remote ? {code:0,stdout:'"ssh://host"',stderr:""} : args[2] === "image" ? {code:1,stdout:"",stderr:""}: undefined); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:docker.process})).rejects.toThrow(); + expect(docker.calls.some((args) => args[2] === "create")).toBe(false); + } +}); +it("rejects malformed completion, missing artifact, running state and inconsistent exit evidence", async () => { + const f = await setup(); + for (const text of ["not json", '{"status":"completed","index":"/etc/passwd"}', '{"status":"completed","index":"/run/training/output/missing.json"}', '{"status":"pending","index":"/run/training/output/index.json"}']) { + const docker = dockerFixture(async (args, options) => { if (args[2] === "start") {options.stdout?.(text);return {code:0,stdout:"",stderr:""};} return undefined; }); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:docker.process})).rejects.toThrow(); + expect(docker.calls.some((args) => args[2] === "rm")).toBe(true); + } + for (const state of [{Running:true,ExitCode:0},{Running:false,ExitCode:1}]) { + const docker = dockerFixture(async (args) => args[4] === "{{json .State}}" ? {code:0,stdout:JSON.stringify(state),stderr:""}:undefined); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:docker.process})).rejects.toThrow("final state"); + } +}); +it("refuses foreign ownership and reports unverified cleanup", async () => { + const f = await setup(); + const foreign = dockerFixture(async (args) => args[2] === "inspect" ? { code:0,stdout:[id,"/foreign",image,{}].map((value) => JSON.stringify(value)).join("\n"),stderr:"" }:undefined); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:foreign.process})).rejects.toThrow(); + expect(foreign.calls.some((args) => args[2] === "rm")).toBe(false); + const failed = dockerFixture(async (args) => args[2] === "container" ? {code:0,stdout:id,stderr:""}:undefined); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:failed.process})).rejects.toThrow("cleanup is unverified"); +}); +it("cancellation and deadlines stop the real owned container, including a lost create response", async () => { + const f = await setup(); + for (const phase of ["create", "start"]) { + const controller = new AbortController(); + const docker = dockerFixture(async (args) => { if (args[2] === phase) { controller.abort(); if (phase === "start") throw Error("cancelled"); } return undefined; }); + expect(await launchTrainingContainer({...f,image,timeoutMs:1000,signal:controller.signal,streams:streams(),process:docker.process})).toBe(130); + expect(docker.calls.some((args) => args[2] === "rm")).toBe(true); + } + const docker = dockerFixture(async (args, options) => { + if (args[2] === "start") await new Promise((_resolve,reject) => options.signal!.addEventListener("abort",()=>reject(Error("deadline")),{once:true})); + return undefined; + }); + await expect(launchTrainingContainer({...f,image,timeoutMs:50,streams:streams(),process:docker.process})).rejects.toThrow("deadline"); + expect(docker.calls.some((args) => args[2] === "rm")).toBe(true); + const already = new AbortController();already.abort(); + expect(await launchTrainingContainer({...f,image,timeoutMs:1000,signal:already.signal,streams:streams(),process:docker.process})).toBe(130); +}); +it("refuses oversized launch configuration", async () => { + const f = await setup();await writeFile(f.configPath," ".repeat(1024*1024+1)); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams()})).rejects.toThrow("1 MiB"); +}); +it("preserves non-success exit codes and proves absence when create leaves no container", async () => { + const f = await setup(); + const exited = dockerFixture(async (args) => args[2] === "start" ? {code:2,stdout:"",stderr:""} : args[4] === "{{json .State}}" ? {code:0,stdout:'{"Running":false,"ExitCode":2}',stderr:""}:undefined); + expect(await launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:exited.process})).toBe(2); + for(const remains of ["",id]) { + const missing = dockerFixture(async (args)=> args[2] === "create" ? {code:1,stdout:"",stderr:""} : args[2] === "inspect" ? {code:1,stdout:"",stderr:""}: args[2] === "container" ? {code:0,stdout:remains,stderr:""}:undefined); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:missing.process})).rejects.toThrow(remains ? "closure is unknown" : "valid training container"); + } +}); + +it("refuses a root caller before any Docker invocation", async () => { + const f=await setup(),docker=dockerFixture();const uid=vi.spyOn(process as {getuid:()=>number},"getuid").mockReturnValue(0); + try { await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:docker.process})).rejects.toThrow("non-root");expect(docker.calls).toEqual([]); } finally {uid.mockRestore();} +}); + +it("observes cancellation that arrives while preparing filesystem inputs", async () => { + const f=await setup(),docker=dockerFixture(),controller=new AbortController(); + const pending=launchTrainingContainer({...f,image,timeoutMs:1000,signal:controller.signal,streams:streams(),process:docker.process}); + queueMicrotask(()=>controller.abort()); + expect(await pending).toBe(130);expect(docker.calls.every((args)=>args[2]!=="create")).toBe(true); +}); +it("cleans a verified container even when create returned a truncated ID",async()=>{ + const f=await setup();let name=""; + const docker=dockerFixture(async(args)=>{ + if(args[2]==="create"){name=args[args.indexOf("--name")+1]!;return {code:0,stdout:id.slice(0,12),stderr:""};} + if(args[2]==="inspect")return {code:0,stdout:[id,`/${name}`,image,{"com.spawnfile.training.owner":name}].map((v)=>JSON.stringify(v)).join("\n"),stderr:""}; + return undefined; + }); + await expect(launchTrainingContainer({...f,image,timeoutMs:1000,streams:streams(),process:docker.process})).rejects.toThrow("valid training container identity"); + expect(docker.calls.some((args)=>args[2]==="rm"&&args.at(-1)===id)).toBe(true); +}); + +it("publishes the live cockpit only on host loopback and removes its container on cancellation",async()=>{ + const f=await setup(),controller=new AbortController(); + const docker=dockerFixture(async(args)=>{if(args[2]==="start"){controller.abort();throw Error("cancelled");}return undefined;}); + expect(await launchTrainingContainer({...f,args:[...f.args,"--view","53484"],image,timeoutMs:1000,signal:controller.signal,streams:streams(),process:docker.process})).toBe(130); + const create=docker.calls.find((args)=>args[2]==="create")!; + expect(create[create.indexOf("--publish")+1]).toBe("127.0.0.1:53484:53484"); + expect(create).not.toContain("0.0.0.0:53484:53484");expect(docker.calls.some((args)=>args[2]==="rm")).toBe(true); +}); + + +it("stages private readonly context beside the shared output, never in host temporary storage", async () => { + const f = await setup(); let scratch = ""; + const tmp = vi.spyOn(os, "tmpdir").mockImplementation(() => { throw Error("Host temp cannot be staged for Docker"); }); + const docker = dockerFixture(async (args) => { + if (args[2] === "create") { + const mount = args.find((entry) => entry.includes("dst=/run/paideia/context.json"))!; + expect(mount.endsWith(",readonly")).toBe(true); + const file = mount.split("src=")[1]!.split(",")[0]!; scratch = path.dirname(file); + expect(path.dirname(scratch)).toBe(path.dirname(f.output)); + expect(scratch.startsWith(f.output + path.sep)).toBe(false); + expect(f.config.inputs.some((entry) => scratch === entry.source || scratch.startsWith(entry.source + path.sep))).toBe(false); + expect((await stat(scratch)).mode & 0o777).toBe(0o700); + expect((await stat(file)).mode & 0o777).toBe(0o444); + } + return undefined; + }); + expect(await launchTrainingContainer({ ...f, image, timeoutMs: 1000, streams: streams(), process: docker.process })).toBe(0); + expect(tmp).not.toHaveBeenCalled(); + await expect(stat(scratch)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("retains bounded useful Docker create errors while redacting declared auth paths", async () => { + const f = await setup(); + const docker = dockerFixture(async (args) => args[2] === "create" ? { code: 1, stdout: "", + stderr: `invalid mount config: bind source path does not exist: /shared/context.json\ncredential=${f.config.auth[0]!.source} ${"x".repeat(4000)}` } + : args[2] === "inspect" ? { code: 1, stdout: "", stderr: "No container" } : undefined); + const error = await launchTrainingContainer({ ...f, image, timeoutMs: 1000, streams: streams(), process: docker.process }).catch((failure: unknown) => failure); + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain("bind source path does not exist: /shared/context.json"); + expect(message).toContain("[auth source]"); expect(message).not.toContain(f.config.auth[0]!.source); + expect(message).not.toContain("\n"); expect(message.length).toBeLessThan(2200); +}); diff --git a/src/compiler/training/container/launch.ts b/src/compiler/training/container/launch.ts new file mode 100644 index 00000000..525b2c54 --- /dev/null +++ b/src/compiler/training/container/launch.ts @@ -0,0 +1,118 @@ +import { randomUUID } from "node:crypto"; +import { chmod, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS } from "../../../shared/index.js"; +import { parseDetachedContainerInspect } from "../../runProjectDocker.js"; +import type { TrainingContext } from "../contract.js"; +import { trainingImageSchema } from "./contract.js"; +import { prepareTrainingContainer } from "./prepare.js"; +import { runTrainingDocker, type TrainingDockerProcess } from "./process.js"; + +export interface LaunchTrainingContainerOptions { + image: string; configPath: string; context: TrainingContext; args: readonly string[]; + timeoutMs: number; signal?: AbortSignal; + streams: { stdout(line: string): void; stderr(line: string): void }; + process?: TrainingDockerProcess; +} +const inspectFormat = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}"; +export const launchTrainingContainer = async (options: LaunchTrainingContainerOptions): Promise => { + if (options.signal?.aborted) return 130; + const image = trainingImageSchema.parse(options.image); + const uid = process.getuid?.(), gid = process.getgid?.(); + if (uid === undefined || gid === undefined || uid === 0) throw Error("Training requires an explicit non-root local user"); + const configBytes = await readFile(options.configPath, "utf8"); + if (Buffer.byteLength(configBytes) > 1024 * 1024) throw Error("Training launch config exceeds 1 MiB"); + const prepared = await prepareTrainingContainer(JSON.parse(configBytes), options.context, options.args); + const execute = options.process ?? runTrainingDocker; + const controller = new AbortController(), deadline = Date.now() + options.timeoutMs; + let interrupted = false, timedOut = false; + const interrupt = (): void => { interrupted = true; controller.abort(); }; + const timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs); + options.signal?.addEventListener("abort", interrupt, { once: true }); + process.once("SIGINT", interrupt); process.once("SIGTERM", interrupt); + if (options.signal?.aborted) interrupt(); + const prefix = ["--context", prepared.config.dockerContext]; + const call = async (args: string[], cleanup = false, stream = false) => { + if (!cleanup && controller.signal.aborted) throw Error("Container operation cancelled"); + return execute([...prefix, ...args], { + timeoutMs: cleanup ? 10_000 : Math.max(1, deadline - Date.now()), + ...(cleanup ? {} : { signal: controller.signal }), ...(stream ? options.streams : {}) + }); }; + const name = `spawnfile-training-${randomUUID()}`; + const labels = { "com.spawnfile.training.owner": name }; + let privateRoot: string | undefined, containerId: string | undefined, creationAttempted = false; + let expectedImage: string | undefined, lastLine = "", closureVerified = false; + try { + const endpoint = await execute(["context", "inspect", prepared.config.dockerContext, "--format", "{{json .Endpoints.docker.Host}}"], { timeoutMs: 10_000, signal: controller.signal }); + if (endpoint.code !== 0 || !/^unix:\/\//u.test(JSON.parse(endpoint.stdout))) throw Error("Training requires an explicitly selected local Unix Docker context"); + const inspected = await call(["image", "inspect", image, "--format", "{{.Id}}"]); + expectedImage = inspected.stdout.trim(); + if (inspected.code !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(expectedImage) || (image.startsWith("sha256:") && image !== expectedImage)) throw Error("Training image must be locally available with an immutable identity"); + privateRoot = await realpath(await mkdtemp(path.join(path.dirname(prepared.config.output.source), ".spawnfile-training-container-"))); + await chmod(privateRoot, 0o700); + const contextFile = path.join(privateRoot, "context.json"); + await writeFile(contextFile, JSON.stringify(prepared.context), { mode: 0o444, flag: "wx" }); + const mounts = [...prepared.config.inputs.map((entry) => `type=bind,src=${entry.source},dst=${entry.destination},readonly`), + `type=bind,src=${prepared.config.output.source},dst=/run/training/output`, + `type=bind,src=${contextFile},dst=/run/paideia/context.json,readonly`, + ...prepared.config.auth.map((entry) => `type=bind,src=${entry.source},dst=/run/paideia-auth/${entry.provider},readonly`)]; + const args = ["create", "--name", name, "--label", `com.spawnfile.training.owner=${name}`, + "--init", "--read-only", "--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + ...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS, "--pids-limit", "512", + "--tmpfs", `/tmp:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=1777`, "--tmpfs", `/work:rw,nosuid,nodev,size=4g,uid=${uid},gid=${gid},mode=700`, "--tmpfs", `/home/training:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=700`, + "--env", "HOME=/home/training", "--workdir", "/work", "--entrypoint", "/opt/training/bin/train", + ...mounts.flatMap((mount) => ["--mount", mount]), + ...(prepared.viewerPort === undefined ? [] : ["--publish", `127.0.0.1:${prepared.viewerPort}:${prepared.viewerPort}`]), expectedImage, + "train", "--spawnfile-context", "/run/paideia/context.json", ...prepared.args]; + creationAttempted = true; + const created = await call(args); + containerId = created.stdout.trim(); + if (created.code !== 0 || !/^[a-f0-9]{64}$/u.test(containerId)) { + let diagnostic = created.stderr; + for (const auth of prepared.config.auth) diagnostic = diagnostic.replaceAll(auth.source, "[auth source]"); + diagnostic = diagnostic.replace(/\p{Cc}/gu, " ").trim().slice(0, 2048); + throw Error(`Docker did not return a valid training container identity${diagnostic ? `: ${diagnostic}` : ""}`); + } + const identity = await call(["inspect", "--format", inspectFormat, containerId]); + if (identity.code !== 0 || parseDetachedContainerInspect(identity.stdout, containerId, labels, name).imageId !== expectedImage) throw Error("Training container identity mismatch"); + if (controller.signal.aborted) throw Error("Container operation cancelled"); + const result = await execute([...prefix, "start", "--attach", containerId], { + timeoutMs: Math.max(1, deadline - Date.now()), signal: controller.signal, + stdout: (line) => { if (line.trim()) lastLine = line; options.streams.stdout(line); }, stderr: options.streams.stderr + }); + const terminal = await call(["inspect", "--format", "{{json .State}}", containerId]); + const state = JSON.parse(terminal.stdout) as { Running?: unknown; ExitCode?: unknown }; + if (terminal.code !== 0 || state.Running !== false || !Number.isInteger(state.ExitCode) || result.code !== state.ExitCode) throw Error("Training container final state is unverified"); + if (state.ExitCode !== 0 && state.ExitCode !== 1) return state.ExitCode as number; + const receipt = JSON.parse(lastLine) as { status?: unknown; index?: unknown }; + if (receipt.status !== "completed" || typeof receipt.index !== "string" || !receipt.index.startsWith("/run/training/output/") || path.posix.normalize(receipt.index) !== receipt.index) throw Error("Training exited without a valid final receipt"); + const artifact = path.join(prepared.config.output.source, receipt.index.slice("/run/training/output/".length)); + if (await realpath(artifact) !== artifact || !(await stat(artifact)).isFile()) throw Error("Training completion artifact is missing or escapes output"); + return state.ExitCode as number; + } catch (error) { + if (interrupted) return 130; + if (timedOut) throw Error("Container training exceeded command deadline"); + throw error; + } finally { + clearTimeout(timer); options.signal?.removeEventListener("abort", interrupt); + process.removeListener("SIGINT", interrupt); process.removeListener("SIGTERM", interrupt); + try { + if (creationAttempted) { + // A timed-out create may still have created our uniquely labelled container. + const found = await call(["inspect", "--format", inspectFormat, name], true); + if (found.code === 0) { + const id: unknown = JSON.parse(found.stdout.split("\n")[0]!); + if (typeof id !== "string" || parseDetachedContainerInspect(found.stdout, id, labels, name).imageId !== expectedImage || (containerId && /^[a-f0-9]{64}$/u.test(containerId) && id !== containerId)) throw Error("Refusing cleanup of unverified training container"); + const removed = await call(["rm", "--force", id], true); + const remaining = await call(["container", "ls", "--all", "--no-trunc", "--filter", `id=${id}`, "--format", "{{.ID}}"], true); + if (removed.code !== 0 || remaining.code !== 0 || remaining.stdout.trim()) throw Error("Training container cleanup is unverified"); + closureVerified = true; + } else { + const remaining = await call(["container", "ls", "--all", "--filter", `name=^/${name}$`, "--format", "{{.ID}}"], true); + if (remaining.code !== 0 || remaining.stdout.trim()) throw Error("Training container closure is unknown"); + closureVerified = true; + } + } + } finally { if (privateRoot && (!creationAttempted || closureVerified)) await rm(privateRoot, { recursive: true, force: true }); } + } +}; diff --git a/src/compiler/training/container/prepare.test.ts b/src/compiler/training/container/prepare.test.ts new file mode 100644 index 00000000..78adac76 --- /dev/null +++ b/src/compiler/training/container/prepare.test.ts @@ -0,0 +1,86 @@ +import { mkdir, rename, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import { fixture } from "./fixtures.test-helper.js"; +import { prepareTrainingContainer } from "./prepare.js"; +import { trainingContainerConfigSchema, trainingImageSchema } from "./contract.js"; +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const setup = async () => { const value = await fixture(); roots.push(value.root); return value; }; +it("maps canonical sources and CLI paths, leaving installed bridge and project-relative editables intact", async () => { + const f = await setup(); + const result = await prepareTrainingContainer(f.config, f.context, [...f.args, "--resource", `archive=${f.project}`, "--bridge-command", "/opt/training/bin/bridge", "--editable", "a.md"]); + expect(result.context.project.root).toBe("/run/training/inputs/project"); + expect(result.context.sources[0]?.sourcePath).toBe("/run/training/inputs/project/Spawnfile"); + expect(result.context.documents[0]?.sourcePath).toBe("/run/training/inputs/project/Spawnfile"); + expect(result.context.skills[0]?.sourcePath).toBe("/run/training/inputs/project/Spawnfile"); + expect(result.args).toEqual(["--train", "/run/training/inputs/project/train.yaml", "--out", "/run/training/output", "--resource", "archive=/run/training/inputs/project", "--bridge-command", "/opt/training/bin/bridge", "--editable", "a.md"]); +}); +it("rejects mutable images, extra authority and overlapping destinations", async () => { + const f = await setup(); + for (const image of ["latest", "image:tag", "sha256:bad", "a@sha256:"+"a".repeat(63)]) expect(trainingImageSchema.safeParse(image).success).toBe(false); + for (const value of [{ ...f.config, command: "shell" }, { ...f.config, auth: [f.config.auth[0], { ...f.config.auth[0] }] }, + { ...f.config, inputs: [...f.config.inputs, { source: f.output, destination: "/run/training/inputs/project/sub" }] }, + { ...f.config, inputs: [{ source: f.project, destination: "/run/training/inputs/../run" }] }]) expect(trainingContainerConfigSchema.safeParse(value).success).toBe(false); +}); +it("rejects unmapped paths, output under readonly mounts, malformed resources and host executables/viewers", async () => { + const f = await setup(); + for (const args of [["--train", "/missing"], ["--out", f.project], ["--train"], ["--resource", "bad"], ["--resource"], ["--bridge-command", "/usr/bin/bridge"], ["--bridge-command"], ["--view", "0"]]) { + await expect(prepareTrainingContainer(f.config, f.context, args)).rejects.toThrow(); + } +}); +it("rejects symlink aliases, whole auth directories and overlapping writable mounts", async () => { + const f = await setup(); const alias = path.join(f.root, "alias"); await symlink(f.project, alias); + for (const config of [{ ...f.config, inputs: [{ source: alias, destination: "/run/training/inputs/project" }] }, + { ...f.config, auth: [{ provider: "grok", source: f.project }] }, + { ...f.config, output: { source: f.project, destination: "/run/training/output" } }, + { ...f.config, output: { source: path.join(f.root, "auth-leaf"), destination: "/run/training/output" } }, + { ...f.config, inputs: [{ source: f.root, destination: "/run/training/inputs/project" }] }]) await expect(prepareTrainingContainer(config, f.context, f.args)).rejects.toThrow(); + vi.spyOn(os, "homedir").mockReturnValue(f.root); + const cliHome = path.join(f.root, ".grok"); await mkdir(cliHome); await writeFile(path.join(cliHome,"auth.json"), "fixture"); + await expect(prepareTrainingContainer({ ...f.config, inputs: [{ source: cliHome, destination: "/run/training/inputs/project" }] }, f.context, f.args)).rejects.toThrow("Host homes"); +}); + +it("rejects noncanonical raw spellings and overlapping host input roots before remapping",async()=>{ + const f=await setup();const inner=path.join(f.project,"inner");await mkdir(inner);const other=path.join(f.root,"other");await mkdir(other);const leaf=path.join(f.project,"credential");await writeFile(leaf,"fixture"); + await expect(prepareTrainingContainer({...f.config,auth:[{provider:"grok",source:other+"/../project/credential"}]},f.context,f.args)).rejects.toThrow("canonical"); + await expect(prepareTrainingContainer({...f.config,inputs:[...f.config.inputs,{source:inner,destination:"/run/training/inputs/inner"}]},f.context,f.args)).rejects.toThrow("source roots must not overlap"); + await expect(prepareTrainingContainer({...f.config,output:{source:f.output+"/../output",destination:"/run/training/output"}},f.context,f.args)).rejects.toThrow("canonical"); +}); + +it("accepts one explicit viewer port and rejects ephemeral, invalid or duplicate publication",async()=>{ + const f=await setup();const prepared=await prepareTrainingContainer(f.config,f.context,[...f.args,"--view","53484"]); + expect(prepared.viewerPort).toBe(53484);expect(prepared.args.slice(-2)).toEqual(["--view","53484"]); + for(const values of [["0"],["65536"],["-1"],["1.5"],["127.0.0.1:3"],[""]])await expect(prepareTrainingContainer(f.config,f.context,[...f.args,"--view",...values])).rejects.toThrow("explicit port"); + await expect(prepareTrainingContainer(f.config,f.context,[...f.args,"--view","1234","--view","1234"])).rejects.toThrow("explicit port"); +}); + + +it("accepts real nested project worktrees without treating their .claude directory as the host profile", async () => { + const f = await setup(); + vi.spyOn(os, "homedir").mockReturnValue(f.root); + const worktree = path.join(f.root, "Documents", "project", ".claude", "worktrees", "training"); + await mkdir(path.dirname(worktree), { recursive: true }); + await rename(f.project, worktree); + const remap = (value: T): T => JSON.parse(JSON.stringify(value).replaceAll(f.project, worktree)) as T; + const result = await prepareTrainingContainer(remap(f.config), remap(f.context), remap(f.args)); + expect(result.context.project.root).toBe("/run/training/inputs/project"); + expect(result.config.inputs[0]?.source).toBe(worktree); + expect(result.args[1]).toBe("/run/training/inputs/project/train.yaml"); +}); + +it("still rejects exact host roots, global profile subtrees and explicit auth exposure", async () => { + const f = await setup(); + const home = path.join(f.root, "home"); await mkdir(home); + vi.spyOn(os, "homedir").mockReturnValue(home); + const denied = [home, "/"]; + for (const name of [".codex", ".claude", ".grok", ".ssh", ".config"]) { + const root = path.join(home, name), nested = path.join(root, "nested"); + await mkdir(nested, { recursive: true }); denied.push(root, nested); + } + for (const source of denied) { + await expect(prepareTrainingContainer({ ...f.config, inputs: [{ ...f.config.inputs[0], source }] }, f.context, f.args)).rejects.toThrow("Host homes"); + } + await expect(prepareTrainingContainer({ ...f.config, inputs: [{ ...f.config.inputs[0], source: f.config.auth[0]!.source }] }, f.context, f.args)).rejects.toThrow("Auth must not be exposed"); +}); diff --git a/src/compiler/training/container/prepare.ts b/src/compiler/training/container/prepare.ts new file mode 100644 index 00000000..fd042044 --- /dev/null +++ b/src/compiler/training/container/prepare.ts @@ -0,0 +1,66 @@ +import { lstat, realpath } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { trainingContextSchema, type TrainingContext } from "../contract.js"; +import { trainingContainerConfigSchema, type TrainingContainerConfig } from "./contract.js"; + +const inside = (root: string, value: string): boolean => value === root || value.startsWith(`${root}/`); +export const prepareTrainingContainer = async (raw: unknown, context: TrainingContext, args: readonly string[]): Promise<{ + config: TrainingContainerConfig; context: TrainingContext; args: string[]; viewerPort?: number; +}> => { + const config = trainingContainerConfigSchema.parse(raw); + const entries = [...config.inputs, config.output]; + const home = path.resolve(os.homedir()); + const globalConfigRoots = [".codex", ".claude", ".grok", ".ssh", ".config"].map((name) => path.join(home, name)); + for (const entry of [...entries, ...config.auth]) { + const canonical = await realpath(entry.source); + if (canonical !== entry.source) throw Error("Training bind sources must be canonical, without symlink aliases"); + const stat = await lstat(canonical); + if (stat.isSymbolicLink() || (!stat.isFile() && !stat.isDirectory())) throw Error("Training binds require regular files or directories"); + if (config.auth.includes(entry as TrainingContainerConfig["auth"][number]) && !stat.isFile()) throw Error("Auth must be a single regular leaf file"); + if (entries.includes(entry as TrainingContainerConfig["output"])) { + if (["/", "/etc", "/var", "/run", "/tmp", "/opt", "/usr", "/Users", "/home", home].includes(canonical) || globalConfigRoots.some((root) => inside(root, canonical))) throw Error("Host homes, configuration and system roots cannot be training inputs"); + if (config.auth.some((auth) => inside(canonical, auth.source))) throw Error("Auth must not be exposed through input or output mounts"); + } + } + if (config.inputs.some((entry, index) => config.inputs.some((other, otherIndex) => index !== otherIndex && inside(entry.source, other.source)))) throw Error("Training input source roots must not overlap"); + if (!(await lstat(config.output.source)).isDirectory()) throw Error("Training output must be an existing directory"); + if (config.inputs.some((entry) => inside(entry.source, config.output.source) || inside(config.output.source, entry.source))) throw Error("Training output and inputs must not overlap"); + const map = (value: string): string => { + const absolute = path.resolve(value); + const entry = entries.find((item) => inside(item.source, absolute)); + if (!entry) throw Error(`Training path is not covered by a declared mount: ${value}`); + return path.posix.join(entry.destination, path.relative(entry.source, absolute).split(path.sep).join("/")); + }; + const source = (entry: T): T => ({ ...entry, sourcePath: map(entry.sourcePath) }); + const mapped = trainingContextSchema.parse({ ...context, + project: { ...context.project, root: map(context.project.root), manifest: map(context.project.manifest) }, + agent: { ...context.agent, source: map(context.agent.source) }, + sources: context.sources.map(source), documents: context.documents.map(source), skills: context.skills.map(source) + }); + const mappedArgs = [...args]; + let viewerPort: number | undefined; + for (let index = 0; index < mappedArgs.length; index++) { + const flag = mappedArgs[index]; + if (["--train", "--test", "--out", "--cost-config"].includes(flag!)) { + const value = mappedArgs[++index]; + if (!value) throw Error(`Missing ${flag} path`); + mappedArgs[index] = map(value); + if (flag === "--out" && !inside("/run/training/output", mappedArgs[index]!)) throw Error("Training --out must use the writable output mount"); + } else if (flag === "--resource") { + const value = mappedArgs[++index] ?? "", equals = value.indexOf("="); + if (equals < 1) throw Error("Expected resource=id path mapping"); + mappedArgs[index] = `${value.slice(0, equals)}=${map(value.slice(equals + 1))}`; + } else if (flag === "--bridge-command") { + const value = mappedArgs[++index] ?? ""; + if (!/^\/opt\/training\/[A-Za-z0-9._/-]+$/u.test(value) || path.posix.normalize(value) !== value) throw Error("Bridge must be an installed executable under /opt/training"); + } else if (flag === "--view") { + const value = mappedArgs[++index] ?? ""; + const port = Number(value); + if (viewerPort !== undefined || !/^\d+$/u.test(value) || !Number.isSafeInteger(port) || port < 1 || port > 65535) throw Error("Container --view requires one explicit port from 1 to 65535"); + viewerPort = port; + mappedArgs[index] = String(port); + } + } + return { config, context: mapped, args: mappedArgs, ...(viewerPort === undefined ? {} : { viewerPort }) }; +}; diff --git a/src/compiler/training/container/process.test.ts b/src/compiler/training/container/process.test.ts new file mode 100644 index 00000000..17f5feb2 --- /dev/null +++ b/src/compiler/training/container/process.test.ts @@ -0,0 +1,40 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { afterEach, expect, it, vi } from "vitest"; +const spawn = vi.hoisted(() => vi.fn()); +vi.mock("node:child_process", () => ({ spawn })); +import { runTrainingDocker } from "./process.js"; +afterEach(() => { vi.clearAllMocks(); }); +const child = () => { + const value = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(), kill: vi.fn() }); + value.kill.mockImplementation(() => { queueMicrotask(() => value.emit("close", null)); return true; }); + spawn.mockReturnValue(value); return value; +}; +it("runs only Docker without a shell, streams complete lines, and retains bounded tail", async () => { + const native = child(), output = vi.fn(), errors = vi.fn(); + const result = runTrainingDocker(["start","--attach","owned"],{timeoutMs:1000,stdout:output,stderr:errors}); + native.stdout.write("one\ntw");native.stdout.write("o\nfinal");native.stderr.write("warning\nlast");native.emit("close",0); + expect(await result).toEqual({code:0,stdout:"final",stderr:"last"}); + expect(output.mock.calls.flat()).toEqual(["one","two","final"]);expect(errors.mock.calls.flat()).toEqual(["warning","last"]); + expect(spawn).toHaveBeenCalledWith("docker",["start","--attach","owned"],{shell:false,stdio:["ignore","pipe","pipe"]}); +}); +it("captures short inspection output and preserves exit code",async()=>{ + const native=child();const pending=runTrainingDocker(["inspect"],{timeoutMs:1000});native.stdout.write("first\nlast");native.stderr.write("error\ntail");native.emit("close",2);expect(await pending).toEqual({code:2,stdout:"first\nlast",stderr:"error\ntail"}); +}); +it("cancels pending operations and refuses already cancelled launches",async()=>{ + const native=child(),controller=new AbortController();const pending=runTrainingDocker(["start"],{timeoutMs:1000,signal:controller.signal});controller.abort();await expect(pending).rejects.toThrow("cancelled");expect(native.kill).toHaveBeenCalledWith("SIGKILL"); + spawn.mockClear();await expect(runTrainingDocker([],{timeoutMs:1000,signal:controller.signal})).rejects.toThrow("cancelled");expect(spawn).not.toHaveBeenCalled(); +}); +it("bounds hangs and oversized line/cumulative output and surfaces launch errors",async()=>{ + let native=child();let pending=runTrainingDocker([],{timeoutMs:10});await expect(pending).rejects.toThrow("deadline"); + native=child();pending=runTrainingDocker([],{timeoutMs:1000});native.stdout.write("x".repeat(1024*1024+1));await expect(pending).rejects.toThrow("line size"); + native=child();pending=runTrainingDocker([],{timeoutMs:1000});for(let i=0;i<4;i++)native.stdout.write("x".repeat(800000)+"\n");await expect(pending).rejects.toThrow("capture size"); + native=child();pending=runTrainingDocker([],{timeoutMs:1000});native.emit("error",Error("missing docker"));native.emit("close",null);await expect(pending).rejects.toThrow("missing docker"); +}); + +it("releases a stuck Docker client after kill so owned-container cleanup can proceed",async()=>{ + const native=child();native.kill.mockImplementation(()=>true); + const controller=new AbortController(),pending=runTrainingDocker([],{timeoutMs:5000,signal:controller.signal}); + controller.abort();await expect(pending).rejects.toThrow("cancelled");expect(native.stdout.destroyed).toBe(true); + native.emit("close",0); +}); diff --git a/src/compiler/training/container/process.ts b/src/compiler/training/container/process.ts new file mode 100644 index 00000000..20daa7e5 --- /dev/null +++ b/src/compiler/training/container/process.ts @@ -0,0 +1,50 @@ +import { spawn } from "node:child_process"; + +export interface TrainingDockerProcess { + (args: readonly string[], options: { timeoutMs: number; signal?: AbortSignal; stdout?: (line: string) => void; stderr?: (line: string) => void }): Promise<{ code: number; stdout: string; stderr: string }>; +} +/** Docker client only; no model executable runs on the host. */ +export const runTrainingDocker: TrainingDockerProcess = (args, options) => new Promise((resolve, reject) => { + if (options.signal?.aborted) { reject(Error("Training Docker operation cancelled")); return; } + const child = spawn("docker", [...args], { shell: false, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = "", stderr = "", failure: Error | undefined; + let forceClose: ReturnType | undefined; + let settled = false; + const pending = { stdout: "", stderr: "" }; + const stop = (message: string): void => { + failure ??= Error(message); child.kill("SIGKILL"); + forceClose ??= setTimeout(() => { + // Releasing a stuck client lets the owner independently stop/verify the container. + child.stdout.destroy(); child.stderr.destroy(); finish(null); + }, 1000); + }; + const abort = (): void => stop("Training Docker operation cancelled"); + const timer = setTimeout(() => stop("Training Docker operation exceeded deadline"), options.timeoutMs); + options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) abort(); + const consume = (channel: "stdout" | "stderr", chunk: string): void => { + pending[channel] += chunk; + if (Buffer.byteLength(pending[channel]) > 1024 * 1024) { stop("Docker output exceeded bounded line size"); return; } + const lines = pending[channel].split("\n"); pending[channel] = lines.pop()!; + for (const line of lines) { + options[channel]?.(line); + if (channel === "stdout") stdout = options.stdout ? line : stdout + line + "\n"; + else stderr = options.stderr ? line : stderr + line + "\n"; + if (stdout.length + stderr.length > 2 * 1024 * 1024) stop("Docker output exceeded bounded capture size"); + } + }; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => consume("stdout", chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => consume("stderr", chunk)); + child.once("error", (error) => { failure = error; }); + const finish = (code: number | null): void => { + if (settled) return; settled = true; + clearTimeout(forceClose); clearTimeout(timer); options.signal?.removeEventListener("abort", abort); + for (const channel of ["stdout", "stderr"] as const) if (pending[channel]) { + options[channel]?.(pending[channel]); + if (channel === "stdout") stdout = options.stdout ? pending[channel] : stdout + pending[channel]; + else stderr = options.stderr ? pending[channel] : stderr + pending[channel]; + } + if (failure) reject(failure); else resolve({ code: code ?? 1, stdout, stderr }); + }; + child.once("close", finish); +}); From c8c26df224feeff3d4e9a5e66f32581271b6e4f0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Mon, 14 Sep 2026 15:13:08 +0200 Subject: [PATCH 06/84] fix: register training Python libraries for clean environments --- runtime-images/training/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 1a1582a7..741e06a4 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -6,7 +6,9 @@ FROM ${NATIVE_IMAGE} AS training # Both parents must use the same architecture and Debian release. COPY --from=python /usr/local /opt/python COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/ -ENV LD_LIBRARY_PATH=/opt/python/lib +# The bridge intentionally clears ambient environment variables. Register native +# libraries in the image so Python extensions also load in that clean process. +RUN echo /opt/python/lib > /etc/ld.so.conf.d/training-python.conf && ldconfig WORKDIR /opt/training/paideia COPY paideia/package.json paideia/package-lock.json ./ From 10afc1235fee2864404d291860cbb87e0da473ad Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 02:53:31 +0200 Subject: [PATCH 07/84] feat: checkpoint declarative training preparation --- package.json | 10 +- src/cli/paideiaDelegation.ts | 14 +++ src/cli/trainCommand.ts | 4 +- src/compiler/training/container/launch.ts | 7 ++ src/compiler/training/index.ts | 2 + src/compiler/training/preparation/AGENTS.md | 12 ++ src/compiler/training/preparation/CLAUDE.md | 1 + src/compiler/training/preparation/contract.ts | 61 ++++++++++ .../training/preparation/copyAssets.ts | 9 ++ src/compiler/training/preparation/files.ts | 77 +++++++++++++ .../preparation/fixtures.test-helper.ts | 64 +++++++++++ src/compiler/training/preparation/image.ts | 83 ++++++++++++++ src/compiler/training/preparation/index.ts | 3 + src/compiler/training/preparation/inputs.ts | 66 +++++++++++ .../training/preparation/prepare.test.ts | 91 +++++++++++++++ src/compiler/training/preparation/prepare.ts | 107 ++++++++++++++++++ tsconfig.json | 3 +- 17 files changed, 608 insertions(+), 6 deletions(-) create mode 100644 src/compiler/training/preparation/AGENTS.md create mode 120000 src/compiler/training/preparation/CLAUDE.md create mode 100644 src/compiler/training/preparation/contract.ts create mode 100644 src/compiler/training/preparation/copyAssets.ts create mode 100644 src/compiler/training/preparation/files.ts create mode 100644 src/compiler/training/preparation/fixtures.test-helper.ts create mode 100644 src/compiler/training/preparation/image.ts create mode 100644 src/compiler/training/preparation/index.ts create mode 100644 src/compiler/training/preparation/inputs.ts create mode 100644 src/compiler/training/preparation/prepare.test.ts create mode 100644 src/compiler/training/preparation/prepare.ts diff --git a/package.json b/package.json index 688356f0..19d59ed7 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "scripts": { "compile:explicit-test-mcp": "node --experimental-strip-types scripts/compile-explicit-test-mcp.ts", - "build": "rm -rf dist && tsc --project tsconfig.build.json && chmod +x dist/cli/index.js && node --experimental-strip-types ./src/evidenceExportHelper/copyAssets.ts && node --experimental-strip-types ./src/runtime/copyScaffoldAssets.ts && node --experimental-strip-types ./src/deployment/native/copyArtifacts.ts", + "build": "rm -rf dist && tsc --project tsconfig.build.json && chmod +x dist/cli/index.js && node --experimental-strip-types ./src/evidenceExportHelper/copyAssets.ts && node --experimental-strip-types ./src/runtime/copyScaffoldAssets.ts && node --experimental-strip-types ./src/deployment/native/copyArtifacts.ts && node --experimental-strip-types ./src/compiler/training/preparation/copyAssets.ts", "build:native": "node --experimental-strip-types ./src/deployment/native/build.ts", "clean": "rm -rf coverage dist", "coverage": "vitest run --coverage", @@ -55,7 +55,7 @@ "test:verdict": "node -e \"const fs=require('fs'); const file='coverage/vitest-results.json'; if (!fs.existsSync(file)) { console.error('VERDICT: results file missing'); process.exit(1); } let result; try { result=JSON.parse(fs.readFileSync(file, 'utf8')); } catch { console.error('VERDICT: results file unparseable'); process.exit(1); } const total=result.numTotalTests; const totalSuites=result.numTotalTestSuites; const passedTests=result.numPassedTests; const failedTests=result.numFailedTests; const pendingTests=result.numPendingTests; const todoTests=result.numTodoTests; const failedSuites=result.numFailedTestSuites; if (typeof result.success !== 'boolean' || !Number.isInteger(passedTests) || passedTests < 0 || !Number.isInteger(failedTests) || failedTests < 0 || !Number.isInteger(pendingTests) || pendingTests < 0 || !Number.isInteger(todoTests) || todoTests < 0 || !Number.isInteger(failedSuites) || failedSuites < 0 || !Number.isInteger(totalSuites) || totalSuites < 0) { console.error('VERDICT: results schema unrecognised'); process.exit(1); } if (!Number.isInteger(total) || total <= 0) { console.error('VERDICT: no tests found'); process.exit(1); } if (passedTests + failedTests + pendingTests + todoTests !== total) { console.error('VERDICT: results counters do not reconcile (tests='+total+', passed='+passedTests+', failed='+failedTests+', skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } const failed=result.success === false || failedTests > 0 || failedSuites > 0; if (failed) { console.error('VERDICT: test failures found (tests='+failedTests+', suites='+failedSuites+', success='+result.success+')'); process.exit(2); } if (passedTests === 0) { console.error('VERDICT: no tests actually ran (skipped='+pendingTests+', todo='+todoTests+')'); process.exit(1); } console.log('VERDICT: tests passed (tests='+total+', suites='+totalSuites+', passed='+passedTests+', skipped='+pendingTests+', todo='+todoTests+')');\"", "test:coverage-verdict": "node --import tsx -e \"import fs from 'node:fs'; const file='coverage/coverage-summary.json'; const fail=(message)=>{ console.error('COVERAGE VERDICT: '+message); process.exit(1); }; let summary; try { summary=JSON.parse(fs.readFileSync(file, 'utf8')); } catch (error) { fail(error && error.code === 'ENOENT' ? 'summary file missing' : 'summary file unparseable'); } import('./vitest.config.ts').then(({default:config})=>{ const thresholds=config.test && config.test.coverage && config.test.coverage.thresholds; if (!thresholds || typeof thresholds !== 'object' || !summary || typeof summary !== 'object' || !summary.total || typeof summary.total !== 'object') fail('summary or threshold schema unrecognised'); const metricNames=['branches','functions','lines','statements']; const metrics=metricNames.filter((metric)=>Object.prototype.hasOwnProperty.call(thresholds, metric)); if (metrics.length === 0) fail('no threshold metrics configured'); let shortfall=false; for (const metric of metrics) { const threshold=thresholds[metric]; const pct=summary.total[metric] && summary.total[metric].pct; if (typeof threshold !== 'number' || !Number.isFinite(threshold) || typeof pct !== 'number' || !Number.isFinite(pct)) fail('missing or non-numeric value for '+metric); console.log('COVERAGE: '+metric+' '+pct+'% (threshold '+threshold+'%)'); if (pct < threshold) shortfall=true; } if (shortfall) { console.error('COVERAGE VERDICT: thresholds not met'); process.exit(2); } console.log('COVERAGE VERDICT: thresholds met'); }).catch((error)=>fail('could not load vitest config: '+error.message));\"", "test:node": "mkdir -p coverage || exit 1; rm -f coverage/node-test.tap || exit 1; node --import tsx --test --test-reporter=tap src/runtime/pi/appControlDeliveryMetadata.test.ts > coverage/node-test.tap 2>&1; node_status=$?; cat coverage/node-test.tap; cat_status=$?; npm run test:node-verdict; verdict_status=$?; if [ \"$verdict_status\" -eq 2 ]; then exit 2; fi; if [ \"$verdict_status\" -ne 0 ]; then exit 1; fi; if [ \"$cat_status\" -ne 0 ]; then echo 'NODE VERDICT: could not read TAP output' >&2; exit 1; fi; if [ \"$node_status\" -ne 0 ]; then echo \"NODE VERDICT: node:test runner exited $node_status despite a passing TAP summary\" >&2; exit 1; fi; exit 0", - "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' \u2014 cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", + "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' — cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", "test:product-state-volume": "node --experimental-strip-types scripts/product-state-volume-integration.test.ts", "test:boundaries": "vitest run --coverage.enabled=false src/ownership/rootOwnershipBoundary.test.ts src/deployment/providerRuntimeBoundary.test.ts src/ownership/mnemePublicImportBoundary.test.ts src/target/containerBundleArchive.test.ts", "test:causal-conformance": "tsx src/ledger/causalConformanceCli.ts", @@ -87,6 +87,10 @@ "types": "./dist/auth/index.d.ts", "import": "./dist/auth/index.js" }, - "./*": "./*" + "./*": "./*", + "./training": { + "types": "./dist/compiler/training/index.d.ts", + "import": "./dist/compiler/training/index.js" + } } } diff --git a/src/cli/paideiaDelegation.ts b/src/cli/paideiaDelegation.ts index 82439fe4..b9f4c090 100644 --- a/src/cli/paideiaDelegation.ts +++ b/src/cli/paideiaDelegation.ts @@ -7,6 +7,9 @@ import { fileURLToPath } from "node:url"; import { trainingContextSchema, type TrainingContext } from "../compiler/training/index.js"; import { launchTrainingContainer } from "../compiler/training/container/index.js"; +import { runTrainingDocker } from "../compiler/training/container/process.js"; +import { prepareTraining } from "../compiler/training/preparation/index.js"; +import { readBoundedJson } from "../compiler/training/preparation/inputs.js"; import { SpawnfileError } from "../shared/index.js"; import type { PaideiaProcessOutcome } from "./paideiaSupervisor.js"; import type { CliStreams } from "./runCli.js"; @@ -130,6 +133,17 @@ const runChild = (options: DelegatePaideiaTrainingOptions, contextPath: string): /** Delegates through the public CLI, never importing Paideia or selecting a fallback adapter. */ export const delegatePaideiaTraining = async (options: DelegatePaideiaTrainingOptions): Promise => { if (options.signal?.aborted) return 130; + if (options.trainingConfig) { + const config = await readBoundedJson(options.trainingConfig) as { version?: unknown }; + if (config.version === "spawnfile.training-container.v2") { + if (options.trainingImage) throw failure("V2 owns its image declaration; --training-image is only for v1"); + const prepared = await prepareTraining({ configPath: options.trainingConfig, context: options.context, args: options.args, + dryRun: options.dryRun, process: runTrainingDocker, timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams }); + if (!("dryRun" in prepared)) return launchTrainingContainer({ ...prepared, + timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams }); + options.streams.stderr(`Training preparation plan ${prepared.digest}; no Docker, auth or filesystem mutations`); + } + } if (!options.dryRun) { if (!options.trainingImage || !options.trainingConfig) throw failure("Actual training requires --training-image and --training-config; host execution is disabled"); return launchTrainingContainer({ image: options.trainingImage, configPath: options.trainingConfig, diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index bee119da..0c857aa7 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -35,8 +35,8 @@ export const registerTrainCommand = ( if (options.dryRun !== true && typeof options.out !== "string") { throw new SpawnfileError("validation_error", "Actual training requires --out; dry-run does not write an output directory"); } - if (options.dryRun !== true && (typeof options.trainingImage !== "string" || typeof options.trainingConfig !== "string")) { - throw new SpawnfileError("validation_error", "Actual training requires --training-image and --training-config; host execution is disabled"); + if (options.dryRun !== true && typeof options.trainingConfig !== "string") { + throw new SpawnfileError("validation_error", "Actual training requires --training-config; v1 additionally requires --training-image; host execution is disabled"); } const timeout = options.timeoutMs === undefined ? 3_600_000 : Number(options.timeoutMs); if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 3_600_000 || diff --git a/src/compiler/training/container/launch.ts b/src/compiler/training/container/launch.ts index 525b2c54..fb979386 100644 --- a/src/compiler/training/container/launch.ts +++ b/src/compiler/training/container/launch.ts @@ -7,12 +7,14 @@ import type { TrainingContext } from "../contract.js"; import { trainingImageSchema } from "./contract.js"; import { prepareTrainingContainer } from "./prepare.js"; import { runTrainingDocker, type TrainingDockerProcess } from "./process.js"; +import { parseTrainingMappedPreparation } from "../preparation/contract.js"; export interface LaunchTrainingContainerOptions { image: string; configPath: string; context: TrainingContext; args: readonly string[]; timeoutMs: number; signal?: AbortSignal; streams: { stdout(line: string): void; stderr(line: string): void }; process?: TrainingDockerProcess; + preparationPath?: string; } const inspectFormat = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}"; export const launchTrainingContainer = async (options: LaunchTrainingContainerOptions): Promise => { @@ -23,6 +25,10 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp const configBytes = await readFile(options.configPath, "utf8"); if (Buffer.byteLength(configBytes) > 1024 * 1024) throw Error("Training launch config exceeds 1 MiB"); const prepared = await prepareTrainingContainer(JSON.parse(configBytes), options.context, options.args); + if (options.preparationPath) { + if (await realpath(options.preparationPath) !== options.preparationPath) throw Error("Preparation receipt path must be canonical"); + parseTrainingMappedPreparation(JSON.parse(await readFile(options.preparationPath, "utf8"))); + } const execute = options.process ?? runTrainingDocker; const controller = new AbortController(), deadline = Date.now() + options.timeoutMs; let interrupted = false, timedOut = false; @@ -55,6 +61,7 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp const mounts = [...prepared.config.inputs.map((entry) => `type=bind,src=${entry.source},dst=${entry.destination},readonly`), `type=bind,src=${prepared.config.output.source},dst=/run/training/output`, `type=bind,src=${contextFile},dst=/run/paideia/context.json,readonly`, + ...options.preparationPath ? [`type=bind,src=${options.preparationPath},dst=/run/paideia/preparation.json,readonly`] : [], ...prepared.config.auth.map((entry) => `type=bind,src=${entry.source},dst=/run/paideia-auth/${entry.provider},readonly`)]; const args = ["create", "--name", name, "--label", `com.spawnfile.training.owner=${name}`, "--init", "--read-only", "--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", diff --git a/src/compiler/training/index.ts b/src/compiler/training/index.ts index e904fe1a..e7fcf643 100644 --- a/src/compiler/training/index.ts +++ b/src/compiler/training/index.ts @@ -1,2 +1,4 @@ export * from "./contract.js"; export * from "./context.js"; +export { parseTrainingMappedPreparation } from "./preparation/contract.js"; +export type { TrainingMappedPreparation } from "./preparation/contract.js"; diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md new file mode 100644 index 00000000..3286961d --- /dev/null +++ b/src/compiler/training/preparation/AGENTS.md @@ -0,0 +1,12 @@ +# Training preparation + +Owns the v2 declaration, local pinned inputs, packaged image recipe and verified +build reuse before the existing container launcher. Never invoke a model, host +project script or sibling implementation. Project fixture meaning stays in the +installed integration. Credentials remain explicit runtime leaves, never image +inputs. Dry-run performs reads only; resume validates the preserved preparation. + +`contract.ts` declares authoring and runtime receipts; `files.ts` safely seals +declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; +`prepare.ts` combines those operations. Keep tests adjacent and files below 400 +lines. Source/lock/recipe mutation must invalidate cache and exact resume. diff --git a/src/compiler/training/preparation/CLAUDE.md b/src/compiler/training/preparation/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/preparation/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/preparation/contract.ts b/src/compiler/training/preparation/contract.ts new file mode 100644 index 00000000..2efe241c --- /dev/null +++ b/src/compiler/training/preparation/contract.ts @@ -0,0 +1,61 @@ +import path from "node:path"; +import { z } from "zod"; +import { trainingImageSchema } from "../container/contract.js"; + +const local = z.string().min(1).refine(value => !/[,\r\n\0]/u.test(value)); +const relative = local.refine(value => !path.isAbsolute(value) && !value.includes("\\") && value.split("/").every(part => part !== "" && part !== "." && part !== ".." && part !== ".git")); +const reference = z.object({ input: z.string().min(1), path: z.union([z.literal("."), relative]) }).strict(); +const sha = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +const destination = z.string().regex(/^\/run\/training\/inputs\/[A-Za-z0-9._/-]+$/u).refine(value => path.posix.normalize(value) === value && !value.endsWith("/")); +export const trainingBuildSchema = z.object({ + recipe: z.literal("daimon-dspy.v1"), nativeImage: trainingImageSchema, pythonImage: trainingImageSchema, + platform: z.enum(["linux/arm64", "linux/amd64"]), + paideia: local, bridge: local, claude: local, + grok: z.object({ source: local, sha256: sha }).strict(), + integration: z.object({ source: local, entry: relative.refine(value => /^[A-Za-z0-9._/-]+$/u.test(value)) }).strict(), bootstrap: local +}).strict(); +export const trainingPreparationSchema = z.object({ + version: z.literal("spawnfile.training-container.v2"), + dockerContext: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), + image: z.union([z.object({ ref: trainingImageSchema }).strict(), z.object({ build: trainingBuildSchema }).strict()]), + integration: z.object({ settings: reference }).strict(), + inputs: z.array(z.object({ + id: z.string().regex(/^[A-Za-z][A-Za-z0-9_-]{0,63}$/u), source: local, destination, + git: z.object({ revision: z.string().regex(/^[a-f0-9]{40}$/u), + overlays: z.array(z.object({ source: local, path: relative, sha256: sha }).strict()).max(256).default([]) + }).strict().optional() + }).strict()).min(1).max(64), + output: z.object({ source: local, destination: z.literal("/run/training/output") }).strict(), + auth: z.array(z.object({ source: local, provider: z.enum(["codex", "grok", "claude"]) }).strict()).max(3) +}).strict().superRefine((value, context) => { + if (new Set(value.inputs.map(input => input.id)).size !== value.inputs.length) context.addIssue({ code: "custom", message: "Input IDs must be unique" }); + if (new Set(value.auth.map(auth => auth.provider)).size !== value.auth.length) context.addIssue({ code: "custom", message: "Auth providers must be unique" }); + if (!value.inputs.some(input => input.id === value.integration.settings.input)) context.addIssue({ code: "custom", message: "Integration settings require a declared input" }); +}); +export type TrainingPreparationConfig = z.infer; +export type TrainingImageBuild = z.infer; + +/** Image integration reads this protected, container-addressed receipt, never host paths. */ +export interface TrainingMappedPreparation { + version: "spawnfile.training-preparation.v1"; + preparationDigest: string; + imageId: string; + bindings: { inputId: string; destination: string }[]; + outputRoot: "/run/training/output"; + packagePaths: { spawnfile: string; paideia: string; bridge: string; nativeWorker: string; integration: string; bootstrap: string }; + integration: { settings: { input: string; path: string } }; +} + +const containerPath = z.string().regex(/^\/(?:run|opt)\/[A-Za-z0-9._/-]+$/u).refine(value => path.posix.normalize(value) === value); +const mappedSchema = z.object({ + version: z.literal("spawnfile.training-preparation.v1"), preparationDigest: sha, imageId: trainingImageSchema, + bindings: z.array(z.object({ inputId: z.string().min(1), destination }).strict()).min(1).max(64), + outputRoot: z.literal("/run/training/output"), + packagePaths: z.object({ spawnfile: containerPath, paideia: containerPath, bridge: containerPath, + nativeWorker: containerPath, integration: containerPath, bootstrap: containerPath }).strict(), + integration: z.object({ settings: reference }).strict() +}).strict().superRefine((value, context) => { + if (new Set(value.bindings.map(binding => binding.inputId)).size !== value.bindings.length || + !value.bindings.some(binding => binding.inputId === value.integration.settings.input)) context.addIssue({ code: "custom", message: "Invalid preparation binding IDs" }); +}); +export const parseTrainingMappedPreparation = (value: unknown): TrainingMappedPreparation => mappedSchema.parse(value); diff --git a/src/compiler/training/preparation/copyAssets.ts b/src/compiler/training/preparation/copyAssets.ts new file mode 100644 index 00000000..1f28e6e7 --- /dev/null +++ b/src/compiler/training/preparation/copyAssets.ts @@ -0,0 +1,9 @@ +import { cp, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../../../../", import.meta.url)); +const target = path.join(root, "dist/compiler/training/preparation/assets"); +await mkdir(target, { recursive: true }); +await cp(path.join(root, "runtime-images/training/Dockerfile"), path.join(target, "Dockerfile")); +await cp(path.join(root, "package-lock.json"), path.join(target, "package-lock.json")); diff --git a/src/compiler/training/preparation/files.ts b/src/compiler/training/preparation/files.ts new file mode 100644 index 00000000..9a5cdcbf --- /dev/null +++ b/src/compiler/training/preparation/files.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { copyFile, lstat, mkdir, readdir, realpath, chmod } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +export const within = (root: string, file: string): boolean => file === root || file.startsWith(root + path.sep); +export const hashJson = (value: unknown): string => `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; +export interface SealedFile { source: string; destination: string; sha256: string; mode: number; size: number } +const ignored = new Set(["node_modules", ".venv", ".git", "__pycache__", "coverage", ".runtime", "AGENTS.md", "CLAUDE.md"]); + +export function assertInputRoot(source: string, auth: readonly string[]): void { + const home = os.homedir(); + if (["/", "/etc", "/var", "/run", "/tmp", "/opt", "/usr", "/Users", "/home", home].includes(source) || + [".codex", ".claude", ".grok", ".ssh", ".config"].some(name => within(path.join(home, name), source)) || + auth.some(leaf => within(source, leaf) || within(leaf, source))) throw Error("Training input exposes a protected host root or auth leaf"); +} + +export async function exactPath(source: string): Promise { + const absolute = path.resolve(source); + if (await realpath(absolute) !== absolute || (await lstat(absolute)).isSymbolicLink()) throw Error("Training source must be canonical, without symlink aliases"); + return absolute; +} + +export async function sealFile(source: string, destination: string): Promise { + await exactPath(source); + const before = await lstat(source, { bigint: true }); + if (!before.isFile() || before.size > 536_870_912n) throw Error("Training source must be a regular file no larger than 512 MiB"); + const digest = createHash("sha256"); let size = 0; + for await (const chunk of createReadStream(source)) { + size += chunk.length; + if (size > 536_870_912) throw Error("Training source exceeded its size limit"); + digest.update(chunk); + } + const after = await lstat(source, { bigint: true }); + if (BigInt(size) !== before.size || after.size !== before.size || after.ino !== before.ino || after.ctimeNs !== before.ctimeNs || after.dev !== before.dev) throw Error("Training source changed while hashing"); + return { source, destination, sha256: `sha256:${digest.digest("hex")}`, mode: Number(before.mode & 0o777n), size }; +} + +/** Only explicitly selected trees are traversed. Symlinks never enter Docker context. */ +export async function sealTree(source: string, destination: string, options: { ignoreDevelopment?: boolean; ignoreGit?: boolean; internalSymlinks?: boolean } = {}): Promise { + await exactPath(source); + const files: SealedFile[] = []; + const walk = async (root: string, target: string, depth: number): Promise => { + if (depth > 32 || files.length > 10000) throw Error("Training source tree exceeds bounds"); + let stat = await lstat(root); + if (stat.isSymbolicLink()) { + const actual = await realpath(root); + if (!options.internalSymlinks || !within(source, actual)) throw Error("Training source trees must not contain escaping symlinks"); + root = actual; stat = await lstat(root); + } + if (stat.isDirectory()) { + for (const entry of (await readdir(root)).sort()) { + if (options.ignoreGit && entry === ".git") continue; + if (options.ignoreDevelopment && (ignored.has(entry) || /(?:\.test\.[cm]?[jt]s|_test\.py|\.pyc)$/u.test(entry))) continue; + await walk(path.join(root, entry), path.posix.join(target, entry), depth + 1); + } + } else files.push(await sealFile(root, target)); + }; + await walk(source, destination, 0); + if (files.length > 10000 || files.reduce((sum, file) => sum + file.size, 0) > 1_073_741_824) throw Error("Training source tree exceeds bounds"); + return files; +} + +export async function copySealed(files: readonly SealedFile[], root: string): Promise { + for (const file of files) { + const target = path.resolve(root, file.destination); + if (!within(root, target) || target === root) throw Error("Training destination escapes owned staging"); + await mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + await copyFile(file.source, target, 1); + await chmod(target, file.mode); + const copied = await sealFile(target, file.destination); + if (copied.sha256 !== file.sha256 || copied.size !== file.size) throw Error("Training source changed during staging"); + } +} + +export const fileIdentity = (files: readonly SealedFile[]) => files.map(({ destination, sha256, mode, size }) => ({ destination, sha256, mode, size })); diff --git a/src/compiler/training/preparation/fixtures.test-helper.ts b/src/compiler/training/preparation/fixtures.test-helper.ts new file mode 100644 index 00000000..608507eb --- /dev/null +++ b/src/compiler/training/preparation/fixtures.test-helper.ts @@ -0,0 +1,64 @@ +import { mkdtemp, mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { createHash } from "node:crypto"; +import type { TrainingPreparationConfig } from "./contract.js"; +import type { TrainingContext } from "../contract.js"; +import type { TrainingDockerProcess } from "../container/process.js"; + +export const image = `sha256:${"a".repeat(64)}`; +export const sha = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +export async function preparationFixture() { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-preparation-"))); + const put = async (file: string, value = "fixture") => { await mkdir(path.dirname(path.join(root, file)), { recursive: true }); await writeFile(path.join(root, file), value); }; + await put("project/Spawnfile", 'spawnfile_version: "0.1"\nkind: agent\nname: author\nruntime: daimon\n'); + await put("project/train.yaml"); await put("settings/settings.json", "{}"); await put("auth", "fake-subscription"); + for (const target of ["own", "paideia", "claude"]) { + const manifest = { name: target, version: "1.0.0", dependencies: {} }; + await put(`${target}/package.json`, JSON.stringify(manifest)); + await put(`${target}/package-lock.json`, JSON.stringify({ lockfileVersion: 3, packages: { "": manifest } })); + } + await put("own/dist/cli/index.js"); await put("own/runtimes.yaml"); await put("own/moltnet-releases.json"); + await put("own/runtime-images/training/Dockerfile", "ARG NATIVE_IMAGE\nFROM ${NATIVE_IMAGE}\nCOPY train /opt/training/bin/train\n"); + await put("paideia/dist/src/cli/main.js"); await put("bridge/pyproject.toml"); await put("bridge/requirements.lock"); + await put("bridge/paideia_dspy/__init__.py"); await put("integration/entry.ts"); await put("bootstrap/start.ts"); await put("grok", "native-binary"); + const config: TrainingPreparationConfig = { + version: "spawnfile.training-container.v2", dockerContext: "local", + image: { build: { recipe: "daimon-dspy.v1", nativeImage: image, pythonImage: image, platform: "linux/arm64", + paideia: "paideia", bridge: "bridge", claude: "claude", grok: { source: "grok", sha256: sha("native-binary") }, + integration: { source: "integration", entry: "entry.ts" }, bootstrap: "bootstrap" } }, + integration: { settings: { input: "settings", path: "settings.json" } }, + inputs: [{ id: "project", source: "project", destination: "/run/training/inputs/project" }, { id: "settings", source: "settings", destination: "/run/training/inputs/settings" }], + output: { source: "output", destination: "/run/training/output" }, auth: [{ source: "auth", provider: "claude" }] + }; + const sourcePath = path.join(root, "project/Spawnfile"); + const source = { sourcePath, destinationPath: "Spawnfile", sha256: sha(await readFile(sourcePath, "utf8")) }; + const context: TrainingContext = { version: "spawnfile.training-context.v1", producer: { package: "spawnfile", version: "test" }, + project: { root: path.join(root, "project"), manifest: sourcePath, sourceDigest: source.sha256 }, + agent: { id: "agent:author", name: "author", source: sourcePath, runtime: "daimon", engine: null, model: null }, + sources: [source], documents: [], skills: [], resources: [], requirements: { nativeCompilation: true, isolatedPreparation: true } }; + const configPath = path.join(root, "training.json"); + const save = () => writeFile(configPath, JSON.stringify(config)); await save(); + return { root, config, configPath, context, put, save, + args: ["--train", path.join(root, "project/train.yaml"), "--out", path.join(root, "output")] }; +} + +export function imageDocker() { + const calls: string[][] = []; const images = new Map(); let builtContext: string | undefined; + const process: TrainingDockerProcess = async args => { + calls.push([...args]); + if (args[0] === "context") return { code: 0, stdout: JSON.stringify("unix:///socket"), stderr: "" }; + if (args[2] === "build") { + builtContext = args.at(-1)!; + const dockerfile = await readFile(path.join(builtContext, "Dockerfile"), "utf8"); + const digest = JSON.parse(dockerfile.split("LABEL com.spawnfile.training.recipe=")[1]!.trim()); + images.set(args[args.indexOf("--tag") + 1]!, digest); + return { code: 0, stdout: "built", stderr: "" }; + } + const target = args[4]!; + if (target === image) return { code: 0, stdout: image, stderr: "" }; + const digest = images.get(target); + return digest ? { code: 0, stdout: `${JSON.stringify(image)}\n${JSON.stringify(digest)}`, stderr: "" } : { code: 1, stdout: "", stderr: "missing" }; + }; + return { calls, images, process, get builtContext() { return builtContext; } }; +} diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts new file mode 100644 index 00000000..6ed66f18 --- /dev/null +++ b/src/compiler/training/preparation/image.ts @@ -0,0 +1,83 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { TrainingDockerProcess } from "../container/process.js"; +import type { TrainingImageBuild } from "./contract.js"; +import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; + +const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); +export const trainingAssets = path.extname(fileURLToPath(import.meta.url)) === ".ts" + ? path.join(packageRoot, "runtime-images/training") : fileURLToPath(new URL("./assets/", import.meta.url)); +export interface TrainingImagePlan { digest: string; files: SealedFile[]; dockerfile: string; entry: string; build: TrainingImageBuild } + +async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json")): Promise { + const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")); + const lock = JSON.parse(await readFile(lockSource, "utf8")); + if (![2, 3].includes(lock.lockfileVersion) || !lock.packages?.[""] || + JSON.stringify(manifest.dependencies ?? {}) !== JSON.stringify(lock.packages[""].dependencies ?? {})) throw Error(`Training package manifest/lock mismatch: ${target}`); + if (Object.values(lock.packages).some(entry => { + const item = entry as { link?: boolean; resolved?: string }; + return item.link || item.resolved?.startsWith("file:") || item.resolved?.startsWith("../"); + })) throw Error(`Training package ${target} has unsupported local dependencies; provide a complete registry-locked distribution`); + return [await sealFile(path.join(root, "package.json"), `${target}/package.json`), + await sealFile(lockSource, `${target}/package-lock.json`), + ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true }) : []]; +} + +export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { + const resolve = (value: string) => path.resolve(root, value); + for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, build.bootstrap]) assertInputRoot(resolve(source), auth); + const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); + const dockerfile = await readFile(path.join(assets, "Dockerfile"), "utf8"); + const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot + ? path.join(ownRoot, "package-lock.json") : path.join(assets, "package-lock.json"); + const files = [ + ...await packageFiles(resolve(build.paideia), "paideia", true), + ...await packageFiles(ownRoot, "spawnfile", true, ownLock), + ...await packageFiles(resolve(build.claude), "claude", false), + ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), + ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), + ...await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }), + await sealFile(resolve(build.grok.source), "grok"), + await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), + await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json") + ]; + if (files.find(file => file.destination === "grok")!.sha256 !== build.grok.sha256) throw Error("Grok executable digest mismatch"); + for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js"]) { + if (!files.some(file => file.destination === required)) throw Error(`Training distribution is missing ${required}`); + } + const entry = `#!/bin/sh\nexec /usr/local/bin/node --experimental-strip-types ${JSON.stringify(`/opt/training/integration/${build.integration.entry}`)} "$@"\n`; + const digest = hashJson({ recipe: build.recipe, nativeImage: build.nativeImage, pythonImage: build.pythonImage, platform: build.platform, + files: fileIdentity(files), dockerfile, entry }); + return { digest, files, dockerfile, entry, build }; +} + +/** Cache is content-addressed and still requires a matching immutable image and label. */ +export async function buildTrainingImage(plan: TrainingImagePlan, options: { + parent: string; dockerContext: string; process: TrainingDockerProcess; timeoutMs: number; signal?: AbortSignal; + streams: { stdout(line: string): void; stderr(line: string): void }; +}): Promise<{ imageId: string; cached: boolean }> { + const tag = `spawnfile-training:${plan.digest.slice(7)}`; + const call = (args: string[], stream = false) => options.process(["--context", options.dockerContext, ...args], { + timeoutMs: options.timeoutMs, signal: options.signal, ...stream ? options.streams : {} + }); + const inspect = async (): Promise => { + const result = await call(["image", "inspect", tag, "--format", '{{json .Id}}\n{{json (index .Config.Labels "com.spawnfile.training.recipe")}}']); + if (result.code !== 0) return undefined; + try { const [id, digest] = result.stdout.trim().split("\n").map(line => JSON.parse(line)); + return /^sha256:[a-f0-9]{64}$/u.test(id) && digest === plan.digest ? id : undefined; + } catch { return undefined; } + }; + const cached = await inspect(); if (cached) return { imageId: cached, cached: true }; + const staging = await mkdtemp(path.join(options.parent, ".spawnfile-training-image-")); + try { + await copySealed(plan.files, staging); + await writeFile(path.join(staging, "Dockerfile"), `${plan.dockerfile}\nLABEL com.spawnfile.training.recipe=${JSON.stringify(plan.digest)}\n`, { mode: 0o600 }); + await writeFile(path.join(staging, "train"), plan.entry, { mode: 0o755 }); + const result = await call(["build", "--platform", plan.build.platform, "--build-arg", `NATIVE_IMAGE=${plan.build.nativeImage}`, + "--build-arg", `PYTHON_IMAGE=${plan.build.pythonImage}`, "--tag", tag, staging], true); + if (result.code !== 0) throw Error("Training image build failed; inspect the streamed build diagnostic"); + const imageId = await inspect(); if (!imageId) throw Error("Training image build did not produce a verified immutable image"); + return { imageId, cached: false }; + } finally { await rm(staging, { recursive: true, force: true }); } +} diff --git a/src/compiler/training/preparation/index.ts b/src/compiler/training/preparation/index.ts new file mode 100644 index 00000000..1352a64d --- /dev/null +++ b/src/compiler/training/preparation/index.ts @@ -0,0 +1,3 @@ +export { prepareTraining } from "./prepare.js"; +export { trainingPreparationSchema, parseTrainingMappedPreparation } from "./contract.js"; +export type { TrainingMappedPreparation, TrainingPreparationConfig } from "./contract.js"; diff --git a/src/compiler/training/preparation/inputs.ts b/src/compiler/training/preparation/inputs.ts new file mode 100644 index 00000000..9cc894c5 --- /dev/null +++ b/src/compiler/training/preparation/inputs.ts @@ -0,0 +1,66 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; +import type { TrainingPreparationConfig } from "./contract.js"; +import { assertInputRoot, copySealed, exactPath, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; + +const execute = promisify(execFile); +const git = async (cwd: string, args: string[]) => (await execute("git", args, { cwd, timeout: 120000, maxBuffer: 8 * 1024 * 1024 })).stdout; +export interface PlannedInput { + id: string; source: string; destination: string; digest: string; + git?: { revision: string; tree: string; common: string; overlays: SealedFile[] }; +} + +export async function planInputs(config: TrainingPreparationConfig, root: string, auth: string[]): Promise { + return Promise.all(config.inputs.map(async input => { + const source = await exactPath(path.resolve(root, input.source)); + assertInputRoot(source, auth); + if (!input.git) return { id: input.id, source, destination: input.destination, digest: hashJson(fileIdentity(await sealTree(source, "input", { ignoreGit: true, internalSymlinks: true }))) }; + if ((await git(source, ["rev-parse", "--show-prefix"])).trim()) throw Error("Pinned Git input source must be a repository root"); + const common = await exactPath((await git(source, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim()); + const tree = (await git(source, ["rev-parse", `${input.git.revision}^{tree}`])).trim(); + const entries = (await git(source, ["ls-tree", "-rz", "--full-tree", input.git.revision])).split("\0").filter(Boolean); + if (!/^[a-f0-9]{40}$/u.test(tree) || entries.some(entry => !/^(100644|100755) blob [a-f0-9]{40}\t/u.test(entry))) throw Error("Pinned Git inputs require regular files; symlinks and submodules are unsupported"); + const overlays: SealedFile[] = []; + for (const overlay of input.git.overlays) { + const overlaySource = path.resolve(root, overlay.source); assertInputRoot(overlaySource, auth); + const file = await sealFile(overlaySource, overlay.path); + if (file.sha256 !== overlay.sha256 || overlays.some(previous => previous.destination === file.destination)) throw Error("Git overlay digest mismatch or duplicate destination"); + overlays.push(file); + } + return { id: input.id, source, destination: input.destination, git: { revision: input.git.revision, tree, common, overlays }, + digest: hashJson({ revision: input.git.revision, tree, overlays: fileIdentity(overlays) }) }; + })); +} + +/** A real self-contained Git object store, never a copied worktree pointer. */ +export async function stageInput(input: PlannedInput, target: string): Promise { + if (!input.git) return input.source; + await mkdir(target, { mode: 0o700 }); + await git(target, ["init", "--quiet"]); + await git(target, ["-c", "protocol.file.allow=always", "fetch", "--quiet", "--depth=1", pathToFileURL(input.git.common).href, input.git.revision]); + await git(target, ["-c", "advice.detachedHead=false", "checkout", "--quiet", "--detach", input.git.revision]); + if ((await git(target, ["rev-parse", "HEAD^{tree}"])).trim() !== input.git.tree || + (await git(target, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim() !== path.join(target, ".git")) throw Error("Git training snapshot identity mismatch"); + await git(target, ["fsck", "--full", "--no-dangling"]); + await copySealed(input.git.overlays, target); + return target; +} + +export async function verifyCanonicalPins(inputs: PlannedInput[], sources: readonly { sourcePath: string; sha256: string }[], staged: readonly string[]): Promise { + for (const source of sources) { + const index = inputs.findIndex(input => source.sourcePath === input.source || source.sourcePath.startsWith(input.source + path.sep)); + if (index < 0) throw Error("Canonical training source is outside declared inputs"); + if (!inputs[index]!.git) continue; + const file = path.join(staged[index]!, path.relative(inputs[index]!.source, source.sourcePath)); + if ((await sealFile(file, "pin")).sha256 !== source.sha256) throw Error("Pinned Git snapshot differs from the selected canonical agent"); + } +} + +export async function readBoundedJson(file: string): Promise { + const source = await readFile(file, "utf8"); + if (Buffer.byteLength(source) > 1024 * 1024) throw Error("Training preparation JSON exceeds 1 MiB"); + return JSON.parse(source); +} diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts new file mode 100644 index 00000000..dc30b1d3 --- /dev/null +++ b/src/compiler/training/preparation/prepare.test.ts @@ -0,0 +1,91 @@ +import { execFile } from "node:child_process"; +import { lstat, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it } from "vitest"; +import { prepareTraining } from "./prepare.js"; +import { preparationFixture, imageDocker, image } from "./fixtures.test-helper.js"; +import { parseTrainingMappedPreparation } from "./contract.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { const f = await preparationFixture(); roots.push(f.root); const docker = imageDocker(); + return { ...f, docker, options: { configPath: f.configPath, context: f.context, args: f.args, dryRun: false, + process: docker.process, timeoutMs: 30000, packageRoot: path.join(f.root, "own"), streams: { stdout() {}, stderr() {} } } }; } + +it("prepares cold and warm runs, preserves mapped identity and resumes without rebuilding", async () => { + const f = await fixture(); + const first = await prepareTraining(f.options); if ("dryRun" in first) throw Error("actual preparation expected"); + expect(first.image).toBe(image); + expect((await lstat(path.join(f.root, "output"))).isDirectory()).toBe(true); + const mapped = parseTrainingMappedPreparation(JSON.parse(await readFile(first.preparationPath, "utf8"))); + expect(mapped.bindings).toEqual([{ inputId: "project", destination: "/run/training/inputs/project" }, { inputId: "settings", destination: "/run/training/inputs/settings" }]); + expect(JSON.stringify(mapped)).not.toContain(f.root); + const resumed = await prepareTraining({ ...f.options, args: [...f.args, "--resume"] }); + expect(resumed).toEqual({ ...first, args: [...first.args, "--resume"] }); + f.config.output.source = "second"; await f.save(); + await prepareTraining({ ...f.options, args: ["--train", f.args[1]!, "--out", path.join(f.root, "second")] }); + expect(f.docker.calls.filter(args => args[2] === "build")).toHaveLength(1); + expect(await readdir(f.root)).not.toContain(path.basename(f.docker.builtContext!)); +}); + +it("dry-run reads declarations without Docker, auth access, output or staging mutations", async () => { + const f = await fixture(); f.config.auth[0]!.source = "missing-auth"; await f.save(); + const before = await readdir(f.root); + expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); + expect(f.docker.calls).toEqual([]); expect(await readdir(f.root)).toEqual(before); +}); + +it("rejects changed executable, fixture, declaration and saved image on exact resume", async () => { + const f = await fixture(); await prepareTraining(f.options); + await f.put("paideia/dist/src/cli/main.js", "changed"); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("changed"); + await f.put("paideia/dist/src/cli/main.js"); await f.put("project/train.yaml", "changed"); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("changed"); + await f.put("project/train.yaml"); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"], process: async args => args[0] === "context" + ? { code: 0, stdout: '"unix:///socket"', stderr: "" } : { code: 1, stdout: "", stderr: "" } })).rejects.toThrow("image"); +}); + +it("materializes a real pinned worktree with overlays and rejects snapshot tampering", async () => { + const f = await fixture(), git = promisify(execFile), project = f.context.project.root; + const run = (args: string[]) => git("git", args, { cwd: project }); + await run(["init", "-q"]); await run(["add", "."]); + await run(["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "seed"]); + const revision = (await run(["rev-parse", "HEAD"])).stdout.trim(); + const { sha } = await import("./fixtures.test-helper.js"); + await f.put("overlay", "generated"); f.config.inputs[0]!.git = { revision, overlays: [{ source: "overlay", path: "tools.tar", sha256: sha("generated") }] }; await f.save(); + const prepared = await prepareTraining(f.options); if ("dryRun" in prepared) throw Error("actual expected"); + expect(prepared.context.project.root).not.toBe(project); + expect(await readFile(path.join(prepared.context.project.root, "tools.tar"), "utf8")).toBe("generated"); + expect((await lstat(path.join(prepared.context.project.root, ".git"))).isDirectory()).toBe(true); + await prepareTraining({ ...f.options, args: [...f.args, "--resume"] }); + await writeFile(path.join(prepared.context.project.root, "tools.tar"), "tampered"); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("snapshot changed"); +}); + +it("rejects auth/input overlap, missing auth, unsafe context and output mismatch before launch", async () => { + const f = await fixture(); f.config.auth[0]!.source = "project/Spawnfile"; await f.save(); + await expect(prepareTraining(f.options)).rejects.toThrow("auth leaf"); expect(f.docker.calls).toEqual([]); + f.config.auth[0]!.source = "missing"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow(); + f.config.auth[0]!.source = "auth"; f.config.output.source = "project/run"; await f.save(); + await expect(prepareTraining(f.options)).rejects.toThrow("overlaps"); + f.config.output.source = "output"; await f.save(); + await expect(prepareTraining({ ...f.options, process: async () => ({ code: 0, stdout: '"tcp://remote"', stderr: "" }) })).rejects.toThrow("Unix"); + await expect(prepareTraining({ ...f.options, args: ["--out", "/different"] })).rejects.toThrow("match configured output"); +}); + +it("rejects package symlinks, unsupported local locks and malformed Grok pins before building", async () => { + const f = await fixture(); + await symlink(path.join(f.root, "auth"), path.join(f.root, "integration", "leak")); + await expect(prepareTraining(f.options)).rejects.toThrow("symlinks"); await rm(path.join(f.root, "integration", "leak")); + await f.put("claude/package-lock.json", JSON.stringify({ lockfileVersion: 3, packages: { "": { dependencies: {} }, "node_modules/local": { link: true } } })); + await expect(prepareTraining(f.options)).rejects.toThrow("unsupported local"); + expect(f.docker.calls).toEqual([]); +}); + +it("uses an explicit immutable image without package preparation", async () => { + const f = await fixture(); f.config.image = { ref: image }; await f.save(); + const result = await prepareTraining(f.options); expect(result).toMatchObject({ image }); + expect(f.docker.calls.some(args => args[2] === "build")).toBe(false); +}); diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts new file mode 100644 index 00000000..ddf23117 --- /dev/null +++ b/src/compiler/training/preparation/prepare.ts @@ -0,0 +1,107 @@ +import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TrainingContext } from "../contract.js"; +import { trainingContainerConfigSchema } from "../container/contract.js"; +import type { TrainingDockerProcess } from "../container/process.js"; +import { trainingPreparationSchema, parseTrainingMappedPreparation, type TrainingMappedPreparation } from "./contract.js"; +import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } from "./files.js"; +import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; +import { planTrainingImage, buildTrainingImage } from "./image.js"; + +export interface PrepareTrainingOptions { + configPath: string; context: TrainingContext; args: readonly string[]; dryRun: boolean; + process: TrainingDockerProcess; timeoutMs: number; signal?: AbortSignal; + streams: { stdout(line: string): void; stderr(line: string): void }; + /** Test-only package fixture; production always resolves its own installed distribution. */ + packageRoot?: string; +} +export interface PreparedTraining { + digest: string; image: string; configPath: string; preparationPath: string; context: TrainingContext; args: string[]; +} + +/** Reads only until the explicit dry-run boundary; preparation never executes project code. */ +export async function prepareTraining(options: PrepareTrainingOptions): Promise { + const config = trainingPreparationSchema.parse(await readBoundedJson(options.configPath)); + const root = path.dirname(path.resolve(options.configPath)); + const auth = config.auth.map(entry => ({ ...entry, source: path.resolve(root, entry.source) })); + const output = path.resolve(root, config.output.source), parent = path.dirname(output); + assertInputRoot(output, auth.map(entry => entry.source)); + if (await realpath(parent) !== parent) throw Error("Training output parent must be canonical and already exist"); + const inputs = await planInputs(config, root, auth.map(entry => entry.source)); + const roots = inputs.map(input => input.source); + for (let index = 0; index < inputs.length; index++) { + const input = inputs[index]!; + if (within(input.source, output) || within(output, input.source)) throw Error("Training output overlaps readonly input"); + for (const other of inputs.slice(index + 1)) if (within(input.source, other.source) || within(other.source, input.source) || + within(input.destination, other.destination) || within(other.destination, input.destination)) throw Error("Training inputs overlap"); + } + const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot) : undefined; + const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image, + canonical: options.context.project.sourceDigest }); + if (options.dryRun) return { digest, dryRun: true }; + options.signal?.throwIfAborted(); + for (const entry of auth) if (await exactPath(entry.source) !== entry.source || !(await lstat(entry.source)).isFile()) throw Error("Training auth must be a canonical regular leaf"); + const execute = (args: string[]) => options.process(args, { timeoutMs: options.timeoutMs, signal: options.signal }); + const endpoint = await execute(["context", "inspect", config.dockerContext, "--format", "{{json .Endpoints.docker.Host}}"]); + if (endpoint.code !== 0 || !/^unix:\/\//u.test(JSON.parse(endpoint.stdout))) throw Error("Training preparation requires a local Unix Docker context"); + const staging = path.join(parent, `.spawnfile-training-preparation-${hashJson(output).slice(7, 23)}`); + const configPath = path.join(staging, "launch.json"), preparationPath = path.join(staging, "mapped.json"), statePath = path.join(staging, "state.json"); + const resume = options.args.includes("--resume"); + let image: string, staged: string[]; + if (resume) { + const previous = JSON.parse(await readFile(statePath, "utf8")) as { digest: string; image: string; staged: string[]; snapshots: (string | null)[] }; + if (previous.digest !== digest || !/^sha256:[a-f0-9]{64}$/u.test(previous.image) || + JSON.stringify(previous.staged) !== JSON.stringify(inputs.map(input => input.git ? path.join(staging, input.id) : input.source))) throw Error("Training preparation changed; exact resume rejected"); + image = previous.image; staged = previous.staged; + const snapshots = await Promise.all(inputs.map(async (input, index) => input.git ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true }))) : null)); + if (JSON.stringify(previous.snapshots) !== JSON.stringify(snapshots)) throw Error("Persisted training input snapshot changed"); + const mapped = parseTrainingMappedPreparation(await readBoundedJson(preparationPath)); + if (mapped.preparationDigest !== digest || mapped.imageId !== image) throw Error("Persisted training preparation identity mismatch"); + await verifyCanonicalPins(inputs, options.context.sources, staged); + } else { + let owned = false; + try { + await mkdir(staging, { mode: 0o700 }); owned = true; + try { await mkdir(output, { mode: 0o700 }); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !(await lstat(output)).isDirectory() || await realpath(output) !== output) throw error; } + staged = await Promise.all(inputs.map(input => stageInput(input, path.join(staging, input.id)))); + await verifyCanonicalPins(inputs, options.context.sources, staged); + image = imagePlan ? (await buildTrainingImage(imagePlan, { parent, dockerContext: config.dockerContext, process: options.process, + timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams })).imageId : "ref" in config.image ? config.image.ref : ""; + if (!image.startsWith("sha256:")) { + const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]); + if (inspected.code !== 0) throw Error("Training image is unavailable"); image = inspected.stdout.trim(); + } + const mapped: TrainingMappedPreparation = { version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, + bindings: inputs.map(input => ({ inputId: input.id, destination: input.destination })), outputRoot: "/run/training/output", + packagePaths: { spawnfile: "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy", + nativeWorker: "/opt/training/paideia/dist/src/adapters/daimon-native", integration: "/opt/training/integration", bootstrap: "/opt/training/bootstrap" }, integration: config.integration }; + const launch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, + inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth }); + await writeFile(configPath, JSON.stringify(launch), { flag: "wx", mode: 0o600 }); + await writeFile(preparationPath, JSON.stringify(parseTrainingMappedPreparation(mapped)), { flag: "wx", mode: 0o400 }); + const snapshots = await Promise.all(inputs.map(async (input, index) => input.git ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true }))) : null)); + await writeFile(statePath, JSON.stringify({ digest, image, staged, snapshots }), { flag: "wx", mode: 0o600 }); + } catch (error) { if (owned) await rm(staging, { recursive: true, force: true }); throw error; } + } + const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]); + if (inspected.code !== 0 || inspected.stdout.trim() !== image) throw Error("Saved training image is missing or changed"); + const map = (file: string): string => { + const absolute = path.resolve(file); + const index = roots.findIndex(source => within(source, absolute)); + return index < 0 ? absolute : path.join(staged[index]!, path.relative(roots[index]!, absolute)); + }; + const context: TrainingContext = { ...options.context, + project: { ...options.context.project, root: map(options.context.project.root), manifest: map(options.context.project.manifest) }, + agent: { ...options.context.agent, source: map(options.context.agent.source) }, + sources: options.context.sources.map(source => ({ ...source, sourcePath: map(source.sourcePath) })), + documents: options.context.documents.map(source => ({ ...source, sourcePath: map(source.sourcePath) })), + skills: options.context.skills.map(source => ({ ...source, sourcePath: map(source.sourcePath) })) }; + const args = [...options.args]; + for (let index = 0; index < args.length; index++) { + if (["--train", "--test", "--cost-config"].includes(args[index]!)) args[++index] = map(args[index]!); + else if (args[index] === "--resource") { const value = args[++index]!, split = value.indexOf("="); args[index] = `${value.slice(0, split)}=${map(value.slice(split + 1))}`; } + else if (args[index] === "--out" && path.resolve(args[++index]!) !== output) throw Error("Training --out must match configured output"); + } + return { digest, image, configPath, preparationPath, context, args }; +} diff --git a/tsconfig.json b/tsconfig.json index 4cdd75da..dc49e39f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,6 +24,7 @@ "src/deployment/native/build.ts", "src/deployment/native/copyArtifacts.ts", "src/evidenceExportHelper/copyAssets.ts", - "src/runtime/copyScaffoldAssets.ts" + "src/runtime/copyScaffoldAssets.ts", + "src/compiler/training/preparation/copyAssets.ts" ] } From 9ee55086835fb10d9cbcaf5cc16d3f0a396e56d6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 03:30:30 +0200 Subject: [PATCH 08/84] feat: complete declarative training image and input preparation --- runtime-images/training/Dockerfile | 2 +- specs/TRAINING.md | 5 +- specs/TRAINING_CONTAINERS.md | 79 ++++++++++++++++++- src/cli/trainCommand.ts | 2 +- src/cli/trainingPreparation.test.ts | 50 ++++++++++++ .../training/preparation/contract.test.ts | 24 ++++++ src/compiler/training/preparation/contract.ts | 4 +- .../training/preparation/files.test.ts | 32 ++++++++ .../training/preparation/image.test.ts | 78 ++++++++++++++++++ src/compiler/training/preparation/image.ts | 7 +- .../training/preparation/inputs.test.ts | 64 +++++++++++++++ src/compiler/training/preparation/inputs.ts | 28 ++++++- .../training/preparation/prepare.test.ts | 41 +++++++++- src/compiler/training/preparation/prepare.ts | 22 ++++-- 14 files changed, 417 insertions(+), 21 deletions(-) create mode 100644 src/cli/trainingPreparation.test.ts create mode 100644 src/compiler/training/preparation/contract.test.ts create mode 100644 src/compiler/training/preparation/files.test.ts create mode 100644 src/compiler/training/preparation/image.test.ts create mode 100644 src/compiler/training/preparation/inputs.test.ts diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 741e06a4..17286c93 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -37,7 +37,7 @@ COPY integration /opt/training/integration COPY bootstrap /opt/training/bootstrap COPY train /opt/training/bin/train RUN chmod 0555 /opt/training/bin/train /opt/training/bin/grok \ - && chmod -R a+rX /opt/training/bootstrap /opt/training/integration \ + && chmod -R a+rX /opt/training \ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \ && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \ diff --git a/specs/TRAINING.md b/specs/TRAINING.md index 9abe17df..e425c804 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -5,8 +5,9 @@ Paideia CLI. Spawnfile owns canonical source resolution and native compilation; Paideia owns datasets, evaluation, cost planning, optimization and isolated trials. The [container boundary](TRAINING_CONTAINERS.md) runs the complete experiment -inside one immutable image. Actual training requires explicit image and mount -configuration; dry-run remains a host-only estimate. +inside one immutable image. The v2 training configuration prepares its pinned +image and inputs behind the same command; the v1 image/mount path remains +available. Dry-run remains a host-only estimate. ```sh spawnfile train ./Spawnfile --agent agent:writer \ diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 9338270e..95f8f784 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -8,7 +8,7 @@ host-only estimate and does not use Docker or model authentication. ## Launch contract -Actual training requires both `--training-image` (a digest reference or immutable +The advanced v1 path requires both `--training-image` (a digest reference or immutable image ID already installed locally) and `--training-config` (JSON below). `--paideia-command` applies only to dry-run; actual execution always starts the image-owned `/opt/training/bin/train`. No shell or executable from the host is @@ -100,3 +100,80 @@ This is a single-container boundary. Native Daimon sandbox policy still controls individual agent access inside it. The image build, usable private integration, authenticated model run and live terminal receipt require end-to-end verification before calling this deployment ready. + +## Declarative preparation (v2) + +Use the same command with `--training-config evals/training.json`; v2 owns the +image, so omit `--training-image`. `--train`, `--out`, `--resume` and `--view` +retain their meanings. Docker must already be available in the named local +context. The command does not configure or start a machine-global VM. + +```json +{ + "version": "spawnfile.training-container.v2", + "dockerContext": "desktop-linux", + "image": { + "build": { + "recipe": "daimon-dspy.v1", + "nativeImage": "registry.example/native@sha256:<64 hex characters>", + "pythonImage": "docker.io/library/python@sha256:<64 hex characters>", + "platform": "linux/arm64", + "paideia": "./packages/paideia", + "bridge": "./packages/dspy", + "claude": "./packages/claude", + "grok": { "source": "./bin/grok", "sha256": "sha256:<64 hex characters>" }, + "integration": { "source": "./integration", "entry": "container/entry.ts" }, + "bootstrap": "./bootstrap" + } + }, + "integration": { "settings": { "input": "evals", "path": "settings.json" } }, + "inputs": [ + { "id": "project", "source": "../project", "destination": "/run/training/inputs/project" }, + { "id": "evals", "source": ".", "include": ["settings.json", "train.paideia.yaml", "test.paideia.yaml"], "destination": "/run/training/inputs/evals" } + ], + "output": { "source": "../runs/author", "destination": "/run/training/output" }, + "auth": [] +} +``` + +Replace digest placeholders with verified immutable pins. This recipe supports +Daimon with DSPy only; it does not promise other native runtime layouts. +`bootstrap` is optional: omit it when the installed integration builds its own +verified runtime bootstrap from maintained source inside the writable output. An +already-built image can instead use `"image": {"ref": "sha256:..."}`. + +Source paths resolve relative to the config file. The output parent must exist; +the command creates the output and its private preparation directory. Plain +inputs are readonly bindings. `include` stages only declared files/subtrees, +preserving their paths, and avoids mounting historical runs or host dependencies. + +A Git input may declare `"git": {"revision": "", "overlays": +[{"source": "./generated/tools.tar", "path": "tools.tar", "sha256": "sha256:..."}]}`. +Its source must be a repository root. Spawnfile creates a self-contained pinned +Git snapshot and adds new hash-verified generated files. It never copies a +worktree's `.git` pointer or overwrites tracked files. The selected canonical +agent's pinned source bytes must match that snapshot. Confined internal links +are supported; escaping links and submodules are rejected. Git and local +`include` modes are separate. + +Build staging includes explicit distributions, locks, the installed Spawnfile, +and its packaged recipe. Local `file:`/link dependency closures are unsupported +and fail before building; provide complete registry-locked distributions instead. +The recipe supplies the Daimon peer from its native parent. Credentials and +datasets never enter the image context. Cache identity includes actual source, +lock, executable and recipe bytes, parent images, architecture and entrypoint; +reuse also verifies the image ID and recipe label in the selected Docker context. + +The installed integration reads `/run/paideia/preparation.json` using +`parseTrainingMappedPreparation` from `spawnfile/training`. This protected +`spawnfile.training-preparation.v1` receipt contains the preparation digest, +immutable image ID, named container bindings, output root, installed package +paths and the input-relative settings reference. It contains no host paths or +credentials. The integration resolves its own typed settings; Spawnfile never +rewrites arbitrary JSON strings or invokes a host project preparation script. + +Dry-run hashes and validates local inputs but performs no Docker, auth or +preparation writes. Exact resume checks current source identity and preserved +snapshots/receipts, then requires the saved immutable image. It never rebuilds +a replacement under an existing experiment identity. Paideia independently +checks its experiment checkpoint and cumulative budgets. diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index 0c857aa7..d9c6a8bd 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -22,7 +22,7 @@ export const registerTrainCommand = ( .option("--agent ", "Exact canonical agent node id (inferred only for a single-agent project)") .option("--paideia-command ", "Installed Paideia executable; no shell or automatic install", "paideia") .option("--training-image ", "Pinned image containing the complete training environment") - .option("--training-config ", "Explicit local Docker inputs, output and auth leaf bindings") + .option("--training-config ", "V2 image recipe and declared inputs, or advanced v1 local bindings") .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") .option("--resume", "Resume the exact persisted training experiment in --out"); for (const name of forwarded) { diff --git a/src/cli/trainingPreparation.test.ts b/src/cli/trainingPreparation.test.ts new file mode 100644 index 00000000..a1e39f9c --- /dev/null +++ b/src/cli/trainingPreparation.test.ts @@ -0,0 +1,50 @@ +import { chmod, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { imageDocker, image, preparationFixture } from "../compiler/training/preparation/fixtures.test-helper.js"; +import type { TrainingDockerProcess } from "../compiler/training/container/process.js"; +const state = vi.hoisted(() => ({ run: undefined as TrainingDockerProcess | undefined })); +vi.mock("../compiler/training/container/process.js", () => ({ runTrainingDocker: ((args, options) => state.run!(args, options)) satisfies TrainingDockerProcess })); +import { runCli } from "./runCli.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +it("runs cold, warm and exact-resume through the actual public command and existing launcher", async () => { + const f = await preparationFixture(); roots.push(f.root); const docker = imageDocker(); let name = "", output = "", preparation = ""; + const id = "b".repeat(64), calls: string[][] = []; + state.run = async (args, options) => { + calls.push([...args]); + const result = (stdout: string) => ({ code: 0, stdout, stderr: "" }); + if (args[2] === "create") { + name = args[args.indexOf("--name") + 1]!; + output = args.find(value => value.includes("dst=/run/training/output"))!.split(",").find(value => value.startsWith("src="))!.slice(4); + preparation = args.find(value => value.includes("dst=/run/paideia/preparation.json"))!.split(",").find(value => value.startsWith("src="))!.slice(4); + return result(id); + } + if (args[2] === "start") { await writeFile(path.join(output, "index.json"), "{}"); options.stdout?.('{"status":"completed","index":"/run/training/output/index.json"}'); return result(""); } + if (args[2] === "inspect") return result(args[4] === "{{json .State}}" ? JSON.stringify({ Running: false, ExitCode: 0 }) : [id, `/${name}`, image, { "com.spawnfile.training.owner": name }].map(value => JSON.stringify(value)).join("\n")); + if (args[2] === "rm" || args[2] === "container") return result(""); + return docker.process(args, options); + }; + const errors: string[] = []; + const args = () => ["train", f.context.project.root, "--training-config", f.configPath, "--train", f.args[1]!, "--out", path.join(f.root, f.config.output.source)]; + const run = (extra: string[] = []) => runCli([...args(), ...extra], { streams: { stdout() {}, stderr: value => errors.push(value) } }); + expect(await run()).toBe(0); expect(errors).toEqual([]); + expect(JSON.parse(await readFile(preparation, "utf8")).imageId).toBe(image); + expect(await run(["--resume"])).toBe(0); + f.config.output.source = "warm"; await f.save(); expect(await run()).toBe(0); + expect(calls.filter(args => args[2] === "build")).toHaveLength(1); + expect(calls.filter(args => args[2] === "create")).toHaveLength(3); +}); + +it("keeps v2 public dry-run free of Docker, auth and preparation writes", async () => { + const f = await preparationFixture(); roots.push(f.root); f.config.image = { ref: image }; f.config.auth[0]!.source = "missing"; await f.save(); + const command = path.join(f.root, "estimate"); await writeFile(command, '#!/usr/bin/env node\nconsole.log(JSON.stringify({schema:"paideia.training-cost-plan.v1",modelCallsMade:0}));\n'); await chmod(command, 0o755); + state.run = async () => { throw Error("Dry-run must never call Docker"); }; + const output: string[] = []; + expect(await runCli(["train", f.context.project.root, "--train", f.args[1]!, "--training-config", f.configPath, "--paideia-command", command, "--dry-run"], { + streams: { stdout: value => output.push(value), stderr() {} } + })).toBe(0); + expect(JSON.parse(output.at(-1)!)).toMatchObject({ modelCallsMade: 0 }); + await expect(readFile(path.join(f.root, "output/index.json"))).rejects.toThrow(); +}); diff --git a/src/compiler/training/preparation/contract.test.ts b/src/compiler/training/preparation/contract.test.ts new file mode 100644 index 00000000..d2708be4 --- /dev/null +++ b/src/compiler/training/preparation/contract.test.ts @@ -0,0 +1,24 @@ +import { expect, it } from "vitest"; +import { parseTrainingMappedPreparation, trainingPreparationSchema } from "./contract.js"; + +const config = () => ({ version: "spawnfile.training-container.v2", dockerContext: "local", image: { ref: `sha256:${"a".repeat(64)}` }, + integration: { settings: { input: "project", path: "." } }, inputs: [{ id: "project", source: "project", destination: "/run/training/inputs/project" }], + output: { source: "out", destination: "/run/training/output" }, auth: [] }); +it("keeps authoring strict and rejects ambiguous input and settings references", () => { + const base = config(); expect(trainingPreparationSchema.parse(base)).toEqual(base); + expect(() => trainingPreparationSchema.parse({ ...base, inputs: [...base.inputs, ...base.inputs] })).toThrow("unique"); + expect(() => trainingPreparationSchema.parse({ ...base, auth: [{ source: "one", provider: "grok" }, { source: "two", provider: "grok" }] })).toThrow("unique"); + expect(() => trainingPreparationSchema.parse({ ...base, integration: { settings: { input: "missing", path: "settings.json" } } })).toThrow("declared"); + expect(() => trainingPreparationSchema.parse({ ...base, inputs: [{ ...base.inputs[0], include: ["x"], git: { revision: "a".repeat(40) } }] })).toThrow("separate"); + for (const bad of ["../escape", "/absolute", "a\\b", "a//b"]) expect(() => trainingPreparationSchema.parse({ ...base, inputs: [{ ...base.inputs[0], include: [bad] }] })).toThrow(); +}); +it("validates the public mapped receipt without accepting host paths or missing IDs", () => { + const receipt = { version: "spawnfile.training-preparation.v1", preparationDigest: `sha256:${"a".repeat(64)}`, imageId: `sha256:${"b".repeat(64)}`, + bindings: [{ inputId: "project", destination: "/run/training/inputs/project" }], outputRoot: "/run/training/output", + packagePaths: Object.fromEntries(["spawnfile", "paideia", "bridge", "nativeWorker", "integration", "bootstrap"].map(key => [key, `/opt/training/${key}`])), + integration: { settings: { input: "project", path: "." } } }; + expect(parseTrainingMappedPreparation(receipt)).toEqual(receipt); + expect(() => parseTrainingMappedPreparation({ ...receipt, bindings: [...receipt.bindings, ...receipt.bindings] })).toThrow("IDs"); + expect(() => parseTrainingMappedPreparation({ ...receipt, integration: { settings: { input: "absent", path: "." } } })).toThrow("IDs"); + expect(() => parseTrainingMappedPreparation({ ...receipt, packagePaths: { ...receipt.packagePaths, bridge: "/Users/operator/.config" } })).toThrow(); +}); diff --git a/src/compiler/training/preparation/contract.ts b/src/compiler/training/preparation/contract.ts index 2efe241c..1e23e1af 100644 --- a/src/compiler/training/preparation/contract.ts +++ b/src/compiler/training/preparation/contract.ts @@ -12,7 +12,7 @@ export const trainingBuildSchema = z.object({ platform: z.enum(["linux/arm64", "linux/amd64"]), paideia: local, bridge: local, claude: local, grok: z.object({ source: local, sha256: sha }).strict(), - integration: z.object({ source: local, entry: relative.refine(value => /^[A-Za-z0-9._/-]+$/u.test(value)) }).strict(), bootstrap: local + integration: z.object({ source: local, entry: relative.refine(value => /^[A-Za-z0-9._/-]+$/u.test(value)) }).strict(), bootstrap: local.optional() }).strict(); export const trainingPreparationSchema = z.object({ version: z.literal("spawnfile.training-container.v2"), @@ -21,6 +21,7 @@ export const trainingPreparationSchema = z.object({ integration: z.object({ settings: reference }).strict(), inputs: z.array(z.object({ id: z.string().regex(/^[A-Za-z][A-Za-z0-9_-]{0,63}$/u), source: local, destination, + include: z.array(relative).min(1).max(256).optional(), git: z.object({ revision: z.string().regex(/^[a-f0-9]{40}$/u), overlays: z.array(z.object({ source: local, path: relative, sha256: sha }).strict()).max(256).default([]) }).strict().optional() @@ -31,6 +32,7 @@ export const trainingPreparationSchema = z.object({ if (new Set(value.inputs.map(input => input.id)).size !== value.inputs.length) context.addIssue({ code: "custom", message: "Input IDs must be unique" }); if (new Set(value.auth.map(auth => auth.provider)).size !== value.auth.length) context.addIssue({ code: "custom", message: "Auth providers must be unique" }); if (!value.inputs.some(input => input.id === value.integration.settings.input)) context.addIssue({ code: "custom", message: "Integration settings require a declared input" }); + if (value.inputs.some(input => input.git && input.include)) context.addIssue({ code: "custom", message: "Git snapshots and selective local inputs are separate modes" }); }); export type TrainingPreparationConfig = z.infer; export type TrainingImageBuild = z.infer; diff --git a/src/compiler/training/preparation/files.test.ts b/src/compiler/training/preparation/files.test.ts new file mode 100644 index 00000000..ec1d9ac7 --- /dev/null +++ b/src/compiler/training/preparation/files.test.ts @@ -0,0 +1,32 @@ +import { mkdtemp, mkdir, realpath, rm, symlink, truncate, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it } from "vitest"; +import { assertInputRoot, copySealed, exactPath, sealFile, sealTree } from "./files.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-files-"))); roots.push(root); return root; } +it("rejects protected roots, credential containment and leaf aliases", async () => { + for (const root of ["/", os.homedir(), path.join(os.homedir(), ".claude/settings"), "/private/data"]) { + expect(() => assertInputRoot(root, ["/private/data/auth"])).toThrow(); + } + const root = await fixture(); await writeFile(path.join(root, "file"), "bytes"); await symlink(path.join(root, "file"), path.join(root, "alias")); + await expect(exactPath(path.join(root, "alias"))).rejects.toThrow("canonical"); + await expect(sealFile(root, "root")).rejects.toThrow("regular file"); + await truncate(path.join(root, "file"), 536870913); await expect(sealFile(path.join(root, "file"), "file")).rejects.toThrow("512 MiB"); +}); +it("preserves confined documentation links but rejects escaping, cyclic and deep trees", async () => { + const root = await fixture(); await writeFile(path.join(root, "AGENTS.md"), "guide"); await symlink("AGENTS.md", path.join(root, "CLAUDE.md")); + expect(await sealTree(root, "input", { internalSymlinks: true })).toHaveLength(2); + await expect(sealTree(root, "image")).rejects.toThrow("symlinks"); + await symlink("../", path.join(root, "escape")); await expect(sealTree(root, "input", { internalSymlinks: true })).rejects.toThrow("symlinks"); await rm(path.join(root, "escape")); + await symlink(".", path.join(root, "cycle")); await expect(sealTree(root, "input", { internalSymlinks: true })).rejects.toThrow("bounds"); await rm(path.join(root, "cycle")); + let nested = root; for (let index = 0; index < 34; index++) { nested = path.join(nested, "nested"); await mkdir(nested); } + await expect(sealTree(root, "input", { ignoreDevelopment: true })).rejects.toThrow("bounds"); +}); +it("does not allow a crafted sealed destination to write outside its owned staging", async () => { + const root = await fixture(), file = path.join(root, "source"); await writeFile(file, "bytes"); + const sealed = await sealFile(file, "../escape"); + await expect(copySealed([sealed], root)).rejects.toThrow("escapes"); +}); diff --git a/src/compiler/training/preparation/image.test.ts b/src/compiler/training/preparation/image.test.ts new file mode 100644 index 00000000..045f9255 --- /dev/null +++ b/src/compiler/training/preparation/image.test.ts @@ -0,0 +1,78 @@ +import { chmod, mkdir, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { afterEach, expect, it } from "vitest"; +import { planTrainingImage, buildTrainingImage } from "./image.js"; +import { preparationFixture, imageDocker, image } from "./fixtures.test-helper.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { const f = await preparationFixture(); roots.push(f.root); if (!("build" in f.config.image)) throw Error("build expected"); + return { ...f, build: f.config.image.build, own: path.join(f.root, "own") }; } + +it("binds actual runtime bytes, locks, recipe, native parent, platform and entry into image identity", async () => { + const f = await fixture(); const plan = () => planTrainingImage(f.build, f.root, [], f.own); + const initial = await plan(); + for (const file of ["paideia/dist/src/cli/main.js", "bridge/paideia_dspy/__init__.py", "bridge/requirements.lock", "integration/entry.ts", "bootstrap/start.ts", "own/dist/cli/index.js", "own/runtime-images/training/Dockerfile"]) { + const previous = await readFile(path.join(f.root, file)); await f.put(file, "changed"); + expect((await plan()).digest).not.toBe(initial.digest); await writeFile(path.join(f.root, file), previous); + } + for (const patch of [{ platform: "linux/amd64" as const }, { nativeImage: `sha256:${"b".repeat(64)}` }, { pythonImage: `sha256:${"c".repeat(64)}` }]) { + expect((await planTrainingImage({ ...f.build, ...patch }, f.root, [], f.own)).digest).not.toBe(initial.digest); + } + await f.put("bridge/__pycache__/cache.pyc"); await f.put("integration/ignored.test.ts"); await f.put("integration/AGENTS.md"); + expect((await plan()).digest).toBe(initial.digest); +}); + +it("rejects dependency drift, executable pin mismatch, missing required entries and image symlink escapes", async () => { + const f = await fixture(); const plan = () => planTrainingImage(f.build, f.root, [], f.own); + f.build.grok.sha256 = image; await expect(plan()).rejects.toThrow("digest mismatch"); + const { sha } = await import("./fixtures.test-helper.js"); f.build.grok.sha256 = sha("native-binary"); + f.build.integration.entry = "missing.ts"; await expect(plan()).rejects.toThrow("missing integration"); f.build.integration.entry = "entry.ts"; + await f.put("claude/package.json", JSON.stringify({ dependencies: { injected: "1" } })); await expect(plan()).rejects.toThrow("manifest/lock mismatch"); + await f.put("claude/package.json", JSON.stringify({ dependencies: {} })); + await symlink(path.join(f.root, "auth"), path.join(f.root, "integration", "escape.ts")); await expect(plan()).rejects.toThrow("symlinks"); +}); + +it("does not trust a cached tag, empty successful build or changed staged bytes", async () => { + const f = await fixture(), plan = await planTrainingImage(f.build, f.root, [], f.own), docker = imageDocker(); + const options = { parent: f.root, dockerContext: "local", process: docker.process, timeoutMs: 1000, streams: { stdout() {}, stderr() {} } }; + docker.images.set(`spawnfile-training:${plan.digest.slice(7)}`, "wrong-label"); + expect(await buildTrainingImage(plan, options)).toMatchObject({ cached: false, imageId: image }); + expect(await buildTrainingImage(plan, options)).toMatchObject({ cached: true }); + const before = await readdir(f.root); + await expect(buildTrainingImage(plan, { ...options, process: async args => ({ code: args[2] === "build" ? 1 : 0, stdout: "invalid JSON", stderr: "" }) })).rejects.toThrow("build failed"); + await expect(buildTrainingImage(plan, { ...options, process: async () => ({ code: 0, stdout: "", stderr: "" }) })).rejects.toThrow("verified immutable"); + expect(await readdir(f.root)).toEqual(before); + await f.put("integration/entry.ts", "changed after plan"); + await expect(buildTrainingImage(plan, { ...options, process: async () => ({ code: 1, stdout: "", stderr: "" }) })).rejects.toThrow("changed during staging"); +}); + +it("makes privately staged runtime directories traversable for the configured nonroot user", async () => { + const f = await fixture(); + const packageDirs = ["spawnfile/dist/compiler/training", "paideia/dist/src/cli", "paideia/bridges/dspy", "integration", "bootstrap"]; + for (const directory of packageDirs) { + const target = path.join(f.root, "runtime", directory); + await mkdir(target, { recursive: true, mode: 0o700 }); + await chmod(target, 0o700); + await writeFile(path.join(target, "index.js"), "export const fixture = true;", { mode: 0o600 }); + } + const dockerfile = await readFile(new URL("../../../../runtime-images/training/Dockerfile", import.meta.url), "utf8"); + const command = dockerfile.match(/chmod -R a\+rX ([^\\\n]+)/u)![1]!.trim().split(/\s+/u); + execFileSync("chmod", ["-R", "a+rX", ...command.map(target => target.replace("/opt/training", path.join(f.root, "runtime")))]); + for (const directory of packageDirs) { + expect((await stat(path.join(f.root, "runtime", directory))).mode & 0o005).toBe(0o005); + expect((await stat(path.join(f.root, "runtime", directory, "index.js"))).mode & 0o004).toBe(0o004); + } +}); + +it("supports integration-owned bootstrap generation without a host snapshot", async () => { + const f = await fixture(); delete f.build.bootstrap; + const plan = await planTrainingImage(f.build, f.root, [], f.own), docker = imageDocker(); + expect(plan.files.some(file => file.destination.startsWith("bootstrap/"))).toBe(false); + await buildTrainingImage(plan, { parent: f.root, dockerContext: "local", timeoutMs: 1000, + streams: { stdout() {}, stderr() {} }, process: async (args, options) => { + if (args[2] === "build") expect(await readdir(path.join(args.at(-1)!, "bootstrap"))).toEqual([]); + return docker.process(args, options); + } }); +}); diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 6ed66f18..cffeeb83 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import type { TrainingDockerProcess } from "../container/process.js"; @@ -26,7 +26,7 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { const resolve = (value: string) => path.resolve(root, value); - for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, build.bootstrap]) assertInputRoot(resolve(source), auth); + for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); const dockerfile = await readFile(path.join(assets, "Dockerfile"), "utf8"); const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot @@ -37,7 +37,7 @@ export async function planTrainingImage(build: TrainingImageBuild, root: string, ...await packageFiles(resolve(build.claude), "claude", false), ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), - ...await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }), + ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }) : [], await sealFile(resolve(build.grok.source), "grok"), await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json") @@ -72,6 +72,7 @@ export async function buildTrainingImage(plan: TrainingImagePlan, options: { const staging = await mkdtemp(path.join(options.parent, ".spawnfile-training-image-")); try { await copySealed(plan.files, staging); + await mkdir(path.join(staging, "bootstrap"), { recursive: true, mode: 0o700 }); await writeFile(path.join(staging, "Dockerfile"), `${plan.dockerfile}\nLABEL com.spawnfile.training.recipe=${JSON.stringify(plan.digest)}\n`, { mode: 0o600 }); await writeFile(path.join(staging, "train"), plan.entry, { mode: 0o755 }); const result = await call(["build", "--platform", plan.build.platform, "--build-arg", `NATIVE_IMAGE=${plan.build.nativeImage}`, diff --git a/src/compiler/training/preparation/inputs.test.ts b/src/compiler/training/preparation/inputs.test.ts new file mode 100644 index 00000000..cc387b93 --- /dev/null +++ b/src/compiler/training/preparation/inputs.test.ts @@ -0,0 +1,64 @@ +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it, vi } from "vitest"; +import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; +import { preparationFixture, sha } from "./fixtures.test-helper.js"; + +const roots: string[] = []; +afterEach(async () => { vi.unstubAllEnvs(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { const f = await preparationFixture(); roots.push(f.root); return f; } +it("copies only selected inputs, ignoring historical runs, and seals their contents", async () => { + const f = await fixture(); await f.put("project/history/old-run", "not selected"); + f.config.inputs[0]!.include = ["Spawnfile", "train.yaml"]; + const inputs = await planInputs(f.config, f.root, []), target = path.join(f.root, "snapshot"); + expect(await stageInput(inputs[0]!, target)).toBe(target); + expect(await readFile(path.join(target, "train.yaml"), "utf8")).toBe("fixture"); + await expect(readFile(path.join(target, "history/old-run"))).rejects.toThrow(); + await verifyCanonicalPins(inputs, f.context.sources, [target, "unused"]); + await writeFile(path.join(target, "Spawnfile"), "changed"); + await expect(verifyCanonicalPins(inputs, f.context.sources, [target, "unused"])).rejects.toThrow("differs"); + await expect(verifyCanonicalPins(inputs, [{ sourcePath: "/not-declared", sha256: sha("x") }], [target, "unused"])).rejects.toThrow("outside"); + f.config.inputs[0]!.include = ["Spawnfile", "Spawnfile"]; + await expect(planInputs(f.config, f.root, [])).rejects.toThrow("overlap"); +}); + +it("accepts confined tracked links and rejects escaping Git links and overlay drift", async () => { + const f = await fixture(), cwd = f.context.project.root, execute = promisify(execFile); + const git = (args: string[]) => execute("git", args, { cwd }); + await f.put("project/AGENTS.md", "guide"); await symlink("AGENTS.md", path.join(cwd, "CLAUDE.md")); + await git(["init", "-q"]); await git(["add", "."]); + const commit = async () => { await git(["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "seed"]); return (await git(["rev-parse", "HEAD"])).stdout.trim(); }; + f.config.inputs[0]!.git = { revision: await commit(), overlays: [] }; + const inputs = await planInputs(f.config, f.root, []), target = path.join(f.root, "snapshot"); + await stageInput(inputs[0]!, target); expect(await readFile(path.join(target, "CLAUDE.md"), "utf8")).toBe("guide"); + await symlink("../../auth", path.join(cwd, "escape")); await git(["add", "."]); f.config.inputs[0]!.git!.revision = await commit(); + await expect(planInputs(f.config, f.root, [])).rejects.toThrow("symlink escapes"); + await git(["rm", "escape"]); f.config.inputs[0]!.git!.revision = await commit(); + f.config.inputs[0]!.git!.overlays = [{ source: "grok", path: "extra", sha256: sha("wrong") }]; + await expect(planInputs(f.config, f.root, [])).rejects.toThrow("digest mismatch"); + f.config.inputs[0]!.source = "project/nested"; await mkdir(path.join(cwd, "nested")); + await expect(planInputs(f.config, f.root, [])).rejects.toThrow("repository root"); +}); + +it("bounds retained JSON metadata", async () => { + const f = await fixture(); expect(await readBoundedJson(f.configPath)).toEqual(f.config); + await f.put("large.json", "x".repeat(1048577)); await expect(readBoundedJson(path.join(f.root, "large.json"))).rejects.toThrow("1 MiB"); +}); + +it("does not execute ambient Git filters while checking out a pinned repository", async () => { + const f = await fixture(), cwd = f.context.project.root, execute = promisify(execFile); + const git = (args: string[]) => execute("git", args, { cwd }); + await f.put("project/.gitattributes", "train.yaml filter=probe\n"); + await git(["init", "-q"]); await git(["add", "."]); + await git(["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "seed"]); + f.config.inputs[0]!.git = { revision: (await git(["rev-parse", "HEAD"])).stdout.trim(), overlays: [] }; + const marker = path.join(f.root, "host-filter-ran"), config = path.join(f.root, "global.gitconfig"); + await writeFile(config, `[filter "probe"]\n smudge = touch ${marker}\n`); + vi.stubEnv("GIT_CONFIG_GLOBAL", config); + const inputs = await planInputs(f.config, f.root, []); + await stageInput(inputs[0]!, path.join(f.root, "isolated")); + await expect(access(marker)).rejects.toThrow(); + expect(await readFile(path.join(f.root, "isolated/train.yaml"), "utf8")).toBe("fixture"); +}); diff --git a/src/compiler/training/preparation/inputs.ts b/src/compiler/training/preparation/inputs.ts index 9cc894c5..a672c824 100644 --- a/src/compiler/training/preparation/inputs.ts +++ b/src/compiler/training/preparation/inputs.ts @@ -7,9 +7,16 @@ import type { TrainingPreparationConfig } from "./contract.js"; import { assertInputRoot, copySealed, exactPath, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; const execute = promisify(execFile); -const git = async (cwd: string, args: string[]) => (await execute("git", args, { cwd, timeout: 120000, maxBuffer: 8 * 1024 * 1024 })).stdout; +const git = async (cwd: string, args: string[]) => { + const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_"))); + return (await execute("git", ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", ...args], { + cwd, env: { ...env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_ATTR_NOSYSTEM: "1", GIT_TERMINAL_PROMPT: "0" }, + timeout: 120000, maxBuffer: 8 * 1024 * 1024 + })).stdout; +}; export interface PlannedInput { id: string; source: string; destination: string; digest: string; + files?: SealedFile[]; git?: { revision: string; tree: string; common: string; overlays: SealedFile[] }; } @@ -17,12 +24,24 @@ export async function planInputs(config: TrainingPreparationConfig, root: string return Promise.all(config.inputs.map(async input => { const source = await exactPath(path.resolve(root, input.source)); assertInputRoot(source, auth); + if (input.include) { + const files: SealedFile[] = []; + for (const selected of input.include) files.push(...await sealTree(path.join(source, selected), selected, { internalSymlinks: true })); + if (new Set(files.map(file => file.destination)).size !== files.length) throw Error("Selected training input paths overlap"); + return { id: input.id, source, destination: input.destination, files, digest: hashJson(fileIdentity(files)) }; + } if (!input.git) return { id: input.id, source, destination: input.destination, digest: hashJson(fileIdentity(await sealTree(source, "input", { ignoreGit: true, internalSymlinks: true }))) }; if ((await git(source, ["rev-parse", "--show-prefix"])).trim()) throw Error("Pinned Git input source must be a repository root"); const common = await exactPath((await git(source, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim()); const tree = (await git(source, ["rev-parse", `${input.git.revision}^{tree}`])).trim(); const entries = (await git(source, ["ls-tree", "-rz", "--full-tree", input.git.revision])).split("\0").filter(Boolean); - if (!/^[a-f0-9]{40}$/u.test(tree) || entries.some(entry => !/^(100644|100755) blob [a-f0-9]{40}\t/u.test(entry))) throw Error("Pinned Git inputs require regular files; symlinks and submodules are unsupported"); + if (!/^[a-f0-9]{40}$/u.test(tree) || entries.some(entry => !/^(100644|100755|120000) blob [a-f0-9]{40}\t/u.test(entry))) throw Error("Pinned Git inputs require regular files or confined links; submodules are unsupported"); + for (const entry of entries.filter(entry => entry.startsWith("120000"))) { + const split = entry.indexOf("\t"), file = entry.slice(split + 1), object = entry.slice(0, split).split(" ")[2]!; + const link = (await git(source, ["cat-file", "blob", object])).trim(); + const target = path.posix.normalize(path.posix.join(path.posix.dirname(file), link)); + if (!link || path.posix.isAbsolute(link) || link.includes("\\") || target === ".." || target.startsWith("../") || target.split("/").includes(".git")) throw Error("Git symlink escapes the pinned input tree"); + } const overlays: SealedFile[] = []; for (const overlay of input.git.overlays) { const overlaySource = path.resolve(root, overlay.source); assertInputRoot(overlaySource, auth); @@ -37,9 +56,10 @@ export async function planInputs(config: TrainingPreparationConfig, root: string /** A real self-contained Git object store, never a copied worktree pointer. */ export async function stageInput(input: PlannedInput, target: string): Promise { + if (input.files) { await mkdir(target, { mode: 0o700 }); await copySealed(input.files, target); return target; } if (!input.git) return input.source; await mkdir(target, { mode: 0o700 }); - await git(target, ["init", "--quiet"]); + await git(target, ["init", "--quiet", "--template="]); await git(target, ["-c", "protocol.file.allow=always", "fetch", "--quiet", "--depth=1", pathToFileURL(input.git.common).href, input.git.revision]); await git(target, ["-c", "advice.detachedHead=false", "checkout", "--quiet", "--detach", input.git.revision]); if ((await git(target, ["rev-parse", "HEAD^{tree}"])).trim() !== input.git.tree || @@ -53,7 +73,7 @@ export async function verifyCanonicalPins(inputs: PlannedInput[], sources: reado for (const source of sources) { const index = inputs.findIndex(input => source.sourcePath === input.source || source.sourcePath.startsWith(input.source + path.sep)); if (index < 0) throw Error("Canonical training source is outside declared inputs"); - if (!inputs[index]!.git) continue; + if (!inputs[index]!.git && !inputs[index]!.files) continue; const file = path.join(staged[index]!, path.relative(inputs[index]!.source, source.sourcePath)); if ((await sealFile(file, "pin")).sha256 !== source.sha256) throw Error("Pinned Git snapshot differs from the selected canonical agent"); } diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index dc30b1d3..6447b69f 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { lstat, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import { chmod, lstat, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; import { afterEach, expect, it } from "vitest"; @@ -89,3 +89,42 @@ it("uses an explicit immutable image without package preparation", async () => { const result = await prepareTraining(f.options); expect(result).toMatchObject({ image }); expect(f.docker.calls.some(args => args[2] === "build")).toBe(false); }); + +it("validates preserved launch and mapped receipts instead of trusting their declared digest", async () => { + const f = await fixture(); const prepared = await prepareTraining(f.options); if ("dryRun" in prepared) throw Error("actual expected"); + const original = await readFile(prepared.preparationPath, "utf8"), mapped = JSON.parse(original); + await chmod(prepared.preparationPath, 0o600); + mapped.bindings[0].destination = "/run/training/inputs/elsewhere"; + await writeFile(prepared.preparationPath, JSON.stringify(mapped)); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("mapped receipt changed"); + await writeFile(prepared.preparationPath, original); + mapped.imageId = `sha256:${"b".repeat(64)}`; await writeFile(prepared.preparationPath, JSON.stringify(mapped)); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("identity mismatch"); + await writeFile(prepared.preparationPath, original); + const launch = JSON.parse(await readFile(prepared.configPath, "utf8")); launch.auth = []; + await writeFile(prepared.configPath, JSON.stringify(launch)); + await expect(prepareTraining({ ...f.options, args: [...f.args, "--resume"] })).rejects.toThrow("launch or mapped receipt changed"); +}); + +it("stages selective input closure and forwards resource paths through the snapshot on resume", async () => { + const f = await fixture(); f.config.inputs[0]!.include = ["Spawnfile", "train.yaml"]; await f.save(); + const args = [...f.args, "--resource", `archive=${f.context.project.root}`, "--test", f.args[1]!, "--cost-config", f.args[1]!]; + const prepared = await prepareTraining({ ...f.options, args }); if ("dryRun" in prepared) throw Error("actual expected"); + expect(prepared.args).toContain(`archive=${prepared.context.project.root}`); + expect(prepared.args.filter(value => value === path.join(prepared.context.project.root, "train.yaml"))).toHaveLength(3); + await prepareTraining({ ...f.options, args: [...args, "--resume"] }); +}); + +it("resolves immutable repository references and rejects missing images, overlapping inputs and preexisting files", async () => { + const f = await fixture(); f.config.image = { ref: `registry.invalid/image@${image}` }; await f.save(); + const prepared = await prepareTraining({ ...f.options, process: async args => args[0] === "context" + ? { code: 0, stdout: '"unix:///socket"', stderr: "" } : { code: 0, stdout: image, stderr: "" } }); + expect(prepared).toMatchObject({ image }); + f.config.output.source = "missing-image"; await f.save(); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "missing-image")], process: async args => args[0] === "context" + ? { code: 0, stdout: '"unix:///socket"', stderr: "" } : { code: 1, stdout: "", stderr: "" } })).rejects.toThrow("unavailable"); + f.config.inputs[1]!.source = "project"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow("inputs overlap"); + f.config.inputs[1]!.source = "settings"; f.config.output.source = "file"; await f.put("file"); await f.save(); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "file")] })).rejects.toThrow(); + f.config.auth[0]!.source = "own"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow("regular leaf"); +}); diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index ddf23117..c223d898 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -8,6 +8,13 @@ import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; import { planTrainingImage, buildTrainingImage } from "./image.js"; +function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { + return parseTrainingMappedPreparation({ version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, + bindings: config.inputs.map(input => ({ inputId: input.id, destination: input.destination })), outputRoot: "/run/training/output", + packagePaths: { spawnfile: "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy", + nativeWorker: "/opt/training/paideia/dist/src/adapters/daimon-native", integration: "/opt/training/integration", bootstrap: "/opt/training/bootstrap" }, integration: config.integration }); +} + export interface PrepareTrainingOptions { configPath: string; context: TrainingContext; args: readonly string[]; dryRun: boolean; process: TrainingDockerProcess; timeoutMs: number; signal?: AbortSignal; @@ -51,9 +58,9 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< if (resume) { const previous = JSON.parse(await readFile(statePath, "utf8")) as { digest: string; image: string; staged: string[]; snapshots: (string | null)[] }; if (previous.digest !== digest || !/^sha256:[a-f0-9]{64}$/u.test(previous.image) || - JSON.stringify(previous.staged) !== JSON.stringify(inputs.map(input => input.git ? path.join(staging, input.id) : input.source))) throw Error("Training preparation changed; exact resume rejected"); + JSON.stringify(previous.staged) !== JSON.stringify(inputs.map(input => input.git || input.files ? path.join(staging, input.id) : input.source))) throw Error("Training preparation changed; exact resume rejected"); image = previous.image; staged = previous.staged; - const snapshots = await Promise.all(inputs.map(async (input, index) => input.git ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true }))) : null)); + const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true, internalSymlinks: true }))) : null)); if (JSON.stringify(previous.snapshots) !== JSON.stringify(snapshots)) throw Error("Persisted training input snapshot changed"); const mapped = parseTrainingMappedPreparation(await readBoundedJson(preparationPath)); if (mapped.preparationDigest !== digest || mapped.imageId !== image) throw Error("Persisted training preparation identity mismatch"); @@ -72,18 +79,19 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]); if (inspected.code !== 0) throw Error("Training image is unavailable"); image = inspected.stdout.trim(); } - const mapped: TrainingMappedPreparation = { version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, - bindings: inputs.map(input => ({ inputId: input.id, destination: input.destination })), outputRoot: "/run/training/output", - packagePaths: { spawnfile: "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy", - nativeWorker: "/opt/training/paideia/dist/src/adapters/daimon-native", integration: "/opt/training/integration", bootstrap: "/opt/training/bootstrap" }, integration: config.integration }; + const mapped = mappedReceipt(digest, image, config); const launch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth }); await writeFile(configPath, JSON.stringify(launch), { flag: "wx", mode: 0o600 }); await writeFile(preparationPath, JSON.stringify(parseTrainingMappedPreparation(mapped)), { flag: "wx", mode: 0o400 }); - const snapshots = await Promise.all(inputs.map(async (input, index) => input.git ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true }))) : null)); + const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true, internalSymlinks: true }))) : null)); await writeFile(statePath, JSON.stringify({ digest, image, staged, snapshots }), { flag: "wx", mode: 0o600 }); } catch (error) { if (owned) await rm(staging, { recursive: true, force: true }); throw error; } } + const expectedLaunch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, + inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth }); + if (hashJson(await readBoundedJson(configPath)) !== hashJson(expectedLaunch) || + hashJson(await readBoundedJson(preparationPath)) !== hashJson(mappedReceipt(digest, image, config))) throw Error("Saved training launch or mapped receipt changed"); const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]); if (inspected.code !== 0 || inspected.stdout.trim() !== image) throw Error("Saved training image is missing or changed"); const map = (file: string): string => { From e3244e3fd5ab43c405afe4f2611789d8c335b10a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 03:48:50 +0200 Subject: [PATCH 09/84] fix: reject training output mismatch before preparation --- .../training/preparation/prepare.test.ts | 27 ++++++++++++++++--- src/compiler/training/preparation/prepare.ts | 6 ++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index 6447b69f..a3683ffe 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -69,7 +69,7 @@ it("rejects auth/input overlap, missing auth, unsafe context and output mismatch await expect(prepareTraining(f.options)).rejects.toThrow("auth leaf"); expect(f.docker.calls).toEqual([]); f.config.auth[0]!.source = "missing"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow(); f.config.auth[0]!.source = "auth"; f.config.output.source = "project/run"; await f.save(); - await expect(prepareTraining(f.options)).rejects.toThrow("overlaps"); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "project/run")] })).rejects.toThrow("overlaps"); f.config.output.source = "output"; await f.save(); await expect(prepareTraining({ ...f.options, process: async () => ({ code: 0, stdout: '"tcp://remote"', stderr: "" }) })).rejects.toThrow("Unix"); await expect(prepareTraining({ ...f.options, args: ["--out", "/different"] })).rejects.toThrow("match configured output"); @@ -123,8 +123,29 @@ it("resolves immutable repository references and rejects missing images, overlap f.config.output.source = "missing-image"; await f.save(); await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "missing-image")], process: async args => args[0] === "context" ? { code: 0, stdout: '"unix:///socket"', stderr: "" } : { code: 1, stdout: "", stderr: "" } })).rejects.toThrow("unavailable"); - f.config.inputs[1]!.source = "project"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow("inputs overlap"); + f.config.inputs[1]!.source = "project"; await f.save(); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "missing-image")] })).rejects.toThrow("inputs overlap"); f.config.inputs[1]!.source = "settings"; f.config.output.source = "file"; await f.put("file"); await f.save(); await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "file")] })).rejects.toThrow(); - f.config.auth[0]!.source = "own"; await f.save(); await expect(prepareTraining(f.options)).rejects.toThrow("regular leaf"); + f.config.auth[0]!.source = "own"; await f.save(); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "file")] })).rejects.toThrow("regular leaf"); +}); + +it("rejects mismatched output before Docker or staging for cold, dry-run and exact resume", async () => { + const f = await fixture(); + const before = await readdir(f.root); + for (const args of [["--out", path.join(f.root, "other")], ["--out"]]) { + for (const mode of [{ dryRun: false }, { dryRun: true }, { dryRun: false, resume: true }]) { + await expect(prepareTraining({ ...f.options, dryRun: mode.dryRun, args: [...args, ...("resume" in mode ? ["--resume"] : [])] })) + .rejects.toThrow("Training --out must match configured output"); + expect(f.docker.calls).toEqual([]); + expect(await readdir(f.root)).toEqual(before); + } + } + const prepared = await prepareTraining(f.options); + if ("dryRun" in prepared) throw Error("actual preparation expected"); + const calls = f.docker.calls.length, saved = await readFile(prepared.preparationPath); + await expect(prepareTraining({ ...f.options, args: ["--out", path.join(f.root, "other")] })).rejects.toThrow("match configured output"); + expect(f.docker.calls).toHaveLength(calls); + expect(await readFile(prepared.preparationPath)).toEqual(saved); }); diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index c223d898..586357f4 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -32,6 +32,11 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< const root = path.dirname(path.resolve(options.configPath)); const auth = config.auth.map(entry => ({ ...entry, source: path.resolve(root, entry.source) })); const output = path.resolve(root, config.output.source), parent = path.dirname(output); + for (let index = 0; index < options.args.length; index++) { + if (options.args[index] !== "--out") continue; + const declared = options.args[++index]; + if (declared === undefined || path.resolve(declared) !== output) throw Error("Training --out must match configured output"); + } assertInputRoot(output, auth.map(entry => entry.source)); if (await realpath(parent) !== parent) throw Error("Training output parent must be canonical and already exist"); const inputs = await planInputs(config, root, auth.map(entry => entry.source)); @@ -109,7 +114,6 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< for (let index = 0; index < args.length; index++) { if (["--train", "--test", "--cost-config"].includes(args[index]!)) args[++index] = map(args[index]!); else if (args[index] === "--resource") { const value = args[++index]!, split = value.indexOf("="); args[index] = `${value.slice(0, split)}=${map(value.slice(split + 1))}`; } - else if (args[index] === "--out" && path.resolve(args[++index]!) !== output) throw Error("Training --out must match configured output"); } return { digest, image, configPath, preparationPath, context, args }; } From bbc5ec76f2c6530441536e332fef836da4a7fdf3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 05:17:25 +0200 Subject: [PATCH 10/84] feat: authorize captured measurement repair forks --- runtime-images/training/Dockerfile | 6 + specs/TRAINING_CONTAINERS.md | 39 ++++++ src/cli/paideiaDelegation.ts | 5 + src/cli/trainCommand.test.ts | 10 ++ src/cli/trainCommand.ts | 3 + src/compiler/training/container/launch.ts | 11 ++ .../training/container/repair.test.ts | 42 ++++++ src/compiler/training/preparation/AGENTS.md | 7 + src/compiler/training/preparation/contract.ts | 2 +- .../training/preparation/files.test.ts | 8 ++ src/compiler/training/preparation/files.ts | 2 +- .../training/preparation/image.test.ts | 19 +++ src/compiler/training/preparation/image.ts | 10 +- src/compiler/training/preparation/prepare.ts | 35 ++++- src/compiler/training/repair/AGENTS.md | 10 ++ src/compiler/training/repair/CLAUDE.md | 1 + src/compiler/training/repair/compatibility.ts | 35 +++++ src/compiler/training/repair/contract.ts | 34 +++++ src/compiler/training/repair/index.ts | 3 + src/compiler/training/repair/prepare.ts | 65 +++++++++ src/compiler/training/repair/repair.test.ts | 130 ++++++++++++++++++ src/compiler/training/repair/witness.ts | 44 ++++++ 22 files changed, 513 insertions(+), 8 deletions(-) create mode 100644 src/compiler/training/container/repair.test.ts create mode 100644 src/compiler/training/repair/AGENTS.md create mode 120000 src/compiler/training/repair/CLAUDE.md create mode 100644 src/compiler/training/repair/compatibility.ts create mode 100644 src/compiler/training/repair/contract.ts create mode 100644 src/compiler/training/repair/index.ts create mode 100644 src/compiler/training/repair/prepare.ts create mode 100644 src/compiler/training/repair/repair.test.ts create mode 100644 src/compiler/training/repair/witness.ts diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 17286c93..b41e4a00 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -20,6 +20,10 @@ WORKDIR /opt/training/spawnfile COPY spawnfile/package.json spawnfile/package-lock.json ./ RUN npm ci --omit=dev --ignore-scripts +WORKDIR /opt/training/compiler +COPY compiler/package.json compiler/package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts + COPY bridge /opt/training/paideia/bridges/dspy RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir -r /opt/training/paideia/bridges/dspy/requirements.lock \ @@ -32,6 +36,8 @@ COPY grok /opt/training/bin/grok COPY paideia/dist /opt/training/paideia/dist COPY spawnfile/dist /opt/training/spawnfile/dist +COPY compiler/dist /opt/training/compiler/dist +COPY compiler/runtimes.yaml compiler/moltnet-releases.json /opt/training/compiler/ COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/ COPY integration /opt/training/integration COPY bootstrap /opt/training/bootstrap diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 95f8f784..67af9775 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -177,3 +177,42 @@ preparation writes. Exact resume checks current source identity and preserved snapshots/receipts, then requires the saved immutable image. It never rebuilds a replacement under an existing experiment identity. Paideia independently checks its experiment checkpoint and cumulative budgets. + +## Explicit measurement repair fork + +`spawnfile train PROJECT --training-config evals/training.json --train evals/train.paideia.yaml --out runs/repair --repair-measurements runs/parent` +creates a new experiment from captured work. The declaration's output must match +`--out` and be fresh/disjoint. It does not weaken ordinary `--resume`. To resume +this child, retain the same repair arguments and add `--resume`. + +V2 builds now seal an automatic `spawnfile.training-witness.v1` beside their +preparation state, including exact image input bytes, recipe, input identities and +canonical context. Legacy parents require `--repair-witness /path/manifest.json`: +its envelope, complete copied image closure, original preparation digest and +installed parent image's recipe label must all verify. A caller's unverified image +name or compatibility assertion is insufficient. Image-reference-only declarations +cannot establish this compatibility and reject repair. + +The `daimon-dspy.v1` recipe supports optional `image.build.compiler`, a complete +registry-locked Spawnfile distribution used only for native compilation under +`/opt/training/compiler`. Omission uses the current installed compiler. Repair may +pin the parent's original compiler while using corrected evaluation/launch code. +The mapped preparation receipt supplies that exact compiler executable path. + +`spawnfile.daimon-dspy-compatibility.v1` requires unchanged native/Python images, +platform, compiler closure, installed integration, native adapter/trial code, +canonical source and every declared input digest/binding. Optimizer Python and +locks stay pinned; only checkpoint/protocol import plumbing and documentation may +change. Generated `.coverage`, `.pytest_cache` and `coverage.json` are excluded +explicitly, never arbitrary dotfiles. Paideia separately verifies candidate, +criteria, budgets, splits, native capture closure and repair eligibility. + +Only `runs/`, `blobs/` and the four command/training/host/optimizer checkpoint JSON +files are copied into a sealed read-only parent projection. Runtime homes, auth, +mutable caches and invocation databases are excluded. The exact projection manifest +and checkpoint bytes are hashed. `/run/paideia/repair.json` is a protected read-only +`paideia.measurement-repair.v1` receipt; the launcher forwards its fixed path via +`--repair-context`. It contains container paths and digests, no host credentials. +Paideia owns error-only rescoring, retaining successful historical pass/fail results, +paired optimizer import, cumulative accounting and the new experiment lineage. +A repair receipt does not itself assert that any judgment or continuation succeeded. diff --git a/src/cli/paideiaDelegation.ts b/src/cli/paideiaDelegation.ts index b9f4c090..de58a3c3 100644 --- a/src/cli/paideiaDelegation.ts +++ b/src/cli/paideiaDelegation.ts @@ -19,6 +19,7 @@ export interface DelegatePaideiaTrainingOptions { command: string; args: readonly string[]; dryRun: boolean; + repairMeasurements?: string; repairWitness?: string; trainingImage?: string; trainingConfig?: string; timeoutMs: number; @@ -133,17 +134,21 @@ const runChild = (options: DelegatePaideiaTrainingOptions, contextPath: string): /** Delegates through the public CLI, never importing Paideia or selecting a fallback adapter. */ export const delegatePaideiaTraining = async (options: DelegatePaideiaTrainingOptions): Promise => { if (options.signal?.aborted) return 130; + if (options.repairWitness && !options.repairMeasurements) throw failure("--repair-witness requires --repair-measurements"); if (options.trainingConfig) { const config = await readBoundedJson(options.trainingConfig) as { version?: unknown }; if (config.version === "spawnfile.training-container.v2") { if (options.trainingImage) throw failure("V2 owns its image declaration; --training-image is only for v1"); const prepared = await prepareTraining({ configPath: options.trainingConfig, context: options.context, args: options.args, + repairMeasurements: options.repairMeasurements, repairWitness: options.repairWitness, dryRun: options.dryRun, process: runTrainingDocker, timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams }); if (!("dryRun" in prepared)) return launchTrainingContainer({ ...prepared, timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams }); options.streams.stderr(`Training preparation plan ${prepared.digest}; no Docker, auth or filesystem mutations`); + options = { ...options, repairMeasurements: undefined, repairWitness: undefined }; } } + if (options.repairMeasurements) throw failure("Measurement repair requires the v2 preparation declaration"); if (!options.dryRun) { if (!options.trainingImage || !options.trainingConfig) throw failure("Actual training requires --training-image and --training-config; host execution is disabled"); return launchTrainingContainer({ image: options.trainingImage, configPath: options.trainingConfig, diff --git a/src/cli/trainCommand.test.ts b/src/cli/trainCommand.test.ts index 6f54b9fc..9a0ee377 100644 --- a/src/cli/trainCommand.test.ts +++ b/src/cli/trainCommand.test.ts @@ -115,3 +115,13 @@ describe("spawnfile train", () => { "--judge-citation-repairs", literal, "--judge-citation-repairs", literal, "--out", "local output", "--dry-run"]); }); }); + +it("keeps public repair authority separate from forwarded optimizer flags", async () => { + const delegate = vi.fn(async (_options: DelegatePaideiaTrainingOptions) => 0); + expect(await runCli(["train", await project(), ...base, "--training-config", "/recipe.json", "--repair-measurements", "/parent", "--repair-witness", "/witness.json"], { + handlers: { delegatePaideiaTraining: delegate }, streams: { stdout() {}, stderr() {} } + })).toBe(0); + expect(delegate.mock.calls[0]![0]).toMatchObject({ repairMeasurements: "/parent", repairWitness: "/witness.json" }); + expect(delegate.mock.calls[0]![0].args).not.toContain("--repair-measurements"); + expect(delegate.mock.calls[0]![0].args).not.toContain("--repair-context"); +}); diff --git a/src/cli/trainCommand.ts b/src/cli/trainCommand.ts index d9c6a8bd..0ca0823c 100644 --- a/src/cli/trainCommand.ts +++ b/src/cli/trainCommand.ts @@ -24,6 +24,8 @@ export const registerTrainCommand = ( .option("--training-image ", "Pinned image containing the complete training environment") .option("--training-config ", "V2 image recipe and declared inputs, or advanced v1 local bindings") .option("--dry-run", "Validate and estimate without compiling, authenticating or starting models") + .option("--repair-measurements ", "Fork captured work into a fresh output for authorized measurement repair") + .option("--repair-witness ", "Explicit verified image witness for a legacy parent") .option("--resume", "Resume the exact persisted training experiment in --out"); for (const name of forwarded) { const flag = `--${name} `; @@ -55,6 +57,7 @@ export const registerTrainCommand = ( if (options.dryRun === true) args.push("--dry-run"); if (options.resume === true) args.push("--resume"); setExitCode(await handlers.delegatePaideiaTraining({ context, args, + repairMeasurements: options.repairMeasurements as string | undefined, repairWitness: options.repairWitness as string | undefined, trainingImage: options.trainingImage as string | undefined, trainingConfig: options.trainingConfig as string | undefined, command: options.paideiaCommand as string, dryRun: options.dryRun === true, timeoutMs: timeout + 5000, streams, signal })); diff --git a/src/compiler/training/container/launch.ts b/src/compiler/training/container/launch.ts index fb979386..6eb5984b 100644 --- a/src/compiler/training/container/launch.ts +++ b/src/compiler/training/container/launch.ts @@ -7,6 +7,8 @@ import type { TrainingContext } from "../contract.js"; import { trainingImageSchema } from "./contract.js"; import { prepareTrainingContainer } from "./prepare.js"; import { runTrainingDocker, type TrainingDockerProcess } from "./process.js"; +import { repairEnvelopeSchema } from "../repair/contract.js"; +import { hashJson } from "../preparation/files.js"; import { parseTrainingMappedPreparation } from "../preparation/contract.js"; export interface LaunchTrainingContainerOptions { @@ -15,6 +17,7 @@ export interface LaunchTrainingContainerOptions { streams: { stdout(line: string): void; stderr(line: string): void }; process?: TrainingDockerProcess; preparationPath?: string; + repairPath?: string; } const inspectFormat = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}"; export const launchTrainingContainer = async (options: LaunchTrainingContainerOptions): Promise => { @@ -29,6 +32,13 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp if (await realpath(options.preparationPath) !== options.preparationPath) throw Error("Preparation receipt path must be canonical"); parseTrainingMappedPreparation(JSON.parse(await readFile(options.preparationPath, "utf8"))); } + if (options.repairPath) { + if (await realpath(options.repairPath) !== options.repairPath) throw Error("Repair receipt path must be canonical"); + const raw = JSON.parse(await readFile(options.repairPath, "utf8")); + const envelope = repairEnvelopeSchema.parse(raw); + if (hashJson(raw.receipt) !== envelope.digest || envelope.receipt.current.imageId !== image || + !prepared.config.inputs.some(input => input.destination === envelope.receipt.parent.root)) throw Error("Repair receipt launch identity mismatch"); + } const execute = options.process ?? runTrainingDocker; const controller = new AbortController(), deadline = Date.now() + options.timeoutMs; let interrupted = false, timedOut = false; @@ -62,6 +72,7 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp `type=bind,src=${prepared.config.output.source},dst=/run/training/output`, `type=bind,src=${contextFile},dst=/run/paideia/context.json,readonly`, ...options.preparationPath ? [`type=bind,src=${options.preparationPath},dst=/run/paideia/preparation.json,readonly`] : [], + ...options.repairPath ? [`type=bind,src=${options.repairPath},dst=/run/paideia/repair.json,readonly`] : [], ...prepared.config.auth.map((entry) => `type=bind,src=${entry.source},dst=/run/paideia-auth/${entry.provider},readonly`)]; const args = ["create", "--name", name, "--label", `com.spawnfile.training.owner=${name}`, "--init", "--read-only", "--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", diff --git a/src/compiler/training/container/repair.test.ts b/src/compiler/training/container/repair.test.ts new file mode 100644 index 00000000..15ee8a46 --- /dev/null +++ b/src/compiler/training/container/repair.test.ts @@ -0,0 +1,42 @@ +import { mkdir, rm, writeFile, symlink } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { launchTrainingContainer } from "./launch.js"; +import { fixture, dockerFixture, image } from "./fixtures.test-helper.js"; +import { hashJson } from "../preparation/files.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function setup() { + const f = await fixture(); roots.push(f.root); const parent = path.join(f.root, "parent"); await mkdir(parent); + f.config.inputs.push({ source: parent, destination: "/run/training/inputs/repair-parent" }); + await writeFile(f.configPath, JSON.stringify(f.config)); + const receipt = { version: "paideia.measurement-repair.v1", parent: { root: "/run/training/inputs/repair-parent", imageId: image, + experimentId: "00000000-0000-4000-8000-000000000001", executionIdentity: "a".repeat(64), manifestDigest: image, + checkpoints: { command: image, training: image, host: image, optimizer: image }, trainingIdentity: "b".repeat(64) }, + current: { imageId: image, canonicalSourceDigest: image, adapterId: "daimon-native" }, + compatibility: { version: "spawnfile.daimon-dspy-compatibility.v1", components: { compiler: image }, inputs: { input: image }, digest: image } }; + const envelope = { receipt, digest: hashJson(receipt) }, repairPath = path.join(f.root, "repair.json"); + await writeFile(repairPath, JSON.stringify(envelope)); const docker = dockerFixture(); + return { ...f, parent, envelope, repairPath, docker, options: { image, configPath: f.configPath, context: f.context, + args: [...f.args, "--repair-context", "/run/paideia/repair.json"], repairPath, timeoutMs: 10000, + process: docker.process, streams: { stdout() {}, stderr() {} } } }; +} +it("mounts receipt and projected parent readonly while retaining verified container cleanup", async () => { + const f = await setup(); expect(await launchTrainingContainer(f.options)).toBe(0); + const create = f.docker.calls.find(args => args[2] === "create")!; + expect(create).toContain(`type=bind,src=${f.parent},dst=/run/training/inputs/repair-parent,readonly`); + expect(create).toContain(`type=bind,src=${f.repairPath},dst=/run/paideia/repair.json,readonly`); + expect(create).toContain("--repair-context"); + expect(f.docker.calls.some(args => args[2] === "rm")).toBe(true); +}); +it.each(["digest", "image", "mount", "symlink"])("rejects wrong repair %s before container operation", async kind => { + const f = await setup(); + if (kind === "digest") f.envelope.digest = `sha256:${"f".repeat(64)}`; + if (kind === "image") { f.envelope.receipt.current.imageId = `sha256:${"f".repeat(64)}`; f.envelope.digest = hashJson(f.envelope.receipt); } + if (kind === "mount") { f.config.inputs.pop(); await writeFile(f.configPath, JSON.stringify(f.config)); } + await writeFile(f.repairPath, JSON.stringify(f.envelope)); + if (kind === "symlink") { const link = path.join(f.root, "alias"); await symlink(f.repairPath, link); f.options.repairPath = link; } + await expect(launchTrainingContainer(f.options)).rejects.toThrow(); + expect(f.docker.calls).toEqual([]); +}); diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index 3286961d..92b617f3 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -10,3 +10,10 @@ inputs. Dry-run performs reads only; resume validates the preserved preparation. declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 lines. Source/lock/recipe mutation must invalidate cache and exact resume. + +- Built preparations seal original image bytes in a protected witness for explicit + captured-work repair. Optional compiler selects a separate full pinned native + compiler distribution; the current package still owns launch/auth operations. +- Exclude explicit generated Python coverage/cache files, not arbitrary dotfiles. +- Repair transport/compatibility lives in sibling `repair/`; it never changes + candidate, criteria or checkpoint semantics owned by Paideia. diff --git a/src/compiler/training/preparation/contract.ts b/src/compiler/training/preparation/contract.ts index 1e23e1af..6ed2a584 100644 --- a/src/compiler/training/preparation/contract.ts +++ b/src/compiler/training/preparation/contract.ts @@ -10,7 +10,7 @@ const destination = z.string().regex(/^\/run\/training\/inputs\/[A-Za-z0-9._/-]+ export const trainingBuildSchema = z.object({ recipe: z.literal("daimon-dspy.v1"), nativeImage: trainingImageSchema, pythonImage: trainingImageSchema, platform: z.enum(["linux/arm64", "linux/amd64"]), - paideia: local, bridge: local, claude: local, + paideia: local, bridge: local, claude: local, compiler: local.optional(), grok: z.object({ source: local, sha256: sha }).strict(), integration: z.object({ source: local, entry: relative.refine(value => /^[A-Za-z0-9._/-]+$/u.test(value)) }).strict(), bootstrap: local.optional() }).strict(); diff --git a/src/compiler/training/preparation/files.test.ts b/src/compiler/training/preparation/files.test.ts index ec1d9ac7..c45e44c0 100644 --- a/src/compiler/training/preparation/files.test.ts +++ b/src/compiler/training/preparation/files.test.ts @@ -30,3 +30,11 @@ it("does not allow a crafted sealed destination to write outside its owned stagi const sealed = await sealFile(file, "../escape"); await expect(copySealed([sealed], root)).rejects.toThrow("escapes"); }); + +it("omits only explicit generated Python test state, preserving dotfiles and executable bytes", async () => { + const root = await fixture(); + await mkdir(path.join(root, ".pytest_cache")); + for (const name of [".coverage", "coverage.json", ".pytest_cache/nodeids", ".runtime-policy", "optimizer.py"]) await writeFile(path.join(root, name), "data"); + const files = await sealTree(root, "bridge", { ignoreDevelopment: true }); + expect(files.map(file => file.destination)).toEqual(["bridge/.runtime-policy", "bridge/optimizer.py"]); +}); diff --git a/src/compiler/training/preparation/files.ts b/src/compiler/training/preparation/files.ts index 9a5cdcbf..888916b2 100644 --- a/src/compiler/training/preparation/files.ts +++ b/src/compiler/training/preparation/files.ts @@ -7,7 +7,7 @@ import path from "node:path"; export const within = (root: string, file: string): boolean => file === root || file.startsWith(root + path.sep); export const hashJson = (value: unknown): string => `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; export interface SealedFile { source: string; destination: string; sha256: string; mode: number; size: number } -const ignored = new Set(["node_modules", ".venv", ".git", "__pycache__", "coverage", ".runtime", "AGENTS.md", "CLAUDE.md"]); +const ignored = new Set(["node_modules", ".venv", ".git", "__pycache__", "coverage", "coverage.json", ".coverage", ".pytest_cache", ".runtime", "AGENTS.md", "CLAUDE.md"]); export function assertInputRoot(source: string, auth: readonly string[]): void { const home = os.homedir(); diff --git a/src/compiler/training/preparation/image.test.ts b/src/compiler/training/preparation/image.test.ts index 045f9255..2c09852d 100644 --- a/src/compiler/training/preparation/image.test.ts +++ b/src/compiler/training/preparation/image.test.ts @@ -76,3 +76,22 @@ it("supports integration-owned bootstrap generation without a host snapshot", as return docker.process(args, options); } }); }); + + +it("pins an explicitly preserved compiler independently of the current launcher", async () => { + const f = await fixture(); + const { cp } = await import("node:fs/promises"); + await cp(f.own, path.join(f.root, "old-compiler"), { recursive: true }); + f.build.compiler = "old-compiler"; + const plan = () => planTrainingImage(f.build, f.root, [], f.own); + const old = await plan(); + const compiler = old.files.find(file => file.destination === "compiler/dist/cli/index.js")!; + await f.put("own/dist/cli/index.js", "new launcher"); + const changed = await plan(); + expect(changed.digest).not.toBe(old.digest); + expect(changed.files.find(file => file.destination === compiler.destination)?.sha256).toBe(compiler.sha256); + await f.put("old-compiler/dist/cli/index.js", "changed compiler"); + expect((await plan()).files.find(file => file.destination === compiler.destination)?.sha256).not.toBe(compiler.sha256); + await f.put("old-compiler/package-lock.json", JSON.stringify({ lockfileVersion: 3, packages: { "": {}, injected: { resolved: "file:../host" } } })); + await expect(plan()).rejects.toThrow("unsupported local dependencies"); +}); diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index cffeeb83..5ff11252 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -26,7 +26,7 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { const resolve = (value: string) => path.resolve(root, value); - for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); + for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); const dockerfile = await readFile(path.join(assets, "Dockerfile"), "utf8"); const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot @@ -34,16 +34,20 @@ export async function planTrainingImage(build: TrainingImageBuild, root: string, const files = [ ...await packageFiles(resolve(build.paideia), "paideia", true), ...await packageFiles(ownRoot, "spawnfile", true, ownLock), + ...await packageFiles(build.compiler ? resolve(build.compiler) : ownRoot, "compiler", true, + build.compiler ? path.join(resolve(build.compiler), "package-lock.json") : ownLock), ...await packageFiles(resolve(build.claude), "claude", false), ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }) : [], await sealFile(resolve(build.grok.source), "grok"), await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), - await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json") + await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json"), + ...await Promise.all(["runtimes.yaml", "moltnet-releases.json"].map(name => + sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`))) ]; if (files.find(file => file.destination === "grok")!.sha256 !== build.grok.sha256) throw Error("Grok executable digest mismatch"); - for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js"]) { + for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js", "compiler/dist/cli/index.js"]) { if (!files.some(file => file.destination === required)) throw Error(`Training distribution is missing ${required}`); } const entry = `#!/bin/sh\nexec /usr/local/bin/node --experimental-strip-types ${JSON.stringify(`/opt/training/integration/${build.integration.entry}`)} "$@"\n`; diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index 586357f4..fb4e9aa5 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -7,11 +7,13 @@ import { trainingPreparationSchema, parseTrainingMappedPreparation, type Trainin import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } from "./files.js"; import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; import { planTrainingImage, buildTrainingImage } from "./image.js"; +import { planMeasurementRepair, stageMeasurementRepair, writeTrainingWitness } from "../repair/index.js"; function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { return parseTrainingMappedPreparation({ version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, bindings: config.inputs.map(input => ({ inputId: input.id, destination: input.destination })), outputRoot: "/run/training/output", - packagePaths: { spawnfile: "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy", + packagePaths: { spawnfile: "build" in config.image && config.image.build.compiler + ? "/opt/training/compiler/dist/cli/index.js" : "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy", nativeWorker: "/opt/training/paideia/dist/src/adapters/daimon-native", integration: "/opt/training/integration", bootstrap: "/opt/training/bootstrap" }, integration: config.integration }); } @@ -21,9 +23,10 @@ export interface PrepareTrainingOptions { streams: { stdout(line: string): void; stderr(line: string): void }; /** Test-only package fixture; production always resolves its own installed distribution. */ packageRoot?: string; + repairMeasurements?: string; repairWitness?: string; } export interface PreparedTraining { - digest: string; image: string; configPath: string; preparationPath: string; context: TrainingContext; args: string[]; + digest: string; image: string; configPath: string; preparationPath: string; repairPath?: string; context: TrainingContext; args: string[]; } /** Reads only until the explicit dry-run boundary; preparation never executes project code. */ @@ -48,14 +51,26 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< within(input.destination, other.destination) || within(other.destination, input.destination)) throw Error("Training inputs overlap"); } const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot) : undefined; + if (options.repairWitness && !options.repairMeasurements) throw Error("A repair witness requires --repair-measurements"); + if (options.repairMeasurements && !imagePlan) throw Error("Measurement repair requires a verifiable image build recipe"); + const repair = options.repairMeasurements ? await planMeasurementRepair({ parent: options.repairMeasurements, + witness: options.repairWitness, output, auth: auth.map(entry => entry.source), image: imagePlan!, inputs, + context: options.context, resume: options.args.includes("--resume") }) : undefined; + if (repair && hashJson(config.integration) !== hashJson(repair.witness.manifest.config.integration)) throw Error("Repair integration settings binding changed"); const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image, - canonical: options.context.project.sourceDigest }); + canonical: options.context.project.sourceDigest, ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); if (options.dryRun) return { digest, dryRun: true }; options.signal?.throwIfAborted(); for (const entry of auth) if (await exactPath(entry.source) !== entry.source || !(await lstat(entry.source)).isFile()) throw Error("Training auth must be a canonical regular leaf"); const execute = (args: string[]) => options.process(args, { timeoutMs: options.timeoutMs, signal: options.signal }); const endpoint = await execute(["context", "inspect", config.dockerContext, "--format", "{{json .Endpoints.docker.Host}}"]); if (endpoint.code !== 0 || !/^unix:\/\//u.test(JSON.parse(endpoint.stdout))) throw Error("Training preparation requires a local Unix Docker context"); + if (repair) { + const label = await execute(["--context", config.dockerContext, "image", "inspect", repair.witness.manifest.parentImage, + "--format", '{{json .Id}}\n{{json (index .Config.Labels "com.spawnfile.training.recipe")}}']); + const values = label.stdout.trim().split("\n").map(value => JSON.parse(value)); + if (label.code !== 0 || values[0] !== repair.witness.manifest.parentImage || values[1] !== repair.witness.manifest.imagePlanDigest) throw Error("Parent image recipe witness is unverified"); + } const staging = path.join(parent, `.spawnfile-training-preparation-${hashJson(output).slice(7, 23)}`); const configPath = path.join(staging, "launch.json"), preparationPath = path.join(staging, "mapped.json"), statePath = path.join(staging, "state.json"); const resume = options.args.includes("--resume"); @@ -91,6 +106,9 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< await writeFile(preparationPath, JSON.stringify(parseTrainingMappedPreparation(mapped)), { flag: "wx", mode: 0o400 }); const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true, internalSymlinks: true }))) : null)); await writeFile(statePath, JSON.stringify({ digest, image, staged, snapshots }), { flag: "wx", mode: 0o600 }); + if (imagePlan) await writeTrainingWitness({ directory: staging, digest, image, config, context: options.context, + args: options.args, inputs, staged, snapshots, plan: imagePlan, + ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); } catch (error) { if (owned) await rm(staging, { recursive: true, force: true }); throw error; } } const expectedLaunch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, @@ -115,5 +133,16 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< if (["--train", "--test", "--cost-config"].includes(args[index]!)) args[++index] = map(args[index]!); else if (args[index] === "--resource") { const value = args[++index]!, split = value.indexOf("="); args[index] = `${value.slice(0, split)}=${map(value.slice(split + 1))}`; } } + if (repair) { + const projected = await stageMeasurementRepair(repair, staging, image, options.context.project.sourceDigest, resume); + const repairLaunch = { ...expectedLaunch, inputs: [...expectedLaunch.inputs, + { source: projected.root, destination: "/run/training/inputs/repair-parent" }] }; + const repairConfig = path.join(staging, "repair-launch.json"); + if (resume) { + if (hashJson(await readBoundedJson(repairConfig)) !== hashJson(repairLaunch)) throw Error("Saved repair launch changed"); + } else await writeFile(repairConfig, JSON.stringify(repairLaunch), { flag: "wx", mode: 0o600 }); + return { digest, image, configPath: repairConfig, preparationPath, repairPath: projected.repairPath, context, + args: [...args, "--repair-context", "/run/paideia/repair.json"] }; + } return { digest, image, configPath, preparationPath, context, args }; } diff --git a/src/compiler/training/repair/AGENTS.md b/src/compiler/training/repair/AGENTS.md new file mode 100644 index 00000000..92777da9 --- /dev/null +++ b/src/compiler/training/repair/AGENTS.md @@ -0,0 +1,10 @@ +# Measurement repair transport + +Owns trusted preparation witnesses, subject-environment compatibility checks and +read-only captured-work projection for an explicit fresh training fork. Paideia +owns checkpoint parsing, repair eligibility, scoring and optimizer continuation. +Never reinterpret measurements, expose runtime homes/auth, or weaken normal +resume. Compare pinned compiler/native/integration/input closures before a receipt. +Legacy witnesses require complete verified image files and original recipe label; +new preparation records this witness automatically. Keep tests adjacent and guards +mutation-tested. No model calls or project scripts belong here. diff --git a/src/compiler/training/repair/CLAUDE.md b/src/compiler/training/repair/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/repair/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/repair/compatibility.ts b/src/compiler/training/repair/compatibility.ts new file mode 100644 index 00000000..7dd030cf --- /dev/null +++ b/src/compiler/training/repair/compatibility.ts @@ -0,0 +1,35 @@ +import type { TrainingContext } from "../contract.js"; +import type { TrainingImagePlan } from "../preparation/image.js"; +import type { PlannedInput } from "../preparation/inputs.js"; +import { fileIdentity, hashJson } from "../preparation/files.js"; +import type { TrainingWitness } from "./contract.js"; + +/** Subject and optimizer semantics remain pinned; only evaluation/fork plumbing may change. */ +export function verifyTrainingCompatibility(witness: TrainingWitness, current: TrainingImagePlan, + inputs: PlannedInput[], context: TrainingContext) { + if (context.agent.runtime !== "daimon" || context.project.sourceDigest !== witness.context.project.sourceDigest || + JSON.stringify(context.agent) !== JSON.stringify(witness.context.agent)) throw Error("Repair canonical agent changed"); + const old = witness.image.files, next = fileIdentity(current.files); + const selection = (files: typeof old, prefix: string, excludes: string[] = []) => { + const values = files.filter(file => file.destination.startsWith(prefix) && !excludes.some(value => file.destination === value || (value.endsWith("/") && file.destination.startsWith(value)))) + .map(file => ({ ...file, destination: file.destination.slice(prefix.length) })).sort((a, b) => a.destination.localeCompare(b.destination)); + if (!values.length) throw Error(`Repair lacks compatibility evidence for ${prefix}`); + return hashJson(values); + }; + const components: Record = {}; + const compare = (name: string, previous: string, updated: string) => { + if (previous !== updated) throw Error(`Repair subject compatibility changed: ${name}`); + components[name] = updated; + }; + for (const field of ["nativeImage", "pythonImage", "platform"] as const) compare(field, + hashJson(witness.image.build[field]), hashJson(current.build[field])); + compare("compiler", selection(old, witness.image.build.compiler ? "compiler/" : "spawnfile/"), selection(next, "compiler/")); + for (const [name, prefix] of Object.entries({ integration: "integration/", native: "paideia/dist/src/adapters/daimon-native/", + trials: "paideia/dist/src/experiments/trials/" })) compare(name, selection(old, prefix), selection(next, prefix)); + const excluded = ["bridge/paideia_dspy/checkpoint.py", "bridge/paideia_dspy/protocol.py", "bridge/README.md", "bridge/protocol.md", "bridge/.coverage", "bridge/.pytest_cache/", "bridge/coverage.json"]; + compare("optimizer", selection(old, "bridge/", excluded), selection(next, "bridge/", excluded)); + const previousInputs = witness.inputs.map(input => ({ id: input.id, destination: input.destination, digest: input.digest })); + const currentInputs = inputs.map(input => ({ id: input.id, destination: input.destination, digest: input.digest })); + if (hashJson(previousInputs) !== hashJson(currentInputs)) throw Error("Repair evidence or dataset inputs changed"); + return { components, inputs: Object.fromEntries(currentInputs.map(input => [input.id, input.digest])) }; +} diff --git a/src/compiler/training/repair/contract.ts b/src/compiler/training/repair/contract.ts new file mode 100644 index 00000000..58810a2b --- /dev/null +++ b/src/compiler/training/repair/contract.ts @@ -0,0 +1,34 @@ +import path from "node:path"; +import { z } from "zod"; +import { trainingContextSchema } from "../contract.js"; +import { trainingPreparationSchema, trainingBuildSchema } from "../preparation/contract.js"; + +export const sha = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +const relative = z.string().min(1).refine(value => !path.posix.isAbsolute(value) && !value.includes("\\") && + value.split("/").every(part => part !== "." && part !== ".." && part !== "")); +export const fileSchema = z.object({ destination: relative, sha256: sha, mode: z.number().int().min(0).max(511), + size: z.number().int().min(0).max(536870912) }).strict(); +export const witnessSchema = z.object({ + schema: z.enum(["spawnfile.training-legacy-witness.v1", "spawnfile.training-witness.v1"]), + parentImage: sha, parentPreparationDigest: sha, imagePlanDigest: sha, + config: trainingPreparationSchema, context: trainingContextSchema, + command: z.object({ args: z.array(z.string()), env: z.record(z.string(), z.string()).optional() }).strict(), + inputs: z.array(z.object({ id: z.string(), source: z.string(), destination: z.string(), digest: sha, + staged: z.string(), snapshotDigest: sha.nullable() }).strict()), + repair: z.object({ witness: sha, parent: sha }).strict().optional(), + image: z.object({ build: trainingBuildSchema, dockerfile: z.string(), entry: z.string(), files: z.array(fileSchema).min(1).max(10000) }).strict() +}).strict(); +export type TrainingWitness = z.infer; +export const witnessEnvelopeSchema = z.object({ manifest: witnessSchema, digest: sha }).strict(); +export const repairReceiptSchema = z.object({ + version: z.literal("paideia.measurement-repair.v1"), + parent: z.object({ root: z.literal("/run/training/inputs/repair-parent"), imageId: sha, + experimentId: z.string().uuid(), executionIdentity: z.string().regex(/^[a-f0-9]{64}$/u), manifestDigest: sha, + checkpoints: z.object({ command: sha, training: sha, host: sha, optimizer: sha }).strict(), + trainingIdentity: z.string().regex(/^[a-f0-9]{64}$/u) }).strict(), + current: z.object({ imageId: sha, canonicalSourceDigest: sha, adapterId: z.literal("daimon-native") }).strict(), + compatibility: z.object({ version: z.literal("spawnfile.daimon-dspy-compatibility.v1"), digest: sha, + components: z.record(z.string(), sha), inputs: z.record(z.string(), sha) }).strict() +}).strict(); +export const repairEnvelopeSchema = z.object({ receipt: repairReceiptSchema, digest: sha }).strict(); +export type RepairEnvelope = z.infer; diff --git a/src/compiler/training/repair/index.ts b/src/compiler/training/repair/index.ts new file mode 100644 index 00000000..2e567eff --- /dev/null +++ b/src/compiler/training/repair/index.ts @@ -0,0 +1,3 @@ +export { planMeasurementRepair, stageMeasurementRepair } from "./prepare.js"; +export { readTrainingWitness, writeTrainingWitness } from "./witness.js"; +export { repairEnvelopeSchema } from "./contract.js"; diff --git a/src/compiler/training/repair/prepare.ts b/src/compiler/training/repair/prepare.ts new file mode 100644 index 00000000..b96fa8d0 --- /dev/null +++ b/src/compiler/training/repair/prepare.ts @@ -0,0 +1,65 @@ +import { lstat, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import type { TrainingContext } from "../contract.js"; +import type { TrainingImagePlan } from "../preparation/image.js"; +import type { PlannedInput } from "../preparation/inputs.js"; +import { assertInputRoot, copySealed, exactPath, hashJson, sealFile, sealTree, within } from "../preparation/files.js"; +import { readTrainingWitness } from "./witness.js"; +import { verifyTrainingCompatibility } from "./compatibility.js"; +import { repairEnvelopeSchema } from "./contract.js"; + +const commandSchema = z.object({ schema: z.literal("paideia.command-checkpoint.v1"), id: z.string().uuid(), identity: z.string().regex(/^[a-f0-9]{64}$/u) }).strict(); +export async function planMeasurementRepair(options: { parent: string; witness?: string; output: string; + auth: string[]; image: TrainingImagePlan; inputs: PlannedInput[]; context: TrainingContext; resume: boolean }) { + const parent = await exactPath(options.parent); + assertInputRoot(parent, options.auth); + if (within(parent, options.output) || within(options.output, parent)) throw Error("Repair output must be disjoint from parent"); + if (!options.resume) { + try { await lstat(options.output); throw Error("Measurement repair requires a fresh output directory"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + } + const staging = path.join(path.dirname(parent), `.spawnfile-training-preparation-${hashJson(parent).slice(7, 23)}`); + const witness = await readTrainingWitness(options.witness ?? path.join(staging, "witness/manifest.json")); + const state = JSON.parse(await readFile(path.join(staging, "state.json"), "utf8")) as { digest: string; image: string }; + if (state.digest !== witness.manifest.parentPreparationDigest || state.image !== witness.manifest.parentImage) throw Error("Repair parent preparation does not match witness"); + const compatibility = verifyTrainingCompatibility(witness.manifest, options.image, options.inputs, options.context); + compatibility.components["parent-witness"] = witness.digest; + const files = [...await sealTree(path.join(parent, "runs"), "runs"), ...await sealTree(path.join(parent, "blobs"), "blobs")]; + const checkpoints: Record = {}; + for (const name of ["command", "training", "host", "optimizer"]) { + const file = await sealFile(path.join(parent, "checkpoint", `${name}.json`), `checkpoint/${name}.json`); + files.push(file); checkpoints[name] = file.sha256; + } + const command = commandSchema.parse(JSON.parse(await readFile(path.join(parent, "checkpoint/command.json"), "utf8"))); + const training = JSON.parse(await readFile(path.join(parent, "checkpoint/training.json"), "utf8")) as { state?: { identity?: unknown } }; + const trainingIdentity = z.string().regex(/^[a-f0-9]{64}$/u).parse(training.state?.identity); + const manifest = { version: "paideia.repair-parent.v1", files: files.map(file => ({ path: file.destination, sha256: file.sha256, size: file.size })) }; + return { parent, witness, compatibility, files, command, trainingIdentity, checkpoints, + manifest, manifestDigest: hashJson(manifest) }; +} +export type MeasurementRepairPlan = Awaited>; + +export async function stageMeasurementRepair(plan: MeasurementRepairPlan, staging: string, image: string, + canonicalSourceDigest: string, resume: boolean) { + const root = path.join(staging, "repair-parent"), repairPath = path.join(staging, "repair.json"); + const compatibility = { version: "spawnfile.daimon-dspy-compatibility.v1", ...plan.compatibility, + digest: hashJson(plan.compatibility) }; + const receipt = repairEnvelopeSchema.shape.receipt.parse({ version: "paideia.measurement-repair.v1", + parent: { root: "/run/training/inputs/repair-parent", imageId: plan.witness.manifest.parentImage, + experimentId: plan.command.id, executionIdentity: plan.command.identity, manifestDigest: plan.manifestDigest, + checkpoints: plan.checkpoints, trainingIdentity: plan.trainingIdentity }, + current: { imageId: image, canonicalSourceDigest, adapterId: "daimon-native" }, compatibility }); + const envelope = { receipt, digest: hashJson(receipt) }; + if (resume) { + if (await exactPath(repairPath) !== repairPath || hashJson(JSON.parse(await readFile(repairPath, "utf8"))) !== hashJson(envelope)) throw Error("Saved measurement repair receipt changed"); + if ((await sealFile(path.join(root, "projection-manifest.json"), "manifest")).sha256 !== plan.manifestDigest) throw Error("Saved repair projection changed"); + for (const file of plan.files) if ((await sealFile(path.join(root, file.destination), file.destination)).sha256 !== file.sha256) throw Error("Saved repair capture changed"); + } else { + await mkdir(root, { mode: 0o700 }); + await copySealed(plan.files, root); + await writeFile(path.join(root, "projection-manifest.json"), JSON.stringify(plan.manifest), { mode: 0o400, flag: "wx" }); + await writeFile(repairPath, JSON.stringify(envelope), { mode: 0o400, flag: "wx" }); + } + return { repairPath, root }; +} diff --git a/src/compiler/training/repair/repair.test.ts b/src/compiler/training/repair/repair.test.ts new file mode 100644 index 00000000..2ce28068 --- /dev/null +++ b/src/compiler/training/repair/repair.test.ts @@ -0,0 +1,130 @@ +import { chmod, mkdir, readFile, readdir, rm, writeFile, symlink } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { preparationFixture, imageDocker, image } from "../preparation/fixtures.test-helper.js"; +import { prepareTraining } from "../preparation/prepare.js"; +import { readTrainingWitness } from "./witness.js"; +import { hashJson } from "../preparation/files.js"; +import { repairEnvelopeSchema } from "./contract.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { + const f = await preparationFixture(); roots.push(f.root); + await f.put("paideia/dist/src/adapters/daimon-native/adapter.js", "native"); + await f.put("paideia/dist/src/experiments/trials/trial.js", "subject"); + await f.put("bridge/paideia_dspy/checkpoint.py", "checkpoint-v1"); + await f.put("bridge/paideia_dspy/protocol.py", "wire-v1"); + await f.put("bridge/paideia_dspy/optimizer.py", "gepa"); + const docker = imageDocker(); + const options = { configPath: f.configPath, context: f.context, args: f.args, dryRun: false, + process: docker.process, timeoutMs: 30000, packageRoot: path.join(f.root, "own"), streams: { stdout() {}, stderr() {} } }; + const previous = await prepareTraining(options); if ("dryRun" in previous) throw Error("actual"); + const parent = path.join(f.root, "output"), staging = path.dirname(previous.preparationPath); + await f.put("output/checkpoint/command.json", JSON.stringify({ schema: "paideia.command-checkpoint.v1", id: "00000000-0000-4000-8000-000000000001", identity: "a".repeat(64) })); + await f.put("output/checkpoint/training.json", JSON.stringify({ state: { identity: "b".repeat(64) } })); + await f.put("output/checkpoint/host.json", "{}"); await f.put("output/checkpoint/optimizer.json", "{}"); + await f.put("output/runs/record/events.jsonl", "event\n"); await f.put("output/blobs/digest", "article"); + await f.put("output/trials/private-home/auth", "DO-NOT-COPY"); await f.put("output/protected-judge-cache/cache", "DO-NOT-COPY"); + f.config.output.source = "child"; await f.save(); + const witnessPath = path.join(staging, "witness/manifest.json"), witness = await readTrainingWitness(witnessPath); + const process: typeof docker.process = async (args, opts) => args[2] === "image" && args[4] === image && args.at(-1)!.includes("Config.Labels") + ? { code: 0, stdout: `${JSON.stringify(image)}\n${JSON.stringify(witness.manifest.imagePlanDigest)}`, stderr: "" } + : docker.process(args, opts); + return { ...f, options: { ...options, process, args: ["--train", f.args[1]!, "--out", path.join(f.root, "child")], repairMeasurements: parent }, + parent, staging, witnessPath, witness, docker }; +} + +it("seals future witness automatically and projects repair captures with exact identities and no native homes", async () => { + const f = await fixture(); + // Only evaluator and checkpoint transport changes are compatible. + await f.put("paideia/dist/src/cli/main.js", "fixed evaluator"); + await f.put("bridge/paideia_dspy/checkpoint.py", "fork-support"); + await f.put("bridge/paideia_dspy/protocol.py", "fork-wire"); + const parentBytes = await readFile(path.join(f.parent, "checkpoint/host.json")); + const prepared = await prepareTraining(f.options); if ("dryRun" in prepared) throw Error("actual"); + const envelope = repairEnvelopeSchema.parse(JSON.parse(await readFile(prepared.repairPath!, "utf8"))); + expect(envelope.digest).toBe(hashJson(envelope.receipt)); + expect(envelope.receipt.parent.executionIdentity).toBe("a".repeat(64)); + expect(envelope.receipt.parent.trainingIdentity).toBe("b".repeat(64)); + expect(envelope.receipt.compatibility.components["parent-witness"]).toBe(f.witness.digest); + const launch = JSON.parse(await readFile(prepared.configPath, "utf8")); + const mounted = launch.inputs.find((input: { destination: string }) => input.destination.endsWith("repair-parent")); + expect((await readdir(mounted.source)).sort()).toEqual(["blobs", "checkpoint", "projection-manifest.json", "runs"]); + expect(await readdir(path.join(mounted.source, "checkpoint"))).toHaveLength(4); + const manifest = JSON.parse(await readFile(path.join(mounted.source, "projection-manifest.json"), "utf8")); + expect(hashJson(manifest)).toBe(envelope.receipt.parent.manifestDigest); + expect(manifest.files).toHaveLength(6); + expect(prepared.args.slice(-2)).toEqual(["--repair-context", "/run/paideia/repair.json"]); + expect(await readFile(path.join(f.parent, "checkpoint/host.json"))).toEqual(parentBytes); + expect(await prepareTraining({ ...f.options, args: [...f.options.args, "--resume"] })).toMatchObject({ repairPath: prepared.repairPath }); + await writeFile(path.join(mounted.source, "blobs/digest"), "tamper"); + await expect(prepareTraining({ ...f.options, args: [...f.options.args, "--resume"] })).rejects.toThrow("capture changed"); +}); + +it.each(["native", "trials", "compiler", "integration", "optimizer", "input", "source", "image", "settings"])("rejects changed %s before Docker or output writes", async kind => { + const f = await fixture(); + const changes: Record = { native: "paideia/dist/src/adapters/daimon-native/adapter.js", trials: "paideia/dist/src/experiments/trials/trial.js", + compiler: "own/dist/cli/index.js", integration: "integration/entry.ts", optimizer: "bridge/paideia_dspy/optimizer.py", input: "project/train.yaml" }; + if (changes[kind]) await f.put(changes[kind], "changed"); + if (kind === "source") f.options.context = { ...f.context, project: { ...f.context.project, sourceDigest: `sha256:${"f".repeat(64)}` } }; + if (kind === "image" && "build" in f.config.image) { f.config.image.build.nativeImage = `sha256:${"f".repeat(64)}`; await f.save(); } + if (kind === "settings") { f.config.integration.settings.path = "."; await f.save(); } + const count = f.docker.calls.length; + await expect(prepareTraining(f.options)).rejects.toThrow(/changed/u); + expect(f.docker.calls).toHaveLength(count); + await expect(readFile(path.join(f.root, "child/checkpoint/command.json"))).rejects.toThrow(); +}); + +it("rejects tampered witness or sealed compiler bytes and unavailable parent image labels", async () => { + const f = await fixture(), original = await readFile(f.witnessPath, "utf8"); + await chmod(f.witnessPath, 0o600); + const raw = JSON.parse(original); raw.manifest.parentImage = `sha256:${"f".repeat(64)}`; + await writeFile(f.witnessPath, JSON.stringify(raw)); + await expect(prepareTraining(f.options)).rejects.toThrow("witness digest"); + await writeFile(f.witnessPath, original); + await f.put(path.relative(f.root, path.join(f.staging, "witness/image-files/spawnfile/dist/cli/index.js")), "corrupted"); + await expect(prepareTraining(f.options)).rejects.toThrow("image bytes changed"); + await f.put(path.relative(f.root, path.join(f.staging, "witness/image-files/spawnfile/dist/cli/index.js"))); + await expect(prepareTraining({ ...f.options, process: f.docker.process })).rejects.toThrow(); +}); + +it("validates repair dry-run without Docker, auth reads, projection or output and requires fresh/disjoint output", async () => { + const f = await fixture(); const before = await readdir(f.root), calls = f.docker.calls.length; + f.config.auth[0]!.source = "missing-auth"; await f.save(); + expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); + expect(f.docker.calls).toHaveLength(calls); expect(await readdir(f.root)).toEqual(before); + await mkdir(path.join(f.root, "child")); + await expect(prepareTraining({ ...f.options, dryRun: true })).rejects.toThrow("fresh output"); + await expect(prepareTraining({ ...f.options, repairMeasurements: path.join(f.root, "child") })).rejects.toThrow("disjoint"); +}); + +it("rejects parent symlinks and changed preparation binding", async () => { + const f = await fixture(); await symlink(path.join(f.root, "auth"), path.join(f.parent, "blobs/leak")); + await expect(prepareTraining(f.options)).rejects.toThrow("symlinks"); + await rm(path.join(f.parent, "blobs/leak")); + const state = JSON.parse(await readFile(path.join(f.staging, "state.json"), "utf8")); state.digest = `sha256:${"d".repeat(64)}`; + await writeFile(path.join(f.staging, "state.json"), JSON.stringify(state)); + await expect(prepareTraining(f.options)).rejects.toThrow("does not match witness"); +}); + +it("checks recipe/preparation identities independently of the witness envelope and rejects oversize", async () => { + const f = await fixture(), original = await readFile(f.witnessPath, "utf8"); await chmod(f.witnessPath, 0o600); + for (const field of ["imagePlanDigest", "parentPreparationDigest"]) { + const envelope = JSON.parse(original); envelope.manifest[field] = `sha256:${"f".repeat(64)}`; + envelope.digest = hashJson(envelope.manifest); await writeFile(f.witnessPath, JSON.stringify(envelope)); + await expect(readTrainingWitness(f.witnessPath)).rejects.toThrow(/recipe identity|preparation identity/u); + } + await writeFile(f.witnessPath, " ".repeat(8 * 1024 * 1024 + 1)); + await expect(readTrainingWitness(f.witnessPath)).rejects.toThrow("8 MiB"); +}); + +it("records a valid future witness for a repaired child and accepts verified legacy schema", async () => { + const f = await fixture(); + const envelope = JSON.parse(await readFile(f.witnessPath, "utf8")); envelope.manifest.schema = "spawnfile.training-legacy-witness.v1"; + envelope.digest = hashJson(envelope.manifest); await chmod(f.witnessPath, 0o600); await writeFile(f.witnessPath, JSON.stringify(envelope)); + const prepared = await prepareTraining({ ...f.options, repairWitness: f.witnessPath }); if ("dryRun" in prepared) throw Error("actual"); + const future = await readTrainingWitness(path.join(path.dirname(prepared.preparationPath), "witness/manifest.json")); + expect(future.manifest.repair?.witness).toBe(envelope.digest); + expect(future.manifest.parentPreparationDigest).toBe(prepared.digest); +}); diff --git a/src/compiler/training/repair/witness.ts b/src/compiler/training/repair/witness.ts new file mode 100644 index 00000000..e5559319 --- /dev/null +++ b/src/compiler/training/repair/witness.ts @@ -0,0 +1,44 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { TrainingContext } from "../contract.js"; +import type { TrainingPreparationConfig } from "../preparation/contract.js"; +import type { TrainingImagePlan } from "../preparation/image.js"; +import { copySealed, exactPath, fileIdentity, hashJson, sealTree } from "../preparation/files.js"; +import type { PlannedInput } from "../preparation/inputs.js"; +import { witnessEnvelopeSchema, type TrainingWitness } from "./contract.js"; + +export async function readTrainingWitness(file: string) { + const canonical = await exactPath(file), bytes = await readFile(canonical, "utf8"); + if (Buffer.byteLength(bytes) > 8 * 1024 * 1024) throw Error("Training witness exceeds 8 MiB"); + const raw = JSON.parse(bytes) as { manifest: unknown; digest: string }; + if (hashJson(raw.manifest) !== raw.digest) throw Error("Training witness digest mismatch"); + const parsed = witnessEnvelopeSchema.parse(raw), witness = parsed.manifest; + const files = await sealTree(path.join(path.dirname(canonical), "image-files"), ""); + const sorted = (value: ReturnType) => [...value].sort((a, b) => a.destination.localeCompare(b.destination)); + if (hashJson(sorted(fileIdentity(files))) !== hashJson(sorted(witness.image.files))) throw Error("Training witness image bytes changed"); + const { recipe, nativeImage, pythonImage, platform } = witness.image.build; + if (hashJson({ recipe, nativeImage, pythonImage, platform, files: witness.image.files, + dockerfile: witness.image.dockerfile, entry: witness.image.entry }) !== witness.imagePlanDigest) throw Error("Training witness recipe identity mismatch"); + const expected = hashJson({ config: raw.manifest && (raw.manifest as TrainingWitness).config, + sources: witness.inputs.map(input => ({ id: input.id, digest: input.digest })), + image: witness.imagePlanDigest, canonical: witness.context.project.sourceDigest, ...(witness.repair ? { repair: witness.repair } : {}) }); + if (expected !== witness.parentPreparationDigest) throw Error("Training witness preparation identity mismatch"); + return { ...parsed, path: canonical }; +} + +/** Future repairs need sealed original bytes, not a mutable checkout or caller assertion. */ +export async function writeTrainingWitness(options: { directory: string; digest: string; image: string; + config: TrainingPreparationConfig; context: TrainingContext; args: readonly string[]; + inputs: PlannedInput[]; staged: string[]; snapshots: (string | null)[]; plan: TrainingImagePlan; repair?: { witness: string; parent: string } }) { + const directory = path.join(options.directory, "witness"); + await mkdir(directory, { mode: 0o700 }); + await copySealed(options.plan.files, path.join(directory, "image-files")); + const manifest: TrainingWitness = { schema: "spawnfile.training-witness.v1", parentImage: options.image, + parentPreparationDigest: options.digest, imagePlanDigest: options.plan.digest, config: options.config, + ...(options.repair ? { repair: options.repair } : {}), + context: options.context, command: { args: [...options.args] }, + inputs: options.inputs.map((input, index) => ({ id: input.id, source: input.source, destination: input.destination, + digest: input.digest, staged: options.staged[index]!, snapshotDigest: options.snapshots[index]! })), + image: { build: options.plan.build, files: fileIdentity(options.plan.files), dockerfile: options.plan.dockerfile, entry: options.plan.entry } }; + await writeFile(path.join(directory, "manifest.json"), JSON.stringify({ manifest, digest: hashJson(manifest) }), { flag: "wx", mode: 0o400 }); +} From afddd27656a84dfde6b57ccf3e33e8325d9874e9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 05:34:27 +0200 Subject: [PATCH 11/84] fix: forward paired measurement repair arguments --- specs/TRAINING_CONTAINERS.md | 2 +- src/compiler/training/preparation/prepare.ts | 2 +- src/compiler/training/repair/repair.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 67af9775..030a013a 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -212,7 +212,7 @@ files are copied into a sealed read-only parent projection. Runtime homes, auth, mutable caches and invocation databases are excluded. The exact projection manifest and checkpoint bytes are hashed. `/run/paideia/repair.json` is a protected read-only `paideia.measurement-repair.v1` receipt; the launcher forwards its fixed path via -`--repair-context`. It contains container paths and digests, no host credentials. +`--repair-context` together with `--repair-measurements /run/training/inputs/repair-parent`. Both refer to the verified read-only projection; no host paths or credentials are forwarded. Paideia owns error-only rescoring, retaining successful historical pass/fail results, paired optimizer import, cumulative accounting and the new experiment lineage. A repair receipt does not itself assert that any judgment or continuation succeeded. diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index fb4e9aa5..10e2e278 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -142,7 +142,7 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< if (hashJson(await readBoundedJson(repairConfig)) !== hashJson(repairLaunch)) throw Error("Saved repair launch changed"); } else await writeFile(repairConfig, JSON.stringify(repairLaunch), { flag: "wx", mode: 0o600 }); return { digest, image, configPath: repairConfig, preparationPath, repairPath: projected.repairPath, context, - args: [...args, "--repair-context", "/run/paideia/repair.json"] }; + args: [...args, "--repair-measurements", "/run/training/inputs/repair-parent", "--repair-context", "/run/paideia/repair.json"] }; } return { digest, image, configPath, preparationPath, context, args }; } diff --git a/src/compiler/training/repair/repair.test.ts b/src/compiler/training/repair/repair.test.ts index 2ce28068..b1c93339 100644 --- a/src/compiler/training/repair/repair.test.ts +++ b/src/compiler/training/repair/repair.test.ts @@ -55,7 +55,7 @@ it("seals future witness automatically and projects repair captures with exact i const manifest = JSON.parse(await readFile(path.join(mounted.source, "projection-manifest.json"), "utf8")); expect(hashJson(manifest)).toBe(envelope.receipt.parent.manifestDigest); expect(manifest.files).toHaveLength(6); - expect(prepared.args.slice(-2)).toEqual(["--repair-context", "/run/paideia/repair.json"]); + expect(prepared.args.slice(-4)).toEqual(["--repair-measurements", "/run/training/inputs/repair-parent", "--repair-context", "/run/paideia/repair.json"]); expect(await readFile(path.join(f.parent, "checkpoint/host.json"))).toEqual(parentBytes); expect(await prepareTraining({ ...f.options, args: [...f.options.args, "--resume"] })).toMatchObject({ repairPath: prepared.repairPath }); await writeFile(path.join(mounted.source, "blobs/digest"), "tamper"); From 47050bea9237c47f2d886d5bc347e040e5ac9129 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Tue, 15 Sep 2026 05:50:50 +0200 Subject: [PATCH 12/84] fix: accept verified lineage in chained training repairs --- src/compiler/training/repair/AGENTS.md | 4 +++ src/compiler/training/repair/prepare.ts | 5 ++- src/compiler/training/repair/repair.test.ts | 40 ++++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/compiler/training/repair/AGENTS.md b/src/compiler/training/repair/AGENTS.md index 92777da9..47b45374 100644 --- a/src/compiler/training/repair/AGENTS.md +++ b/src/compiler/training/repair/AGENTS.md @@ -8,3 +8,7 @@ resume. Compare pinned compiler/native/integration/input closures before a recei Legacy witnesses require complete verified image files and original recipe label; new preparation records this witness automatically. Keep tests adjacent and guards mutation-tested. No model calls or project scripts belong here. + +- Parent command checkpoints may contain the documented optional strict lineage + fields. Preserve those bytes and cumulative rescores when chaining repairs; + automatically use the immediate parent witness, never rewrite identities. diff --git a/src/compiler/training/repair/prepare.ts b/src/compiler/training/repair/prepare.ts index b96fa8d0..3a3f75a2 100644 --- a/src/compiler/training/repair/prepare.ts +++ b/src/compiler/training/repair/prepare.ts @@ -9,7 +9,10 @@ import { readTrainingWitness } from "./witness.js"; import { verifyTrainingCompatibility } from "./compatibility.js"; import { repairEnvelopeSchema } from "./contract.js"; -const commandSchema = z.object({ schema: z.literal("paideia.command-checkpoint.v1"), id: z.string().uuid(), identity: z.string().regex(/^[a-f0-9]{64}$/u) }).strict(); +const commandSchema = z.object({ schema: z.literal("paideia.command-checkpoint.v1"), id: z.string().uuid(), + identity: z.string().regex(/^[a-f0-9]{64}$/u), + lineage: z.object({ parentId: z.string().uuid(), parentIdentity: z.string().regex(/^[a-f0-9]{64}$/u), + receiptDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/u) }).strict().optional() }).strict(); export async function planMeasurementRepair(options: { parent: string; witness?: string; output: string; auth: string[]; image: TrainingImagePlan; inputs: PlannedInput[]; context: TrainingContext; resume: boolean }) { const parent = await exactPath(options.parent); diff --git a/src/compiler/training/repair/repair.test.ts b/src/compiler/training/repair/repair.test.ts index b1c93339..3e3edcc5 100644 --- a/src/compiler/training/repair/repair.test.ts +++ b/src/compiler/training/repair/repair.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, readFile, readdir, rm, writeFile, symlink } from "node:fs/promises"; +import { chmod, mkdir, readFile, readdir, cp, rm, writeFile, symlink } from "node:fs/promises"; import path from "node:path"; import { afterEach, expect, it } from "vitest"; import { preparationFixture, imageDocker, image } from "../preparation/fixtures.test-helper.js"; @@ -128,3 +128,41 @@ it("records a valid future witness for a repaired child and accepts verified leg expect(future.manifest.repair?.witness).toBe(envelope.digest); expect(future.manifest.parentPreparationDigest).toBe(prepared.digest); }); + + +it("chains a repair from its automatic witness while preserving completed captures and rescores", async () => { + const f = await fixture(); + const first = await prepareTraining(f.options); if ("dryRun" in first) throw Error("actual"); + const parent = path.join(f.root, "child"), childId = "00000000-0000-4000-8000-000000000002"; + const firstReceipt = JSON.parse(await readFile(first.repairPath!, "utf8")); + const lineage = { parentId: firstReceipt.receipt.parent.experimentId, + parentIdentity: firstReceipt.receipt.parent.executionIdentity, receiptDigest: firstReceipt.digest }; + await f.put("child/checkpoint/command.json", JSON.stringify({ schema: "paideia.command-checkpoint.v1", id: childId, identity: "c".repeat(64), lineage })); + await f.put("child/checkpoint/training.json", JSON.stringify({ state: { identity: "d".repeat(64), recoveries: 1 } })); + for (const name of ["host", "optimizer"]) await f.put(`child/checkpoint/${name}.json`, "{}"); + await cp(path.join(f.parent, "runs"), path.join(parent, "runs"), { recursive: true }); + await cp(path.join(f.parent, "blobs"), path.join(parent, "blobs"), { recursive: true }); + await f.put("child/runs/record/evaluations/rescore.json", "retained successful rescore"); + f.config.output.source = "grandchild"; await f.save(); + const next = await prepareTraining({ ...f.options, repairMeasurements: parent, + args: ["--train", f.args[1]!, "--out", path.join(f.root, "grandchild")] }); + if ("dryRun" in next) throw Error("actual"); + const receipt = JSON.parse(await readFile(next.repairPath!, "utf8")); + expect(receipt.receipt.parent.experimentId).toBe(childId); + expect(receipt.receipt.parent.executionIdentity).toBe("c".repeat(64)); + const witness = await readTrainingWitness(path.join(path.dirname(first.preparationPath), "witness/manifest.json")); + expect(receipt.receipt.compatibility.components["parent-witness"]).toBe(witness.digest); + const projected = path.join(path.dirname(next.repairPath!), "repair-parent"); + expect(await readFile(path.join(projected, "checkpoint/command.json"))).toEqual(await readFile(path.join(parent, "checkpoint/command.json"))); + expect(await readFile(path.join(projected, "runs/record/evaluations/rescore.json"), "utf8")).toBe("retained successful rescore"); +}); + +it.each([{}, { parentId: "not-uuid", parentIdentity: "a".repeat(64), receiptDigest: `sha256:${"b".repeat(64)}` }, + { parentId: "00000000-0000-4000-8000-000000000001", parentIdentity: "invalid", receiptDigest: "invalid" }, + { parentId: "00000000-0000-4000-8000-000000000001", parentIdentity: "a".repeat(64), receiptDigest: `sha256:${"b".repeat(64)}`, extra: true }])( + "rejects malformed or unknown parent lineage fields", async lineage => { + const f = await fixture(); + const file = path.join(f.parent, "checkpoint/command.json"), original = JSON.parse(await readFile(file, "utf8")); + await writeFile(file, JSON.stringify({ ...original, lineage })); + await expect(prepareTraining({ ...f.options, dryRun: true })).rejects.toThrow(); + }); From 6c2bd45b32d474386eef46e8b67234c2ec82cc0b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:20:13 +0200 Subject: [PATCH 13/84] perf: memoize training image seal digests by file identity --- specs/TRAINING_CONTAINERS.md | 11 +++ src/compiler/training/preparation/AGENTS.md | 5 +- src/compiler/training/preparation/files.ts | 43 ++++++--- .../preparation/fixtures.test-helper.ts | 4 + src/compiler/training/preparation/image.ts | 34 +++---- .../training/preparation/prepare.test.ts | 9 ++ src/compiler/training/preparation/prepare.ts | 7 +- .../training/preparation/sealMemo.test.ts | 89 +++++++++++++++++++ src/compiler/training/preparation/sealMemo.ts | 86 ++++++++++++++++++ 9 files changed, 257 insertions(+), 31 deletions(-) create mode 100644 src/compiler/training/preparation/sealMemo.test.ts create mode 100644 src/compiler/training/preparation/sealMemo.ts diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 030a013a..837894fa 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -164,6 +164,17 @@ datasets never enter the image context. Cache identity includes actual source, lock, executable and recipe bytes, parent images, architecture and entrypoint; reuse also verifies the image ID and recipe label in the selected Docker context. +Recipe planning may reuse a private same-user digest memo at +`$SPAWNFILE_HOME/cache/training-seal.v1.json` (default `~/.spawnfile`; mode 0600, +atomic replace, at most 20,000 entries). An entry is keyed by the canonical absolute +path, device, inode, size, and nanosecond mtime and ctime; any difference rehashes. +Digests are recorded only when the file was unchanged on the open handle while +hashing and its newest timestamp predates hashing by at least two seconds. A corrupt, +foreign-owned, group/world-accessible or unwritable memo is ignored and never fails +preparation. Dry-run reads but never writes it. The memo only shortens plan-side +hashing: staged build-context copies and preparation/resume/repair verification are +always rehashed in full, and image reuse still requires the recipe label. + The installed integration reads `/run/paideia/preparation.json` using `parseTrainingMappedPreparation` from `spawnfile/training`. This protected `spawnfile.training-preparation.v1` receipt contains the preparation digest, diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index 92b617f3..670ea90c 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -8,7 +8,7 @@ inputs. Dry-run performs reads only; resume validates the preserved preparation. `contract.ts` declares authoring and runtime receipts; `files.ts` safely seals declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; -`prepare.ts` combines those operations. Keep tests adjacent and files below 400 +`sealMemo.ts` is the private plan-side digest memo; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Built preparations seal original image bytes in a protected witness for explicit @@ -17,3 +17,6 @@ lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Exclude explicit generated Python coverage/cache files, not arbitrary dotfiles. - Repair transport/compatibility lives in sibling `repair/`; it never changes candidate, criteria or checkpoint semantics owned by Paideia. +- The seal memo (`sealMemo.ts`) is used only by `planTrainingImage`. Never pass it + to `copySealed`, input snapshots, resume, pins or repair verification; those + always rehash. Its key must keep ctime (content rewrites that restore mtime). diff --git a/src/compiler/training/preparation/files.ts b/src/compiler/training/preparation/files.ts index 888916b2..3726a6c8 100644 --- a/src/compiler/training/preparation/files.ts +++ b/src/compiler/training/preparation/files.ts @@ -1,8 +1,9 @@ import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { copyFile, lstat, mkdir, readdir, realpath, chmod } from "node:fs/promises"; +import { constants, type BigIntStats } from "node:fs"; +import { copyFile, lstat, mkdir, open, readdir, realpath, chmod } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import type { SealMemo } from "./sealMemo.js"; export const within = (root: string, file: string): boolean => file === root || file.startsWith(root + path.sep); export const hashJson = (value: unknown): string => `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; @@ -22,23 +23,39 @@ export async function exactPath(source: string): Promise { return absolute; } -export async function sealFile(source: string, destination: string): Promise { +export async function sealFile(source: string, destination: string, memo?: SealMemo): Promise { await exactPath(source); const before = await lstat(source, { bigint: true }); if (!before.isFile() || before.size > 536_870_912n) throw Error("Training source must be a regular file no larger than 512 MiB"); + const mode = Number(before.mode & 0o777n); + const remembered = memo?.lookup(source, before); + if (remembered !== undefined) return { source, destination, sha256: remembered, mode, size: Number(before.size) }; + const hashStartedNs = BigInt(Date.now() + 1) * 1_000_000n; + const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW); const digest = createHash("sha256"); let size = 0; - for await (const chunk of createReadStream(source)) { - size += chunk.length; - if (size > 536_870_912) throw Error("Training source exceeded its size limit"); - digest.update(chunk); - } - const after = await lstat(source, { bigint: true }); - if (BigInt(size) !== before.size || after.size !== before.size || after.ino !== before.ino || after.ctimeNs !== before.ctimeNs || after.dev !== before.dev) throw Error("Training source changed while hashing"); - return { source, destination, sha256: `sha256:${digest.digest("hex")}`, mode: Number(before.mode & 0o777n), size }; + let opened: BigIntStats, after: BigIntStats; + try { + opened = await handle.stat({ bigint: true }); + if (!sameFile(opened, before)) throw Error("Training source changed while hashing"); + for await (const chunk of handle.createReadStream({ autoClose: false, start: 0 })) { + size += chunk.length; + if (size > 536_870_912) throw Error("Training source exceeded its size limit"); + digest.update(chunk); + } + after = await handle.stat({ bigint: true }); + } finally { await handle.close(); } + const current = await lstat(source, { bigint: true }); + if (BigInt(size) !== before.size || !sameFile(after, before) || !sameFile(current, before)) throw Error("Training source changed while hashing"); + const sha256 = `sha256:${digest.digest("hex")}`; + memo?.record(source, before, sha256, hashStartedNs); + return { source, destination, sha256, mode, size }; } +const sameFile = (left: BigIntStats, right: BigIntStats): boolean => left.size === right.size && left.ino === right.ino && + left.dev === right.dev && left.ctimeNs === right.ctimeNs && left.mtimeNs === right.mtimeNs; + /** Only explicitly selected trees are traversed. Symlinks never enter Docker context. */ -export async function sealTree(source: string, destination: string, options: { ignoreDevelopment?: boolean; ignoreGit?: boolean; internalSymlinks?: boolean } = {}): Promise { +export async function sealTree(source: string, destination: string, options: { ignoreDevelopment?: boolean; ignoreGit?: boolean; internalSymlinks?: boolean; memo?: SealMemo } = {}): Promise { await exactPath(source); const files: SealedFile[] = []; const walk = async (root: string, target: string, depth: number): Promise => { @@ -55,7 +72,7 @@ export async function sealTree(source: string, destination: string, options: { i if (options.ignoreDevelopment && (ignored.has(entry) || /(?:\.test\.[cm]?[jt]s|_test\.py|\.pyc)$/u.test(entry))) continue; await walk(path.join(root, entry), path.posix.join(target, entry), depth + 1); } - } else files.push(await sealFile(root, target)); + } else files.push(await sealFile(root, target, options.memo)); }; await walk(source, destination, 0); if (files.length > 10000 || files.reduce((sum, file) => sum + file.size, 0) > 1_073_741_824) throw Error("Training source tree exceeds bounds"); diff --git a/src/compiler/training/preparation/fixtures.test-helper.ts b/src/compiler/training/preparation/fixtures.test-helper.ts index 608507eb..7afc7113 100644 --- a/src/compiler/training/preparation/fixtures.test-helper.ts +++ b/src/compiler/training/preparation/fixtures.test-helper.ts @@ -2,10 +2,14 @@ import { mkdtemp, mkdir, readFile, realpath, writeFile } from "node:fs/promises" import path from "node:path"; import os from "node:os"; import { createHash } from "node:crypto"; +import { mkdtempSync } from "node:fs"; import type { TrainingPreparationConfig } from "./contract.js"; import type { TrainingContext } from "../contract.js"; import type { TrainingDockerProcess } from "../container/process.js"; +// Preparation tests must never touch the developer's real seal memo cache. +process.env.SPAWNFILE_HOME = mkdtempSync(path.join(os.tmpdir(), "spawnfile-preparation-home-")); + export const image = `sha256:${"a".repeat(64)}`; export const sha = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; export async function preparationFixture() { diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 5ff11252..31f3e34d 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -4,13 +4,14 @@ import { fileURLToPath } from "node:url"; import type { TrainingDockerProcess } from "../container/process.js"; import type { TrainingImageBuild } from "./contract.js"; import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; +import type { SealMemo } from "./sealMemo.js"; const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); export const trainingAssets = path.extname(fileURLToPath(import.meta.url)) === ".ts" ? path.join(packageRoot, "runtime-images/training") : fileURLToPath(new URL("./assets/", import.meta.url)); export interface TrainingImagePlan { digest: string; files: SealedFile[]; dockerfile: string; entry: string; build: TrainingImageBuild } -async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json")): Promise { +async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json"), memo?: SealMemo): Promise { const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")); const lock = JSON.parse(await readFile(lockSource, "utf8")); if (![2, 3].includes(lock.lockfileVersion) || !lock.packages?.[""] || @@ -19,12 +20,13 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc const item = entry as { link?: boolean; resolved?: string }; return item.link || item.resolved?.startsWith("file:") || item.resolved?.startsWith("../"); })) throw Error(`Training package ${target} has unsupported local dependencies; provide a complete registry-locked distribution`); - return [await sealFile(path.join(root, "package.json"), `${target}/package.json`), - await sealFile(lockSource, `${target}/package-lock.json`), - ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true }) : []]; + return [await sealFile(path.join(root, "package.json"), `${target}/package.json`, memo), + await sealFile(lockSource, `${target}/package-lock.json`, memo), + ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true, memo }) : []]; } -export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { +/** `memo` only skips plan-side rehashing; staged context copies are always fully rehashed before a build. */ +export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot, memo?: SealMemo): Promise { const resolve = (value: string) => path.resolve(root, value); for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); @@ -32,19 +34,19 @@ export async function planTrainingImage(build: TrainingImageBuild, root: string, const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot ? path.join(ownRoot, "package-lock.json") : path.join(assets, "package-lock.json"); const files = [ - ...await packageFiles(resolve(build.paideia), "paideia", true), - ...await packageFiles(ownRoot, "spawnfile", true, ownLock), + ...await packageFiles(resolve(build.paideia), "paideia", true, undefined, memo), + ...await packageFiles(ownRoot, "spawnfile", true, ownLock, memo), ...await packageFiles(build.compiler ? resolve(build.compiler) : ownRoot, "compiler", true, - build.compiler ? path.join(resolve(build.compiler), "package-lock.json") : ownLock), - ...await packageFiles(resolve(build.claude), "claude", false), - ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), - ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), - ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }) : [], - await sealFile(resolve(build.grok.source), "grok"), - await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), - await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json"), + build.compiler ? path.join(resolve(build.compiler), "package-lock.json") : ownLock, memo), + ...await packageFiles(resolve(build.claude), "claude", false, undefined, memo), + ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true, memo }), + ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true, memo }), + ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true, memo }) : [], + await sealFile(resolve(build.grok.source), "grok", memo), + await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml", memo), + await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json", memo), ...await Promise.all(["runtimes.yaml", "moltnet-releases.json"].map(name => - sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`))) + sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`, memo))) ]; if (files.find(file => file.destination === "grok")!.sha256 !== build.grok.sha256) throw Error("Grok executable digest mismatch"); for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js", "compiler/dist/cli/index.js"]) { diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index a3683ffe..4fed9a72 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -34,6 +34,15 @@ it("dry-run reads declarations without Docker, auth access, output or staging mu const before = await readdir(f.root); expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); expect(f.docker.calls).toEqual([]); expect(await readdir(f.root)).toEqual(before); + const memo = path.join(process.env.SPAWNFILE_HOME!, "cache", "training-seal.v1.json"); + await expect(lstat(memo)).rejects.toThrow(); + f.config.auth[0]!.source = "auth"; await f.save(); + // Outlive the racy-clean window so an actual preparation would record digests. + await new Promise(resolve => setTimeout(resolve, 2100)); + expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); + await expect(lstat(memo)).rejects.toThrow(); + await prepareTraining(f.options); + expect((await lstat(memo)).mode & 0o777).toBe(0o600); }); it("rejects changed executable, fixture, declaration and saved image on exact resume", async () => { diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index 10e2e278..c874b55e 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -7,6 +7,7 @@ import { trainingPreparationSchema, parseTrainingMappedPreparation, type Trainin import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } from "./files.js"; import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; import { planTrainingImage, buildTrainingImage } from "./image.js"; +import { openSealMemo } from "./sealMemo.js"; import { planMeasurementRepair, stageMeasurementRepair, writeTrainingWitness } from "../repair/index.js"; function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { @@ -23,6 +24,8 @@ export interface PrepareTrainingOptions { streams: { stdout(line: string): void; stderr(line: string): void }; /** Test-only package fixture; production always resolves its own installed distribution. */ packageRoot?: string; + /** Plan-side digest memo; defaults to the private Spawnfile cache. Dry-run never writes it. */ + sealMemoPath?: string; repairMeasurements?: string; repairWitness?: string; } export interface PreparedTraining { @@ -50,7 +53,8 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< for (const other of inputs.slice(index + 1)) if (within(input.source, other.source) || within(other.source, input.source) || within(input.destination, other.destination) || within(other.destination, input.destination)) throw Error("Training inputs overlap"); } - const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot) : undefined; + const sealMemo = "build" in config.image ? await openSealMemo(options.sealMemoPath) : undefined; + const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot, sealMemo) : undefined; if (options.repairWitness && !options.repairMeasurements) throw Error("A repair witness requires --repair-measurements"); if (options.repairMeasurements && !imagePlan) throw Error("Measurement repair requires a verifiable image build recipe"); const repair = options.repairMeasurements ? await planMeasurementRepair({ parent: options.repairMeasurements, @@ -60,6 +64,7 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image, canonical: options.context.project.sourceDigest, ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); if (options.dryRun) return { digest, dryRun: true }; + await sealMemo?.save(); options.signal?.throwIfAborted(); for (const entry of auth) if (await exactPath(entry.source) !== entry.source || !(await lstat(entry.source)).isFile()) throw Error("Training auth must be a canonical regular leaf"); const execute = (args: string[]) => options.process(args, { timeoutMs: options.timeoutMs, signal: options.signal }); diff --git a/src/compiler/training/preparation/sealMemo.test.ts b/src/compiler/training/preparation/sealMemo.test.ts new file mode 100644 index 00000000..13aaa6eb --- /dev/null +++ b/src/compiler/training/preparation/sealMemo.test.ts @@ -0,0 +1,89 @@ +import { chmod, mkdtemp, readFile, realpath, rm, stat, utimes, writeFile } from "node:fs/promises"; +import * as fsPromises from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { sealFile, sealTree } from "./files.js"; +import { openSealMemo, trainingSealMemoPath } from "./sealMemo.js"; + +vi.mock("node:fs/promises", async original => { + const actual = await original(); + return { ...actual, open: vi.fn(actual.open) }; +}); + +const roots: string[] = []; +afterEach(async () => { vi.mocked(fsPromises.open).mockClear(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-seal-memo-"))); roots.push(root); return root; } +const opensOf = (file: string) => vi.mocked(fsPromises.open).mock.calls.filter(call => call[0] === file).length; +const memoOptions = { racySafetyNs: 0n }; + +it("reuses a persisted digest without opening or reading the source bytes", async () => { + const root = await fixture(), source = path.join(root, "grok"), memoPath = path.join(root, "cache", "training-seal.v1.json"); + await writeFile(source, "native-binary"); + const first = await openSealMemo(memoPath, memoOptions); + const sealed = await sealFile(source, "grok", first); await first.save(); + expect(opensOf(source)).toBe(1); + expect((await stat(memoPath)).mode & 0o777).toBe(0o600); + const second = await openSealMemo(memoPath, memoOptions); + expect(await sealFile(source, "grok", second)).toEqual(sealed); + expect(opensOf(source)).toBe(1); + expect(await sealFile(source, "grok")).toEqual(sealed); + expect(opensOf(source)).toBe(2); +}); + +it("rehashes when content changes under a restored mtime because ctime is part of the key", async () => { + const root = await fixture(), source = path.join(root, "grok"), memoPath = path.join(root, "memo.json"); + await writeFile(source, "AAAA"); + // Whole-second times restore exactly, so only ctime can distinguish the rewrite. + const pinned = new Date("2020-01-01T00:00:00Z"); await utimes(source, pinned, pinned); + const memo = await openSealMemo(memoPath, memoOptions); + const original = await sealFile(source, "grok", memo); await memo.save(); + const before = await stat(source, { bigint: true }); + await writeFile(source, "BBBB"); await utimes(source, pinned, pinned); + const after = await stat(source, { bigint: true }); + expect([after.mtimeNs, after.size, after.ino, after.dev]).toEqual([before.mtimeNs, before.size, before.ino, before.dev]); + expect(after.ctimeNs).not.toBe(before.ctimeNs); + const reopened = await openSealMemo(memoPath, memoOptions); + const changed = await sealFile(source, "grok", reopened); + expect(changed.sha256).not.toBe(original.sha256); + expect(changed.sha256).toBe((await sealFile(source, "grok")).sha256); +}); + +it("does not memoize racily fresh files and tolerates corrupt, foreign-mode or unwritable memos", async () => { + const root = await fixture(), source = path.join(root, "file"), memoPath = path.join(root, "memo.json"); + await writeFile(source, "fresh"); + const fresh = await openSealMemo(memoPath); await sealFile(source, "file", fresh); await fresh.save(); + await expect(stat(memoPath)).rejects.toThrow(); + for (const [body, mode] of [["{not json", 0o600], [JSON.stringify({ version: "other", entries: [] }), 0o600], + [JSON.stringify({ version: "spawnfile.training-seal-memo.v1", entries: [[`["${source}"]`, "sha256:bad"], "x"] }), 0o600], + [JSON.stringify({ version: "spawnfile.training-seal-memo.v1", entries: {} }), 0o600]] as const) { + await writeFile(memoPath, body); await chmod(memoPath, mode); + const memo = await openSealMemo(memoPath, memoOptions); + expect((await sealFile(source, "file", memo)).sha256).toMatch(/^sha256:/u); + await memo.save(); + expect(JSON.parse(await readFile(memoPath, "utf8")).entries).toHaveLength(1); + } + await chmod(memoPath, 0o644); + const opened = opensOf(source); + await sealFile(source, "file", await openSealMemo(memoPath, memoOptions)); + expect(opensOf(source)).toBe(opened + 1); + const blocked = path.join(root, "blocked"); await writeFile(blocked, "not a directory"); + const unwritable = await openSealMemo(path.join(blocked, "memo.json"), memoOptions); + await sealFile(source, "file", unwritable); + await expect(unwritable.save()).resolves.toBeUndefined(); +}); + +it("bounds retained entries and defaults to the private Spawnfile cache", async () => { + const root = await fixture(), memoPath = path.join(root, "memo.json"); + for (const name of ["a", "b", "c", "d"]) await writeFile(path.join(root, "tree", name).replace("/tree/", "/tree-"), name); + const memo = await openSealMemo(memoPath, { ...memoOptions, maxEntries: 2 }); + for (const name of ["a", "b", "c", "d"]) await sealFile(path.join(root, `tree-${name}`), name, memo); + await memo.save(); + expect(JSON.parse(await readFile(memoPath, "utf8")).entries).toHaveLength(2); + expect(trainingSealMemoPath()).toBe(path.join(path.resolve(process.env.SPAWNFILE_HOME ?? path.join(os.homedir(), ".spawnfile")), "cache", "training-seal.v1.json")); + const tree = path.join(root, "dir"); await fsPromises.mkdir(tree); await writeFile(path.join(tree, "x"), "x"); + const treeMemo = await openSealMemo(memoPath, memoOptions); + const sealed = await sealTree(tree, "dir", { memo: treeMemo }); + expect(await sealTree(tree, "dir", { memo: treeMemo })).toEqual(sealed); + expect(opensOf(path.join(tree, "x"))).toBe(1); +}); diff --git a/src/compiler/training/preparation/sealMemo.ts b/src/compiler/training/preparation/sealMemo.ts new file mode 100644 index 00000000..ad799689 --- /dev/null +++ b/src/compiler/training/preparation/sealMemo.ts @@ -0,0 +1,86 @@ +import { randomBytes } from "node:crypto"; +import type { BigIntStats } from "node:fs"; +import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises"; +import path from "node:path"; +import { resolveSpawnfileHome } from "../../../auth/paths.js"; + +const VERSION = "spawnfile.training-seal-memo.v1"; +const DIGEST = /^sha256:[a-f0-9]{64}$/u; +const MAX_FILE_BYTES = 16 * 1024 * 1024; + +/** + * Private same-user cache of plan-side source digests. It never replaces the full + * rehash of staged build-context copies, so image bytes stay verified on every build. + */ +export interface SealMemo { + lookup(source: string, stat: BigIntStats): string | undefined; + /** Stores only racily-clean digests: metadata older than the safety window at hash start. */ + record(source: string, stat: BigIntStats, sha256: string, hashStartedNs: bigint): void; + save(): Promise; +} + +export interface SealMemoOptions { maxEntries?: number; racySafetyNs?: bigint } + +export const trainingSealMemoPath = (): string => path.join(resolveSpawnfileHome(), "cache", "training-seal.v1.json"); + +export const sealMemoKey = (source: string, stat: BigIntStats): string => + JSON.stringify([source, stat.dev.toString(), stat.ino.toString(), stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString()]); + +async function readTrusted(file: string): Promise> { + const entries = new Map(); + try { + const stat = await lstat(file); + if (!stat.isFile() || stat.size > MAX_FILE_BYTES || (stat.mode & 0o077) !== 0 || + (typeof process.getuid === "function" && stat.uid !== process.getuid())) return entries; + const parsed = JSON.parse(await readFile(file, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object" || (parsed as { version?: unknown }).version !== VERSION) return entries; + const raw = (parsed as { entries?: unknown }).entries; + if (!Array.isArray(raw)) return entries; + for (const item of raw) { + if (Array.isArray(item) && item.length === 2 && typeof item[0] === "string" && typeof item[1] === "string" && DIGEST.test(item[1])) { + entries.set(item[0], item[1]); + } + } + } catch { return new Map(); } + return entries; +} + +export async function openSealMemo(file = trainingSealMemoPath(), options: SealMemoOptions = {}): Promise { + const maxEntries = options.maxEntries ?? 20_000; + const racySafetyNs = options.racySafetyNs ?? 2_000_000_000n; + const previous = await readTrusted(file); + const touched = new Map(); + let dirty = false; + return { + lookup(source, stat) { + const key = sealMemoKey(source, stat); + const found = touched.get(key) ?? previous.get(key); + if (found !== undefined) touched.set(key, found); + return found; + }, + record(source, stat, sha256, hashStartedNs) { + if (!DIGEST.test(sha256)) return; + const newest = stat.ctimeNs > stat.mtimeNs ? stat.ctimeNs : stat.mtimeNs; + if (newest > hashStartedNs - racySafetyNs) return; + touched.set(sealMemoKey(source, stat), sha256); dirty = true; + }, + async save() { + const retained = [...touched]; + for (const entry of [...previous].reverse()) { + if (retained.length >= maxEntries) break; + if (!touched.has(entry[0])) retained.push(entry); + } + if (!dirty && retained.length === previous.size) return; + const bounded = retained.slice(0, maxEntries).reverse(); + const temporary = `${file}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; + try { + await mkdir(path.dirname(file), { recursive: true, mode: 0o700 }); + const handle = await open(temporary, "wx", 0o600); + try { await handle.writeFile(JSON.stringify({ version: VERSION, entries: bounded })); await handle.sync(); } + finally { await handle.close(); } + await rename(temporary, file); + dirty = false; + } catch { await rm(temporary, { force: true }).catch(() => undefined); } + } + }; +} From be3dc7fc8818d77cd3e3ba70ebbb7a654547c0a2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:23:43 +0200 Subject: [PATCH 14/84] perf: replace recursive training image chmod with staged modes and ordered layers --- runtime-images/training/AGENTS.md | 24 ++++ runtime-images/training/Dockerfile | 43 +++++--- specs/TRAINING_CONTAINERS.md | 7 ++ src/compiler/training/preparation/AGENTS.md | 2 +- .../training/preparation/contextModes.test.ts | 103 ++++++++++++++++++ .../training/preparation/contextModes.ts | 34 ++++++ .../training/preparation/image.test.ts | 21 +--- src/compiler/training/preparation/image.ts | 2 + 8 files changed, 198 insertions(+), 38 deletions(-) create mode 100644 src/compiler/training/preparation/contextModes.test.ts create mode 100644 src/compiler/training/preparation/contextModes.ts diff --git a/runtime-images/training/AGENTS.md b/runtime-images/training/AGENTS.md index 749d10fe..888c791b 100644 --- a/runtime-images/training/AGENTS.md +++ b/runtime-images/training/AGENTS.md @@ -9,3 +9,27 @@ verified native executables and an installed integration entrypoint. Credentials cases and results are runtime mounts, never image contents. Native runtime and Python base images must be immutable. Do not download unpinned CLI installers. The recipe is opt-in during incubation; no published runtime is implied. + +## Layer order and modes + +Layers run from least to most frequently changing: Python parent copies, locked +Claude/Paideia/Spawnfile/compiler dependencies, the lock-keyed DSPy venv, one +layout RUN, bridge source plus its editable install, then pinned executables and +distributions. Nothing RUNs after the late COPY layers, so a changed distribution +rebuilds only its own COPY layers and the label. + +In-image modes are unchanged from the former `chmod 0555 grok train && chmod -R +a+rX /opt/training`, and owners stay COPY/RUN defaults (root): + +- Staged context (`src/compiler/training/preparation/contextModes.ts`): every + directory gains 0555 (private staging dirs become 0755); every file gains 0444 + plus 0111 when any execute bit exists (0600 -> 0644, 0700/0744 -> 0755, + 0400 -> 0444); `grok` and `train` are exactly 0555. Staged mtimes are fixed at + 2000-01-01T00:00:00Z. +- RUN outputs: a `find ... -exec chmod a+rX` closure that selects only entries the + recursive chmod would change, so closed lower-layer files are never copied up. + It covers `/opt/training` after dependencies and the bridge after `pip -e`. The + integration `node_modules` layout is created after the closure, as before. +- `contextModes.test.ts` runs the rendered closure against real `chmod -R a+rX` + and asserts staged modes, recipe order and symlinks. Build-cache and timing + effects require a Docker-enabled measurement. diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index b41e4a00..a4138c8a 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -10,6 +10,12 @@ COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/ # libraries in the image so Python extensions also load in that clean process. RUN echo /opt/python/lib > /etc/ld.so.conf.d/training-python.conf && ldconfig +# Layers run from least to most frequently changing: locked dependencies, the +# DSPy venv, bridge source, pinned executables, then distributions. Staged context +# modes are already `a+rX`-closed (grok/train 0555), so late COPY layers need no RUN. +COPY claude/package.json claude/package-lock.json /opt/training/claude/ +RUN cd /opt/training/claude && npm ci --omit=dev + WORKDIR /opt/training/paideia COPY paideia/package.json paideia/package-lock.json ./ RUN npm ci --omit=dev --omit=peer --ignore-scripts \ @@ -24,26 +30,15 @@ WORKDIR /opt/training/compiler COPY compiler/package.json compiler/package-lock.json ./ RUN npm ci --omit=dev --ignore-scripts -COPY bridge /opt/training/paideia/bridges/dspy +COPY bridge/requirements.lock /opt/training/paideia/bridges/dspy/requirements.lock RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir -r /opt/training/paideia/bridges/dspy/requirements.lock \ - && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir setuptools==80.9.0 \ - && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy - -COPY claude/package.json claude/package-lock.json /opt/training/claude/ -RUN cd /opt/training/claude && npm ci --omit=dev -COPY grok /opt/training/bin/grok + && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir setuptools==80.9.0 -COPY paideia/dist /opt/training/paideia/dist -COPY spawnfile/dist /opt/training/spawnfile/dist -COPY compiler/dist /opt/training/compiler/dist -COPY compiler/runtimes.yaml compiler/moltnet-releases.json /opt/training/compiler/ -COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/ -COPY integration /opt/training/integration -COPY bootstrap /opt/training/bootstrap -COPY train /opt/training/bin/train -RUN chmod 0555 /opt/training/bin/train /opt/training/bin/grok \ - && chmod -R a+rX /opt/training \ +# Exactly `chmod -R a+rX /opt/training`, but only entries that need a change are +# touched, so unchanged lower-layer files are never copied up into a new layer. +RUN find /opt/training \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} + \ + && mkdir -m 0755 /opt/training/bin /opt/training/integration \ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \ && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \ @@ -51,6 +46,20 @@ RUN chmod 0555 /opt/training/bin/train /opt/training/bin/grok \ && ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia \ && ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile \ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon + +COPY bridge /opt/training/paideia/bridges/dspy +RUN /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy \ + && find /opt/training/paideia/bridges/dspy \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} + + +COPY grok /opt/training/bin/grok +COPY train /opt/training/bin/train +COPY compiler/runtimes.yaml compiler/moltnet-releases.json /opt/training/compiler/ +COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/ +COPY compiler/dist /opt/training/compiler/dist +COPY integration /opt/training/integration +COPY bootstrap /opt/training/bootstrap +COPY paideia/dist /opt/training/paideia/dist +COPY spawnfile/dist /opt/training/spawnfile/dist ENV PATH=/opt/training/bin:/opt/training/paideia/bridges/dspy/.venv/bin:/opt/spawnfile/runtime-installs/daimon/bin:/usr/local/bin:/usr/bin:/bin ENV HOME=/home/training WORKDIR /work diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 837894fa..629eeef2 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -175,6 +175,13 @@ preparation. Dry-run reads but never writes it. The memo only shortens plan-side hashing: staged build-context copies and preparation/resume/repair verification are always rehashed in full, and image reuse still requires the recipe label. +Staged build contexts normalize modes and times before `docker build`: directories +0755, files `a+rX`-closed, `grok` and the entrypoint 0555, mtimes fixed. Together +with the recipe's change-only `a+rX` closure this reproduces the former recursive +chmod's in-image modes without a whole-tree RUN after every distribution copy +(`runtime-images/training/AGENTS.md`). The recipe text is part of the image digest, +so this change rebuilds existing training images once. + The installed integration reads `/run/paideia/preparation.json` using `parseTrainingMappedPreparation` from `spawnfile/training`. This protected `spawnfile.training-preparation.v1` receipt contains the preparation digest, diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index 670ea90c..58861dc1 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -8,7 +8,7 @@ inputs. Dry-run performs reads only; resume validates the preserved preparation. `contract.ts` declares authoring and runtime receipts; `files.ts` safely seals declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; -`sealMemo.ts` is the private plan-side digest memo; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 +`sealMemo.ts` is the private plan-side digest memo; `contextModes.ts` normalizes staged build-context modes/times; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Built preparations seal original image bytes in a protected witness for explicit diff --git a/src/compiler/training/preparation/contextModes.test.ts b/src/compiler/training/preparation/contextModes.test.ts new file mode 100644 index 00000000..28ff17b9 --- /dev/null +++ b/src/compiler/training/preparation/contextModes.test.ts @@ -0,0 +1,103 @@ +import { execFileSync } from "node:child_process"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { normalizeTrainingContext, readableClosure, TRAINING_CONTEXT_MTIME } from "./contextModes.js"; +import { buildTrainingImage, planTrainingImage } from "./image.js"; +import { imageDocker, preparationFixture } from "./fixtures.test-helper.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function temporary() { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-context-modes-"))); roots.push(root); return root; } +const recipe = () => readFile(new URL("../../../../runtime-images/training/Dockerfile", import.meta.url), "utf8"); + +/** Modes staged before normalization: private dirs from `mkdir 0700` and assorted source file modes. */ +async function assortedTree(root: string): Promise { + const files: [string, number][] = [["grok", 0o700], ["train", 0o600], ["bridge/requirements.lock", 0o600], ["bridge/pkg/tool.py", 0o640], + ["bridge/pkg/run.sh", 0o710], ["integration/entry.ts", 0o400], ["integration/bin/partial", 0o100], ["paideia/dist/main.js", 0o644], + ["paideia/dist/exec.js", 0o755], ["paideia/dist/owner-exec.js", 0o744], ["bootstrap/start.ts", 0o604], ["sticky/file", 0o4750]]; + for (const [file, mode] of files) { + await mkdir(path.dirname(path.join(root, file)), { recursive: true, mode: 0o700 }); + await writeFile(path.join(root, file), file); await chmod(path.join(root, file), mode); + } + for (const [directory, mode] of [["bridge", 0o700], ["bridge/pkg", 0o711], ["integration/bin", 0o750], ["sticky", 0o2700]] as const) { + await chmod(path.join(root, directory), mode); + } +} +async function modes(root: string): Promise> { + const found: Record = {}; + const walk = async (directory: string) => { + for (const name of (await readdir(directory)).sort()) { + const entry = path.join(directory, name), stat = await lstat(entry); + found[path.relative(root, entry)] = (stat.mode & 0o7777).toString(8); + if (stat.isDirectory()) await walk(entry); + } + }; + await walk(root); return found; +} + +it("renders a closure that is exactly `chmod -R a+rX` for every staged mode shape", async () => { + const dockerfile = await recipe(); + const idioms = [...dockerfile.matchAll(/find (\S+) (\\\( \\\( -type d .*? -exec chmod a\+rX \{\} \+)/gu)]; + expect(idioms.map(match => match[1])).toEqual(["/opt/training", "/opt/training/paideia/bridges/dspy"]); + expect(new Set(idioms.map(match => match[2])).size).toBe(1); + const root = await temporary(), reference = path.join(root, "reference"), candidate = path.join(root, "candidate"); + await assortedTree(reference); await assortedTree(candidate); + execFileSync("chmod", ["-R", "a+rX", reference]); + execFileSync("/bin/sh", ["-c", `find ${JSON.stringify(candidate)} ${idioms[0]![2]!.replaceAll("\\(", "'('").replaceAll("\\)", "')'")}`]); + expect(await modes(candidate)).toEqual(await modes(reference)); +}); + +it("stages COPY content with the modes the former chmod 0555 grok/train plus chmod -R a+rX produced", async () => { + const root = await temporary(), reference = path.join(root, "reference"), candidate = path.join(root, "candidate"); + await assortedTree(reference); await assortedTree(candidate); + await chmod(path.join(reference, "grok"), 0o555); await chmod(path.join(reference, "train"), 0o555); + execFileSync("chmod", ["-R", "a+rX", reference]); + await normalizeTrainingContext(candidate); + const expected = await modes(reference); + expect(await modes(candidate)).toEqual(expected); + // Documented expectation: directories 0755, readable files 0644, executables 0755, grok/train 0555. + expect(expected).toMatchObject({ bridge: "755", "bridge/pkg": "755", "bridge/requirements.lock": "644", "bridge/pkg/run.sh": "755", + "integration/entry.ts": "444", "integration/bin/partial": "555", grok: "555", train: "555", sticky: "2755", "sticky/file": "4755" }); + expect((await lstat(path.join(candidate, "bridge/pkg/tool.py"))).mtime).toEqual(TRAINING_CONTEXT_MTIME); + expect(readableClosure(0o600, false)).toBe(0o644); +}); + +it("hands Docker a normalized context and never runs a layer after the distribution copies", async () => { + const f = await preparationFixture(); roots.push(f.root); if (!("build" in f.config.image)) throw Error("build expected"); + await chmod(path.join(f.root, "integration/entry.ts"), 0o600); await chmod(path.join(f.root, "grok"), 0o700); + const plan = await planTrainingImage(f.config.image.build, f.root, [], path.join(f.root, "own")), docker = imageDocker(); + let staged: Record = {}, mtimes = new Set(); + await buildTrainingImage(plan, { parent: f.root, dockerContext: "local", timeoutMs: 1000, streams: { stdout() {}, stderr() {} }, + process: async (args, options) => { + if (args[2] === "build") { + const context = args.at(-1)!; staged = await modes(context); + mtimes = new Set(await Promise.all(Object.keys(staged).filter(file => file !== "Dockerfile").map(async file => (await lstat(path.join(context, file))).mtimeMs))); + } + return docker.process(args, options); + } }); + expect(staged).toMatchObject({ grok: "555", train: "555", integration: "755", "integration/entry.ts": "644", bridge: "755", "paideia/dist/src/cli": "755" }); + expect(Object.entries(staged).filter(([, mode]) => !["755", "644", "555"].includes(mode))).toEqual([]); + expect([...mtimes]).toEqual([TRAINING_CONTEXT_MTIME.getTime()]); + + const dockerfile = await recipe(); + const instructions = dockerfile.split(/\n(?! )/u).map(line => line.trim()).filter(line => line && !line.startsWith("#")); + expect(instructions.join("\n")).not.toMatch(/chmod (-R|0)/u); + const firstLate = instructions.indexOf("COPY grok /opt/training/bin/grok"); + expect(instructions.slice(firstLate).map(line => line.split(" ")[0])).toEqual(["COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "ENV", "ENV", "WORKDIR", "ENTRYPOINT"]); + const copies = instructions.filter(line => line.startsWith("COPY ") && !line.startsWith("COPY --from")); + expect(copies.flatMap(line => line.split(/\s+/u).slice(1, -1)).sort()).toEqual(["bootstrap", "bridge", "bridge/requirements.lock", "claude/package-lock.json", "claude/package.json", + "compiler/dist", "compiler/moltnet-releases.json", "compiler/package-lock.json", "compiler/package.json", "compiler/runtimes.yaml", "grok", "integration", + "paideia/dist", "paideia/package-lock.json", "paideia/package.json", "spawnfile/dist", "spawnfile/moltnet-releases.json", "spawnfile/package-lock.json", + "spawnfile/package.json", "spawnfile/runtimes.yaml", "train"]); + expect(instructions.filter(line => line.startsWith("COPY --from"))).toHaveLength(2); + expect(dockerfile.match(/ln -s \S+ \S+/gu)).toEqual([ + "ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon node_modules/@noopolis/daimon", + "ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude", + "ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia", + "ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile", + "ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia", + "ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile", + "ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon"]); +}); diff --git a/src/compiler/training/preparation/contextModes.ts b/src/compiler/training/preparation/contextModes.ts new file mode 100644 index 00000000..05032898 --- /dev/null +++ b/src/compiler/training/preparation/contextModes.ts @@ -0,0 +1,34 @@ +import { chmod, lstat, readdir, utimes } from "node:fs/promises"; +import path from "node:path"; + +/** Fixed staged timestamp: deterministic context bytes, and still valid in ZIP-based Python builds. */ +export const TRAINING_CONTEXT_MTIME = new Date("2000-01-01T00:00:00Z"); + +/** Executables the recipe previously forced to 0555 before its recursive closure. */ +const EXACT_MODES: Readonly> = { grok: 0o555, train: 0o555 }; + +/** + * `chmod a+rX` closure: every directory gains 0555; every other entry gains 0444, + * plus 0111 when any execute bit is already present. Symlinks are never staged. + */ +export const readableClosure = (mode: number, directory: boolean): number => + directory ? (mode & 0o7777) | 0o555 : (mode & 0o7777) | 0o444 | ((mode & 0o111) !== 0 ? 0o111 : 0); + +/** + * Fix staged build-context modes and times so COPY produces exactly the modes the + * former in-image `chmod 0555 grok train && chmod -R a+rX /opt/training` produced. + */ +export async function normalizeTrainingContext(staging: string): Promise { + const visit = async (entry: string, relative: string): Promise => { + const stat = await lstat(entry); + if (stat.isSymbolicLink()) throw Error("Training build context must not contain symlinks"); + if (stat.isDirectory()) { + for (const name of (await readdir(entry)).sort()) await visit(path.join(entry, name), path.posix.join(relative, name)); + if (relative) await chmod(entry, readableClosure(stat.mode, true)); + } else { + await chmod(entry, EXACT_MODES[relative] ?? readableClosure(stat.mode, false)); + } + if (relative) await utimes(entry, TRAINING_CONTEXT_MTIME, TRAINING_CONTEXT_MTIME); + }; + await visit(staging, ""); +} diff --git a/src/compiler/training/preparation/image.test.ts b/src/compiler/training/preparation/image.test.ts index 2c09852d..1f2aa6fe 100644 --- a/src/compiler/training/preparation/image.test.ts +++ b/src/compiler/training/preparation/image.test.ts @@ -1,6 +1,5 @@ -import { chmod, mkdir, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; import path from "node:path"; -import { execFileSync } from "node:child_process"; import { afterEach, expect, it } from "vitest"; import { planTrainingImage, buildTrainingImage } from "./image.js"; import { preparationFixture, imageDocker, image } from "./fixtures.test-helper.js"; @@ -48,24 +47,6 @@ it("does not trust a cached tag, empty successful build or changed staged bytes" await expect(buildTrainingImage(plan, { ...options, process: async () => ({ code: 1, stdout: "", stderr: "" }) })).rejects.toThrow("changed during staging"); }); -it("makes privately staged runtime directories traversable for the configured nonroot user", async () => { - const f = await fixture(); - const packageDirs = ["spawnfile/dist/compiler/training", "paideia/dist/src/cli", "paideia/bridges/dspy", "integration", "bootstrap"]; - for (const directory of packageDirs) { - const target = path.join(f.root, "runtime", directory); - await mkdir(target, { recursive: true, mode: 0o700 }); - await chmod(target, 0o700); - await writeFile(path.join(target, "index.js"), "export const fixture = true;", { mode: 0o600 }); - } - const dockerfile = await readFile(new URL("../../../../runtime-images/training/Dockerfile", import.meta.url), "utf8"); - const command = dockerfile.match(/chmod -R a\+rX ([^\\\n]+)/u)![1]!.trim().split(/\s+/u); - execFileSync("chmod", ["-R", "a+rX", ...command.map(target => target.replace("/opt/training", path.join(f.root, "runtime")))]); - for (const directory of packageDirs) { - expect((await stat(path.join(f.root, "runtime", directory))).mode & 0o005).toBe(0o005); - expect((await stat(path.join(f.root, "runtime", directory, "index.js"))).mode & 0o004).toBe(0o004); - } -}); - it("supports integration-owned bootstrap generation without a host snapshot", async () => { const f = await fixture(); delete f.build.bootstrap; const plan = await planTrainingImage(f.build, f.root, [], f.own), docker = imageDocker(); diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 31f3e34d..5496d9a8 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -5,6 +5,7 @@ import type { TrainingDockerProcess } from "../container/process.js"; import type { TrainingImageBuild } from "./contract.js"; import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; import type { SealMemo } from "./sealMemo.js"; +import { normalizeTrainingContext } from "./contextModes.js"; const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); export const trainingAssets = path.extname(fileURLToPath(import.meta.url)) === ".ts" @@ -81,6 +82,7 @@ export async function buildTrainingImage(plan: TrainingImagePlan, options: { await mkdir(path.join(staging, "bootstrap"), { recursive: true, mode: 0o700 }); await writeFile(path.join(staging, "Dockerfile"), `${plan.dockerfile}\nLABEL com.spawnfile.training.recipe=${JSON.stringify(plan.digest)}\n`, { mode: 0o600 }); await writeFile(path.join(staging, "train"), plan.entry, { mode: 0o755 }); + await normalizeTrainingContext(staging); const result = await call(["build", "--platform", plan.build.platform, "--build-arg", `NATIVE_IMAGE=${plan.build.nativeImage}`, "--build-arg", `PYTHON_IMAGE=${plan.build.pythonImage}`, "--tag", tag, staging], true); if (result.code !== 0) throw Error("Training image build failed; inspect the streamed build diagnostic"); From 3c399a5ad7a43d6444925d8a22ed23d1a1a47d13 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:26:10 +0200 Subject: [PATCH 15/84] test: isolate training dry-run seal memo assertion --- src/compiler/training/preparation/prepare.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index 4fed9a72..0798db0b 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -34,14 +34,15 @@ it("dry-run reads declarations without Docker, auth access, output or staging mu const before = await readdir(f.root); expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); expect(f.docker.calls).toEqual([]); expect(await readdir(f.root)).toEqual(before); - const memo = path.join(process.env.SPAWNFILE_HOME!, "cache", "training-seal.v1.json"); + // A dedicated memo: other tests in this file may already have written the shared one. + const memo = path.join(f.root, "..", `${path.basename(f.root)}-memo`, "training-seal.v1.json"); roots.push(path.dirname(memo)); await expect(lstat(memo)).rejects.toThrow(); f.config.auth[0]!.source = "auth"; await f.save(); // Outlive the racy-clean window so an actual preparation would record digests. await new Promise(resolve => setTimeout(resolve, 2100)); - expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); + expect(await prepareTraining({ ...f.options, dryRun: true, sealMemoPath: memo })).toMatchObject({ dryRun: true }); await expect(lstat(memo)).rejects.toThrow(); - await prepareTraining(f.options); + await prepareTraining({ ...f.options, sealMemoPath: memo }); expect((await lstat(memo)).mode & 0o777).toBe(0o600); }); From dcd874d60fac07d067c0acd3f5518a753f6647ab Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:42:24 +0200 Subject: [PATCH 16/84] revert: remove training seal digest memo --- specs/TRAINING_CONTAINERS.md | 11 --- src/compiler/training/preparation/AGENTS.md | 5 +- src/compiler/training/preparation/files.ts | 43 +++------ .../preparation/fixtures.test-helper.ts | 4 - src/compiler/training/preparation/image.ts | 34 ++++--- .../training/preparation/prepare.test.ts | 10 --- src/compiler/training/preparation/prepare.ts | 7 +- .../training/preparation/sealMemo.test.ts | 89 ------------------- src/compiler/training/preparation/sealMemo.ts | 86 ------------------ 9 files changed, 31 insertions(+), 258 deletions(-) delete mode 100644 src/compiler/training/preparation/sealMemo.test.ts delete mode 100644 src/compiler/training/preparation/sealMemo.ts diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 629eeef2..dd77c9b4 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -164,17 +164,6 @@ datasets never enter the image context. Cache identity includes actual source, lock, executable and recipe bytes, parent images, architecture and entrypoint; reuse also verifies the image ID and recipe label in the selected Docker context. -Recipe planning may reuse a private same-user digest memo at -`$SPAWNFILE_HOME/cache/training-seal.v1.json` (default `~/.spawnfile`; mode 0600, -atomic replace, at most 20,000 entries). An entry is keyed by the canonical absolute -path, device, inode, size, and nanosecond mtime and ctime; any difference rehashes. -Digests are recorded only when the file was unchanged on the open handle while -hashing and its newest timestamp predates hashing by at least two seconds. A corrupt, -foreign-owned, group/world-accessible or unwritable memo is ignored and never fails -preparation. Dry-run reads but never writes it. The memo only shortens plan-side -hashing: staged build-context copies and preparation/resume/repair verification are -always rehashed in full, and image reuse still requires the recipe label. - Staged build contexts normalize modes and times before `docker build`: directories 0755, files `a+rX`-closed, `grok` and the entrypoint 0555, mtimes fixed. Together with the recipe's change-only `a+rX` closure this reproduces the former recursive diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index 58861dc1..f233b557 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -8,7 +8,7 @@ inputs. Dry-run performs reads only; resume validates the preserved preparation. `contract.ts` declares authoring and runtime receipts; `files.ts` safely seals declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; -`sealMemo.ts` is the private plan-side digest memo; `contextModes.ts` normalizes staged build-context modes/times; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 +`contextModes.ts` normalizes staged build-context modes/times; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Built preparations seal original image bytes in a protected witness for explicit @@ -17,6 +17,3 @@ lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Exclude explicit generated Python coverage/cache files, not arbitrary dotfiles. - Repair transport/compatibility lives in sibling `repair/`; it never changes candidate, criteria or checkpoint semantics owned by Paideia. -- The seal memo (`sealMemo.ts`) is used only by `planTrainingImage`. Never pass it - to `copySealed`, input snapshots, resume, pins or repair verification; those - always rehash. Its key must keep ctime (content rewrites that restore mtime). diff --git a/src/compiler/training/preparation/files.ts b/src/compiler/training/preparation/files.ts index 3726a6c8..888916b2 100644 --- a/src/compiler/training/preparation/files.ts +++ b/src/compiler/training/preparation/files.ts @@ -1,9 +1,8 @@ import { createHash } from "node:crypto"; -import { constants, type BigIntStats } from "node:fs"; -import { copyFile, lstat, mkdir, open, readdir, realpath, chmod } from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { copyFile, lstat, mkdir, readdir, realpath, chmod } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import type { SealMemo } from "./sealMemo.js"; export const within = (root: string, file: string): boolean => file === root || file.startsWith(root + path.sep); export const hashJson = (value: unknown): string => `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; @@ -23,39 +22,23 @@ export async function exactPath(source: string): Promise { return absolute; } -export async function sealFile(source: string, destination: string, memo?: SealMemo): Promise { +export async function sealFile(source: string, destination: string): Promise { await exactPath(source); const before = await lstat(source, { bigint: true }); if (!before.isFile() || before.size > 536_870_912n) throw Error("Training source must be a regular file no larger than 512 MiB"); - const mode = Number(before.mode & 0o777n); - const remembered = memo?.lookup(source, before); - if (remembered !== undefined) return { source, destination, sha256: remembered, mode, size: Number(before.size) }; - const hashStartedNs = BigInt(Date.now() + 1) * 1_000_000n; - const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW); const digest = createHash("sha256"); let size = 0; - let opened: BigIntStats, after: BigIntStats; - try { - opened = await handle.stat({ bigint: true }); - if (!sameFile(opened, before)) throw Error("Training source changed while hashing"); - for await (const chunk of handle.createReadStream({ autoClose: false, start: 0 })) { - size += chunk.length; - if (size > 536_870_912) throw Error("Training source exceeded its size limit"); - digest.update(chunk); - } - after = await handle.stat({ bigint: true }); - } finally { await handle.close(); } - const current = await lstat(source, { bigint: true }); - if (BigInt(size) !== before.size || !sameFile(after, before) || !sameFile(current, before)) throw Error("Training source changed while hashing"); - const sha256 = `sha256:${digest.digest("hex")}`; - memo?.record(source, before, sha256, hashStartedNs); - return { source, destination, sha256, mode, size }; + for await (const chunk of createReadStream(source)) { + size += chunk.length; + if (size > 536_870_912) throw Error("Training source exceeded its size limit"); + digest.update(chunk); + } + const after = await lstat(source, { bigint: true }); + if (BigInt(size) !== before.size || after.size !== before.size || after.ino !== before.ino || after.ctimeNs !== before.ctimeNs || after.dev !== before.dev) throw Error("Training source changed while hashing"); + return { source, destination, sha256: `sha256:${digest.digest("hex")}`, mode: Number(before.mode & 0o777n), size }; } -const sameFile = (left: BigIntStats, right: BigIntStats): boolean => left.size === right.size && left.ino === right.ino && - left.dev === right.dev && left.ctimeNs === right.ctimeNs && left.mtimeNs === right.mtimeNs; - /** Only explicitly selected trees are traversed. Symlinks never enter Docker context. */ -export async function sealTree(source: string, destination: string, options: { ignoreDevelopment?: boolean; ignoreGit?: boolean; internalSymlinks?: boolean; memo?: SealMemo } = {}): Promise { +export async function sealTree(source: string, destination: string, options: { ignoreDevelopment?: boolean; ignoreGit?: boolean; internalSymlinks?: boolean } = {}): Promise { await exactPath(source); const files: SealedFile[] = []; const walk = async (root: string, target: string, depth: number): Promise => { @@ -72,7 +55,7 @@ export async function sealTree(source: string, destination: string, options: { i if (options.ignoreDevelopment && (ignored.has(entry) || /(?:\.test\.[cm]?[jt]s|_test\.py|\.pyc)$/u.test(entry))) continue; await walk(path.join(root, entry), path.posix.join(target, entry), depth + 1); } - } else files.push(await sealFile(root, target, options.memo)); + } else files.push(await sealFile(root, target)); }; await walk(source, destination, 0); if (files.length > 10000 || files.reduce((sum, file) => sum + file.size, 0) > 1_073_741_824) throw Error("Training source tree exceeds bounds"); diff --git a/src/compiler/training/preparation/fixtures.test-helper.ts b/src/compiler/training/preparation/fixtures.test-helper.ts index 7afc7113..608507eb 100644 --- a/src/compiler/training/preparation/fixtures.test-helper.ts +++ b/src/compiler/training/preparation/fixtures.test-helper.ts @@ -2,14 +2,10 @@ import { mkdtemp, mkdir, readFile, realpath, writeFile } from "node:fs/promises" import path from "node:path"; import os from "node:os"; import { createHash } from "node:crypto"; -import { mkdtempSync } from "node:fs"; import type { TrainingPreparationConfig } from "./contract.js"; import type { TrainingContext } from "../contract.js"; import type { TrainingDockerProcess } from "../container/process.js"; -// Preparation tests must never touch the developer's real seal memo cache. -process.env.SPAWNFILE_HOME = mkdtempSync(path.join(os.tmpdir(), "spawnfile-preparation-home-")); - export const image = `sha256:${"a".repeat(64)}`; export const sha = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; export async function preparationFixture() { diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 5496d9a8..50a980f6 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from "node:url"; import type { TrainingDockerProcess } from "../container/process.js"; import type { TrainingImageBuild } from "./contract.js"; import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; -import type { SealMemo } from "./sealMemo.js"; import { normalizeTrainingContext } from "./contextModes.js"; const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); @@ -12,7 +11,7 @@ export const trainingAssets = path.extname(fileURLToPath(import.meta.url)) === " ? path.join(packageRoot, "runtime-images/training") : fileURLToPath(new URL("./assets/", import.meta.url)); export interface TrainingImagePlan { digest: string; files: SealedFile[]; dockerfile: string; entry: string; build: TrainingImageBuild } -async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json"), memo?: SealMemo): Promise { +async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json")): Promise { const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")); const lock = JSON.parse(await readFile(lockSource, "utf8")); if (![2, 3].includes(lock.lockfileVersion) || !lock.packages?.[""] || @@ -21,13 +20,12 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc const item = entry as { link?: boolean; resolved?: string }; return item.link || item.resolved?.startsWith("file:") || item.resolved?.startsWith("../"); })) throw Error(`Training package ${target} has unsupported local dependencies; provide a complete registry-locked distribution`); - return [await sealFile(path.join(root, "package.json"), `${target}/package.json`, memo), - await sealFile(lockSource, `${target}/package-lock.json`, memo), - ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true, memo }) : []]; + return [await sealFile(path.join(root, "package.json"), `${target}/package.json`), + await sealFile(lockSource, `${target}/package-lock.json`), + ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true }) : []]; } -/** `memo` only skips plan-side rehashing; staged context copies are always fully rehashed before a build. */ -export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot, memo?: SealMemo): Promise { +export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { const resolve = (value: string) => path.resolve(root, value); for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); @@ -35,19 +33,19 @@ export async function planTrainingImage(build: TrainingImageBuild, root: string, const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot ? path.join(ownRoot, "package-lock.json") : path.join(assets, "package-lock.json"); const files = [ - ...await packageFiles(resolve(build.paideia), "paideia", true, undefined, memo), - ...await packageFiles(ownRoot, "spawnfile", true, ownLock, memo), + ...await packageFiles(resolve(build.paideia), "paideia", true), + ...await packageFiles(ownRoot, "spawnfile", true, ownLock), ...await packageFiles(build.compiler ? resolve(build.compiler) : ownRoot, "compiler", true, - build.compiler ? path.join(resolve(build.compiler), "package-lock.json") : ownLock, memo), - ...await packageFiles(resolve(build.claude), "claude", false, undefined, memo), - ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true, memo }), - ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true, memo }), - ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true, memo }) : [], - await sealFile(resolve(build.grok.source), "grok", memo), - await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml", memo), - await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json", memo), + build.compiler ? path.join(resolve(build.compiler), "package-lock.json") : ownLock), + ...await packageFiles(resolve(build.claude), "claude", false), + ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), + ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), + ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }) : [], + await sealFile(resolve(build.grok.source), "grok"), + await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), + await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json"), ...await Promise.all(["runtimes.yaml", "moltnet-releases.json"].map(name => - sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`, memo))) + sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`))) ]; if (files.find(file => file.destination === "grok")!.sha256 !== build.grok.sha256) throw Error("Grok executable digest mismatch"); for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js", "compiler/dist/cli/index.js"]) { diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index 0798db0b..a3683ffe 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -34,16 +34,6 @@ it("dry-run reads declarations without Docker, auth access, output or staging mu const before = await readdir(f.root); expect(await prepareTraining({ ...f.options, dryRun: true })).toMatchObject({ dryRun: true }); expect(f.docker.calls).toEqual([]); expect(await readdir(f.root)).toEqual(before); - // A dedicated memo: other tests in this file may already have written the shared one. - const memo = path.join(f.root, "..", `${path.basename(f.root)}-memo`, "training-seal.v1.json"); roots.push(path.dirname(memo)); - await expect(lstat(memo)).rejects.toThrow(); - f.config.auth[0]!.source = "auth"; await f.save(); - // Outlive the racy-clean window so an actual preparation would record digests. - await new Promise(resolve => setTimeout(resolve, 2100)); - expect(await prepareTraining({ ...f.options, dryRun: true, sealMemoPath: memo })).toMatchObject({ dryRun: true }); - await expect(lstat(memo)).rejects.toThrow(); - await prepareTraining({ ...f.options, sealMemoPath: memo }); - expect((await lstat(memo)).mode & 0o777).toBe(0o600); }); it("rejects changed executable, fixture, declaration and saved image on exact resume", async () => { diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index c874b55e..10e2e278 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -7,7 +7,6 @@ import { trainingPreparationSchema, parseTrainingMappedPreparation, type Trainin import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } from "./files.js"; import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; import { planTrainingImage, buildTrainingImage } from "./image.js"; -import { openSealMemo } from "./sealMemo.js"; import { planMeasurementRepair, stageMeasurementRepair, writeTrainingWitness } from "../repair/index.js"; function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { @@ -24,8 +23,6 @@ export interface PrepareTrainingOptions { streams: { stdout(line: string): void; stderr(line: string): void }; /** Test-only package fixture; production always resolves its own installed distribution. */ packageRoot?: string; - /** Plan-side digest memo; defaults to the private Spawnfile cache. Dry-run never writes it. */ - sealMemoPath?: string; repairMeasurements?: string; repairWitness?: string; } export interface PreparedTraining { @@ -53,8 +50,7 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< for (const other of inputs.slice(index + 1)) if (within(input.source, other.source) || within(other.source, input.source) || within(input.destination, other.destination) || within(other.destination, input.destination)) throw Error("Training inputs overlap"); } - const sealMemo = "build" in config.image ? await openSealMemo(options.sealMemoPath) : undefined; - const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot, sealMemo) : undefined; + const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot) : undefined; if (options.repairWitness && !options.repairMeasurements) throw Error("A repair witness requires --repair-measurements"); if (options.repairMeasurements && !imagePlan) throw Error("Measurement repair requires a verifiable image build recipe"); const repair = options.repairMeasurements ? await planMeasurementRepair({ parent: options.repairMeasurements, @@ -64,7 +60,6 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image, canonical: options.context.project.sourceDigest, ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); if (options.dryRun) return { digest, dryRun: true }; - await sealMemo?.save(); options.signal?.throwIfAborted(); for (const entry of auth) if (await exactPath(entry.source) !== entry.source || !(await lstat(entry.source)).isFile()) throw Error("Training auth must be a canonical regular leaf"); const execute = (args: string[]) => options.process(args, { timeoutMs: options.timeoutMs, signal: options.signal }); diff --git a/src/compiler/training/preparation/sealMemo.test.ts b/src/compiler/training/preparation/sealMemo.test.ts deleted file mode 100644 index 13aaa6eb..00000000 --- a/src/compiler/training/preparation/sealMemo.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { chmod, mkdtemp, readFile, realpath, rm, stat, utimes, writeFile } from "node:fs/promises"; -import * as fsPromises from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, expect, it, vi } from "vitest"; -import { sealFile, sealTree } from "./files.js"; -import { openSealMemo, trainingSealMemoPath } from "./sealMemo.js"; - -vi.mock("node:fs/promises", async original => { - const actual = await original(); - return { ...actual, open: vi.fn(actual.open) }; -}); - -const roots: string[] = []; -afterEach(async () => { vi.mocked(fsPromises.open).mockClear(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); -async function fixture() { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-seal-memo-"))); roots.push(root); return root; } -const opensOf = (file: string) => vi.mocked(fsPromises.open).mock.calls.filter(call => call[0] === file).length; -const memoOptions = { racySafetyNs: 0n }; - -it("reuses a persisted digest without opening or reading the source bytes", async () => { - const root = await fixture(), source = path.join(root, "grok"), memoPath = path.join(root, "cache", "training-seal.v1.json"); - await writeFile(source, "native-binary"); - const first = await openSealMemo(memoPath, memoOptions); - const sealed = await sealFile(source, "grok", first); await first.save(); - expect(opensOf(source)).toBe(1); - expect((await stat(memoPath)).mode & 0o777).toBe(0o600); - const second = await openSealMemo(memoPath, memoOptions); - expect(await sealFile(source, "grok", second)).toEqual(sealed); - expect(opensOf(source)).toBe(1); - expect(await sealFile(source, "grok")).toEqual(sealed); - expect(opensOf(source)).toBe(2); -}); - -it("rehashes when content changes under a restored mtime because ctime is part of the key", async () => { - const root = await fixture(), source = path.join(root, "grok"), memoPath = path.join(root, "memo.json"); - await writeFile(source, "AAAA"); - // Whole-second times restore exactly, so only ctime can distinguish the rewrite. - const pinned = new Date("2020-01-01T00:00:00Z"); await utimes(source, pinned, pinned); - const memo = await openSealMemo(memoPath, memoOptions); - const original = await sealFile(source, "grok", memo); await memo.save(); - const before = await stat(source, { bigint: true }); - await writeFile(source, "BBBB"); await utimes(source, pinned, pinned); - const after = await stat(source, { bigint: true }); - expect([after.mtimeNs, after.size, after.ino, after.dev]).toEqual([before.mtimeNs, before.size, before.ino, before.dev]); - expect(after.ctimeNs).not.toBe(before.ctimeNs); - const reopened = await openSealMemo(memoPath, memoOptions); - const changed = await sealFile(source, "grok", reopened); - expect(changed.sha256).not.toBe(original.sha256); - expect(changed.sha256).toBe((await sealFile(source, "grok")).sha256); -}); - -it("does not memoize racily fresh files and tolerates corrupt, foreign-mode or unwritable memos", async () => { - const root = await fixture(), source = path.join(root, "file"), memoPath = path.join(root, "memo.json"); - await writeFile(source, "fresh"); - const fresh = await openSealMemo(memoPath); await sealFile(source, "file", fresh); await fresh.save(); - await expect(stat(memoPath)).rejects.toThrow(); - for (const [body, mode] of [["{not json", 0o600], [JSON.stringify({ version: "other", entries: [] }), 0o600], - [JSON.stringify({ version: "spawnfile.training-seal-memo.v1", entries: [[`["${source}"]`, "sha256:bad"], "x"] }), 0o600], - [JSON.stringify({ version: "spawnfile.training-seal-memo.v1", entries: {} }), 0o600]] as const) { - await writeFile(memoPath, body); await chmod(memoPath, mode); - const memo = await openSealMemo(memoPath, memoOptions); - expect((await sealFile(source, "file", memo)).sha256).toMatch(/^sha256:/u); - await memo.save(); - expect(JSON.parse(await readFile(memoPath, "utf8")).entries).toHaveLength(1); - } - await chmod(memoPath, 0o644); - const opened = opensOf(source); - await sealFile(source, "file", await openSealMemo(memoPath, memoOptions)); - expect(opensOf(source)).toBe(opened + 1); - const blocked = path.join(root, "blocked"); await writeFile(blocked, "not a directory"); - const unwritable = await openSealMemo(path.join(blocked, "memo.json"), memoOptions); - await sealFile(source, "file", unwritable); - await expect(unwritable.save()).resolves.toBeUndefined(); -}); - -it("bounds retained entries and defaults to the private Spawnfile cache", async () => { - const root = await fixture(), memoPath = path.join(root, "memo.json"); - for (const name of ["a", "b", "c", "d"]) await writeFile(path.join(root, "tree", name).replace("/tree/", "/tree-"), name); - const memo = await openSealMemo(memoPath, { ...memoOptions, maxEntries: 2 }); - for (const name of ["a", "b", "c", "d"]) await sealFile(path.join(root, `tree-${name}`), name, memo); - await memo.save(); - expect(JSON.parse(await readFile(memoPath, "utf8")).entries).toHaveLength(2); - expect(trainingSealMemoPath()).toBe(path.join(path.resolve(process.env.SPAWNFILE_HOME ?? path.join(os.homedir(), ".spawnfile")), "cache", "training-seal.v1.json")); - const tree = path.join(root, "dir"); await fsPromises.mkdir(tree); await writeFile(path.join(tree, "x"), "x"); - const treeMemo = await openSealMemo(memoPath, memoOptions); - const sealed = await sealTree(tree, "dir", { memo: treeMemo }); - expect(await sealTree(tree, "dir", { memo: treeMemo })).toEqual(sealed); - expect(opensOf(path.join(tree, "x"))).toBe(1); -}); diff --git a/src/compiler/training/preparation/sealMemo.ts b/src/compiler/training/preparation/sealMemo.ts deleted file mode 100644 index ad799689..00000000 --- a/src/compiler/training/preparation/sealMemo.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { randomBytes } from "node:crypto"; -import type { BigIntStats } from "node:fs"; -import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises"; -import path from "node:path"; -import { resolveSpawnfileHome } from "../../../auth/paths.js"; - -const VERSION = "spawnfile.training-seal-memo.v1"; -const DIGEST = /^sha256:[a-f0-9]{64}$/u; -const MAX_FILE_BYTES = 16 * 1024 * 1024; - -/** - * Private same-user cache of plan-side source digests. It never replaces the full - * rehash of staged build-context copies, so image bytes stay verified on every build. - */ -export interface SealMemo { - lookup(source: string, stat: BigIntStats): string | undefined; - /** Stores only racily-clean digests: metadata older than the safety window at hash start. */ - record(source: string, stat: BigIntStats, sha256: string, hashStartedNs: bigint): void; - save(): Promise; -} - -export interface SealMemoOptions { maxEntries?: number; racySafetyNs?: bigint } - -export const trainingSealMemoPath = (): string => path.join(resolveSpawnfileHome(), "cache", "training-seal.v1.json"); - -export const sealMemoKey = (source: string, stat: BigIntStats): string => - JSON.stringify([source, stat.dev.toString(), stat.ino.toString(), stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString()]); - -async function readTrusted(file: string): Promise> { - const entries = new Map(); - try { - const stat = await lstat(file); - if (!stat.isFile() || stat.size > MAX_FILE_BYTES || (stat.mode & 0o077) !== 0 || - (typeof process.getuid === "function" && stat.uid !== process.getuid())) return entries; - const parsed = JSON.parse(await readFile(file, "utf8")) as unknown; - if (!parsed || typeof parsed !== "object" || (parsed as { version?: unknown }).version !== VERSION) return entries; - const raw = (parsed as { entries?: unknown }).entries; - if (!Array.isArray(raw)) return entries; - for (const item of raw) { - if (Array.isArray(item) && item.length === 2 && typeof item[0] === "string" && typeof item[1] === "string" && DIGEST.test(item[1])) { - entries.set(item[0], item[1]); - } - } - } catch { return new Map(); } - return entries; -} - -export async function openSealMemo(file = trainingSealMemoPath(), options: SealMemoOptions = {}): Promise { - const maxEntries = options.maxEntries ?? 20_000; - const racySafetyNs = options.racySafetyNs ?? 2_000_000_000n; - const previous = await readTrusted(file); - const touched = new Map(); - let dirty = false; - return { - lookup(source, stat) { - const key = sealMemoKey(source, stat); - const found = touched.get(key) ?? previous.get(key); - if (found !== undefined) touched.set(key, found); - return found; - }, - record(source, stat, sha256, hashStartedNs) { - if (!DIGEST.test(sha256)) return; - const newest = stat.ctimeNs > stat.mtimeNs ? stat.ctimeNs : stat.mtimeNs; - if (newest > hashStartedNs - racySafetyNs) return; - touched.set(sealMemoKey(source, stat), sha256); dirty = true; - }, - async save() { - const retained = [...touched]; - for (const entry of [...previous].reverse()) { - if (retained.length >= maxEntries) break; - if (!touched.has(entry[0])) retained.push(entry); - } - if (!dirty && retained.length === previous.size) return; - const bounded = retained.slice(0, maxEntries).reverse(); - const temporary = `${file}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; - try { - await mkdir(path.dirname(file), { recursive: true, mode: 0o700 }); - const handle = await open(temporary, "wx", 0o600); - try { await handle.writeFile(JSON.stringify({ version: VERSION, entries: bounded })); await handle.sync(); } - finally { await handle.close(); } - await rename(temporary, file); - dirty = false; - } catch { await rm(temporary, { force: true }).catch(() => undefined); } - } - }; -} From 566308fe434f2118d17620a0385b1819129388c1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:42:28 +0200 Subject: [PATCH 17/84] docs: state that a changed training distribution rebuilds later COPY layers --- runtime-images/training/AGENTS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime-images/training/AGENTS.md b/runtime-images/training/AGENTS.md index 888c791b..71be1657 100644 --- a/runtime-images/training/AGENTS.md +++ b/runtime-images/training/AGENTS.md @@ -16,7 +16,8 @@ Layers run from least to most frequently changing: Python parent copies, locked Claude/Paideia/Spawnfile/compiler dependencies, the lock-keyed DSPy venv, one layout RUN, bridge source plus its editable install, then pinned executables and distributions. Nothing RUNs after the late COPY layers, so a changed distribution -rebuilds only its own COPY layers and the label. +rebuilds its own COPY layer, every later COPY layer and the label, but no RUN +layer (no npm, pip or recursive chmod work). In-image modes are unchanged from the former `chmod 0555 grok train && chmod -R a+rX /opt/training`, and owners stay COPY/RUN defaults (root): From 353b4f7b94df269ac5c1f0b96ff1d1fa1f832c6a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:44:34 +0200 Subject: [PATCH 18/84] test: add docker-deferred training image mode equivalence check --- package.json | 3 +- scripts/README.md | 1 + scripts/training-image-modes.test.ts | 43 +++++++++++++ scripts/training-image-modes.ts | 51 +++++++++++++++ scripts/verify-training-image-modes.ts | 88 ++++++++++++++++++++++++++ 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 scripts/training-image-modes.test.ts create mode 100644 scripts/training-image-modes.ts create mode 100644 scripts/verify-training-image-modes.ts diff --git a/package.json b/package.json index 19d59ed7..91c23f24 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "test:coverage-verdict": "node --import tsx -e \"import fs from 'node:fs'; const file='coverage/coverage-summary.json'; const fail=(message)=>{ console.error('COVERAGE VERDICT: '+message); process.exit(1); }; let summary; try { summary=JSON.parse(fs.readFileSync(file, 'utf8')); } catch (error) { fail(error && error.code === 'ENOENT' ? 'summary file missing' : 'summary file unparseable'); } import('./vitest.config.ts').then(({default:config})=>{ const thresholds=config.test && config.test.coverage && config.test.coverage.thresholds; if (!thresholds || typeof thresholds !== 'object' || !summary || typeof summary !== 'object' || !summary.total || typeof summary.total !== 'object') fail('summary or threshold schema unrecognised'); const metricNames=['branches','functions','lines','statements']; const metrics=metricNames.filter((metric)=>Object.prototype.hasOwnProperty.call(thresholds, metric)); if (metrics.length === 0) fail('no threshold metrics configured'); let shortfall=false; for (const metric of metrics) { const threshold=thresholds[metric]; const pct=summary.total[metric] && summary.total[metric].pct; if (typeof threshold !== 'number' || !Number.isFinite(threshold) || typeof pct !== 'number' || !Number.isFinite(pct)) fail('missing or non-numeric value for '+metric); console.log('COVERAGE: '+metric+' '+pct+'% (threshold '+threshold+'%)'); if (pct < threshold) shortfall=true; } if (shortfall) { console.error('COVERAGE VERDICT: thresholds not met'); process.exit(2); } console.log('COVERAGE VERDICT: thresholds met'); }).catch((error)=>fail('could not load vitest config: '+error.message));\"", "test:node": "mkdir -p coverage || exit 1; rm -f coverage/node-test.tap || exit 1; node --import tsx --test --test-reporter=tap src/runtime/pi/appControlDeliveryMetadata.test.ts > coverage/node-test.tap 2>&1; node_status=$?; cat coverage/node-test.tap; cat_status=$?; npm run test:node-verdict; verdict_status=$?; if [ \"$verdict_status\" -eq 2 ]; then exit 2; fi; if [ \"$verdict_status\" -ne 0 ]; then exit 1; fi; if [ \"$cat_status\" -ne 0 ]; then echo 'NODE VERDICT: could not read TAP output' >&2; exit 1; fi; if [ \"$node_status\" -ne 0 ]; then echo \"NODE VERDICT: node:test runner exited $node_status despite a passing TAP summary\" >&2; exit 1; fi; exit 0", "test:node-verdict": "node -e \"const fs=require('fs'); const text=fs.readFileSync('coverage/node-test.tap', 'utf8'); const tests=text.match(/^# tests ([0-9]+)\\r?$/m); const suites=text.match(/^# suites ([0-9]+)\\r?$/m); const passed=text.match(/^# pass ([0-9]+)\\r?$/m); const failed=text.match(/^# fail ([0-9]+)\\r?$/m); const skipped=text.match(/^# skipped ([0-9]+)\\r?$/m); const todo=text.match(/^# todo ([0-9]+)\\r?$/m); const cancelled=text.match(/^# cancelled ([0-9]+)\\r?$/m); if (!tests || !suites || !passed || !failed || !skipped || !todo || !cancelled) { console.error('NODE VERDICT: TAP summary unrecognised'); process.exit(1); } const testCount=Number(tests[1]); const suiteCount=Number(suites[1]); const passCount=Number(passed[1]); const failCount=Number(failed[1]); const skippedCount=Number(skipped[1]); const todoCount=Number(todo[1]); const cancelledCount=Number(cancelled[1]); if (failCount > 0) { console.error('NODE VERDICT: test failures found (tests='+testCount+', failed='+failCount+')'); process.exit(2); } if (cancelledCount > 0) { console.error('NODE VERDICT: incomplete run (cancelled='+cancelledCount+')'); process.exit(1); } if (passCount + skippedCount + todoCount + cancelledCount !== testCount) { console.error('NODE VERDICT: TAP counts do not reconcile (tests='+testCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', cancelled='+cancelledCount+')'); process.exit(1); } if (testCount < 10) { console.error('NODE VERDICT: expected at least 10 node:test cases, found '+testCount+' — cases were removed or the file was gutted'); process.exit(1); } if (passCount < 10) { console.error('NODE VERDICT: expected at least 10 passing node:test cases, found '+passCount+' (skipped='+skippedCount+', todo='+todoCount+')'); process.exit(1); } if (suiteCount < 1) { console.error('NODE VERDICT: expected at least 1 node:test suite, found '+suiteCount); process.exit(1); } console.log('NODE VERDICT: tests passed (tests='+testCount+', suites='+suiteCount+', passed='+passCount+', skipped='+skippedCount+', todo='+todoCount+', failed=0)');\"", + "verify:training-image-modes": "tsx scripts/verify-training-image-modes.ts", "test:product-state-volume": "node --experimental-strip-types scripts/product-state-volume-integration.test.ts", "test:boundaries": "vitest run --coverage.enabled=false src/ownership/rootOwnershipBoundary.test.ts src/deployment/providerRuntimeBoundary.test.ts src/ownership/mnemePublicImportBoundary.test.ts src/target/containerBundleArchive.test.ts", "test:causal-conformance": "tsx src/ledger/causalConformanceCli.ts", @@ -63,7 +64,7 @@ "test:e2e:distribution-image": "tsx src/e2e/cli.ts distribution-image", "test:e2e:distribution-roundtrip": "tsx src/e2e/cli.ts distribution-roundtrip", "test:e2e:daimon-org": "tsx src/e2e/cli.ts daimon-org", - "test:scripts": "node --experimental-strip-types --test scripts/build-local-daimon-runtime.test.ts scripts/build-local-moltnet.test.ts scripts/compile-explicit-test-mcp.test.ts scripts/native-helper-workflows.test.ts scripts/typescript-policy.test.ts", + "test:scripts": "node --experimental-strip-types --test scripts/build-local-daimon-runtime.test.ts scripts/build-local-moltnet.test.ts scripts/compile-explicit-test-mcp.test.ts scripts/native-helper-workflows.test.ts scripts/typescript-policy.test.ts scripts/training-image-modes.test.ts", "test:unit": "npm run test:vitest; vitest_status=$?; npm run test:verdict; verdict_status=$?; npm run test:coverage-verdict; coverage_status=$?; npm run test:node; node_status=$?; failed_lanes=''; if [ \"$verdict_status\" -eq 2 ]; then failed_lanes='vitest test failures'; fi; if [ \"$node_status\" -eq 2 ]; then if [ -n \"$failed_lanes\" ]; then failed_lanes=\"$failed_lanes; node:test suite\"; else failed_lanes='node:test suite'; fi; fi; result_status=0; if [ -n \"$failed_lanes\" ]; then echo \"FAIL(tests): $failed_lanes\"; result_status=1; fi; if [ \"$verdict_status\" -ne 0 ] && [ \"$verdict_status\" -ne 2 ]; then echo 'FAIL(vitest-verdict): could not determine the vitest result'; result_status=1; fi; if [ \"$node_status\" -ne 0 ] && [ \"$node_status\" -ne 2 ]; then echo 'FAIL(node-lane): could not determine the node:test result'; result_status=1; fi; if [ \"$result_status\" -ne 0 ]; then exit 1; elif [ \"$vitest_status\" -ne 0 ]; then echo \"FAIL(vitest): exited $vitest_status with no test failure\"; exit 1; elif [ \"$coverage_status\" -eq 2 ]; then echo 'FAIL(coverage): thresholds not met (tests all passed)'; exit 2; elif [ \"$coverage_status\" -ne 0 ]; then echo 'FAIL(coverage-verdict): could not evaluate coverage (tests all passed)'; exit 1; else echo 'PASS'; fi", "test": "npm run test:unit && npm run test:scripts" }, diff --git a/scripts/README.md b/scripts/README.md index cc91c3eb..961b2d3f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,6 +12,7 @@ in the npm package. Scripts run as native TypeScript on Node 22.19+. | `create-linux-amd64-dependency-closure.ts` | `npm run prepare:linux-amd64-closure` | Prepare reviewed npm dependencies/cache in the pinned build container | | `create-linux-amd64-go-closure.ts` | `npm run prepare:linux-amd64-go-closure` | Prepare the pinned Go module-cache closure | | `compile-explicit-test-mcp.ts` | `npm run compile:explicit-test-mcp` | Lower bounded test MCP declarations against a compiled report; build the CLI first | +| `verify-training-image-modes.ts` | `npm run verify:training-image-modes -- --build-config ` | Build the staged-mode training image and a control with the old `chmod -R a+rX`, then fail on any mode/owner/type/link difference under `/opt/training` (incl. the bridge venv); Docker required, not run in CI | | `verify-package-closure.ts` | `npm run verify:package-closure` | Verify the packed CLI and runtime closure | | `product-state-volume-integration.test.ts` | `npm run test:product-state-volume`, CI | Test real volume preseed; Docker and host volume access required | | `source-provenance-bundle.integration.test.ts` | `npm run test:source-provenance-docker` | Test the real offline Daimon archive build | diff --git a/scripts/training-image-modes.test.ts b/scripts/training-image-modes.test.ts new file mode 100644 index 00000000..897bf003 --- /dev/null +++ b/scripts/training-image-modes.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes } from "./training-image-modes.ts"; + +const row = (entry: string, mode = "755", type = "d", target = "") => `${entry}\t${mode}\troot\troot\t${type}\t${target}`; +const base = [row("/opt/training"), row("/opt/training/paideia/bridges/dspy/.venv"), row("/opt/training/bin/grok", "555", "f"), + row("/opt/training/bin/claude", "777", "l", "/opt/training/claude/node_modules/.bin/claude")]; + +test("identical listings in any order compare equal", () => { + assert.equal(identicalModes(diffModeListings(base.join("\n"), [...base].reverse().join("\n") + "\n")), true); +}); + +test("mode, owner, type and link target drift are each reported", () => { + const drifts: [number, string][] = [[2, row("/opt/training/bin/grok", "755", "f")], [2, base[2]!.replace("\troot\troot", "\ttraining\troot")], + [3, row("/opt/training/bin/claude", "777", "l", "/elsewhere")], [2, row("/opt/training/bin/grok", "555", "l")]]; + for (const [index, drifted] of drifts) { + const control = [...base]; control[index] = drifted; + const diff = diffModeListings(base.join("\n"), control.join("\n")); + assert.equal(identicalModes(diff), false); + assert.equal(diff.changed.length, 1); + } +}); + +test("entries present on only one side are reported", () => { + const diff = diffModeListings([...base, row("/opt/training/extra", "644", "f")].join("\n"), base.slice(0, 3).join("\n")); + assert.deepEqual(diff.onlyNew, ["/opt/training/bin/claude", "/opt/training/extra"]); + assert.deepEqual(diff.onlyControl, []); +}); + +test("empty, malformed, duplicated or rootless listings fail instead of comparing equal", () => { + assert.throws(() => diffModeListings("", ""), /missing a required root/u); + assert.throws(() => diffModeListings(base.join("\n"), "/opt/training 755"), /Malformed/u); + assert.throws(() => diffModeListings(base.slice(0, 1).join("\n"), base.join("\n")), /missing a required root/u); + assert.throws(() => diffModeListings([...base, base[2]!].join("\n"), base.join("\n")), /Duplicate/u); +}); + +test("recipes must actually differ in the recursive closure", () => { + assert.doesNotThrow(() => assertControlRecipe("RUN chmod 0555 x \\\n && chmod -R a+rX /opt/training")); + assert.throws(() => assertControlRecipe("RUN find /opt/training"), /Control recipe/u); + assert.throws(() => assertNewRecipe("RUN chmod -R a+rX /opt/training"), /still runs/u); + assert.throws(() => assertControlRecipe("# chmod -R a+rX /opt/training\nRUN true"), /Control recipe/u); + assert.doesNotThrow(() => assertNewRecipe("# Exactly `chmod -R a+rX /opt/training`, but change-only\nRUN find /opt/training")); +}); diff --git a/scripts/training-image-modes.ts b/scripts/training-image-modes.ts new file mode 100644 index 00000000..c8c26889 --- /dev/null +++ b/scripts/training-image-modes.ts @@ -0,0 +1,51 @@ +/** Pure helpers for the Docker-deferred training image mode comparison. */ + +/** One `find -printf` record per entry: path, octal mode, owner, group, type, link target. */ +export const MODE_LISTING_FORMAT = "%p\\t%m\\t%u\\t%g\\t%y\\t%l\\n"; +export const MODE_ROOTS = ["/opt/training", "/opt/training/paideia/bridges/dspy/.venv"] as const; +/** The former recipe's whole-tree closure; a control recipe without it is not a control. */ +export const CONTROL_CLOSURE = "chmod -R a+rX /opt/training"; + +export interface ModeListingDiff { onlyNew: string[]; onlyControl: string[]; changed: { path: string; next: string; control: string }[] } + +/** Instruction text only: Dockerfile comments may mention the closure without running it. */ +const instructions = (dockerfile: string): string => dockerfile.split("\n").filter(line => !line.trimStart().startsWith("#")).join("\n"); + +export function assertControlRecipe(dockerfile: string): void { + if (!instructions(dockerfile).includes(CONTROL_CLOSURE)) throw Error(`Control recipe must run "${CONTROL_CLOSURE}"`); +} + +export function assertNewRecipe(dockerfile: string): void { + if (instructions(dockerfile).includes(CONTROL_CLOSURE)) throw Error("New recipe still runs the recursive chmod; nothing would be compared"); +} + +function parse(listing: string, label: string): Map { + const entries = new Map(); + for (const line of listing.split("\n")) { + if (!line) continue; + const [entry, ...fields] = line.split("\t"); + if (!entry || fields.length !== 5) throw Error(`Malformed ${label} listing line: ${line}`); + if (entries.has(entry)) throw Error(`Duplicate ${label} listing entry: ${entry}`); + entries.set(entry, fields.join("\t")); + } + if (!MODE_ROOTS.every(root => entries.has(root))) throw Error(`${label} listing is missing a required root (${MODE_ROOTS.join(", ")})`); + return entries; +} + +/** Compares modes, owners, groups, types and link targets; mtimes are deliberately excluded. */ +export function diffModeListings(next: string, control: string): ModeListingDiff { + const left = parse(next, "new"), right = parse(control, "control"); + const diff: ModeListingDiff = { onlyNew: [], onlyControl: [], changed: [] }; + for (const [entry, fields] of left) { + const other = right.get(entry); + if (other === undefined) diff.onlyNew.push(entry); + else if (other !== fields) diff.changed.push({ path: entry, next: fields, control: other }); + } + for (const entry of right.keys()) if (!left.has(entry)) diff.onlyControl.push(entry); + for (const list of [diff.onlyNew, diff.onlyControl]) list.sort(); + diff.changed.sort((a, b) => a.path.localeCompare(b.path)); + return diff; +} + +export const identicalModes = (diff: ModeListingDiff): boolean => + diff.onlyNew.length === 0 && diff.onlyControl.length === 0 && diff.changed.length === 0; diff --git a/scripts/verify-training-image-modes.ts b/scripts/verify-training-image-modes.ts new file mode 100644 index 00000000..9b2a1283 --- /dev/null +++ b/scripts/verify-training-image-modes.ts @@ -0,0 +1,88 @@ +/** + * Docker-deferred check that the staged-mode training recipe reproduces the former + * recursive-chmod image exactly. Builds both images from the same sealed plan and + * compares mode, owner, group, type and link target of every entry. + * + * npx tsx scripts/verify-training-image-modes.ts --build-config \ + * [--root ] [--control-ref 47050be] [--docker-context default] [--keep-images] + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, MODE_LISTING_FORMAT, MODE_ROOTS } from "./training-image-modes.ts"; + +type ImageModule = typeof import("../src/compiler/training/preparation/image.js"); +type FilesModule = typeof import("../src/compiler/training/preparation/files.js"); +type ModesModule = typeof import("../src/compiler/training/preparation/contextModes.js"); +type ContractModule = typeof import("../src/compiler/training/preparation/contract.js"); + +const repository = fileURLToPath(new URL("../", import.meta.url)); + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function docker(context: string, args: string[], capture = false): string { + const result = spawnSync("docker", ["--context", context, ...args], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024, + stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit" }); + if (result.status !== 0) throw Error(`docker ${args[0]} failed with exit ${result.status}`); + return capture ? result.stdout : ""; +} + +async function main(): Promise { + const configPath = option("--build-config"); + if (!configPath) throw Error("--build-config is required"); + const controlRef = option("--control-ref") ?? "47050be"; + const dockerContext = option("--docker-context") ?? "default"; + const { planTrainingImage } = await import("../src/compiler/training/preparation/image.js") as ImageModule; + const { copySealed } = await import("../src/compiler/training/preparation/files.js") as FilesModule; + const { normalizeTrainingContext } = await import("../src/compiler/training/preparation/contextModes.js") as ModesModule; + const { trainingBuildSchema } = await import("../src/compiler/training/preparation/contract.js") as ContractModule; + + const raw = JSON.parse(await readFile(configPath, "utf8")) as { image?: { build?: unknown } }; + const build = trainingBuildSchema.parse(raw.image?.build ?? raw); + const root = path.resolve(option("--root") ?? path.dirname(configPath)); + const plan = await planTrainingImage(build, root, [], repository); + const controlRecipe = execFileSync("git", ["-C", repository, "show", `${controlRef}:runtime-images/training/Dockerfile`], { encoding: "utf8" }); + assertNewRecipe(plan.dockerfile); + assertControlRecipe(controlRecipe); + + const parent = await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-modes-")); + const tags = { next: "spawnfile-training-modes:new", control: "spawnfile-training-modes:control" }; + try { + // Mirror buildTrainingImage staging; only the new side normalizes modes, as in each recipe's era. + const stage = async (name: string, dockerfile: string, normalize: boolean) => { + const staging = path.join(parent, name); + await mkdir(staging, { mode: 0o700 }); + await copySealed(plan.files, staging); + await mkdir(path.join(staging, "bootstrap"), { recursive: true, mode: 0o700 }); + await writeFile(path.join(staging, "Dockerfile"), dockerfile, { mode: 0o600 }); + await writeFile(path.join(staging, "train"), plan.entry, { mode: 0o755 }); + if (normalize) await normalizeTrainingContext(staging); + return staging; + }; + const buildArgs = ["--platform", build.platform, "--build-arg", `NATIVE_IMAGE=${build.nativeImage}`, "--build-arg", `PYTHON_IMAGE=${build.pythonImage}`]; + docker(dockerContext, ["build", ...buildArgs, "--tag", tags.next, await stage("new", plan.dockerfile, true)]); + docker(dockerContext, ["build", ...buildArgs, "--tag", tags.control, await stage("control", controlRecipe, false)]); + const listing = (tag: string) => docker(dockerContext, ["run", "--rm", "--platform", build.platform, "--user", "0:0", "--network", "none", + "--entrypoint", "find", tag, ...MODE_ROOTS, "-printf", MODE_LISTING_FORMAT], true); + const diff = diffModeListings(listing(tags.next), listing(tags.control)); + if (!identicalModes(diff)) { + for (const entry of diff.onlyNew.slice(0, 50)) console.error(`only in new: ${entry}`); + for (const entry of diff.onlyControl.slice(0, 50)) console.error(`only in control: ${entry}`); + for (const change of diff.changed.slice(0, 50)) console.error(`differs: ${change.path}\n new ${change.next}\n control ${change.control}`); + console.error(`MODE VERDICT: FAIL (onlyNew=${diff.onlyNew.length}, onlyControl=${diff.onlyControl.length}, changed=${diff.changed.length})`); + process.exitCode = 1; + return; + } + console.log("MODE VERDICT: PASS (modes, owners, groups, types and link targets identical)"); + } finally { + await rm(parent, { recursive: true, force: true }); + if (!process.argv.includes("--keep-images")) spawnSync("docker", ["--context", dockerContext, "image", "rm", "-f", tags.next, tags.control], { stdio: "ignore" }); + } +} + +main().catch(error => { console.error(`MODE VERDICT: ERROR ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; }); From ba4c77c37ec3f698d13906894cb94298914adf32 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:01:05 +0200 Subject: [PATCH 19/84] feat: vendor the Daimon Grok 1.0.34 broker contract manifest and pin the Grok CLI from it --- runtime-images/daimon/Dockerfile | 8 ++- scripts/build-local-daimon-runtime.test.ts | 22 +++++++ scripts/build-local-daimon-runtime.ts | 25 ++++++++ ...urce-provenance-bundle.integration.test.ts | 20 ++++-- src/runtime/container.ts | 6 +- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 62 +++++++++++++++++-- 8 files changed, 130 insertions(+), 17 deletions(-) diff --git a/runtime-images/daimon/Dockerfile b/runtime-images/daimon/Dockerfile index f2768443..834a264e 100644 --- a/runtime-images/daimon/Dockerfile +++ b/runtime-images/daimon/Dockerfile @@ -139,10 +139,14 @@ RUN test -n "${GROK_CLI_VERSION}" \ && test -x ${RUNTIME_ROOT}/node_modules/@openai/codex/bin/codex.js \ && test -f ${RUNTIME_ROOT}/node_modules/@noopolis/daimon/dist/runtime/cli.js \ && case "${TARGETARCH}" in \ - amd64) broker_arch=x64; broker_sha=e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd ;; \ - arm64) broker_arch=arm64; broker_sha=ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d ;; \ + amd64) broker_arch=x64 ;; \ + arm64) broker_arch=arm64 ;; \ *) echo "Unsupported Daimon engine-broker architecture: ${TARGETARCH}" >&2; exit 1 ;; \ esac \ + && package_manifest=${RUNTIME_ROOT}/node_modules/@noopolis/daimon/dist/runtime/contract-manifest.json \ + && test "$(sha256sum "${package_manifest}" | awk '{print "sha256:" $1}')" = "${DAIMON_MANIFEST_SHA256}" \ + && broker_sha="$(node -e 'const m=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).grokEngineBroker; const v=m?.artifacts?.[process.argv[2]+"Sha256"]; if(typeof v!=="string"||!/^[a-f0-9]{64}$/.test(v))process.exit(1); process.stdout.write(v)' "${package_manifest}" "${broker_arch}")" \ + && node -e 'const m=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).grokEngineBroker, a=m?.grokCliArtifacts?.[process.argv[2]]; if(!a||m.grokCliVersion!==process.argv[3]||a.url!==process.argv[4]||a.sha256!==process.argv[5].replace(/^sha256:/,"")){console.error("Grok CLI pin does not match the Daimon contract manifest ("+(m?.grokCliVersion)+")");process.exit(1)}' "${package_manifest}" "${broker_arch}" "${GROK_CLI_VERSION}" "${GROK_CLI_URL}" "${GROK_CLI_SHA256}" \ && broker_source=${RUNTIME_ROOT}/node_modules/@noopolis/daimon/dist/runtime/native/daimon-engine-broker \ && test -x "${broker_source}" \ && test "$(sha256sum "${broker_source}" | awk '{print $1}')" = "${broker_sha}" \ diff --git a/scripts/build-local-daimon-runtime.test.ts b/scripts/build-local-daimon-runtime.test.ts index 0762af47..3da558cc 100644 --- a/scripts/build-local-daimon-runtime.test.ts +++ b/scripts/build-local-daimon-runtime.test.ts @@ -5,8 +5,10 @@ import path from "node:path"; import test from "node:test"; import { + assertPinnedGrokCli, createLocalDaimonCapabilityReceipt, readDaimonCliArtifactPins, + readPinnedGrokCli, resolveDaimonSourceMode, resolveLocalBuildArchitecture, resolveLocalImageTag, @@ -355,3 +357,23 @@ test("Daimon Dockerfile stage graph preserves cache and offline-network boundari assert.deepEqual(stagesContaining(/\bapt-get\b/u), ["base_registry"]); assert.deepEqual(stagesContaining(/\bcurl\s+-/u), ["agy_source_registry", "grok_source_registry"]); }); + +test("local Daimon builds accept only the Grok CLI build the vendored contract manifest pins", () => { + const amd64 = readPinnedGrokCli("amd64"), arm64 = readPinnedGrokCli("arm64"); + assert.equal(amd64.version, "1.0.34"); + assert.equal(amd64.url, "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"); + assert.equal(amd64.sha256, "sha256:be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d"); + assert.equal(arm64.sha256, "sha256:39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94"); + const pinned = { executable_sha256: amd64.sha256, url: amd64.url, version: amd64.version }; + assert.doesNotThrow(() => assertPinnedGrokCli(pinned, amd64)); + assert.throws(() => assertPinnedGrokCli(pinned, arm64), /pinned Grok CLI 1\.0\.34/u); + assert.throws(() => assertPinnedGrokCli({ ...pinned, version: "1.0.13" }, amd64), /pinned Grok CLI/u); + assert.throws(() => assertPinnedGrokCli({ ...pinned, url: "https://example.invalid/grok" }, amd64), /pinned Grok CLI/u); +}); + +test("the runtime image derives broker and Grok pins from the attested Daimon manifest instead of literals", () => { + const dockerfile = readFileSync(new URL("../runtime-images/daimon/Dockerfile", import.meta.url), "utf8"); + assert.doesNotMatch(dockerfile, /broker_sha=[a-f0-9]{64}/u); + assert.match(dockerfile, /grokCliArtifacts\?\.\[process\.argv\[2\]\]/u); + assert.match(dockerfile, /m\.grokCliVersion!==process\.argv\[3\]/u); +}); diff --git a/scripts/build-local-daimon-runtime.ts b/scripts/build-local-daimon-runtime.ts index ad520aff..8e99db3a 100644 --- a/scripts/build-local-daimon-runtime.ts +++ b/scripts/build-local-daimon-runtime.ts @@ -146,6 +146,30 @@ export const readDaimonCliArtifactPins = (env: Record { + const broker = (JSON.parse(readFileSync(manifestPath, "utf8")) as { grokEngineBroker?: { grokCliArtifacts?: Record; grokCliVersion?: unknown } }).grokEngineBroker; + const artifact = broker?.grokCliArtifacts?.[architecture === "amd64" ? "x64" : "arm64"]; + if (typeof broker?.grokCliVersion !== "string" || typeof artifact?.url !== "string" || typeof artifact.sha256 !== "string" || !sha256Digest.test(artifact.sha256)) { + throw new Error("Vendored Daimon contract manifest does not pin a Grok CLI artifact"); + } + return { sha256: `sha256:${artifact.sha256}`, url: artifact.url, version: broker.grokCliVersion }; +}; + +export const assertPinnedGrokCli = (grok: DaimonCliArtifacts["grok"], pin: GrokCliPin): void => { + if (grok.version !== pin.version || grok.url !== pin.url || grok.executable_sha256 !== pin.sha256) { + throw new Error(`GROK_CLI_VERSION/GROK_CLI_URL/GROK_CLI_SHA256 must be the pinned Grok CLI ${pin.version} (${pin.url}, ${pin.sha256})`); + } +}; + export const resolveLocalImageTag = (value: string | undefined): string => { const tag = value?.trim(); const match = tag?.match(/^127\.0\.0\.1:((?:[1-9]\d{0,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))\/noopolis\/spawnfile-runtime-daimon:([A-Za-z0-9_][A-Za-z0-9_.-]{0,127})$/u); @@ -315,6 +339,7 @@ const main = (): void => { const imageTag = resolveLocalImageTag(process.env.SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG); const architecture = resolveLocalBuildArchitecture(process.arch); const artifacts = readDaimonCliArtifactPins(); + assertPinnedGrokCli(artifacts.grok, readPinnedGrokCli(architecture)); const packageDirectory = mkdtempSync(path.join(os.tmpdir(), "spawnfile-daimon-package-")); try { const bundled = sourceMode === "source-bundle" ? stageBundleBuiltDaimon(packageDirectory, artifacts, architecture) : null; diff --git a/scripts/source-provenance-bundle.integration.test.ts b/scripts/source-provenance-bundle.integration.test.ts index 4ed0741f..98a2b5d9 100644 --- a/scripts/source-provenance-bundle.integration.test.ts +++ b/scripts/source-provenance-bundle.integration.test.ts @@ -15,6 +15,14 @@ const { renderRuntimeLinkMaterializer } = await import( const repository = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const digest = (file: string): string => `sha256:${execFileSync("shasum", ["-a", "256", file], { encoding: "utf8" }).split(" ")[0] ?? ""}`; +const pinnedManifest = JSON.parse(readFileSync(path.join(repository, "src", "runtime", "daimon", "contract-manifest.json"), "utf8")) as { grokEngineBroker: { artifacts: { x64Sha256: string }; grokCliArtifacts: { x64: { url: string } }; grokCliVersion: string } }; +// The runtime image refuses any Grok executable but the manifest-pinned build, +// so this test needs the real linux-x86_64 binary (136+ MB) rather than a stub. +const pinnedGrokCli = (): string => { + const file = process.env.SPAWNFILE_GROK_CLI_FILE; + if (!file || !path.isAbsolute(file)) throw new Error("SPAWNFILE_GROK_CLI_FILE must name the pinned linux-x86_64 Grok CLI executable"); + return file; +}; const sha512 = (file: string): string => `sha512:${createHash("sha512").update(readFileSync(file)).digest("hex")}`; test("actual Daimon lock produces a real offline linux/amd64 shipped artifact and rejects tampering", { timeout: 360_000 }, () => { @@ -47,7 +55,7 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an assert.match(execFileSync("tar", ["-tzf", path.join(output, "daimon.tgz")], { encoding: "utf8" }), /package\/dist\/runtime\/contract-manifest\.json/u); assert.match(execFileSync("tar", ["-tvzf", path.join(output, "daimon.tgz")], { encoding: "utf8" }), /-rwxr-xr-x[^\n]*package\/dist\/runtime\/native\/daimon-engine-broker/u); const packedBroker = execFileSync("tar", ["-xOf", path.join(output, "daimon.tgz"), "package/dist/runtime/native/daimon-engine-broker"]); - assert.equal(`sha256:${createHash("sha256").update(packedBroker).digest("hex")}`, "sha256:e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd"); + assert.equal(`sha256:${createHash("sha256").update(packedBroker).digest("hex")}`, `sha256:${pinnedManifest.grokEngineBroker.artifacts.x64Sha256}`); const packageContext = path.join(temporary, "package-context"), probe = path.join(temporary, "probe"); mkdirSync(packageContext); mkdirSync(probe); cpSync(path.join(output, "daimon.tgz"), path.join(packageContext, "daimon.tgz")); cpSync(path.join(output, "runtime-dependencies.tar"), path.join(packageContext, "dependencies.tar")); cpSync(path.join(output, "source-inputs.json"), path.join(packageContext, "source-inputs.json")); execFileSync("docker", ["build", "--network=none", "--platform", "linux/amd64", "--target", "offline_dependency_probe", "--build-context", `daimon_package=${packageContext}`, @@ -55,7 +63,7 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an "--build-arg", `DAIMON_DEPENDENCY_ARCHIVE_SHA256=${digest(path.join(output, "runtime-dependencies.tar"))}`, "-f", path.join(repository, "runtime-images", "daimon", "Dockerfile"), repository], { stdio: "inherit" }); assert.deepEqual(JSON.parse(readFileSync(path.join(probe, "probe", "source-inputs.json"), "utf8")), identity); const grok = path.join(packageContext, "grok"), agyTree = path.join(temporary, "agy-tree"), agy = path.join(agyTree, "antigravity"), agyTar = path.join(packageContext, "agy.tar.gz"); - writeFileSync(grok, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); mkdirSync(agyTree); writeFileSync(agy, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + cpSync(pinnedGrokCli(), grok); mkdirSync(agyTree); writeFileSync(agy, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); execFileSync("tar", ["-czf", agyTar, "-C", agyTree, "antigravity"]); const runtimeArchive = digest(path.join(output, "runtime-dependencies.tar")); const dependencyLock = dependencyReceipt.manifest.dependency_lock; @@ -71,17 +79,17 @@ test("actual Daimon lock produces a real offline linux/amd64 shipped artifact an const port = mapped.slice(mapped.lastIndexOf(":") + 1); const identityPath = path.join(repository, ".local-daimon-runtime-identity.json"), priorIdentity = existsSync(identityPath) ? readFileSync(identityPath) : null; try { - execFileSync("npm", ["run", "--silent", "build:local-daimon"], { cwd: repository, env: { ...process.env, AGY_CLI_SHA256: agySha, AGY_CLI_SHA512: sha512(agyTar), AGY_CLI_URL: "https://invalid.example/agy", AGY_CLI_VERSION: "fixture", CODEX_CLI_SHA256: codexSha, GROK_CLI_SHA256: grokSha, GROK_CLI_URL: "https://invalid.example/grok", GROK_CLI_VERSION: "fixture", SPAWNFILE_AGY_CLI_ARCHIVE: agyTar, SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE: dependencyTar, SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG: `127.0.0.1:${port}/noopolis/spawnfile-runtime-daimon:archive-wrapper`, SPAWNFILE_DAIMON_SOURCE_BUNDLE: sourceTar, SPAWNFILE_GROK_CLI_FILE: grok }, stdio: "ignore" }); + execFileSync("npm", ["run", "--silent", "build:local-daimon"], { cwd: repository, env: { ...process.env, AGY_CLI_SHA256: agySha, AGY_CLI_SHA512: sha512(agyTar), AGY_CLI_URL: "https://invalid.example/agy", AGY_CLI_VERSION: "fixture", CODEX_CLI_SHA256: codexSha, GROK_CLI_SHA256: grokSha, GROK_CLI_URL: pinnedManifest.grokEngineBroker.grokCliArtifacts.x64.url, GROK_CLI_VERSION: pinnedManifest.grokEngineBroker.grokCliVersion, SPAWNFILE_AGY_CLI_ARCHIVE: agyTar, SPAWNFILE_DAIMON_DEPENDENCY_BUNDLE: dependencyTar, SPAWNFILE_DAIMON_LOCAL_IMAGE_TAG: `127.0.0.1:${port}/noopolis/spawnfile-runtime-daimon:archive-wrapper`, SPAWNFILE_DAIMON_SOURCE_BUNDLE: sourceTar, SPAWNFILE_GROK_CLI_FILE: grok }, stdio: "ignore" }); const wrapperIdentity = JSON.parse(readFileSync(identityPath, "utf8")); assert.equal(wrapperIdentity.image_architecture, "amd64"); assert.match(wrapperIdentity.image_reference, new RegExp(`^127\\.0\\.0\\.1:${port}/noopolis/spawnfile-runtime-daimon@sha256:[a-f0-9]{64}$`, "u")); } finally { if (priorIdentity) writeFileSync(identityPath, priorIdentity); else rmSync(identityPath, { force: true }); } - const receipt = { architecture: "amd64", daimon: { package_sha256: packageSha, source_inputs: sourceInputs, source_sha256: digest(path.join(packageContext, "source-inputs.json")) }, engines: { agy: { executable_sha256: agySha }, codex: { executable_sha256: codexSha }, grok: { executable_sha256: grokSha } }, manifest_sha256: manifestSha, provenance: { agy: { archive: { format: "tar.gz", sha512: sha512(agyTar), url: "https://invalid.example/agy", version: "fixture" } }, grok: { executable: { sha256: grokSha, url: "https://invalid.example/grok", version: "fixture" } } }, version: "spawnfile.daimon-runtime-capability-receipt.v1" }; + const receipt = { architecture: "amd64", daimon: { package_sha256: packageSha, source_inputs: sourceInputs, source_sha256: digest(path.join(packageContext, "source-inputs.json")) }, engines: { agy: { executable_sha256: agySha }, codex: { executable_sha256: codexSha }, grok: { executable_sha256: grokSha } }, manifest_sha256: manifestSha, provenance: { agy: { archive: { format: "tar.gz", sha512: sha512(agyTar), url: "https://invalid.example/agy", version: "fixture" } }, grok: { executable: { sha256: grokSha, url: pinnedManifest.grokEngineBroker.grokCliArtifacts.x64.url, version: pinnedManifest.grokEngineBroker.grokCliVersion } } }, version: "spawnfile.daimon-runtime-capability-receipt.v1" }; const shipped = path.join(temporary, "shipped"), shippedTar = path.join(temporary, "shipped.tar"); mkdirSync(shipped); execFileSync("docker", ["build", "--network=none", "--platform", "linux/amd64", "--build-context", `daimon_package=${packageContext}`, "--output", `type=tar,dest=${shippedTar}`, - "--build-arg", `DAIMON_CAPABILITY_RECEIPT_BASE64=${Buffer.from(`${JSON.stringify(receipt)}\n`).toString("base64")}`, "--build-arg", `DAIMON_MANIFEST_SHA256=${manifestSha}`, "--build-arg", `DAIMON_PACKAGE_SHA256=${packageSha}`, "--build-arg", `DAIMON_SOURCE_SHA256=${receipt.daimon.source_sha256}`, "--build-arg", "DAIMON_DEPENDENCY_MODE=offline-bundle", "--build-arg", `DAIMON_DEPENDENCY_ARCHIVE_SHA256=${runtimeArchive}`, "--build-arg", `CODEX_CLI_SHA256=${codexSha}`, "--build-arg", "GROK_CLI_VERSION=fixture", "--build-arg", "GROK_CLI_URL=https://invalid.example/grok", "--build-arg", `GROK_CLI_SHA256=${grokSha.slice(7)}`, "--build-arg", "AGY_CLI_VERSION=fixture", "--build-arg", "AGY_CLI_URL=https://invalid.example/agy", "--build-arg", `AGY_CLI_SHA512=${sha512(agyTar).slice(7)}`, "--build-arg", `AGY_CLI_SHA256=${agySha.slice(7)}`, "-f", path.join(repository, "runtime-images", "daimon", "Dockerfile"), repository], { stdio: "inherit" }); + "--build-arg", `DAIMON_CAPABILITY_RECEIPT_BASE64=${Buffer.from(`${JSON.stringify(receipt)}\n`).toString("base64")}`, "--build-arg", `DAIMON_MANIFEST_SHA256=${manifestSha}`, "--build-arg", `DAIMON_PACKAGE_SHA256=${packageSha}`, "--build-arg", `DAIMON_SOURCE_SHA256=${receipt.daimon.source_sha256}`, "--build-arg", "DAIMON_DEPENDENCY_MODE=offline-bundle", "--build-arg", `DAIMON_DEPENDENCY_ARCHIVE_SHA256=${runtimeArchive}`, "--build-arg", `CODEX_CLI_SHA256=${codexSha}`, "--build-arg", `GROK_CLI_VERSION=${pinnedManifest.grokEngineBroker.grokCliVersion}`, "--build-arg", `GROK_CLI_URL=${pinnedManifest.grokEngineBroker.grokCliArtifacts.x64.url}`, "--build-arg", `GROK_CLI_SHA256=${grokSha.slice(7)}`, "--build-arg", "AGY_CLI_VERSION=fixture", "--build-arg", "AGY_CLI_URL=https://invalid.example/agy", "--build-arg", `AGY_CLI_SHA512=${sha512(agyTar).slice(7)}`, "--build-arg", `AGY_CLI_SHA256=${agySha.slice(7)}`, "-f", path.join(repository, "runtime-images", "daimon", "Dockerfile"), repository], { stdio: "inherit" }); execFileSync("tar", ["-xf", shippedTar, "-C", shipped], { stdio: "inherit" }); assert.deepEqual(JSON.parse(readFileSync(path.join(shipped, "opt", "spawnfile", "runtime-installs", "daimon", "source-inputs.json"), "utf8")), receipt.daimon.source_inputs); const shippedRoot=path.join(shipped,"opt","spawnfile","runtime-installs","daimon"); - assert.equal(digest(path.join(shippedRoot,"bin","daimon-engine-broker")),"sha256:e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd"); + assert.equal(digest(path.join(shippedRoot,"bin","daimon-engine-broker")),`sha256:${pinnedManifest.grokEngineBroker.artifacts.x64Sha256}`); assert.equal(readFileSync(path.join(shippedRoot,"contract-manifest.sha256"),"utf8"),`${manifestSha}\n`); const orgContext = path.join(temporary, "literal-org-context"), orgTag = `spawnfile-literal-org-${Date.now().toString(36)}`; mkdirSync(orgContext); diff --git a/src/runtime/container.ts b/src/runtime/container.ts index 1be8b529..3f2b5784 100644 --- a/src/runtime/container.ts +++ b/src/runtime/container.ts @@ -12,7 +12,7 @@ import { DAIMON_LOCAL_RUNTIME_IDENTITY_ENV, loadLocalDaimonRuntimeIdentity } from "./localDaimonAuthority.js"; -import { DAIMON_CONTRACT_MANIFEST_SHA256 } from "./daimon/contractManifest.js"; +import { DAIMON_CONTRACT_MANIFEST_SHA256, DAIMON_GROK_ENGINE_BROKER } from "./daimon/contractManifest.js"; import { DAIMON_WAKE_FUSE_DIRECTORY, DAIMON_WAKE_FUSE_DIRECTORY_ENV @@ -244,9 +244,9 @@ export const createRuntimeInstallRecipe = async ( `test -f ${installRoot}/contract-manifest.json && test -f ${installRoot}/contract-manifest.sha256 && manifest="$(cat ${installRoot}/contract-manifest.sha256)" && test "$manifest" = ${JSON.stringify(DAIMON_CONTRACT_MANIFEST_SHA256)} && test "$(sha256sum ${installRoot}/contract-manifest.json | awk '{print "sha256:" $1}')" = "$manifest" && node -e 'const fs=require("fs");const r=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));if(r.manifest_sha256!==process.argv[2])process.exit(1)' ${installRoot}/${DAIMON_CAPABILITY_RECEIPT_FILE} "$manifest"`, `ln -sf ${installRoot}/bin/daimon-runtime /usr/local/bin/daimon-runtime`, `ln -sf ${installRoot}/bin/codex /usr/local/bin/codex`, - `install -o root -g root -m 0555 ${installRoot}/bin/grok /usr/local/bin/grok`, + `install -o root -g root -m 0555 ${installRoot}/bin/grok /usr/local/bin/grok && arch="$(dpkg --print-architecture)" && case "$arch" in amd64) expected=${DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.x64.sha256} ;; arm64) expected=${DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256} ;; *) exit 1 ;; esac && test "$(sha256sum /usr/local/bin/grok | awk '{print $1}')" = "$expected"`, `ln -sf ${installRoot}/bin/agy /usr/local/bin/agy`, - `mkdir -p /opt/daimon/bin && install -o root -g root -m 0555 ${installRoot}/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker && arch="$(dpkg --print-architecture)" && case "$arch" in amd64) expected=e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd ;; arm64) expected=ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d ;; *) exit 1 ;; esac && test "$(sha256sum /opt/daimon/bin/daimon-engine-broker | awk '{print $1}')" = "$expected"` + `mkdir -p /opt/daimon/bin && install -o root -g root -m 0555 ${installRoot}/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker && arch="$(dpkg --print-architecture)" && case "$arch" in amd64) expected=${DAIMON_GROK_ENGINE_BROKER.artifacts.x64Sha256} ;; arm64) expected=${DAIMON_GROK_ENGINE_BROKER.artifacts.arm64Sha256} ;; *) exit 1 ;; esac && test "$(sha256sum /opt/daimon/bin/daimon-engine-broker | awk '{print $1}')" = "$expected"` ], copyCommands: [createRuntimeImageCopyCommand(daimonRuntime.image, installRoot)], env: { diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index ead74419..8144e93a 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d","sourceSha256":"bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58","x64Sha256":"e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns"},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9","sourceSha256":"36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","x64Sha256":"36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v1","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index b3f7bdc7..bec82dd2 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:79bc6cd06aad3038ea26937f5b3e02d51abc001cf3629d80f49e377e45047b62 +sha256:c9ea30c600c4844621c64a015e3d42e97f410c9942eb41dd675b5c04add34270 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 3d563c51..8860bd2d 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:79bc6cd06aad3038ea26937f5b3e02d51abc001cf3629d80f49e377e45047b62" as const; + "sha256:c9ea30c600c4844621c64a015e3d42e97f410c9942eb41dd675b5c04add34270" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -31,6 +31,15 @@ export const DAIMON_GROK_SUBSCRIPTION_REALM = { fileMode: 0o600, maxCredentialBytes: 64 * 1024 } as const; +export const DAIMON_GROK_BROKER_MODELS = ["grok-4.6", "grok-4.5", "grok-build"] as const; +export const DAIMON_GROK_BROKER_REASONING_EFFORTS = ["low", "medium", "high"] as const; +export type DaimonGrokBrokerModel = typeof DAIMON_GROK_BROKER_MODELS[number]; +export type DaimonGrokBrokerReasoningEffort = typeof DAIMON_GROK_BROKER_REASONING_EFFORTS[number]; +/** + * Byte mirror of Daimon's `GROK_ENGINE_BROKER` (daimon + * `src/contracts/runtimeContractManifest.ts`), attested key-for-key against the + * vendored `contract-manifest.json` by `parseDaimonContractManifest`. + */ export const DAIMON_GROK_ENGINE_BROKER = { nativeAbiVersion: 2, nativeExecutablePath: "/opt/daimon/bin/daimon-engine-broker", @@ -45,11 +54,56 @@ export const DAIMON_GROK_ENGINE_BROKER = { providerProxy: { host: "127.0.0.1", port: 43_123 }, mcpFacade: { host: "127.0.0.1", port: 43_124, path: "/mcp" }, identities: { organizationUid: 2_000, brokerUid: 2_100, firstWorkerUid: 2_200 }, + grokCliVersion: "1.0.34", + grokCliBuild: "3736acbc8658", + grokCliArtifacts: { + arm64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64", sha256: "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", bytes: 136_090_504 }, + x64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64", sha256: "be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d", bytes: 163_035_648 } + }, + worker: { + modelId: "daimon-broker-grok", + models: DAIMON_GROK_BROKER_MODELS, + reasoningEfforts: DAIMON_GROK_BROKER_REASONING_EFFORTS, + defaultModel: "grok-4.6", + defaultReasoningEffort: "low", + toolIds: ["run_terminal_cmd", "read_file", "grep", "list_dir", "search_tool", "use_tool"], + visibleTools: ["grep", "list_dir", "read_file", "run_terminal_command", "search_tool", "use_tool"], + maxTurns: 48, + systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", + configSha256: { + "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, + "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, + "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } + }, + home: { + directory: { uid: 0, group: "worker", mode: 0o1771 }, + sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, + readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + } + }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + controlProtocolVersion: "noopolis.daimon.engine-broker.v2", + turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], + serviceConfigVersions: ["noopolis.daimon.engine-broker-service.v1", "noopolis.daimon.engine-broker-service.v2"], + turnLimits: { + keys: ["maxRequests", "maxTokens", "timeoutMs"], + v1Defaults: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + bounds: { maxRequests: [1, 48], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, + limitReasons: ["tokens", "requests", "timeout", "none"], + wakeMayOnlyLower: true, + tokenCeilingOvershoot: "at-most-one-request", + maxInFlightRequests: 1, + requestUsageMaxTokens: 500_000, + missingUsageEstimate: { inputBytesPerToken: 2, outputTokens: 4_096 } + }, + wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, + projectionVersion: "noopolis.daimon.grok-broker-projection.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", artifacts: { - sourceSha256: "bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58", - x64Sha256: "e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd", - arm64Sha256: "ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d" + sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", + x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", + arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" } } as const; /** From 1f3e3b78a98e407e4b1b78049ba62cfce5a102c8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:04:18 +0200 Subject: [PATCH 20/84] feat: require a declared xAI model and reasoning effort for brokered Daimon Grok agents --- src/compiler/modelEnv.test.ts | 10 ++++++ src/compiler/modelEnv.ts | 10 ++++-- src/compiler/runProjectAuth.ts | 1 + src/compiler/syncProjectAuth.ts | 6 ++-- src/compiler/types.ts | 1 + src/compiler/updateProjectModels.ts | 1 + src/distribution/preflight.test.ts | 10 ++++++ src/distribution/preflight.ts | 6 +++- src/manifest/executionSchemas.test.ts | 34 +++++++++++++++++++++ src/manifest/executionSchemas.ts | 25 +++++++++++++-- src/runtime/daimon/AGENTS.md | 11 +++++++ src/runtime/daimon/adapter.test.ts | 38 ++++++++++++++++++++++- src/runtime/daimon/adapter.ts | 15 ++++++--- src/runtime/daimon/config.test.ts | 8 +++++ src/runtime/daimon/config.ts | 7 ++++- src/runtime/daimon/grokModel.ts | 44 +++++++++++++++++++++++++++ src/runtime/pi/appTemplateTypes.ts | 2 +- src/shared/types.ts | 2 +- 18 files changed, 215 insertions(+), 16 deletions(-) create mode 100644 src/manifest/executionSchemas.test.ts create mode 100644 src/runtime/daimon/grokModel.ts diff --git a/src/compiler/modelEnv.test.ts b/src/compiler/modelEnv.test.ts index 0f4593be..7f184519 100644 --- a/src/compiler/modelEnv.test.ts +++ b/src/compiler/modelEnv.test.ts @@ -6,6 +6,7 @@ import { listEffectiveExecutionModelTargets, listExecutionModelProviders, listExecutionModelSecretNames, + modelAuthMethodNeedsCliCredential, resolveEffectiveModelTarget, resolveExecutionModelAuthMethods, resolveModelProviderEnvName @@ -203,4 +204,13 @@ describe("modelEnv", () => { it("formats provider env names for unknown providers", () => { expect(resolveModelProviderEnvName("my-proxy")).toBe("MY_PROXY_API_KEY"); }); + + it("never turns Daimon-owned grok auth into a host CLI credential or api key secret", () => { + const execution = { model: { primary: { auth: { method: "grok" as const }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" as const } } }; + expect(listExecutionModelSecretNames(execution)).toEqual([]); + expect(modelAuthMethodNeedsCliCredential("grok")).toBe(false); + expect(modelAuthMethodNeedsCliCredential("codex")).toBe(true); + expect(modelAuthMethodNeedsCliCredential("claude-code")).toBe(true); + expect(resolveEffectiveModelTarget(execution.model.primary, execution)).toEqual({ auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoningEffort: "low" }); + }); }); diff --git a/src/compiler/modelEnv.ts b/src/compiler/modelEnv.ts index 1b739239..8c9141b6 100644 --- a/src/compiler/modelEnv.ts +++ b/src/compiler/modelEnv.ts @@ -15,8 +15,13 @@ const MODEL_PROVIDER_ENV_VARS = new Map([ export const CLI_CREDENTIAL_SECRET_NAME = "SPAWNFILE_CLI_AUTH_JSON"; +/** + * Only the host-imported CLI logins travel as `SPAWNFILE_CLI_AUTH_JSON`. `grok` + * is a Daimon-owned subscription: its credential is the broker's bootstrap + * slot (`runtime/daimon/runAuth.ts`), never this secret. + */ export const modelAuthMethodNeedsCliCredential = (method: ModelAuthMethod): boolean => - method !== "api_key" && method !== "none"; + method === "claude-code" || method === "codex"; const resolveLegacyModelAuthMethod = ( execution: ExecutionBlock | undefined, @@ -122,7 +127,8 @@ export const resolveEffectiveModelTarget = ( }, ...(target.endpoint ? { endpoint: target.endpoint } : {}), name: target.name, - provider: target.provider + provider: target.provider, + ...(target.reasoning_effort ? { reasoningEffort: target.reasoning_effort } : {}) }; }; diff --git a/src/compiler/runProjectAuth.ts b/src/compiler/runProjectAuth.ts index 6d4cba21..573522cd 100644 --- a/src/compiler/runProjectAuth.ts +++ b/src/compiler/runProjectAuth.ts @@ -27,6 +27,7 @@ const MODEL_AUTH_IMPORT_KINDS: Record = { api_key: null, "claude-code": "claude-code", codex: "codex", + grok: null, none: null }; diff --git a/src/compiler/syncProjectAuth.ts b/src/compiler/syncProjectAuth.ts index 31a6ea8e..928d4533 100644 --- a/src/compiler/syncProjectAuth.ts +++ b/src/compiler/syncProjectAuth.ts @@ -7,7 +7,7 @@ import { setAuthProfileEnv } from "../auth/index.js"; import { readUtf8File } from "../filesystem/index.js"; -import { SpawnfileError } from "../shared/index.js"; +import { type ModelAuthMethod, SpawnfileError } from "../shared/index.js"; import { listAgentSurfaceSecretNames } from "./agentSurfaces.js"; import { buildCompilePlan } from "./buildCompilePlan.js"; @@ -28,12 +28,12 @@ export interface SyncProjectAuthOptions { const resolveAuthRequirements = async ( inputPath: string ): Promise<{ - methods: Set<"api_key" | "claude-code" | "codex" | "none">; + methods: Set; optionalEnvNames: Set; requiredEnvNames: Set; }> => { const plan = await buildCompilePlan(inputPath); - const methods = new Set<"api_key" | "claude-code" | "codex" | "none">(); + const methods = new Set(); const optionalEnvNames = new Set(); const requiredEnvNames = new Set(); diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 26dcbcbe..f1ddb32d 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -236,6 +236,7 @@ export interface EffectiveModelTarget { endpoint?: ModelEndpoint; name: string; provider: string; + reasoningEffort?: "high" | "low" | "medium"; } export interface ResolvedSubagentRef { diff --git a/src/compiler/updateProjectModels.ts b/src/compiler/updateProjectModels.ts index 3dac9000..667c4147 100644 --- a/src/compiler/updateProjectModels.ts +++ b/src/compiler/updateProjectModels.ts @@ -50,6 +50,7 @@ const AUTH_METHOD_PROVIDER_HINTS: Record = { "api_key": "use a model provider like openai or anthropic, then pass --auth api_key", "claude-code": "use provider anthropic, then pass --auth claude-code", codex: "use provider openai, then pass --auth codex", + grok: "use provider xai with a Daimon grok engine agent, then pass --auth grok and declare reasoning_effort", none: "use a model provider like local, then pass --auth none" }; diff --git a/src/distribution/preflight.test.ts b/src/distribution/preflight.test.ts index 71b76c66..7ea4e53f 100644 --- a/src/distribution/preflight.test.ts +++ b/src/distribution/preflight.test.ts @@ -118,4 +118,14 @@ describe("runImagePreflight", () => { }) ).not.toThrow(); }); + + it("treats Daimon-owned grok auth as satisfied by the runtime's own credential slot", () => { + const base = importReport(); + const report = { + ...base, + model_auth_methods: { xai: "grok" as const }, + runtime_instances: base.runtime_instances.map((instance) => ({ ...instance, model_auth_methods: { xai: "grok" as const }, runtime: "daimon" })) + }; + expect(() => runImagePreflight({ authValues: { DIST_REQUIRED_TOKEN: "y" }, report })).not.toThrow(); + }); }); diff --git a/src/distribution/preflight.ts b/src/distribution/preflight.ts index 4ea38746..6c7e5b1b 100644 --- a/src/distribution/preflight.ts +++ b/src/distribution/preflight.ts @@ -6,6 +6,10 @@ import type { } from "./types.js"; const ENV_BASED_AUTH_METHODS = new Set(["api_key", "none"]); +// Satisfied by the runtime's own credential slot (the Daimon Grok broker's +// bootstrap realm, checked by `imageRuntimeAuth.ts`), never by an env value +// or a host CLI import. +const RUNTIME_OWNED_AUTH_METHODS = new Set(["grok"]); // Import-based auth methods are satisfiable sourceless when the consumer has the // matching local credential import (their logged-in Claude Code / Codex session). @@ -32,7 +36,7 @@ const collectUnsupportedAuth = ( const unsupported: Array<{ instance: string; method: string; provider: string; runtime: string }> = []; for (const instance of report.runtime_instances) { for (const [provider, method] of Object.entries(instance.model_auth_methods)) { - if (ENV_BASED_AUTH_METHODS.has(method)) { + if (ENV_BASED_AUTH_METHODS.has(method) || RUNTIME_OWNED_AUTH_METHODS.has(method)) { continue; } const importKind = IMPORT_AUTH_METHOD_KINDS[method]; diff --git a/src/manifest/executionSchemas.test.ts b/src/manifest/executionSchemas.test.ts new file mode 100644 index 00000000..86b01cd6 --- /dev/null +++ b/src/manifest/executionSchemas.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { executionSchema } from "./executionSchemas.js"; + +const grokPrimary = { auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" }; + +describe("brokered Grok model declarations", () => { + it("accepts xai grok auth with a closed reasoning effort", () => { + for (const reasoning_effort of ["low", "medium", "high"]) { + expect(executionSchema.safeParse({ model: { primary: { ...grokPrimary, reasoning_effort } } }).success).toBe(true); + } + const { reasoning_effort: _effort, ...withoutEffort } = grokPrimary; + // Presence is enforced per engine by the Daimon adapter; the schema only fences where it may appear. + expect(executionSchema.safeParse({ model: { primary: withoutEffort } }).success).toBe(true); + }); + + it("refuses grok auth off xai, an endpoint, an unknown effort, or an effort on any other auth", () => { + const invalid = [ + { ...grokPrimary, provider: "openai" }, + { ...grokPrimary, provider: "custom", endpoint: { base_url: "http://127.0.0.1/v1", compatibility: "openai" } }, + { ...grokPrimary, reasoning_effort: "xhigh" }, + { ...grokPrimary, reasoning_effort: "minimal" }, + { auth: { method: "codex" }, name: "gpt-5.4", provider: "openai", reasoning_effort: "low" }, + { auth: { method: "api_key" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" }, + { name: "grok-4.6", provider: "xai", reasoning_effort: "low" } + ]; + for (const primary of invalid) { + expect(executionSchema.safeParse({ model: { primary } }).success, JSON.stringify(primary)).toBe(false); + } + // Inherited (model-level) grok auth never carries an effort. + const { auth: _auth, ...inherited } = grokPrimary; + expect(executionSchema.safeParse({ model: { auth: { method: "grok" }, primary: inherited } }).success).toBe(false); + }); +}); diff --git a/src/manifest/executionSchemas.ts b/src/manifest/executionSchemas.ts index bd52129e..3474f38d 100644 --- a/src/manifest/executionSchemas.ts +++ b/src/manifest/executionSchemas.ts @@ -1,6 +1,12 @@ import { z } from "zod"; -const modelAuthMethodSchema = z.enum(["api_key", "claude-code", "codex", "none"]); +const modelAuthMethodSchema = z.enum(["api_key", "claude-code", "codex", "grok", "none"]); +/** + * The closed reasoning-effort vocabulary a brokered Daimon Grok worker accepts + * (`DAIMON_GROK_BROKER_REASONING_EFFORTS`). Only valid on a target that + * declares `auth.method: grok` itself; it is never inherited. + */ +export const MODEL_REASONING_EFFORTS = ["low", "medium", "high"] as const; const modelEndpointCompatibilitySchema = z.enum(["anthropic", "openai"]); const modelAuthSchema = z @@ -66,10 +72,25 @@ export const modelTargetSchema = z auth: modelEntryAuthSchema.optional(), endpoint: modelEndpointSchema.optional(), name: z.string(), - provider: z.string() + provider: z.string(), + reasoning_effort: z.enum(MODEL_REASONING_EFFORTS).optional() }) .strict() .superRefine((value, context) => { + if (value.auth?.method === "grok" && (value.provider !== "xai" || value.endpoint)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + message: "grok auth is only valid for provider xai without an endpoint" + }); + } + + if (value.reasoning_effort !== undefined && value.auth?.method !== "grok") { + context.addIssue({ + code: z.ZodIssueCode.custom, + message: "reasoning_effort is only valid on a model that declares auth.method grok" + }); + } + const usesCustomEndpoint = value.provider === "custom" || value.provider === "local"; if (usesCustomEndpoint && !value.endpoint) { diff --git a/src/runtime/daimon/AGENTS.md b/src/runtime/daimon/AGENTS.md index 52f67263..9e575d9f 100644 --- a/src/runtime/daimon/AGENTS.md +++ b/src/runtime/daimon/AGENTS.md @@ -59,6 +59,17 @@ digests, fixed root/org/broker/worker identities, root-only registrations, and loopback-only provider/MCP endpoints. Container provisioning must match that authority exactly and must not publish either broker port. +Every Grok agent is brokered, so `grokModel.ts` requires its model in full: +`execution.model.primary` with `provider: xai`, a name from the manifest's +closed list (`grok-4.6`, `grok-4.5`, `grok-build`), a target-level +`auth.method: grok`, and `reasoning_effort` (`low`, `medium`, `high`), with no +fallback and no model-level auth. The pair lowers to `engine.model` and +`engine.reasoningEffort` and selects the manifest-pinned worker `config.toml`. +Nothing is defaulted: Grok 1.0.34 silently drops an undeclared effort and its +catalog default for `grok-4.6` is `high`. `grok` auth is Daimon-owned — it +never becomes `SPAWNFILE_CLI_AUTH_JSON`, an api-key secret, or a host import. +AGY still refuses any `execution.model`; Codex refuses Grok auth. + Codex keeps an isolated per-agent credential home. Grok keeps isolated per-agent non-auth state but one durable rotating subscription credential realm; never fan out Grok refresh authority across writable homes. diff --git a/src/runtime/daimon/adapter.test.ts b/src/runtime/daimon/adapter.test.ts index 8f0b101b..c2c3cf01 100644 --- a/src/runtime/daimon/adapter.test.ts +++ b/src/runtime/daimon/adapter.test.ts @@ -25,6 +25,9 @@ const createDaimonNode = (id: string, name = id, engine = "codex") => { source: `/tmp/agent/${id}/Spawnfile` }); if (engine === "codex") return node; + if (engine === "grok") { + return { ...node, execution: { ...node.execution!, model: { primary: { auth: { method: "grok" as const }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" as const } } } }; + } const { model: _model, ...execution } = node.execution!; return { ...node, execution }; }; @@ -192,7 +195,7 @@ describe("daimonAdapter", () => { expect(JSON.parse(target.files.find((file) => file.path === DAIMON_CONFIG_FILE)!.content).agents) .toEqual(expect.arrayContaining([ expect.objectContaining({ engine: { kind: "codex", model: "gpt-5.4-mini" } }), - expect.objectContaining({ engine: { kind: "grok" } }), + expect.objectContaining({ engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } }), expect.objectContaining({ engine: { kind: "agy" } }) ])); expect(target.opaqueMountTargets).toEqual([ @@ -383,11 +386,44 @@ describe("daimonAdapter", () => { ]) }); await expect(daimonAdapter.compileAgent(createPiTestNode({ runtime: { name: "daimon", options: { engine: "grok" } } + }))).rejects.toThrow("must use provider xai with auth.method grok"); + await expect(daimonAdapter.compileAgent(createPiTestNode({ + runtime: { name: "daimon", options: { engine: "agy" } } }))).rejects.toThrow("must omit Spawnfile execution.model"); expect(() => daimonAdapter.assertSupportedSurfaces?.({ moltnet: [{ network: "test" }] } as any)).not.toThrow(); expect(() => daimonAdapter.assertSupportedSurfaces?.({ discord: [{}] } as any)).toThrow("only lowers Moltnet"); }); + it("requires a brokered Grok agent to declare one listed xAI model and its reasoning effort, never inherited", async () => { + const grok = createDaimonNode("grok", "Grok", "grok"); + const withPrimary = (primary: Record, extra: Record = {}) => + ({ ...grok, execution: { ...grok.execution!, model: { primary, ...extra } } }) as typeof grok; + const declared = { auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" }; + await expect(daimonAdapter.compileAgent(grok)).resolves.toBeDefined(); + for (const name of ["grok-4.5", "grok-build"]) { + for (const reasoning_effort of ["medium", "high"]) { + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, name, reasoning_effort }))).resolves.toBeDefined(); + } + } + await expect(daimonAdapter.compileAgent({ ...grok, execution: { sandbox: { mode: "workspace" } } } as typeof grok)) + .rejects.toThrow("must declare its brokered model and reasoning effort"); + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, name: "grok-4" }))).rejects.toThrow("unsupported model grok-4"); + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, name: "grok-code-fast-1" }))).rejects.toThrow("unsupported model"); + const { reasoning_effort: _effort, ...withoutEffort } = declared; + await expect(daimonAdapter.compileAgent(withPrimary(withoutEffort))).rejects.toThrow("must declare reasoning_effort"); + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, reasoning_effort: "xhigh" }))).rejects.toThrow("must declare reasoning_effort"); + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, provider: "openai" }))).rejects.toThrow("must use provider xai"); + await expect(daimonAdapter.compileAgent(withPrimary({ ...declared, auth: undefined }, { auth: { method: "grok" } }))) + .rejects.toThrow("exactly one model with target-level auth"); + await expect(daimonAdapter.compileAgent(withPrimary(declared, { fallback: [declared] }))).rejects.toThrow("exactly one model"); + const codex = createDaimonNode("codex", "Codex", "codex"); + await expect(daimonAdapter.compileAgent({ ...codex, execution: { ...codex.execution!, model: { primary: declared } } } as typeof codex)) + .rejects.toThrow("cannot declare Grok model auth"); + expect(() => daimonAdapter.assertSupportedModelTarget?.({ auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoningEffort: "low" })).not.toThrow(); + expect(() => daimonAdapter.assertSupportedModelTarget?.({ auth: { method: "codex" }, name: "gpt", provider: "openai", reasoningEffort: "low" })).toThrow(/brokered xAI Grok/u); + expect(() => daimonAdapter.assertSupportedModelTarget?.({ auth: { method: "grok" }, name: "grok-4.6", provider: "openai" })).toThrow(/brokered xAI Grok/u); + }); + it("validates the complete public model, option, MCP, and empty-target boundaries", async () => { expect(() => daimonAdapter.assertSupportedModelTarget?.({ auth: { method: "codex" }, provider: "openai" diff --git a/src/runtime/daimon/adapter.ts b/src/runtime/daimon/adapter.ts index 344be639..5bc24261 100644 --- a/src/runtime/daimon/adapter.ts +++ b/src/runtime/daimon/adapter.ts @@ -21,6 +21,7 @@ import { DAIMON_ENGINES, resolveDaimonEngine } from "./config.js"; +import { resolveDaimonGrokModel } from "./grokModel.js"; import { prepareDaimonRuntimeAuth } from "./runAuth.js"; import { hasDaimonScheduleAuthority } from "./scheduleAuthority.js"; import { resolveDaimonAttention } from "./attention.js"; @@ -40,10 +41,11 @@ const assertDaimonSurfaces = (surfaces: ResolvedAgentSurfaces | undefined): void }; const assertDaimonModel = (target: EffectiveModelTarget): void => { - if (target.provider === "openai" && target.auth.method === "codex" && !target.endpoint) return; + if (target.provider === "openai" && target.auth.method === "codex" && !target.endpoint && !target.reasoningEffort) return; + if (target.provider === "xai" && target.auth.method === "grok" && !target.endpoint) return; throw new SpawnfileError( "validation_error", - "Daimon organization runtime v1 accepts only the optional OpenAI Codex subscription intent; Grok and AGY engine auth stays Daimon-owned" + "Daimon organization runtime v1 accepts only the optional OpenAI Codex subscription intent or a brokered xAI Grok declaration; AGY engine auth stays Daimon-owned" ); }; @@ -109,12 +111,17 @@ const unsupportedAgentFeatures = (node: ResolvedAgentNode): void => { if (!server.tools?.length) throw new SpawnfileError("validation_error", `Daimon MCP server ${server.name} requires an explicit tools allowlist`); if (server.transport === "stdio" && !server.command?.startsWith("/")) throw new SpawnfileError("validation_error", `Daimon stdio MCP server ${server.name} requires an absolute command`); } - if (resolveDaimonEngine(node) !== "codex" && node.execution?.model) { + const engine = resolveDaimonEngine(node); + if (engine === "agy" && node.execution?.model) { throw new SpawnfileError( "validation_error", - "Daimon Grok and AGY agents must omit Spawnfile execution.model; their subscription auth and model selection are Daimon-owned" + "Daimon AGY agents must omit Spawnfile execution.model; their subscription auth and model selection are Daimon-owned" ); } + if (engine === "codex" && node.execution?.model?.primary.auth?.method === "grok") { + throw new SpawnfileError("validation_error", `Daimon Codex agent ${node.name} cannot declare Grok model auth`); + } + if (engine === "grok") resolveDaimonGrokModel(node); }; const scheduleCapabilityFor = async ( diff --git a/src/runtime/daimon/config.test.ts b/src/runtime/daimon/config.test.ts index 5817c7c0..9dea841b 100644 --- a/src/runtime/daimon/config.test.ts +++ b/src/runtime/daimon/config.test.ts @@ -162,6 +162,14 @@ describe("Daimon memory lowering", () => { expect(config.agents[0]!.engine).toEqual({ kind: "codex", model: "gpt-5.4-codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } }); }); + it("lowers a brokered Grok agent's declared model and reasoning effort into Daimon's engine config", async () => { + const config = await emitConfig(createDaimonNode({ + execution: { model: { primary: { auth: { method: "grok" }, name: "grok-4.5", provider: "xai", reasoning_effort: "medium" } } }, + runtime: { name: "daimon", options: { engine: "grok" } } + })); + expect(config.agents[0]!.engine).toEqual({ kind: "grok", model: "grok-4.5", reasoningEffort: "medium" }); + }); + it("omits Daimon's engine model selector when no execution model is declared", async () => { const config = await emitConfig(createDaimonNode({ execution: undefined })); diff --git a/src/runtime/daimon/config.ts b/src/runtime/daimon/config.ts index 4f8a56ef..3a730fca 100644 --- a/src/runtime/daimon/config.ts +++ b/src/runtime/daimon/config.ts @@ -20,6 +20,7 @@ import { } from "./memory.js"; import { assertDaimonScheduleAuthority } from "./scheduleAuthority.js"; import { assertDaimonAttentionAuthority, resolveDaimonAttention } from "./attention.js"; +import { resolveDaimonGrokModel } from "./grokModel.js"; export const DAIMON_CODEX_WORKSPACE_NO_NETWORK_POLICY = { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } as const; @@ -100,8 +101,12 @@ export const resolveDaimonEngine = (node: ResolvedAgentNode): DaimonEngine => { const resolveDaimonEngineConfig = ( node: ResolvedAgentNode -): { kind: DaimonEngine; model?: string; codexSandbox?: typeof DAIMON_CODEX_WORKSPACE_NO_NETWORK_POLICY } => { +): { kind: DaimonEngine; model?: string; reasoningEffort?: string; codexSandbox?: typeof DAIMON_CODEX_WORKSPACE_NO_NETWORK_POLICY } => { const kind = resolveDaimonEngine(node); + if (kind === "grok") { + const declared = resolveDaimonGrokModel(node); + return { kind, model: declared.model, reasoningEffort: declared.reasoningEffort }; + } const model = kind === "codex" ? node.execution?.model?.primary?.name : undefined; const codexSandbox = kind === "codex" && node.runtime.options.codex_policy === "workspace-no-network" ? DAIMON_CODEX_WORKSPACE_NO_NETWORK_POLICY diff --git a/src/runtime/daimon/grokModel.ts b/src/runtime/daimon/grokModel.ts new file mode 100644 index 00000000..70eae8ed --- /dev/null +++ b/src/runtime/daimon/grokModel.ts @@ -0,0 +1,44 @@ +import type { ResolvedAgentNode } from "../../compiler/types.js"; +import { SpawnfileError } from "../../shared/index.js"; + +import { + DAIMON_GROK_BROKER_MODELS, + DAIMON_GROK_BROKER_REASONING_EFFORTS, + type DaimonGrokBrokerModel, + type DaimonGrokBrokerReasoningEffort +} from "./contractManifest.js"; + +export interface DaimonGrokModel { + model: DaimonGrokBrokerModel; + reasoningEffort: DaimonGrokBrokerReasoningEffort; +} + +/** + * The declared model of one brokered Daimon Grok agent. + * + * Every Grok agent runs through Daimon's engine broker, whose worker + * `config.toml` bytes are pinned per model x effort and whose provider proxy + * refuses any request body carrying another pair. Nothing may be inherited: Grok + * 1.0.34 silently drops an effort a model does not declare and its catalog + * default for `grok-4.6` is `high`, so the declaration is required in full — + * `execution.model.primary` with `provider: xai`, a name from Daimon's closed + * list, a target-level `auth.method: grok`, and `reasoning_effort` — and there + * is exactly one model (no fallback, no legacy model-level auth). + */ +export const resolveDaimonGrokModel = (node: ResolvedAgentNode): DaimonGrokModel => { + const fail = (detail: string): never => { + throw new SpawnfileError( + "validation_error", + `Daimon Grok agent ${node.name} ${detail}; declare execution.model.primary { provider: xai, name: ${DAIMON_GROK_BROKER_MODELS.join(" | ")}, auth: { method: grok }, reasoning_effort: ${DAIMON_GROK_BROKER_REASONING_EFFORTS.join(" | ")} }` + ); + }; + const declared = node.execution?.model; + if (!declared) return fail("must declare its brokered model and reasoning effort"); + if (declared.fallback?.length || declared.auth) fail("must declare exactly one model with target-level auth"); + const primary = declared.primary; + if (primary.provider !== "xai" || primary.endpoint || primary.auth?.method !== "grok") fail("must use provider xai with auth.method grok"); + if (!(DAIMON_GROK_BROKER_MODELS as readonly string[]).includes(primary.name)) fail(`declares unsupported model ${primary.name}`); + const effort = primary.reasoning_effort; + if (effort === undefined || !(DAIMON_GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(effort)) fail("must declare reasoning_effort"); + return { model: primary.name as DaimonGrokBrokerModel, reasoningEffort: effort as DaimonGrokBrokerReasoningEffort }; +}; diff --git a/src/runtime/pi/appTemplateTypes.ts b/src/runtime/pi/appTemplateTypes.ts index 9f8b9c41..3c0c7afb 100644 --- a/src/runtime/pi/appTemplateTypes.ts +++ b/src/runtime/pi/appTemplateTypes.ts @@ -34,7 +34,7 @@ export interface PiGeneratedAgent { id: string; instructions: string; model: { - auth_method: "api_key" | "claude-code" | "codex" | "none" | "unknown"; + auth_method: "api_key" | "claude-code" | "codex" | "grok" | "none" | "unknown"; name: string; provider: string; }; diff --git a/src/shared/types.ts b/src/shared/types.ts index 3013f75b..c1b7d781 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1,6 +1,6 @@ export type JsonPrimitive = boolean | null | number | string; export type JsonValue = JsonObject | JsonPrimitive | JsonValue[]; -export type ModelAuthMethod = "api_key" | "claude-code" | "codex" | "none"; +export type ModelAuthMethod = "api_key" | "claude-code" | "codex" | "grok" | "none"; export type ModelEndpointCompatibility = "anthropic" | "openai"; export type RuntimeLifecycleStatus = "active" | "deprecated" | "exploratory"; From a9605056228979fd63435732036db3ad83f99016 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:05:23 +0200 Subject: [PATCH 21/84] feat: emit no workspace skill roots for brokered Daimon Grok agents and warn on declared skills --- src/runtime/AGENTS.md | 2 +- src/runtime/daimon/adapter.test.ts | 16 ++++++++++++++++ src/runtime/daimon/adapter.ts | 24 +++++++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 55a5cb45..d94cba70 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -36,7 +36,7 @@ calls it to decide whether to emit an agent `memory` block at all. Keep those tw sides on this one function: a config that points an in-process Mneme runtime at a path the container does not mount fails at its first write instead of degrading. -`common.ts` owns where declared `workspace.skills` are emitted. `createSkillFiles` accepts either one root or a list of roots, and the roots are named constants there: `WORKSPACE_SKILL_BASE_DIRECTORY` (`workspace/skills`) for OpenClaw and PicoClaw, which read that directory with their own skill loaders, and `CLI_ENGINE_SKILL_BASE_DIRECTORIES` (`workspace/.agents/skills` and `workspace/.codex/skills`) for Daimon and Pi, whose skills are discovered by an external coding-agent CLI. Both CLI-engine roots are required and their files are byte-identical on purpose: `.codex/skills` is Codex's own discovery root and `.agents/skills` is the generic root grok, agy, and other file-reading engines use. This mirrors the Moltnet skill install exactly — `resolveMoltnetWorkspaceLayout` in `src/compiler/moltnetClientConfig.ts` runs `moltnet skill install --runtime codex` for these runtimes and Moltnet writes both roots — and it is the reason declared skills now reach an engine at all: a plain `workspace/skills/` root is read by no engine Daimon or Pi can host, so everything emitted there was invisible. +`common.ts` owns where declared `workspace.skills` are emitted. `createSkillFiles` accepts either one root or a list of roots, and the roots are named constants there: `WORKSPACE_SKILL_BASE_DIRECTORY` (`workspace/skills`) for OpenClaw and PicoClaw, which read that directory with their own skill loaders, and `CLI_ENGINE_SKILL_BASE_DIRECTORIES` (`workspace/.agents/skills` and `workspace/.codex/skills`) for Daimon and Pi, whose skills are discovered by an external coding-agent CLI. Both CLI-engine roots are required and their files are byte-identical on purpose: `.codex/skills` is Codex's own discovery root and `.agents/skills` is the generic root grok, agy, and other file-reading engines use. This mirrors the Moltnet skill install exactly — `resolveMoltnetWorkspaceLayout` in `src/compiler/moltnetClientConfig.ts` runs `moltnet skill install --runtime codex` for these runtimes and Moltnet writes both roots — and it is the reason declared skills now reach an engine at all: a plain `workspace/skills/` root is read by no engine Daimon or Pi can host, so everything emitted there was invisible. The one exception is a brokered Daimon Grok agent (`daimonSkillBaseDirectories` in `daimon/adapter.ts`): Grok 1.0.34 discovers `.agents/skills` (never `.codex/skills`) only in a trusted workspace, and Daimon keeps the worker workspace untrusted and overrides the system prompt, so no root is emitted and a compile warning names the declared skills. `common.ts` also owns the `NOOPOLIS_RUN_ID` container env constant (`NOOPOLIS_RUN_ID_ENV` / `resolveNoopolisRunId`); `container.ts` reads it via `createRuntimeContainerEnv` and stamps it into every generated `RuntimeInstallRecipe.env` so every authority container agrees on one run id for causal event envelopes (see `specs/CAUSAL.md`). Never read `run_id` or `principal_id` from model output here. diff --git a/src/runtime/daimon/adapter.test.ts b/src/runtime/daimon/adapter.test.ts index c2c3cf01..d3b73a7a 100644 --- a/src/runtime/daimon/adapter.test.ts +++ b/src/runtime/daimon/adapter.test.ts @@ -76,6 +76,22 @@ describe("daimonAdapter", () => { expect(compiled.files.some((file) => file.path.startsWith("workspace/skills/"))).toBe(false); }); + it("emits no workspace skill root for a brokered Grok agent, never .codex/skills, and reports the declaration", async () => { + const grok = await daimonAdapter.compileAgent(createDaimonNode("grok", "Grok", "grok")); + const skillPaths = grok.files.map((file) => file.path).filter((filePath) => filePath.includes("/skills/")); + expect(skillPaths).toEqual([]); + expect(grok.files.some((file) => file.path.startsWith("workspace/.codex/"))).toBe(false); + expect(grok.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ level: "warn", message: expect.stringContaining("declares workspace skills (note)") }) + ])); + for (const engine of ["codex", "agy"]) { + const compiled = await daimonAdapter.compileAgent(createDaimonNode(engine, engine, engine)); + expect(compiled.files.map((file) => file.path).filter((filePath) => filePath.endsWith("/SKILL.md"))) + .toEqual(["workspace/.agents/skills/note/SKILL.md", "workspace/.codex/skills/note/SKILL.md"]); + expect(compiled.diagnostics.some((diagnostic) => diagnostic.message.includes("declares workspace skills"))).toBe(false); + } + }); + it("emits one strict organization host and no generated engine application", async () => { const plan = await createPlan(); const config = plan.targetFiles.find((file) => file.path === DAIMON_CONFIG_FILE); diff --git a/src/runtime/daimon/adapter.ts b/src/runtime/daimon/adapter.ts index 5bc24261..8d7ebf96 100644 --- a/src/runtime/daimon/adapter.ts +++ b/src/runtime/daimon/adapter.ts @@ -82,6 +82,26 @@ const daimonWorkspaceRestrictionWarning = (node: ResolvedAgentNode): string | un + "the container boundary as this agent's only isolation."; }; +/** + * Workspace skill roots per Daimon engine. + * + * Codex and AGY keep both CLI-engine roots. A brokered Grok worker loads no + * workspace skill at all: Grok 1.0.34 discovers `.agents/skills` (never + * `.codex/skills`) only in a trusted folder, and Daimon keeps the workspace + * untrusted (root-owned empty `trusted_folders.toml`) and replaces the system + * prompt, whose fixed text references no skill. Emitting either root would ship + * files nothing reads, so none are emitted and the declaration is reported. + */ +export const daimonSkillBaseDirectories = (node: ResolvedAgentNode): readonly string[] => + resolveDaimonEngine(node) === "grok" ? [] : CLI_ENGINE_SKILL_BASE_DIRECTORIES; + +const daimonGrokSkillWarning = (node: ResolvedAgentNode): string | undefined => { + if (resolveDaimonEngine(node) !== "grok" || node.skills.length === 0) return undefined; + return `Daimon Grok agent ${node.name} declares workspace skills (${node.skills.map((skill) => skill.name).sort().join(", ")}) ` + + "that its brokered worker never loads: Grok discovers project skills only in a trusted workspace, and Daimon keeps the " + + "workspace untrusted and supplies instructions through the prompt. No skill files are emitted; move the guidance into the agent's docs."; +}; + const daimonCodexPolicyError = (node: ResolvedAgentNode): string | undefined => { if (node.runtime.options.codex_policy === undefined) return undefined; if (node.runtime.options.codex_policy !== "workspace-no-network") { @@ -189,6 +209,7 @@ export const daimonAdapter: RuntimeAdapter = { const memoryVectorWarning = daimonMemoryVectorRecallWarning(node); const workspaceRestrictionWarning = daimonWorkspaceRestrictionWarning(node); const codexPolicyError = daimonCodexPolicyError(node); + const grokSkillWarning = daimonGrokSkillWarning(node); return { capabilities: createAgentCapabilities(node, { mcpOutcome: "supported", @@ -205,11 +226,12 @@ export const daimonAdapter: RuntimeAdapter = { ...(memorySelectionWarning ? [createDiagnostic("warn", memorySelectionWarning)] : []), ...(memoryVectorWarning ? [createDiagnostic("warn", memoryVectorWarning)] : []), ...(workspaceRestrictionWarning ? [createDiagnostic("warn", workspaceRestrictionWarning)] : []), + ...(grokSkillWarning ? [createDiagnostic("warn", grokSkillWarning)] : []), ...(codexPolicyError ? [createDiagnostic("error", codexPolicyError)] : []) ], files: [ ...createDocumentFiles("workspace", node.docs), - ...createSkillFiles(CLI_ENGINE_SKILL_BASE_DIRECTORIES, node.skills) + ...createSkillFiles(daimonSkillBaseDirectories(node), node.skills) ] }; }, From efac0637be05836c9888f3b8d11594d3d34e488f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:08:05 +0200 Subject: [PATCH 22/84] feat: vendor Daimon's Grok worker config renderer bytes and mirror its sandbox profile renderer --- package.json | 1 + scripts/AGENTS.md | 8 ++ scripts/README.md | 1 + scripts/vendor-daimon-grok-contract.ts | 90 +++++++++++++++++++ src/runtime/daimon/grokWorkerConfigBytes.ts | 38 ++++++++ src/runtime/daimon/grokWorkerContract.test.ts | 42 +++++++++ src/runtime/daimon/grokWorkerContract.ts | 68 ++++++++++++++ 7 files changed, 248 insertions(+) create mode 100644 scripts/vendor-daimon-grok-contract.ts create mode 100644 src/runtime/daimon/grokWorkerConfigBytes.ts create mode 100644 src/runtime/daimon/grokWorkerContract.test.ts create mode 100644 src/runtime/daimon/grokWorkerContract.ts diff --git a/package.json b/package.json index ea2122f4..791937c4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "audit:generate": "tsx src/audit/auditCli.ts", "build:local-moltnet": "node --experimental-strip-types ./scripts/build-local-moltnet.ts", "build:local-daimon": "node --experimental-strip-types ./scripts/build-local-daimon-runtime.ts", + "vendor:daimon-contract": "node --import tsx ./scripts/vendor-daimon-grok-contract.ts", "bundle:source-provenance": "node --experimental-strip-types ./scripts/create-source-provenance-bundle.ts", "prepare:linux-amd64-closure": "node --experimental-strip-types ./scripts/create-linux-amd64-dependency-closure.ts", "prepare:linux-amd64-go-closure": "node --experimental-strip-types ./scripts/create-linux-amd64-go-closure.ts", diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 61f5a172..04b40a7e 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -24,6 +24,14 @@ tools live in `../archive/legacy-worktree-tools/` and are not active helpers. - Daimon builds accept only explicit versions, credential-free HTTPS URLs, and executable/archive digest pins. Reject URLs with credentials, queries, or fragments before Docker runs. +- Daimon builds also accept only the Grok CLI build the vendored Daimon + contract manifest pins for the target architecture (`readPinnedGrokCli`). +- `vendor-daimon-grok-contract.ts` (run with `node --import tsx`) is the only + way Daimon's contract enters Spawnfile: it reads a Daimon checkout as data and + writes `src/runtime/daimon/contract-manifest.{json,sha256}` plus + `grokWorkerConfigBytes.ts` (Daimon's own worker `config.toml` renderer bytes + and sandbox-profile samples). `--check` fails on drift. It never builds or + writes inside Daimon. - The local builder pushes only to the fixed loopback development repository. Its generated immutable manifest/receipt identity is ignored and never edits `runtimes.yaml`. Clean-source builds select the native Docker diff --git a/scripts/README.md b/scripts/README.md index cc91c3eb..2d97ef33 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,6 +7,7 @@ in the npm package. Scripts run as native TypeScript on Node 22.19+. | Entrypoint | Caller | Purpose / prerequisites | | --- | --- | --- | | `build-local-daimon-runtime.ts` | `npm run build:local-daimon` | Build a locally sourced runtime; explicit artifact pins, Docker, and loopback registry required | +| `vendor-daimon-grok-contract.ts` | `npm run vendor:daimon-contract [-- --check]` | Vendor Daimon's contract manifest and Grok worker renderer bytes from a Daimon checkout (`SPAWNFILE_DAIMON_SOURCE_DIR`) | | `build-local-moltnet.ts` | `npm run build:local-moltnet` | Build and stamp release binaries from an explicit source checkout | | `create-source-provenance-bundle.ts` | `npm run bundle:source-provenance` | Create deterministic archives with manifests and credential exclusions | | `create-linux-amd64-dependency-closure.ts` | `npm run prepare:linux-amd64-closure` | Prepare reviewed npm dependencies/cache in the pinned build container | diff --git a/scripts/vendor-daimon-grok-contract.ts b/scripts/vendor-daimon-grok-contract.ts new file mode 100644 index 00000000..74d02e56 --- /dev/null +++ b/scripts/vendor-daimon-grok-contract.ts @@ -0,0 +1,90 @@ +#!/usr/bin/env node +// Vendors Daimon's Grok broker contract into Spawnfile from a Daimon checkout: +// the canonical runtime contract manifest (+ digest sidecar) and the exact +// worker `config.toml` bytes Daimon's single renderer produces for every +// declared model x reasoning effort, plus sandbox-profile samples that pin +// Spawnfile's profile mirror to Daimon's renderer. Run with +// `node --import tsx scripts/vendor-daimon-grok-contract.ts [--check]`; +// SPAWNFILE_DAIMON_SOURCE_DIR selects the checkout (default ../daimon). +// Reads Daimon source as data; never builds, writes, or commits in Daimon. + +import { readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const vendorDirectory = path.join(repoRoot, "src", "runtime", "daimon"); +const models = ["grok-4.6", "grok-4.5", "grok-build"] as const; +const efforts = ["low", "medium", "high"] as const; + +/** Deny sets whose rendered bytes the Spawnfile mirror must reproduce exactly. */ +export const PROFILE_SAMPLE_DENY_PATHS: readonly (readonly string[])[] = [ + [], + ["/var/lib/spawnfile/daimon/usage", "/run/daimon-engine-broker", "/run/daimon-engine-broker", "/var/lib/daimon-workers/2201"] +]; + +type DaimonModules = { + contractManifestArtifacts: () => { digest: Buffer; manifest: Buffer }; + renderGrokBrokerWorkerConfig: (policy: { model: string; reasoningEffort: string }) => string; + renderGrokWorkerSandboxProfile: (denyPaths: readonly string[]) => string; +}; + +const loadDaimon = async (daimonDirectory: string): Promise => { + const load = (relative: string): Promise> => import(pathToFileURL(path.join(daimonDirectory, relative)).href); + const [emitter, config, profile] = await Promise.all([ + load("scripts/emitRuntimeContractManifest.ts"), + load("src/runtime/grokBrokerWorkerConfig.ts"), + load("src/runtime/grokWorkerSandboxProfile.ts") + ]); + return { + contractManifestArtifacts: emitter.contractManifestArtifacts as DaimonModules["contractManifestArtifacts"], + renderGrokBrokerWorkerConfig: config.renderGrokBrokerWorkerConfig as DaimonModules["renderGrokBrokerWorkerConfig"], + renderGrokWorkerSandboxProfile: profile.renderGrokWorkerSandboxProfile as DaimonModules["renderGrokWorkerSandboxProfile"] + }; +}; + +export const renderVendoredWorkerModule = (daimon: Pick): string => { + const configs = Object.fromEntries(models.map((model) => [model, Object.fromEntries(efforts.map((reasoningEffort) => + [reasoningEffort, daimon.renderGrokBrokerWorkerConfig({ model, reasoningEffort })]))])); + const samples = PROFILE_SAMPLE_DENY_PATHS.map((denyPaths) => ({ bytes: daimon.renderGrokWorkerSandboxProfile(denyPaths), denyPaths })); + return [ + "/* v8 ignore file -- generated data module */", + "// Generated by scripts/vendor-daimon-grok-contract.ts from Daimon's own renderers. Do not edit by hand.", + "", + "/** Daimon `renderGrokBrokerWorkerConfig({ model, reasoningEffort })` bytes, verified against the manifest pins on use. */", + `export const DAIMON_GROK_WORKER_CONFIG_BYTES = ${JSON.stringify(configs, null, 2)} as const;`, + "", + "/** Daimon `renderGrokWorkerSandboxProfile(denyPaths)` samples; the Spawnfile mirror must reproduce them byte for byte. */", + `export const DAIMON_GROK_WORKER_PROFILE_SAMPLES = ${JSON.stringify(samples, null, 2)} as const;`, + "" + ].join("\n"); +}; + +const main = async (): Promise => { + const args = process.argv.slice(2); + if (args.length > 1 || (args.length === 1 && args[0] !== "--check")) throw new Error("usage: vendor-daimon-grok-contract.ts [--check]"); + const configured = process.env.SPAWNFILE_DAIMON_SOURCE_DIR?.trim(); + if (configured && !path.isAbsolute(configured)) throw new Error("SPAWNFILE_DAIMON_SOURCE_DIR must be absolute"); + const daimon = await loadDaimon(configured || path.resolve(repoRoot, "..", "daimon")); + const artifacts = daimon.contractManifestArtifacts(); + const outputs: Array<[string, Buffer | string]> = [ + [path.join(vendorDirectory, "contract-manifest.json"), artifacts.manifest], + [path.join(vendorDirectory, "contract-manifest.sha256"), artifacts.digest], + [path.join(vendorDirectory, "grokWorkerConfigBytes.ts"), renderVendoredWorkerModule(daimon)] + ]; + if (args[0] === "--check") { + const drifted = outputs.filter(([file, bytes]) => !readFileSync(file).equals(Buffer.from(bytes))); + if (drifted.length > 0) throw new Error(`vendored Daimon contract drifted: ${drifted.map(([file]) => path.relative(repoRoot, file)).join(", ")}`); + process.stdout.write("vendored Daimon Grok contract matches the checkout\n"); + return; + } + for (const [file, bytes] of outputs) writeFileSync(file, bytes); + process.stdout.write(`vendored Daimon Grok contract ${artifacts.digest.toString("ascii").trim()}\n`); +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/src/runtime/daimon/grokWorkerConfigBytes.ts b/src/runtime/daimon/grokWorkerConfigBytes.ts new file mode 100644 index 00000000..24bb788c --- /dev/null +++ b/src/runtime/daimon/grokWorkerConfigBytes.ts @@ -0,0 +1,38 @@ +/* v8 ignore file -- generated data module */ +// Generated by scripts/vendor-daimon-grok-contract.ts from Daimon's own renderers. Do not edit by hand. + +/** Daimon `renderGrokBrokerWorkerConfig({ model, reasoningEffort })` bytes, verified against the manifest pins on use. */ +export const DAIMON_GROK_WORKER_CONFIG_BYTES = { + "grok-4.6": { + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + }, + "grok-4.5": { + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + }, + "grok-build": { + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + } +} as const; + +/** Daimon `renderGrokWorkerSandboxProfile(denyPaths)` samples; the Spawnfile mirror must reproduce them byte for byte. */ +export const DAIMON_GROK_WORKER_PROFILE_SAMPLES = [ + { + "bytes": "[profiles.daimon-strict]\nextends = \"strict\"\nrestrict_network = true\ndeny = []\n", + "denyPaths": [] + }, + { + "bytes": "[profiles.daimon-strict]\nextends = \"strict\"\nrestrict_network = true\ndeny = [\"/run/daimon-engine-broker\", \"/var/lib/daimon-workers/2201\", \"/var/lib/spawnfile/daimon/usage\"]\n", + "denyPaths": [ + "/var/lib/spawnfile/daimon/usage", + "/run/daimon-engine-broker", + "/run/daimon-engine-broker", + "/var/lib/daimon-workers/2201" + ] + } +] as const; diff --git a/src/runtime/daimon/grokWorkerContract.test.ts b/src/runtime/daimon/grokWorkerContract.test.ts new file mode 100644 index 00000000..c8fe548a --- /dev/null +++ b/src/runtime/daimon/grokWorkerContract.test.ts @@ -0,0 +1,42 @@ +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { DAIMON_GROK_BROKER_MODELS, DAIMON_GROK_BROKER_REASONING_EFFORTS, DAIMON_GROK_ENGINE_BROKER } from "./contractManifest.js"; +import { DAIMON_GROK_WORKER_CONFIG_BYTES, DAIMON_GROK_WORKER_PROFILE_SAMPLES } from "./grokWorkerConfigBytes.js"; +import { renderDaimonGrokWorkerSandboxProfile, resolveDaimonGrokWorkerConfig } from "./grokWorkerContract.js"; + +const sha256 = (value: string): string => createHash("sha256").update(value).digest("hex"); + +describe("vendored Daimon Grok worker contract", () => { + it("serves Daimon's renderer bytes for every declared model x effort, each matching the manifest pin", () => { + for (const model of DAIMON_GROK_BROKER_MODELS) { + for (const effort of DAIMON_GROK_BROKER_REASONING_EFFORTS) { + const resolved = resolveDaimonGrokWorkerConfig(model, effort); + expect(resolved.bytes).toBe(DAIMON_GROK_WORKER_CONFIG_BYTES[model][effort]); + expect(sha256(resolved.bytes)).toBe(DAIMON_GROK_ENGINE_BROKER.worker.configSha256[model][effort]); + expect(resolved.bytes).toContain(`model = "${model}"`); + expect(resolved.bytes).toContain(`value = "${effort}"`); + } + } + }); + + it("refuses config bytes that differ from the manifest pin by a single byte", () => { + const tampered = { + ...DAIMON_GROK_WORKER_CONFIG_BYTES, + "grok-4.6": { ...DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"], low: DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"].low.replace("grok-4.6", "grok-4.5") } + }; + expect(() => resolveDaimonGrokWorkerConfig("grok-4.6", "low", tampered)).toThrow(/does not match the contract manifest pin/u); + const swapped = { ...DAIMON_GROK_WORKER_CONFIG_BYTES, "grok-4.6": { ...DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"], low: DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"].medium } }; + expect(() => resolveDaimonGrokWorkerConfig("grok-4.6", "low", swapped)).toThrow(/manifest pin/u); + }); + + it("renders sandbox profile bytes identical to Daimon's renderer", () => { + for (const sample of DAIMON_GROK_WORKER_PROFILE_SAMPLES) { + expect(renderDaimonGrokWorkerSandboxProfile(sample.denyPaths)).toBe(sample.bytes); + } + for (const unsafe of ["relative/path", "/", "/trailing/", "/a/../b", "/quote\"d", "/glob*"]) { + expect(() => renderDaimonGrokWorkerSandboxProfile([unsafe])).toThrow(/deny path/u); + } + }); +}); diff --git a/src/runtime/daimon/grokWorkerContract.ts b/src/runtime/daimon/grokWorkerContract.ts new file mode 100644 index 00000000..5c0f4086 --- /dev/null +++ b/src/runtime/daimon/grokWorkerContract.ts @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { SpawnfileError } from "../../shared/index.js"; + +import { + DAIMON_GROK_ENGINE_BROKER, + type DaimonGrokBrokerModel, + type DaimonGrokBrokerReasoningEffort +} from "./contractManifest.js"; +import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "./grokWorkerConfigBytes.js"; + +export const DAIMON_GROK_WORKER_SANDBOX_PROFILE = "daimon-strict" as const; +export const DAIMON_GROK_WORKER_HOME_DIRECTORY = ".grok" as const; + +const sha256 = (bytes: string): string => createHash("sha256").update(bytes).digest("hex"); + +/** Characters TOML or Grok would reinterpret inside a deny entry (mirrors Daimon's refusal set). */ +const UNSAFE_DENY_PATH_CHARACTER = new RegExp("[\"\\\\\\u0000-\\u001f\\u007f*?[\\]]", "u"); + +/** + * The worker `config.toml` bytes for one declared model x effort: Daimon's own + * renderer output (vendored by `scripts/vendor-daimon-grok-contract.ts`), + * refused unless it hashes to the manifest pin the broker attests every turn. + */ +export const resolveDaimonGrokWorkerConfig = ( + model: DaimonGrokBrokerModel, + reasoningEffort: DaimonGrokBrokerReasoningEffort, + vendored: Readonly>>> = DAIMON_GROK_WORKER_CONFIG_BYTES +): { bytes: string; sha256: string } => { + const pins = DAIMON_GROK_ENGINE_BROKER.worker.configSha256 as Readonly> | undefined>>; + const pin = pins[model]?.[reasoningEffort]; + const bytes = vendored[model]?.[reasoningEffort]; + if (pin === undefined || bytes === undefined || sha256(bytes) !== pin) { + throw new SpawnfileError( + "compile_error", + `Vendored Daimon Grok worker config for ${model}/${reasoningEffort} does not match the contract manifest pin; re-vendor the Daimon contract` + ); + } + return { bytes, sha256: pin }; +}; + +/** + * Byte mirror of Daimon's `renderGrokWorkerSandboxProfile` + * (`daimon/src/runtime/grokWorkerSandboxProfile.ts`), pinned to Daimon's own + * output by `DAIMON_GROK_WORKER_PROFILE_SAMPLES`. Grok 1.0.34 runs every profile + * inside bubblewrap and enforces a non-empty `deny` list; its strict base still + * reads all of `/run`, `/var`, `/tmp` and `/etc`, so this list is what keeps + * protected paths from the worker. + */ +export const renderDaimonGrokWorkerSandboxProfile = (denyPaths: readonly string[]): string => { + const denied = [...new Set(denyPaths)].sort(); + for (const entry of denied) { + if (!path.posix.isAbsolute(entry) || path.posix.normalize(entry) !== entry || entry === "/" || entry.endsWith("/") || UNSAFE_DENY_PATH_CHARACTER.test(entry)) { + throw new SpawnfileError("compile_error", `Invalid Grok worker sandbox deny path: ${JSON.stringify(entry)}`); + } + } + return [ + `[profiles.${DAIMON_GROK_WORKER_SANDBOX_PROFILE}]`, + 'extends = "strict"', + "restrict_network = true", + `deny = [${denied.map((entry) => JSON.stringify(entry)).join(", ")}]`, + "" + ].join("\n"); +}; + +export const daimonGrokWorkerSandboxProfileSha256 = (denyPaths: readonly string[]): string => + sha256(renderDaimonGrokWorkerSandboxProfile(denyPaths)); From bfaec6e2111f450f2e1dc3e5c8c4274a37a38a7c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:14:34 +0200 Subject: [PATCH 23/84] feat: provision brokered Grok workers from Daimon's pinned config bytes with a non-empty deny list and service.json v2 --- src/compiler/AGENTS.md | 19 +- src/compiler/containerArtifactsPlans.test.ts | 1 + src/compiler/containerArtifactsPlans.ts | 1 + src/compiler/containerArtifactsTypes.ts | 2 + .../containerDaimonBrokerRender.test.ts | 112 ++--------- src/compiler/containerDaimonBrokerRender.ts | 124 ++---------- .../containerDaimonCapabilityOrdering.test.ts | 1 + ...tainerDaimonGrokWorkerProvisioning.test.ts | 135 +++++++++++++ .../containerDaimonGrokWorkerProvisioning.ts | 67 +++++++ .../containerDaimonGrokWorkerRender.test.ts | 105 ++++++++++ .../containerDaimonGrokWorkerRender.ts | 182 ++++++++++++++++++ ...tainerDaimonUidEntrypointLifecycle.test.ts | 1 + ...containerDaimonUidEntrypointRender.test.ts | 15 +- src/runtime/daimon/config.ts | 4 + src/runtime/types.ts | 2 + 15 files changed, 556 insertions(+), 215 deletions(-) create mode 100644 src/compiler/containerDaimonGrokWorkerProvisioning.test.ts create mode 100644 src/compiler/containerDaimonGrokWorkerProvisioning.ts create mode 100644 src/compiler/containerDaimonGrokWorkerRender.test.ts create mode 100644 src/compiler/containerDaimonGrokWorkerRender.ts diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 273b82b8..d468cc97 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -33,7 +33,9 @@ src/compiler/ ├── containerPersistentMounts.ts # Durable-mount merge across sources + volume-name uniqueness ├── deploymentLineage.ts # Dev/production lineage namespacing + declared-volume refusal ├── containerEntrypointShell.ts # Shell quoting, recipe env, and CLI credential materialization helpers -├── containerDaimonBrokerRender.ts # Fixed Daimon broker identities, registrations, worker config, and root-launch provisioning +├── containerDaimonBrokerRender.ts # Fixed Daimon broker identities, registrations binary, credential realm, and root-launch provisioning +├── containerDaimonGrokWorkerRender.ts # Brokered Grok registrations: pinned worker config, sandbox deny list, service.json v2 +├── containerDaimonGrokWorkerProvisioning.ts # Root program lines for the attested worker GROK_HOME layout and canonical-path checks ├── containerArtifactsPlans.ts # Environment inventory and runtime target-plan orchestration ├── containerTargetPlanResolution.ts # Per-target paths, packages, auth, secrets, and exposure resolution ├── teamRoster.ts # Context-scoped team roster generation and diagnostics @@ -160,6 +162,21 @@ src/compiler/ existing uid, capability-drop, and `no-new-privileges` posture; Codex owns the per-turn filesystem/network/tool boundary, while trusted MCP servers and Daimon code outside that native boundary remain trusted container processes. +- Brokered Grok workers (`containerDaimonGrokWorkerRender.ts`) take their + `config.toml` bytes only from Daimon's renderer output vendored in + `src/runtime/daimon/grokWorkerConfigBytes.ts`, refused unless they hash to the + manifest pin for the agent's declared model x effort. The sandbox profile's + `deny` list is never empty: Daimon's protected set (realms, bootstrap, peers, + acceptance store) plus the broker's `/etc` and `/run` directories, the usage + ledger and wake fuse, every other worker's home, and the container secret + roots. Entries never nest. Provisioning + (`containerDaimonGrokWorkerProvisioning.ts`) writes Daimon's attested + `GROK_HOME` layout — `root: 1771` home and `sessions/`, `root:root + 0444` config/sandbox/trust/managed/requirements files, events under + `sessions/` — and refuses any registration or deny path that is missing, a + symlink, or not its own realpath. Production registrations all point + `usageLedgerPath` at the one container ledger, because `spawnfile usage` and + Daimon's wake fuse read only that file. - Declared names are checked for uniqueness across EVERY mount source (`containerPersistentMounts.ts`), not just within one source. A resource `name: X` and a store `persistence.name: X` used to compile to two diff --git a/src/compiler/containerArtifactsPlans.test.ts b/src/compiler/containerArtifactsPlans.test.ts index 9b6f915e..4a13cfb6 100644 --- a/src/compiler/containerArtifactsPlans.test.ts +++ b/src/compiler/containerArtifactsPlans.test.ts @@ -205,6 +205,7 @@ describe("runtime target plan source identity", () => { ] as const).map(([slug, engine]) => ({ node: { ...createAgent(), + ...(engine === "grok" ? { execution: { model: { primary: { auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" } } } } : {}), name: slug, runtime: { name: "daimon", options: { engine } } } as ResolvedAgentNode, diff --git a/src/compiler/containerArtifactsPlans.ts b/src/compiler/containerArtifactsPlans.ts index 080fad9c..fb5e8fd7 100644 --- a/src/compiler/containerArtifactsPlans.ts +++ b/src/compiler/containerArtifactsPlans.ts @@ -160,6 +160,7 @@ export const createRuntimeTargetPlans = async ( runtimePlans.push({ configEnvBindings: resolveTargetConfigEnvBindings(adapter.container, target) ?? [], ...(target.engineByNodeId ? { engineByNodeId: target.engineByNodeId } : {}), + ...(target.grokModelByNodeId ? { grokModelByNodeId: target.grokModelByNodeId } : {}), envFiles: resolveTargetEnvFiles(instancePaths.configPath, target), packages: resolveTargetPackages(target, targetInputs), id: target.id, diff --git a/src/compiler/containerArtifactsTypes.ts b/src/compiler/containerArtifactsTypes.ts index c7c0d6b5..b3ad929e 100644 --- a/src/compiler/containerArtifactsTypes.ts +++ b/src/compiler/containerArtifactsTypes.ts @@ -26,6 +26,8 @@ export interface RuntimeTargetPlan { /** Passthrough of `ContainerTarget.engineByNodeId` (see `runtime/types.ts`), for * `containerArtifacts.ts` to stamp onto `ContainerRuntimeInstanceReport.engine_by_node_id`. */ engineByNodeId?: Record; + /** Passthrough of `ContainerTarget.grokModelByNodeId` for the Daimon broker worker render. */ + grokModelByNodeId?: Record; packages?: ResolvedPackage[]; envFiles: Array<{ envName: string; diff --git a/src/compiler/containerDaimonBrokerRender.test.ts b/src/compiler/containerDaimonBrokerRender.test.ts index 2799986a..27d9b99c 100644 --- a/src/compiler/containerDaimonBrokerRender.test.ts +++ b/src/compiler/containerDaimonBrokerRender.test.ts @@ -10,35 +10,33 @@ import { DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifes import { DAIMON_BROKER_REALM, DAIMON_ORGANIZATION_STATE_DIRECTORY, - GROK_SANDBOX_DENY_PATHS, renderDaimonBrokerProvisioning, renderDaimonUsageLedgerProvisioning, renderDaimonWorkspaceResourceSecurity } from "./containerDaimonBrokerRender.js"; const execFile = promisify(execFileCallback); +type Plan = Parameters[0][number]; +const grokPlan = (agentIds: string[], extra: Record = {}, model = { model: "grok-4.6", reasoningEffort: "low" }): Plan => ({ + runtimeName: "daimon", + engineByNodeId: { ...Object.fromEntries(agentIds.map((agentId) => [agentId, "grok"])), ...extra }, + grokModelByNodeId: Object.fromEntries(agentIds.map((agentId) => [agentId, model])), + instancePaths: { instanceRoot: "/var/lib/spawnfile/instances/daimon/daimon-organization", workspacePath: "/var/lib/spawnfile/instances/daimon/daimon-organization/workspace" } +}) as unknown as Plan; const uid = process.getuid?.() ?? 501; const gid = process.getgid?.() ?? 20; const owners = { linkUid: uid, linkGid: gid, readonlyUid: uid, readonlyGid: gid, privilegedUid: uid, privilegedGid: gid }; describe("Daimon broker registration ABI", () => { it("renders the manifest-declared native ABI version", () => { - const plan = { - runtimeName: "daimon", - engineByNodeId: { "agent:grok": "grok" }, - instancePaths: { workspacePath: "/workspace" } - } as unknown as Parameters[0][number]; + const plan = grokPlan(["agent:grok"]); expect(renderDaimonBrokerProvisioning([plan]).join("\n")) .toContain("record.writeUInt32LE(2, 0)"); }); }); describe("Daimon broker registration directory provisioning", () => { - const plan = { - runtimeName: "daimon", - engineByNodeId: { "agent:grok": "grok" }, - instancePaths: { workspacePath: "/workspace" } - } as unknown as Parameters[0][number]; + const plan = grokPlan(["agent:grok"]); it("keeps the broker directory writable until its files are provisioned", () => { const lines = renderDaimonBrokerProvisioning([plan]).join("\n").split("\n"); @@ -74,11 +72,7 @@ describe("Daimon broker registration directory provisioning", () => { }); describe("Daimon broker usage ledger provisioning", () => { - const plan = { - runtimeName: "daimon", - engineByNodeId: { "agent:grok": "grok" }, - instancePaths: { workspacePath: "/workspace" } - } as unknown as Parameters[0][number]; + const plan = grokPlan(["agent:grok"]); it("fixes the usage ledger directory group-writable so Codex/AGY's organization-uid process can also write it, unconditionally (not just alongside the realm)", () => { const { directoryPath } = DAIMON_GROK_TURN_USAGE_LEDGER; @@ -92,86 +86,10 @@ describe("Daimon broker usage ledger provisioning", () => { expect(renderDaimonBrokerProvisioning([])).toEqual([]); }); - it("never lists the usage ledger directory in the rendered sandbox deny list", () => { - // The usage ledger directory is unix-denied to every worker uid - // unconditionally by `renderDaimonUsageLedgerProvisioning` (0770, - // broker:organization — a worker uid never matches either), so Grok - // could never verify a mask over it and would refuse to start if it were - // still listed. See `GROK_SANDBOX_DENY_PATHS`'s doc comment. - const program = renderDaimonBrokerProvisioning([plan]).join("\n"); - const deniedPathsLine = program.split("\n").find((line) => line.includes("const deniedPaths =")); - expect(deniedPathsLine).toBeDefined(); - expect(deniedPathsLine).not.toContain(DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath); - }); -}); - -describe("Grok sandbox deny list: empty, because a non-empty one cannot start", () => { - // Grok 1.0.13 re-execs itself inside bubblewrap whenever `deny` is - // non-empty, then opens every deny-path placeholder to prove the bind-over - // is genuine. It creates that placeholder at mode 000 and the re-exec'd - // process is capability-stripped, so the open returns EACCES and Grok - // refuses to start ("possible __GROK_INSIDE_BWRAP spoof") without ever - // writing a ProfileApplied event — reproduced against the real Linux binary, - // with the deny target proven irrelevant. Every path this list once carried - // is unix-denied to a worker uid unconditionally anyway, the organization - // state directory included now that the ownership guard secures it to 0700 - // (see `containerDaimonUidEntrypointRender.ts`). - const planWithAgents = (agentIds: string[]) => ({ - runtimeName: "daimon", - engineByNodeId: Object.fromEntries(agentIds.map((agentId) => [agentId, "grok"])), - instancePaths: { workspacePath: "/workspace" } - } as unknown as Parameters[0][number]); - - const deniedPathsLineFor = (agentIds: string[]): string | undefined => - renderDaimonBrokerProvisioning([planWithAgents(agentIds)]) - .join("\n").split("\n").find((line) => line.includes("const deniedPaths =")); - - it("renders an empty deny list and a profile that interpolates exactly it", () => { - for (const agentIds of [["agent:solo"], ["agent:cogsworth", "agent:foreman", "agent:graves"]]) { - expect(deniedPathsLineFor(agentIds)).toBe("const deniedPaths = [];"); - } - // Run the rendered program's own profile expression rather than a - // re-typed copy of it, so this asserts the bytes the container writes. - const lines = renderDaimonBrokerProvisioning([planWithAgents(["agent:solo"])]).join("\n").split("\n"); - const source = lines.filter((line) => - line.startsWith("const deniedPaths =") || line.startsWith("const profileFor =")).join("\n"); - expect(source.split("\n")).toHaveLength(2); - const profile = new Function(`${source}\nreturn profileFor();`)() as string; - expect(profile).toBe("[profiles.daimon-strict]\nextends = \"strict\"\nrestrict_network = true\ndeny = []\n"); - }); - - it("never lists a peer worker's home or workspace, the realm, the credential, or the state directory", () => { - const deniedPathsLine = deniedPathsLineFor(["agent:cogsworth", "agent:foreman", "agent:graves"]); - expect(deniedPathsLine).toBeDefined(); - for (const forbidden of [ - "daimon-workers", - "/workspace/agents/cogsworth", - "/workspace/agents/foreman", - "/workspace/agents/graves", - DAIMON_BROKER_REALM, - "/var/lib/spawnfile/daimon/grok-bootstrap-auth", - "/run/daimon-engine-broker", - DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, - DAIMON_ORGANIZATION_STATE_DIRECTORY - ]) { - expect(deniedPathsLine).not.toContain(forbidden); - } - }); - - it("keeps the exported deny-path constant in sync with what gets rendered", () => { - expect(GROK_SANDBOX_DENY_PATHS).toEqual([]); - expect(DAIMON_ORGANIZATION_STATE_DIRECTORY).toBe( - "/var/lib/spawnfile/instances/daimon/daimon-organization/state" - ); - }); }); describe("Daimon root provisioning capability-safe ordering", () => { - const plan = { - runtimeName: "daimon", - engineByNodeId: { "agent:grok": "grok" }, - instancePaths: { workspacePath: "/workspace" } - } as unknown as Parameters[0][number]; + const plan = grokPlan(["agent:grok"]); const render = () => renderDaimonBrokerProvisioning([plan]).join("\n"); it("lets grok write hook state but never replace the sandbox profile", () => { @@ -180,14 +98,14 @@ describe("Daimon root provisioning capability-safe ordering", () => { // Grok creates hook registries under .grok to enforce its deny list, so the worker // needs write access there. The sticky bit means it still cannot unlink or rename the // root-owned sandbox.toml, which is what the profile attestation depends on. - expect(program).toContain("ensureDirectory(configRoot, 0, entry.uid, 0o1771)"); - expect(program).toContain("ensureExactFile(profilePath, profileFor(), 0, 0, 0o444)"); + expect(program).toContain("ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771)"); + expect(program).toContain("ensureExactFile(entry.profilePath, entry.profile, 0o444)"); }); it("tightens the worker config directory after its last file write", () => { const program = render(); - const lastWrite = program.indexOf("ensureEventsFile(eventsPath, entry.uid)"); - const tighten = program.indexOf("ensureDirectory(configRoot, 0, entry.uid, 0o1771)", lastWrite); + const lastWrite = program.indexOf("ensureEventsFile(entry.eventsPath, entry.uid)"); + const tighten = program.indexOf("ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771)", lastWrite); expect(lastWrite).toBeGreaterThanOrEqual(0); expect(tighten).toBeGreaterThan(lastWrite); diff --git a/src/compiler/containerDaimonBrokerRender.ts b/src/compiler/containerDaimonBrokerRender.ts index ddc2ed42..34504095 100644 --- a/src/compiler/containerDaimonBrokerRender.ts +++ b/src/compiler/containerDaimonBrokerRender.ts @@ -1,12 +1,10 @@ -import path from "node:path"; - -import { DAIMON_ORGANIZATION_TARGET_ID } from "../runtime/daimon/config.js"; import { DAIMON_GROK_ENGINE_BROKER, - DAIMON_GROK_TURN_USAGE_LEDGER, - DAIMON_RUNTIME_HOME_ROOT + DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifest.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; +import { resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js"; +import { renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; import { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, @@ -21,75 +19,11 @@ export const DAIMON_BROKER_BACKEND_SOCKET = "/run/daimon-engine-broker/backend.s export const DAIMON_BROKER_LAUNCHER_SOCKET = "/run/daimon-engine-broker/launcher.sock"; export const DAIMON_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; export const DAIMON_BROKER_REALM = "/var/lib/spawnfile/daimon/grok-subscription-realm"; -export const DAIMON_WORKER_ROOT = "/var/lib/daimon-workers"; -/** - * The organization runtime state directory — the parent of the durable wake - * acceptance store (`state/wake-acceptance`) and the runtime readiness - * receipt inside it. - * - * It used to be left at mode 0755 because Docker creates it root-owned and - * world-readable when it materializes the acceptance-store volume mount, and - * nothing tightened it afterwards: the Daimon ownership guard only *chowned* - * ancestor `privateDirectories` and never chmoded them. That made it the one - * path under `/var/lib/spawnfile` a Grok worker uid could actually open, and - * the whole reason a `deny` entry existed at all. - * - * `resolveDaimonUidEntrypointOwnershipPlan` now lists it as a - * `privateModeDirectory`, so the ownership guard secures it to `0700` - * `2000:2000` — the same treatment the acceptance store beneath it already - * had. That is what actually denies the worker, and it does so for every - * Daimon organization, with or without Grok. No non-root, non-organization - * reader loses anything: the only content under it is the acceptance store, - * which was already `0700 2000:2000`, so every reader that works today is - * either the organization uid or a `docker exec` root holding - * `CAP_DAC_READ_SEARCH` (`runProject.ts`), and neither is affected by the - * parent's mode. - */ -export const DAIMON_ORGANIZATION_STATE_DIRECTORY = path.posix.join( - DAIMON_RUNTIME_HOME_ROOT, - DAIMON_ORGANIZATION_TARGET_ID, - "state" -); -/** - * The Grok worker sandbox profile's `deny` list — see `profileFor` below. - * Deliberately empty, which leaves the worker on builtin-`strict` Landlock - * confinement plus `restrict_network`. - * - * A non-empty `deny` list is not a usable mechanism on Grok 1.0.13. Whenever - * one is present, Grok re-execs itself inside bubblewrap and then `open()`s - * every deny-path placeholder to prove its own bind-over is genuine — but it - * creates that placeholder at mode `000` and the re-exec'd process is - * capability-stripped (`--cap-drop ALL`), so the open returns `EACCES` and - * Grok refuses to start: - * - * error: sandbox reports bwrap but required read-deny mounts are not in - * effect (read-deny path could not be opened: Permission denied - * (os error 13)); refusing to start (possible __GROK_INSIDE_BWRAP spoof) - * - * It never reaches `ProfileApplied`, so `sandbox-events.jsonl` stays empty and - * Daimon's attestation never runs either. Reproduced locally against the real - * Linux `grok 1.0.13` binary, with controls confirming the deny *target* is - * irrelevant: a plain directory, a directory with a child mount, and a vanilla - * root-owned home denying an unrelated file all fail identically, while an - * empty deny list exits 0 and emits - * `ProfileApplied {platform: "linux/landlock", enforced: true, - * restrict_network: true}` with no bwrap at all. - * - * Nothing is lost by emptying it. Every path this list ever carried — peer - * worker homes/workspaces, the subscription realm, the bootstrap-auth file, - * the broker's `/run` socket directory, the usage-ledger directory, and now - * `DAIMON_ORGANIZATION_STATE_DIRECTORY` — is unix-denied to a worker uid - * unconditionally, by construction: each is force-chowned/chmoded (never - * merely checked) to a mode whose "other" class carries no read bit, and a - * worker uid never matches the owning uid or gid of any of them (workers run - * under `setresuid`/`setresgid` to a dedicated uid==gid with every - * supplementary group cleared — see `engineBrokerLauncherCore.inc`). - * - * Daimon's `grokWorkerAttestation.ts` still pins the profile by SHA-256 to - * exactly these bytes, so an empty list is not an unconstrained one: the - * worker's profile cannot differ from what is rendered here. - */ -export const GROK_SANDBOX_DENY_PATHS: readonly string[] = []; +export { + DAIMON_ORGANIZATION_STATE_DIRECTORY, + DAIMON_WORKER_ROOT, + resolveDaimonGrokRegistrations +} from "./containerDaimonGrokWorkerRender.js"; interface WorkspaceSecurityResource { backingPath: string; @@ -110,25 +44,6 @@ export const renderDaimonWorkspaceResourceSecurity = ( "const secureWorkspace = (root, uid) => { const visit = (target) => { const info = fs.lstatSync(target); if (info.isSymbolicLink()) { validateResourceLink(target, info); return; } if (info.isDirectory()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o750); for (const name of fs.readdirSync(target)) visit(`${target}/${name}`); fs.chownSync(target, 2000, uid); } else if (info.isFile()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o640); fs.chownSync(target, 2000, uid); } else throw new Error('unsafe worker workspace node'); }; visit(root); };" ]; -const nodeSlug = (nodeId: string): string => nodeId.replace(/^agent:/u, "") - .toLowerCase().replace(/[^a-z0-9]+/gu, "-").replace(/^-+|-+$/gu, ""); - -export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]) => plans - .filter((plan) => plan.runtimeName === "daimon") - .flatMap((plan) => Object.entries(plan.engineByNodeId ?? {}) - .filter(([, engine]) => engine === "grok") - .map(([agentId]) => ({ - agentId, - workspace: path.posix.join(plan.instancePaths.workspacePath, "agents", nodeSlug(agentId)) - }))) - .sort((left, right) => left.agentId.localeCompare(right.agentId)) - .map((entry, slot) => ({ - ...entry, - home: path.posix.join(DAIMON_WORKER_ROOT, String(DAIMON_FIRST_WORKER_UID + slot)), - slot, - uid: DAIMON_FIRST_WORKER_UID + slot - })); - /** * Fixes ownership and mode of the per-turn usage ledger directory for every * Daimon organization, not just ones with a Grok agent. AGY and Codex both @@ -181,9 +96,10 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri .sort((left, right) => left.linkPath.localeCompare(right.linkPath)); const program = [ "const crypto = require('node:crypto'); const fs = require('node:fs');", - `const registrations = ${JSON.stringify(registrations)};`, - "const executable = '/usr/local/bin/grok';", + `const registrations = ${JSON.stringify(registrations.map(({ agentId, home, slot, uid, workspace }) => ({ agentId, home, slot, uid, workspace })))};`, + `const executable = '${DAIMON_GROK_ENGINE_BROKER.grokExecutablePath}';`, "const digest = crypto.createHash('sha256').update(fs.readFileSync(executable)).digest();", + `if (!${JSON.stringify([DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.x64.sha256])}.includes(digest.toString('hex'))) throw new Error('Grok executable is not the manifest-pinned ${DAIMON_GROK_ENGINE_BROKER.grokCliVersion} build');`, "const cString = (buffer, offset, length, value) => { const bytes = Buffer.from(value); if (bytes.length < 1 || bytes.length >= length || bytes.includes(0)) throw new Error('invalid broker registration'); bytes.copy(buffer, offset); };", `const records = registrations.map((entry) => { const record = Buffer.alloc(692); record.writeUInt32LE(${DAIMON_GROK_ENGINE_BROKER.nativeAbiVersion}, 0); record.writeUInt32LE(entry.slot, 4); record.writeUInt32LE(entry.uid, 8); record.writeUInt32LE(entry.uid, 12); cString(record, 16, 129, entry.agentId); cString(record, 145, 256, entry.workspace); cString(record, 401, 256, entry.home); digest.copy(record, 657); return record; });`, "fs.mkdirSync('/etc/daimon-engine-broker', { recursive: true, mode: 0o700 }); fs.chownSync('/etc/daimon-engine-broker', 0, 0); fs.chmodSync('/etc/daimon-engine-broker', 0o700);", @@ -202,24 +118,8 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri `const journalRoot = '${DAIMON_BROKER_REALM}/.daimon-broker'; let journalRootExists = false; try { const info = fs.lstatSync(journalRoot); if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 2100 || info.gid !== 2100 || (info.mode & 0o777) !== 0o700) throw new Error('unsafe broker credential journal directory'); fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); journalRootExists = true; } catch (error) { if (error.code !== 'ENOENT') throw error; }`, `try { const journalPath = '${DAIMON_BROKER_REALM}/.daimon-broker/credential-journal.json'; let journal; try { const raw = readSecure(journalPath, 2100, 'recovery journal'); journal = JSON.parse(raw.toString('utf8')); raw.fill(0); } catch (error) { if (error.code !== 'ENOENT') throw error; } const stale = journal?.version === 'noopolis.daimon.broker-credential-journal.v1' && journal.state === 'stale'; const recover = () => { if (!stale || !Number.isSafeInteger(journal.generation) || journal.generation < 0 || !/^[a-f0-9]{64}$/.test(journal.sourceDigest) || journal.sourceDigest !== journal.promotedDigest || bootstrapDigest === journal.sourceDigest) throw new Error('unsafe broker credential recovery'); atomicOwned(authority, bootstrapBytes); const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } }; if (!existing) { if (stale) recover(); else atomicOwned(authority, bootstrapBytes); } else { const authorityBytes = readSecure(authority, 2100, 'authority'); try { const authorityDigest = crypto.createHash('sha256').update(authorityBytes).digest('hex'); if (stale) { if (authorityDigest !== journal.sourceDigest && authorityDigest !== bootstrapDigest) throw new Error('unsafe broker credential recovery'); if (authorityDigest === bootstrapDigest) { const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } } else recover(); } } finally { authorityBytes.fill(0); } } } finally { bootstrapBytes.fill(0); }`, "if (journalRootExists) { fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); fs.chownSync(journalRoot, 2100, 2100); }", - "const config = '[auth_provider.daimon]\\ntype = \"custom\"\\ncommand = \"/opt/daimon/bin/daimon-engine-broker\"\\nargs = [\"--auth-provider\"]\\n\\n[model.daimon-broker-grok]\\nmodel = \"grok-build\"\\nbase_url = \"http://127.0.0.1:43123/v1\"\\nauth_provider = \"daimon\"\\ncontext_window = 131072\\nsupports_backend_search = false\\n\\n[mcp_servers.daimon]\\nurl = \"http://127.0.0.1:43124/mcp\"\\nheaders = { Authorization = \"Bearer ${DAIMON_MCP_CAPABILITY}\" }\\n';", - `for (const root of ['${DAIMON_WORKER_ROOT}']) { fs.mkdirSync(root, { recursive: true, mode: 0o711 }); fs.chownSync(root, 0, 0); fs.chmodSync(root, 0o711); }`, ...renderDaimonWorkspaceResourceSecurity(workspaceResources), - // Empty on purpose: a non-empty `deny` list makes Grok 1.0.13 refuse to - // start before it ever applies a profile, and every path it used to carry - // is unix-denied to a worker uid unconditionally anyway. See - // `GROK_SANDBOX_DENY_PATHS`'s doc comment. - `const deniedPaths = ${JSON.stringify(GROK_SANDBOX_DENY_PATHS)};`, - "const profileFor = () => `[profiles.daimon-strict]\\nextends = \"strict\"\\nrestrict_network = true\\ndeny = [${deniedPaths.map(JSON.stringify).join(', ')}]\\n`;", - "const ensureDirectory = (target, uid, gid, mode) => { fs.mkdirSync(target, { recursive: true, mode }); const info = fs.lstatSync(target); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('unsafe worker runtime directory'); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", - "const ensureExactFile = (target, content, uid, gid, mode) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, content, { mode, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new Error('unsafe worker runtime file'); const existing = fs.readFileSync(target, 'utf8'); if (existing !== content) throw new Error('worker runtime file identity mismatch'); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", - "const ensureEventsFile = (target, uid) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, '', { mode: 0o640, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.uid !== uid && info.uid !== 0) || (info.gid !== 2100 && info.gid !== 0) || ![0o600,0o640].includes(info.mode & 0o777)) throw new Error('unsafe worker attestation events'); fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o640); fs.chownSync(target, uid, 2100); };", - "// Grok refuses a sandbox profile reached through a symlink ('retargetable'), and", - "// Daimon's attestation requires nlink === 1, so the profile is one plain root-owned", - "// 0444 file written in place: no symlink, and no hard link either.", - `for (const entry of registrations) { for (let ancestor = require('node:path').dirname(entry.workspace); ancestor.startsWith('/var/lib/spawnfile/') && ancestor.length > '/var/lib/spawnfile'.length; ancestor = require('node:path').dirname(ancestor)) fs.chmodSync(ancestor, fs.statSync(ancestor).mode & 0o7777 | 0o011); secureWorkspace(entry.workspace, entry.uid); ensureDirectory(entry.home, 0, 0, 0o700); const configRoot = \`${"${entry.home}"}/.grok\`; ensureDirectory(configRoot, 0, 0, 0o700); const configPath = \`${"${configRoot}"}/config.toml\`; ensureExactFile(configPath, config, 0, 0, 0o444); const profilePath = \`${"${configRoot}"}/sandbox.toml\`, eventsPath = \`${"${configRoot}"}/sandbox-events.jsonl\`; ensureExactFile(profilePath, profileFor(), 0, 0, 0o444); ensureEventsFile(eventsPath, entry.uid); ensureDirectory(configRoot, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); }`, - `const service = { version: 'noopolis.daimon.engine-broker-service.v1', credentialHome: '/var/lib/spawnfile/daimon/grok-subscription-realm', turnStore: '/var/lib/spawnfile/daimon/grok-subscription-realm/turns', registrations: registrations.map((entry) => { const configRoot = \`${"${entry.home}"}/.grok\`, profilePath = \`${"${configRoot}"}/sandbox.toml\`, eventsPath = \`${"${configRoot}"}/sandbox-events.jsonl\`; return { agentId: entry.agentId, slot: entry.slot, workerUid: entry.uid, workspace: entry.workspace, profilePath, eventsPath, profileSha256: crypto.createHash('sha256').update(profileFor()).digest('hex') }; }) };`, - "fs.writeFileSync('/etc/daimon-engine-broker/service.json', `${JSON.stringify(service)}\n`, { mode: 0o440, flag: 'wx' }); fs.chownSync('/etc/daimon-engine-broker/service.json', 0, 2100); fs.chmodSync('/etc/daimon-engine-broker/service.json', 0o440);", + ...renderDaimonGrokWorkerProvisioning(registrations), `fs.chownSync('${DAIMON_BROKER_REALM}', 0, 0); fs.chmodSync('${DAIMON_BROKER_REALM}', 0o700); fs.chownSync('${DAIMON_BROKER_REALM}', 2100, 2100);`, "fs.chmodSync('/etc/daimon-engine-broker', 0o555);" ].join("\n"); diff --git a/src/compiler/containerDaimonCapabilityOrdering.test.ts b/src/compiler/containerDaimonCapabilityOrdering.test.ts index ce35c95c..71b5f612 100644 --- a/src/compiler/containerDaimonCapabilityOrdering.test.ts +++ b/src/compiler/containerDaimonCapabilityOrdering.test.ts @@ -6,6 +6,7 @@ import { createStateOwnershipCommand } from "./containerStateOwnershipRender.js" const daimonPlan = { engineByNodeId: { "agent:grok": "grok" }, + grokModelByNodeId: { "agent:grok": { model: "grok-4.6", reasoningEffort: "low" } }, instancePaths: { configPath: "/var/lib/spawnfile/instances/daimon/organization/daimon/config.json", instanceRoot: "/var/lib/spawnfile/instances/daimon/organization", diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts new file mode 100644 index 00000000..25babd53 --- /dev/null +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -0,0 +1,135 @@ +import crypto from "node:crypto"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; +import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; +import { renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; +import { resolveDaimonGrokRegistrations, type DaimonGrokRegistration } from "./containerDaimonGrokWorkerRender.js"; + +const INSTANCE = "/var/lib/spawnfile/instances/daimon/daimon-organization"; +const BROKER = 2100; + +type Node = { content: string; gid: number; kind: "dir" | "file" | "link"; mode: number; target?: string; uid: number }; + +/** Just enough of `node:fs` to run the rendered root program without root. */ +const memoryFs = (seed: Record>) => { + const nodes = new Map([["/", { content: "", gid: 0, kind: "dir", mode: 0o755, uid: 0 }]]); + const enoent = (target: string) => Object.assign(new Error(`ENOENT: ${target}`), { code: "ENOENT" }); + const put = (target: string, node: Partial) => { + for (let parent = path.posix.dirname(target); !nodes.has(parent); parent = path.posix.dirname(parent)) { + nodes.set(parent, { content: "", gid: 0, kind: "dir", mode: 0o755, uid: 0 }); + } + nodes.set(target, { content: "", gid: 0, kind: "dir", mode: 0o755, uid: 0, ...node }); + }; + for (const [target, node] of Object.entries(seed)) put(target, node); + const get = (target: string): Node => { const node = nodes.get(target); if (!node) throw enoent(target); return node; }; + const stat = (target: string) => { + const node = get(target); + const type = node.kind === "dir" ? 0o040000 : node.kind === "file" ? 0o100000 : 0o120000; + return { gid: node.gid, isDirectory: () => node.kind === "dir", isFile: () => node.kind === "file", isSymbolicLink: () => node.kind === "link", mode: type | node.mode, nlink: 1, uid: node.uid }; + }; + const fs = { + chmodSync: (target: string, mode: number) => { get(target).mode = mode & 0o7777; }, + chownSync: (target: string, uid: number, gid: number) => { const node = get(target); node.uid = uid; node.gid = gid; }, + lstatSync: stat, + mkdirSync: (target: string, options: { mode?: number; recursive?: boolean } = {}) => { + if (nodes.has(target)) { if (options.recursive) return; throw Object.assign(new Error("EEXIST"), { code: "EEXIST" }); } + if (!options.recursive) get(path.posix.dirname(target)); + put(target, { kind: "dir", mode: options.mode ?? 0o755 }); + }, + readFileSync: (target: string) => get(target).content, + realpathSync: (target: string) => { + let resolved = "/"; + for (const segment of target.split("/").filter(Boolean)) { + resolved = path.posix.join(resolved, segment); + const node = get(resolved); + if (node.kind === "link") resolved = node.target!; + } + return resolved; + }, + statSync: stat, + writeFileSync: (target: string, content: string, options: { flag?: string; mode?: number } = {}) => { + if (options.flag === "wx" && nodes.has(target)) throw Object.assign(new Error("EEXIST"), { code: "EEXIST" }); + put(target, { content: String(content), kind: "file", mode: options.mode ?? 0o644 }); + } + }; + return { fs, nodes }; +}; + +const plan = (engines: Record): RuntimeTargetPlan => ({ + engineByNodeId: engines, + grokModelByNodeId: Object.fromEntries(Object.entries(engines).filter(([, engine]) => engine === "grok").map(([id]) => [id, { model: "grok-4.6", reasoningEffort: "low" }])), + instancePaths: { configPath: `${INSTANCE}/daimon/config.json`, instanceRoot: INSTANCE, workspacePath: `${INSTANCE}/workspace` }, + runtimeName: "daimon" +}) as unknown as RuntimeTargetPlan; + +const seedFor = (registrations: readonly DaimonGrokRegistration[], omit: string[] = []) => Object.fromEntries([ + ...registrations.map((entry) => [entry.workspace, { kind: "dir" as const }] as const), + ["/etc/daimon-engine-broker", { kind: "dir" as const, mode: 0o700 }] as const, + ...registrations.flatMap((entry) => entry.denyPaths) + .filter((denied) => !denied.startsWith("/var/lib/daimon-workers/") && !(["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"] as string[]).includes(denied)) + .map((denied) => [denied, { kind: denied.endsWith("grok-bootstrap-auth") ? "file" as const : "dir" as const, mode: 0o700 }] as const) +].filter(([target]) => !omit.includes(target))); + +const run = (registrations: readonly DaimonGrokRegistration[], seed: Record>, lines = renderDaimonGrokWorkerProvisioning(registrations)) => { + const memory = memoryFs(seed); + const secureWorkspace = (): void => undefined; + const program = new Function("fs", "crypto", "secureWorkspace", "require", lines.join("\n")); + program(memory.fs, crypto, secureWorkspace, (name: string) => name === "node:path" ? path.posix : undefined); + return memory.nodes; +}; + +describe("Grok worker home provisioning", () => { + const registrations = resolveDaimonGrokRegistrations([plan({ "agent:a": "grok", "agent:b": "grok", "agent:c": "codex" })]); + + it("provisions Daimon's attested layout: sticky root:worker homes and root-owned read-only turn files", () => { + const nodes = run(registrations, seedFor(registrations)); + for (const entry of registrations) { + expect(nodes.get(entry.home)).toMatchObject({ gid: BROKER, mode: 0o710, uid: entry.uid }); + for (const directory of [entry.grokHome, `${entry.grokHome}/sessions`]) { + expect(nodes.get(directory)).toMatchObject({ gid: entry.uid, kind: "dir", mode: 0o1771, uid: 0 }); + } + expect(nodes.get(`${entry.grokHome}/config.toml`)!.content).toBe(DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"].low); + expect(nodes.get(entry.profilePath)!.content).toBe(entry.profile); + for (const name of ["config.toml", "sandbox.toml", "trusted_folders.toml", "managed_config.toml", "requirements.toml"]) { + const file = nodes.get(`${entry.grokHome}/${name}`)!; + expect(file, name).toMatchObject({ gid: 0, kind: "file", mode: 0o444, uid: 0 }); + // No worker-uid process may write any file that decides a turn, trust included. + expect(file.mode & 0o222, name).toBe(0); + } + for (const name of ["trusted_folders.toml", "managed_config.toml", "requirements.toml"]) expect(nodes.get(`${entry.grokHome}/${name}`)!.content).toBe(""); + expect(nodes.get(entry.eventsPath)).toMatchObject({ gid: BROKER, kind: "file", mode: 0o640, uid: entry.uid }); + expect(nodes.has(`${entry.grokHome}/sandbox-events.jsonl`)).toBe(false); + } + const service = JSON.parse(nodes.get("/etc/daimon-engine-broker/service.json")!.content); + expect(service.version).toBe("noopolis.daimon.engine-broker-service.v2"); + expect(nodes.get("/etc/daimon-engine-broker/service.json")).toMatchObject({ gid: BROKER, mode: 0o440, uid: 0 }); + for (const optional of ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"]) expect(nodes.get(optional)).toMatchObject({ kind: "dir", mode: 0o700, uid: 0 }); + }); + + it("is restart-idempotent over an already provisioned home", () => { + const first = run(registrations, seedFor(registrations)); + const reseeded = Object.fromEntries([...first.entries()].filter(([target]) => target !== "/etc/daimon-engine-broker/service.json")); + expect(() => run(registrations, reseeded)).not.toThrow(); + }); + + it("fails closed on a missing or symlinked deny path, or bytes that drifted from their pins", () => { + const realm = "/var/lib/spawnfile/daimon/grok-subscription-realm"; + expect(() => run(registrations, seedFor(registrations, [realm]))).toThrow(/deny path is missing: \/var\/lib\/spawnfile\/daimon\/grok-subscription-realm/u); + expect(() => run(registrations, { ...seedFor(registrations), [realm]: { kind: "link", target: "/tmp/elsewhere" }, "/tmp/elsewhere": { kind: "dir" } })) + .toThrow(/canonical non-symlink/u); + const workspace = registrations[0]!.workspace; + expect(() => run(registrations, { ...seedFor(registrations), [workspace]: { kind: "link", target: "/tmp/ws" }, "/tmp/ws": { kind: "dir" } })) + .toThrow(/canonical non-symlink/u); + const tampered = registrations.map((entry, index) => index === 0 ? { ...entry, config: entry.config.replace("grok-4.6", "grok-4.5") } : entry); + expect(() => run(tampered, seedFor(tampered))).toThrow(/do not match their pins/u); + const unpinned = registrations.map((entry, index) => index === 0 ? { ...entry, denyPaths: [] } : entry); + expect(() => run(unpinned, seedFor(unpinned))).toThrow(/do not match their pins/u); + const replaced = run(registrations, seedFor(registrations)); + const reseeded = Object.fromEntries([...replaced.entries()].filter(([target]) => target !== "/etc/daimon-engine-broker/service.json")); + reseeded[`${registrations[0]!.grokHome}/trusted_folders.toml`] = { ...reseeded[`${registrations[0]!.grokHome}/trusted_folders.toml`], content: "[trusted]\n" }; + expect(() => run(registrations, reseeded)).toThrow(/identity mismatch/u); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts new file mode 100644 index 00000000..5d3aa8d5 --- /dev/null +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -0,0 +1,67 @@ +import { DAIMON_GROK_ENGINE_BROKER } from "../runtime/daimon/contractManifest.js"; +import { DAIMON_BROKER_UID } from "../runtime/daimon/runtimeIdentity.js"; +import { + DAIMON_GROK_OPTIONAL_DENY_PATHS, + DAIMON_GROK_WORKER_READ_ONLY_FILES, + DAIMON_WORKER_ROOT, + renderDaimonGrokServiceConfig, + type DaimonGrokRegistration +} from "./containerDaimonGrokWorkerRender.js"; + +/** Only the bytes the program writes; everything else it needs is recomputed or checked in place. */ +const programRegistration = (entry: DaimonGrokRegistration) => ({ + agentId: entry.agentId, + config: entry.config, + configSha256: entry.configSha256, + denyPaths: entry.denyPaths, + eventsPath: entry.eventsPath, + grokHome: entry.grokHome, + home: entry.home, + profile: entry.profile, + profilePath: entry.profilePath, + profileSha256: entry.profileSha256, + slot: entry.slot, + uid: entry.uid, + workspace: entry.workspace +}); + +/** + * Root provisioning for every brokered Grok worker, as lines of the broker's + * node provisioning program (which already defines `crypto`, `fs`, and + * `secureWorkspace`). + * + * Layout per Daimon's `GROK_ENGINE_BROKER.worker.home` (attested by the broker + * before every turn): `$GROK_HOME` (`/.grok`) and `$GROK_HOME/sessions` + * are `root: 1771`; `config.toml` (Daimon's renderer bytes for the + * declared model x effort), `sandbox.toml`, and the empty `trusted_folders.toml`, + * `managed_config.toml` and `requirements.toml` are `root:root 0444`, so the + * worker can neither write, rename nor unlink any file that decides a turn — + * trust included; `sessions/sandbox-events.jsonl` is `: 0640`. + * + * Every path handed to Daimon (workspace, home, profile, events, and each deny + * entry) must be canonical: it exists, is not a symlink, and resolves to itself. + * Daimon never resolves these paths, so a symlinked entry would silently mask + * the wrong inode. Root chmods only paths it owns at that moment (no + * `CAP_FOWNER`): reclaim, mode, then hand over. + */ +export const renderDaimonGrokWorkerProvisioning = (registrations: readonly DaimonGrokRegistration[]): string[] => [ + `const grokWorkers = ${JSON.stringify(registrations.map(programRegistration))};`, + `const optionalDenyPaths = new Set(${JSON.stringify(DAIMON_GROK_OPTIONAL_DENY_PATHS)});`, + `const pinnedConfigSha256 = new Set(${JSON.stringify(Object.values(DAIMON_GROK_ENGINE_BROKER.worker.configSha256).flatMap((efforts) => Object.values(efforts)))});`, + "const sha256Hex = (value) => crypto.createHash('sha256').update(value).digest('hex');", + "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !pinnedConfigSha256.has(entry.configSha256) || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", + "const assertCanonical = (target, label) => { const info = fs.lstatSync(target); if (info.isSymbolicLink() || fs.realpathSync(target) !== target) throw new Error(`Grok worker ${label} is not a canonical non-symlink path: ${target}`); return info; };", + "const ensureDirectory = (target, uid, gid, mode) => { fs.mkdirSync(target, { recursive: true, mode: 0o700 }); const info = fs.lstatSync(target); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('unsafe worker runtime directory'); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", + "const ensureExactFile = (target, content, mode) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, content, { mode, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new Error('unsafe worker runtime file'); if (fs.readFileSync(target, 'utf8') !== content) throw new Error(`worker runtime file identity mismatch: ${target}`); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); };", + `const ensureEventsFile = (target, uid) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, '', { mode: 0o640, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.uid !== uid && info.uid !== 0) || ![0, uid, ${DAIMON_BROKER_UID}].includes(info.gid) || ![0o600, 0o640, 0o644].includes(info.mode & 0o777)) throw new Error('unsafe worker attestation events'); fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o640); fs.chownSync(target, uid, ${DAIMON_BROKER_UID}); };`, + `fs.mkdirSync('${DAIMON_WORKER_ROOT}', { recursive: true, mode: 0o711 }); fs.chownSync('${DAIMON_WORKER_ROOT}', 0, 0); fs.chmodSync('${DAIMON_WORKER_ROOT}', 0o711); assertCanonical('${DAIMON_WORKER_ROOT}', 'worker root');`, + // Pass 1: every workspace and home exists, root-held, before any deny list is checked. + `for (const entry of grokWorkers) { for (let ancestor = require('node:path').dirname(entry.workspace); ancestor.startsWith('/var/lib/spawnfile/') && ancestor.length > '/var/lib/spawnfile'.length; ancestor = require('node:path').dirname(ancestor)) fs.chmodSync(ancestor, fs.statSync(ancestor).mode & 0o7777 | 0o011); secureWorkspace(entry.workspace, entry.uid); assertCanonical(entry.workspace, 'workspace'); ensureDirectory(entry.home, 0, 0, 0o700); ensureDirectory(entry.grokHome, 0, 0, 0o700); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, 0, 0o700); for (const target of [entry.home, entry.grokHome]) assertCanonical(target, 'home'); }`, + "for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }", + "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", + // Pass 2: exact root-owned read-only files, the events file, then the final sticky modes. + `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, + `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, + "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", + `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` +]; diff --git a/src/compiler/containerDaimonGrokWorkerRender.test.ts b/src/compiler/containerDaimonGrokWorkerRender.test.ts new file mode 100644 index 00000000..8516ef37 --- /dev/null +++ b/src/compiler/containerDaimonGrokWorkerRender.test.ts @@ -0,0 +1,105 @@ +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifest.js"; +import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; +import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; +import { + DAIMON_GROK_OPTIONAL_DENY_PATHS, + renderDaimonGrokServiceConfig, + resolveDaimonGrokRegistrations +} from "./containerDaimonGrokWorkerRender.js"; + +const INSTANCE = "/var/lib/spawnfile/instances/daimon/daimon-organization"; +const sha256 = (value: string): string => createHash("sha256").update(value).digest("hex"); + +const grokWorkerPlan = ( + engines: Record, + models: Record = Object.fromEntries( + Object.entries(engines).filter(([, engine]) => engine === "grok").map(([id]) => [id, { model: "grok-4.6", reasoningEffort: "low" }]) + ) +): RuntimeTargetPlan => ({ + engineByNodeId: engines, + grokModelByNodeId: models, + instancePaths: { configPath: `${INSTANCE}/daimon/daimon-organization-runtime.json`, instanceRoot: INSTANCE, workspacePath: `${INSTANCE}/workspace` }, + runtimeName: "daimon" +}) as unknown as RuntimeTargetPlan; + +describe("Daimon Grok worker registrations", () => { + it("uses Daimon's pinned worker config bytes for each agent's declared model and reasoning effort", () => { + const registrations = resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok", "agent:b": "grok" }, { + "agent:a": { model: "grok-4.6", reasoningEffort: "low" }, + "agent:b": { model: "grok-build", reasoningEffort: "high" } + })]); + expect(registrations.map((entry) => [entry.agentId, entry.uid, entry.home, entry.grokHome])).toEqual([ + ["agent:a", 2200, "/var/lib/daimon-workers/2200", "/var/lib/daimon-workers/2200/.grok"], + ["agent:b", 2201, "/var/lib/daimon-workers/2201", "/var/lib/daimon-workers/2201/.grok"] + ]); + const [a, b] = registrations; + expect(a!.config).toBe(DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.6"].low); + expect(sha256(a!.config)).toBe(DAIMON_GROK_ENGINE_BROKER.worker.configSha256["grok-4.6"].low); + expect(b!.config).toBe(DAIMON_GROK_WORKER_CONFIG_BYTES["grok-build"].high); + expect(sha256(b!.config)).toBe(DAIMON_GROK_ENGINE_BROKER.worker.configSha256["grok-build"].high); + expect(a!.config).not.toContain("auth_provider"); + expect(a!.profilePath).toBe("/var/lib/daimon-workers/2200/.grok/sandbox.toml"); + expect(a!.eventsPath).toBe("/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl"); + }); + + it("refuses a Grok registration without a declared closed-list model and effort", () => { + expect(() => resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok" }, {})])).toThrow(/no declared broker model/u); + const rootless = { ...grokWorkerPlan({ "agent:a": "grok" }), instancePaths: { configPath: "/c.json", workspacePath: "/w" } } as RuntimeTargetPlan; + expect(() => resolveDaimonGrokRegistrations([rootless])).toThrow(/require an instance root/u); + expect(() => resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok" }, { "agent:a": { model: "grok-4", reasoningEffort: "low" } })])).toThrow(/no declared broker model/u); + expect(() => resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok" }, { "agent:a": { model: "grok-4.6", reasoningEffort: "xhigh" } })])).toThrow(/no declared broker model/u); + }); + + it("denies every realm, broker, ledger, secret, peer, and other-worker path, and never the worker's own", () => { + const [a, b] = resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok", "agent:b": "grok", "agent:c": "codex", "agent:d": "agy" })]); + const denied = a!.denyPaths; + expect(denied.length).toBeGreaterThan(0); + expect(denied).toEqual(expect.arrayContaining([ + "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "/var/lib/spawnfile/daimon/grok-subscription-realm", + "/var/lib/spawnfile/daimon/agy-unlock-secret", + "/var/lib/spawnfile/daimon/agy-subscription-realm", + "/etc/daimon-engine-broker", + "/run/daimon-engine-broker", + DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, + "/var/lib/spawnfile/daimon/wake-fuse", + `${INSTANCE}/state/wake-acceptance`, + `${INSTANCE}/runtime-homes/b`, `${INSTANCE}/workspace/agents/b`, + `${INSTANCE}/runtime-homes/c`, `${INSTANCE}/workspace/agents/c`, + `${INSTANCE}/runtime-homes/d`, `${INSTANCE}/workspace/agents/d`, + "/var/lib/daimon-workers/2201", + ...DAIMON_GROK_OPTIONAL_DENY_PATHS + ])); + for (const own of [`${INSTANCE}/workspace/agents/a`, `${INSTANCE}/runtime-homes/a`, "/var/lib/daimon-workers/2200"]) expect(denied).not.toContain(own); + expect(b!.denyPaths).toContain("/var/lib/daimon-workers/2200"); + expect(denied).toEqual([...denied].sort()); + for (const entry of denied) expect(denied.some((other) => entry.startsWith(`${other}/`))).toBe(false); + expect(a!.profile).toBe(`[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = [${denied.map((entry) => JSON.stringify(entry)).join(", ")}]\n`); + const soloDenied = resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok" })])[0]!.denyPaths; + expect(soloDenied).not.toContain("/var/lib/spawnfile/daimon/agy-subscription-realm"); + expect(soloDenied.length).toBeGreaterThan(8); + }); + + it("renders service.json v2 with per-slot ledger, limits, model, and the profile digest", () => { + const registrations = resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:a": "grok" }, { "agent:a": { model: "grok-4.5", reasoningEffort: "medium" } })]); + const service = renderDaimonGrokServiceConfig(registrations); + expect(Object.keys(service)).toEqual(["version", "credentialHome", "turnStore", "registrations"]); + expect(service.version).toBe("noopolis.daimon.engine-broker-service.v2"); + expect(service.registrations).toEqual([{ + agentId: "agent:a", + slot: 0, + workerUid: 2200, + workspace: `${INSTANCE}/workspace/agents/a`, + profilePath: "/var/lib/daimon-workers/2200/.grok/sandbox.toml", + eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl", + profileSha256: sha256(registrations[0]!.profile), + usageLedgerPath: DAIMON_GROK_TURN_USAGE_LEDGER.filePath, + limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + model: { id: "grok-4.5", reasoningEffort: "medium" } + }]); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts new file mode 100644 index 00000000..3caaae2d --- /dev/null +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -0,0 +1,182 @@ +import path from "node:path"; + +import { SpawnfileError } from "../shared/index.js"; +import { + DAIMON_ORGANIZATION_TARGET_ID, + DAIMON_RUNTIME_ACCEPTANCE_STORE_DIRECTORY, + DAIMON_RUNTIME_HOMES_DIRECTORY, + DAIMON_WAKE_FUSE_DIRECTORY +} from "../runtime/daimon/config.js"; +import { + DAIMON_AGY_SUBSCRIPTION_REALM, + DAIMON_GROK_BROKER_MODELS, + DAIMON_GROK_BROKER_REASONING_EFFORTS, + DAIMON_GROK_ENGINE_BROKER, + DAIMON_GROK_SUBSCRIPTION_REALM, + DAIMON_GROK_TURN_USAGE_LEDGER, + DAIMON_RUNTIME_HOME_ROOT, + type DaimonGrokBrokerModel, + type DaimonGrokBrokerReasoningEffort +} from "../runtime/daimon/contractManifest.js"; +import { + DAIMON_GROK_WORKER_HOME_DIRECTORY, + daimonGrokWorkerSandboxProfileSha256, + renderDaimonGrokWorkerSandboxProfile, + resolveDaimonGrokWorkerConfig +} from "../runtime/daimon/grokWorkerContract.js"; +import { DAIMON_FIRST_WORKER_UID } from "../runtime/daimon/runtimeIdentity.js"; +import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; + +export const DAIMON_WORKER_ROOT = "/var/lib/daimon-workers"; +export const DAIMON_GROK_WORKER_READ_ONLY_FILES = DAIMON_GROK_ENGINE_BROKER.worker.home.readOnlyFiles.names; +/** Container paths no Grok worker may read that nothing else guarantees to exist; provisioning creates them root-owned 0700 when absent. */ +export const DAIMON_GROK_OPTIONAL_DENY_PATHS = ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"] as const; + +/** + * The organization runtime state directory — the parent of the durable wake + * acceptance store. The ownership guard secures it to `0700 2000:2000` for + * every Daimon organization; the acceptance store beneath it is also on every + * Grok worker's sandbox deny list. + */ +export const DAIMON_ORGANIZATION_STATE_DIRECTORY = path.posix.join( + DAIMON_RUNTIME_HOME_ROOT, + DAIMON_ORGANIZATION_TARGET_ID, + "state" +); + +export interface DaimonGrokRegistration { + agentId: string; + config: string; + configSha256: string; + denyPaths: string[]; + eventsPath: string; + grokHome: string; + home: string; + model: DaimonGrokBrokerModel; + profile: string; + profilePath: string; + profileSha256: string; + reasoningEffort: DaimonGrokBrokerReasoningEffort; + slot: number; + uid: number; + usageLedgerPath: string; + workspace: string; +} + +const nodeSlug = (nodeId: string): string => nodeId.replace(/^agent:/u, "") + .toLowerCase().replace(/[^a-z0-9]+/gu, "-").replace(/^-+|-+$/gu, ""); + +const fail = (message: string): never => { + throw new SpawnfileError("compile_error", message); +}; + +const declaredModel = (plan: RuntimeTargetPlan, agentId: string): { model: DaimonGrokBrokerModel; reasoningEffort: DaimonGrokBrokerReasoningEffort } => { + const declared = plan.grokModelByNodeId?.[agentId]; + if (!declared || !(DAIMON_GROK_BROKER_MODELS as readonly string[]).includes(declared.model) + || !(DAIMON_GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(declared.reasoningEffort)) { + return fail(`Daimon Grok agent ${agentId} has no declared broker model and reasoning effort`); + } + return declared as { model: DaimonGrokBrokerModel; reasoningEffort: DaimonGrokBrokerReasoningEffort }; +}; + +/** + * One brokered worker's sandbox deny list. + * + * Daimon's own protected set for the agent (`grokSandboxProtectedPaths`: the + * Grok bootstrap and realm, the AGY realm and unlock secret when any agent is + * AGY, the wake-acceptance store, and every peer's runtime home and workspace) + * plus what this deployment adds: the broker's registration/service directory + * and control/relay socket directory, the usage-ledger and wake-fuse volumes, + * every other worker's home, and the container secret roots. Grok 1.0.34's + * strict base reads all of `/run`, `/var`, `/tmp` and `/etc`, and bind mounts + * from a macOS host ignore unix modes, so this list — not file modes — is the + * boundary. Entries must never nest: one mask inside another cannot be applied. + */ +export const resolveDaimonGrokWorkerDenyPaths = ( + plan: RuntimeTargetPlan, + agentId: string, + workerHomes: readonly string[], + ownHome: string +): string[] => { + const instanceRoot = plan.instancePaths.instanceRoot ?? fail("Daimon Grok registrations require an instance root"); + const agents = Object.entries(plan.engineByNodeId ?? {}); + const peers = agents.filter(([id]) => id !== agentId).map(([id]) => nodeSlug(id)); + const denied = [...new Set([ + DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath, + DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath, + ...(agents.some(([, engine]) => engine === "agy") ? [DAIMON_AGY_SUBSCRIPTION_REALM.unlockMountPath, DAIMON_AGY_SUBSCRIPTION_REALM.durableMountPath] : []), + path.posix.join(instanceRoot, DAIMON_RUNTIME_ACCEPTANCE_STORE_DIRECTORY), + ...peers.flatMap((slug) => [ + path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, slug), + path.posix.join(plan.instancePaths.workspacePath, "agents", slug) + ]), + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), + DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, + DAIMON_WAKE_FUSE_DIRECTORY, + ...workerHomes.filter((home) => home !== ownHome), + ...DAIMON_GROK_OPTIONAL_DENY_PATHS + ])].sort(); + for (const entry of denied) { + const ancestor = denied.find((other) => other !== entry && entry.startsWith(`${other}/`)); + if (ancestor) fail(`Grok worker deny path ${entry} nests inside ${ancestor}`); + } + return denied; +}; + +export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): DaimonGrokRegistration[] => { + const entries = plans + .filter((plan) => plan.runtimeName === "daimon") + .flatMap((plan) => Object.entries(plan.engineByNodeId ?? {}) + .filter(([, engine]) => engine === "grok") + .map(([agentId]) => ({ agentId, plan }))) + .sort((left, right) => left.agentId.localeCompare(right.agentId)); + const homes = entries.map((_, slot) => path.posix.join(DAIMON_WORKER_ROOT, String(DAIMON_FIRST_WORKER_UID + slot))); + return entries.map(({ agentId, plan }, slot) => { + const home = homes[slot]!; + const grokHome = path.posix.join(home, DAIMON_GROK_WORKER_HOME_DIRECTORY); + const { model, reasoningEffort } = declaredModel(plan, agentId); + const config = resolveDaimonGrokWorkerConfig(model, reasoningEffort); + const denyPaths = resolveDaimonGrokWorkerDenyPaths(plan, agentId, homes, home); + const profile = renderDaimonGrokWorkerSandboxProfile(denyPaths); + return { + agentId, + config: config.bytes, + configSha256: config.sha256, + denyPaths, + eventsPath: path.posix.join(grokHome, DAIMON_GROK_ENGINE_BROKER.worker.home.sandboxEvents.relativePath), + grokHome, + home, + model, + profile, + profilePath: path.posix.join(grokHome, "sandbox.toml"), + profileSha256: daimonGrokWorkerSandboxProfileSha256(denyPaths), + reasoningEffort, + slot, + uid: DAIMON_FIRST_WORKER_UID + slot, + // Production keeps one container ledger: `spawnfile usage` and Daimon's + // wake fuse both read it, so a per-slot file would hide Grok spend from both. + usageLedgerPath: DAIMON_GROK_TURN_USAGE_LEDGER.filePath, + workspace: path.posix.join(plan.instancePaths.workspacePath, "agents", nodeSlug(agentId)) + }; + }); +}; + +/** `service.json` v2, exactly the shape Daimon's strict `parseEngineBrokerServiceConfig` accepts. */ +export const renderDaimonGrokServiceConfig = (registrations: readonly DaimonGrokRegistration[]) => ({ + version: DAIMON_GROK_ENGINE_BROKER.serviceConfigVersions[1], + credentialHome: DAIMON_GROK_ENGINE_BROKER.credentialHomePath, + turnStore: DAIMON_GROK_ENGINE_BROKER.turnStorePath, + registrations: registrations.map((entry) => ({ + agentId: entry.agentId, + slot: entry.slot, + workerUid: entry.uid, + workspace: entry.workspace, + profilePath: entry.profilePath, + eventsPath: entry.eventsPath, + profileSha256: entry.profileSha256, + usageLedgerPath: entry.usageLedgerPath, + limits: { ...DAIMON_GROK_ENGINE_BROKER.turnLimits.v1Defaults }, + model: { id: entry.model, reasoningEffort: entry.reasoningEffort } + })) +}); diff --git a/src/compiler/containerDaimonUidEntrypointLifecycle.test.ts b/src/compiler/containerDaimonUidEntrypointLifecycle.test.ts index c76373da..e6a1c5d0 100644 --- a/src/compiler/containerDaimonUidEntrypointLifecycle.test.ts +++ b/src/compiler/containerDaimonUidEntrypointLifecycle.test.ts @@ -30,6 +30,7 @@ const privateStateAncestors = (target: string): string[] => const daimonPlan: RuntimeTargetPlan = { engineByNodeId: { "agent:AGY": "agy", "agent:Codex One": "codex", "agent:Grok Two": "grok" }, + grokModelByNodeId: { "agent:Grok Two": { model: "grok-4.6", reasoningEffort: "low" } }, envFiles: [], id: "daimon-organization", instancePaths: { configPath: "/var/lib/spawnfile/instances/daimon/daimon-organization/daimon/config.json", diff --git a/src/compiler/containerDaimonUidEntrypointRender.test.ts b/src/compiler/containerDaimonUidEntrypointRender.test.ts index e72b3d53..e1cc797e 100644 --- a/src/compiler/containerDaimonUidEntrypointRender.test.ts +++ b/src/compiler/containerDaimonUidEntrypointRender.test.ts @@ -26,6 +26,7 @@ const authorizedUid = 2000; const daimonPlan: RuntimeTargetPlan = { engineByNodeId: { "agent:AGY": "agy", "agent:Codex One": "codex", "agent:Grok Two": "grok" }, + grokModelByNodeId: { "agent:Grok Two": { model: "grok-4.6", reasoningEffort: "low" } }, envFiles: [], id: "daimon-organization", instancePaths: { configPath: "/var/lib/spawnfile/instances/daimon/daimon-organization/daimon/config.json", @@ -210,7 +211,7 @@ describe("renderDaimonUidEntrypoint", () => { expect(rendered).toContain("/var/lib/daimon-workers/"); // Grok refuses a profile that is a symlink or carries a hard-link alias, so it is an // unaliased file in the worker's own read-only .grok directory. - expect(rendered).toContain("ensureExactFile(profilePath, profileFor(), 0, 0, 0o444)"); + expect(rendered).toContain("ensureExactFile(entry.profilePath, entry.profile, 0o444)"); expect(rendered).not.toContain("ensureExactLink"); expect(rendered).toContain("sandbox-events.jsonl"); expect(rendered).toContain("restrict_network = true"); @@ -228,8 +229,11 @@ describe("renderDaimonUidEntrypoint", () => { expect(rendered).toContain("validateResourceLink(target, info); return;"); expect(rendered).toContain("worker runtime file identity mismatch"); expect(rendered).toContain("worker runtime file identity mismatch"); - expect(rendered).toContain("ensureEventsFile(eventsPath, entry.uid)"); - expect(rendered).toContain("noopolis.daimon.engine-broker-service.v1"); + expect(rendered).toContain("ensureEventsFile(entry.eventsPath, entry.uid)"); + expect(rendered).toContain("noopolis.daimon.engine-broker-service.v2"); + expect(rendered).not.toContain("noopolis.daimon.engine-broker-service.v1"); + expect(rendered).toContain("/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl"); + expect(rendered).not.toContain("[auth_provider.daimon]"); expect(rendered).toContain("/etc/daimon-engine-broker/service.json"); expect(rendered).toContain("readSecure(bootstrap, undefined, 'bootstrap')"); expect(rendered).toContain("noopolis.daimon.broker-credential-journal.v1"); @@ -238,7 +242,7 @@ describe("renderDaimonUidEntrypoint", () => { expect(rendered).toContain("generation: journal.generation + 1"); expect(rendered).toContain("state: 'promoted'"); expect(rendered).toContain("bootstrapBytes.fill(0)"); - expect(rendered).toContain("ensureExactFile(profilePath, profileFor(), 0, 0, 0o444)"); + expect(rendered).toContain("ensureExactFile(entry.profilePath, entry.profile, 0o444)"); expect(rendered).toContain("--bounding-set=-all,+chown,+setuid,+setgid -- '/opt/daimon/bin/daimon-engine-broker' &"); expect(rendered).toContain("--bounding-set=-all,+chown,+setuid,+setgid,+setpcap -- '/opt/daimon/bin/daimon-engine-broker' --relay &"); expect(rendered).toContain('"$relay_pid:2100:0000000000000000"'); @@ -394,7 +398,8 @@ describe("renderDaimonUidEntrypoint", () => { ...daimonPlan.instancePaths, instanceRoot: undefined, workspacePath: "relative-workspace" - } + }, + engineByNodeId: { "agent:Codex One": "codex" } }], ["relative-state", "/persisted-state", "/persisted-state"]); expect(rendered).toContain("state_roots=('/persisted-state')"); diff --git a/src/runtime/daimon/config.ts b/src/runtime/daimon/config.ts index 3a730fca..4dba813a 100644 --- a/src/runtime/daimon/config.ts +++ b/src/runtime/daimon/config.ts @@ -209,6 +209,9 @@ export const createDaimonContainerTargets = async ( }) .sort((left, right) => left.id.localeCompare(right.id)); const engineByNodeId = Object.fromEntries(configAgents.map((agent) => [agent.id, agent.engine.kind])); + const grokModelByNodeId = Object.fromEntries(configAgents + .filter((agent) => agent.engine.kind === "grok" && agent.engine.model !== undefined && agent.engine.reasoningEffort !== undefined) + .map((agent) => [agent.id, { model: agent.engine.model!, reasoningEffort: agent.engine.reasoningEffort! }])); const hasAgy = configAgents.some((agent) => agent.engine.kind === "agy"); const hasGrok = configAgents.some((agent) => agent.engine.kind === "grok"); const agyRuntimeHomeMounts = configAgents @@ -253,6 +256,7 @@ export const createDaimonContainerTargets = async ( return [{ engineByNodeId, + ...(hasGrok ? { grokModelByNodeId } : {}), files: [ ...agents.flatMap((input) => input.emittedFiles.map((file) => moveWorkspaceFile(file, input.slug))), { content: serializedConfig, path: DAIMON_CONFIG_FILE }, diff --git a/src/runtime/types.ts b/src/runtime/types.ts index 93e58c55..ed072457 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -58,6 +58,8 @@ export interface ContainerTarget { */ engineByNodeId?: Record; envFiles?: ContainerTargetEnvFile[]; + /** Daimon only: each brokered Grok agent's declared model and reasoning effort, which select its pinned worker config. */ + grokModelByNodeId?: Record; files: EmittedFile[]; id: string; opaqueMountTargets?: string[]; From 038bef096e52e6bb4a448f89eee9e9fd1312b28d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:17:01 +0200 Subject: [PATCH 24/84] feat: run Grok organizations under the pinned bubblewrap seccomp profile and preflight host user namespaces --- src/compiler/AGENTS.md | 10 +++ ...tainerDaimonGrokWorkerProvisioning.test.ts | 29 ++++++- .../containerDaimonGrokWorkerProvisioning.ts | 15 ++++ ...containerDaimonUidEntrypointRender.test.ts | 3 + .../containerDaimonUidEntrypointRender.ts | 8 +- src/compiler/runProjectDaimonCodexDocker.ts | 8 +- src/distribution/consumeImage.ts | 2 +- src/distribution/consumeImageDaimonDocker.ts | 10 ++- src/shared/daimonCodexDocker.test.ts | 57 +++++++++++++ src/shared/daimonCodexDocker.ts | 82 ++++++++++++++++--- src/shared/daimonGrokSeccompProfile.ts | 11 +++ 11 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 src/shared/daimonCodexDocker.test.ts create mode 100644 src/shared/daimonGrokSeccompProfile.ts diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index d468cc97..17eb5a2f 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -162,6 +162,16 @@ src/compiler/ existing uid, capability-drop, and `no-new-privileges` posture; Codex owns the per-turn filesystem/network/tool boundary, while trusted MCP servers and Daimon code outside that native boundary remain trusted container processes. +- Without a strict Codex agent, a Daimon organization with a Grok agent gets + `--security-opt=seccomp=` plus `apparmor=unconfined` instead: + Docker's default seccomp profile with bubblewrap's seven namespace syscalls + (`src/shared/daimonGrokSeccompProfile.ts`, sha-pinned, materialized under + `/container/security/` or the image-up work directory), the narrowest + combination under which Grok 1.0.34's always-on bubblewrap starts. Codex's + fully unconfined options are a superset and win when both engines are + present. The Docker host must allow unprivileged user namespaces + (`kernel.apparmor_restrict_unprivileged_userns=0`); the Daimon entrypoint + refuses to start a Grok organization, naming that sysctl, when it is not. - Brokered Grok workers (`containerDaimonGrokWorkerRender.ts`) take their `config.toml` bytes only from Daimon's renderer output vendored in `src/runtime/daimon/grokWorkerConfigBytes.ts`, refused unless they hash to the diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index 25babd53..a81c01b8 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from "vitest"; import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; -import { renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; +import { renderDaimonGrokHostPreflight, renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; import { resolveDaimonGrokRegistrations, type DaimonGrokRegistration } from "./containerDaimonGrokWorkerRender.js"; const INSTANCE = "/var/lib/spawnfile/instances/daimon/daimon-organization"; @@ -133,3 +133,30 @@ describe("Grok worker home provisioning", () => { expect(() => run(registrations, reseeded)).toThrow(/identity mismatch/u); }); }); + +describe("Grok host user-namespace preflight", () => { + const preflight = async (restrict: string | null, maxNamespaces: string | null) => { + const { mkdtemp, mkdir, rm, writeFile } = await import("node:fs/promises"); + const os = await import("node:os"); + const { spawnSync } = await import("node:child_process"); + const root = await mkdtemp(path.join(os.tmpdir(), "spawnfile-grok-procsys-")); + try { + await mkdir(path.join(root, "kernel")); await mkdir(path.join(root, "user")); + if (restrict !== null) await writeFile(path.join(root, "kernel", "apparmor_restrict_unprivileged_userns"), `${restrict}\n`); + if (maxNamespaces !== null) await writeFile(path.join(root, "user", "max_user_namespaces"), `${maxNamespaces}\n`); + const script = ["set -euo pipefail", ...renderDaimonGrokHostPreflight(root), "echo preflight-ok"].join("\n"); + return spawnSync("bash", ["-c", script], { encoding: "utf8" }); + } finally { + await rm(root, { force: true, recursive: true }); + } + }; + + it("refuses to start when the host restricts unprivileged user namespaces, naming the sysctl", async () => { + const restricted = await preflight("1", "63000"); + expect(restricted.status).toBe(1); + expect(restricted.stderr).toContain("kernel.apparmor_restrict_unprivileged_userns=0"); + expect((await preflight("0", "0")).stderr).toContain("user.max_user_namespaces is 0"); + expect((await preflight("0", "63000")).stdout).toContain("preflight-ok"); + expect((await preflight(null, null)).stdout).toContain("preflight-ok"); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 5d3aa8d5..4082e695 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -65,3 +65,18 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` ]; + +const shellQuote = (value: string): string => `'${value.replace(/'/g, `'"'"'`)}'`; + +/** + * Grok 1.0.34 runs every sandbox profile inside bubblewrap, which needs + * unprivileged user namespaces. Ubuntu 24.04+ hosts (and Colima's default VM) + * ship `kernel.apparmor_restrict_unprivileged_userns=1`, under which bubblewrap + * cannot create them even with the pinned seccomp profile — every worker turn + * would then fail attestation long after startup. The container sees the host + * kernel's sysctls read-only, so this refuses to start with the fix named. + */ +export const renderDaimonGrokHostPreflight = (procSys = "/proc/sys"): string[] => [ + `if [ -r ${shellQuote(`${procSys}/kernel/apparmor_restrict_unprivileged_userns`)} ] && [ "$(cat ${shellQuote(`${procSys}/kernel/apparmor_restrict_unprivileged_userns`)})" != 0 ]; then echo "Daimon Grok workers need unprivileged user namespaces for bubblewrap: set kernel.apparmor_restrict_unprivileged_userns=0 on the Docker host (sysctl -w kernel.apparmor_restrict_unprivileged_userns=0; persist it in /etc/sysctl.d)" >&2; exit 1; fi`, + `if [ -r ${shellQuote(`${procSys}/user/max_user_namespaces`)} ] && [ "$(cat ${shellQuote(`${procSys}/user/max_user_namespaces`)})" = 0 ]; then echo "Daimon Grok workers need unprivileged user namespaces for bubblewrap: user.max_user_namespaces is 0 on the Docker host" >&2; exit 1; fi` +]; diff --git a/src/compiler/containerDaimonUidEntrypointRender.test.ts b/src/compiler/containerDaimonUidEntrypointRender.test.ts index e1cc797e..cc99c401 100644 --- a/src/compiler/containerDaimonUidEntrypointRender.test.ts +++ b/src/compiler/containerDaimonUidEntrypointRender.test.ts @@ -234,6 +234,9 @@ describe("renderDaimonUidEntrypoint", () => { expect(rendered).not.toContain("noopolis.daimon.engine-broker-service.v1"); expect(rendered).toContain("/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl"); expect(rendered).not.toContain("[auth_provider.daimon]"); + const preflight = rendered.indexOf("kernel.apparmor_restrict_unprivileged_userns=0 on the Docker host"); + expect(preflight).toBeGreaterThan(-1); + expect(preflight).toBeLessThan(rendered.indexOf("node <<'SPAWNFILE_DAIMON_BROKER_PROVISION'")); expect(rendered).toContain("/etc/daimon-engine-broker/service.json"); expect(rendered).toContain("readSecure(bootstrap, undefined, 'bootstrap')"); expect(rendered).toContain("noopolis.daimon.broker-credential-journal.v1"); diff --git a/src/compiler/containerDaimonUidEntrypointRender.ts b/src/compiler/containerDaimonUidEntrypointRender.ts index 0a05747a..1dd5a8ca 100644 --- a/src/compiler/containerDaimonUidEntrypointRender.ts +++ b/src/compiler/containerDaimonUidEntrypointRender.ts @@ -17,6 +17,7 @@ import { resolveDaimonVolumeIdentityFiles } from "./containerDaimonOwnershipGuardRender.js"; export { resolveDaimonVolumeIdentityFiles } from "./containerDaimonOwnershipGuardRender.js"; +import { renderDaimonGrokHostPreflight } from "./containerDaimonGrokWorkerProvisioning.js"; import { DAIMON_BROKER_EXECUTABLE, DAIMON_BROKER_BACKEND_SOCKET, @@ -168,9 +169,9 @@ const privateModeDirectories = (runtimePlans: RuntimeTargetPlan[], moltnet?: Ent // creates that parent root-owned and world-readable when it materializes // this mount, and the ancestor pass below only *chowns* it, so it stayed // 0755 and was the one path under `/var/lib/spawnfile` a Grok worker uid - // could open. Securing it to 0700 2000:2000 is what replaces the sandbox - // `deny` entry that Grok 1.0.13 can no longer honour (see - // `GROK_SANDBOX_DENY_PATHS`). Nothing loses access: the only thing + // could open. Securing it to 0700 2000:2000 backs the acceptance store's + // sandbox `deny` entry with unix modes as defense in depth (see + // `containerDaimonGrokWorkerRender.ts`). Nothing loses access: the only thing // beneath it is this store, already 0700 2000:2000, so every reader that // works today is the organization uid or a `docker exec` root holding // CAP_DAC_READ_SEARCH. @@ -331,6 +332,7 @@ export const renderDaimonUidEntrypoint = ( ' if ! getent group "$fixed_uid" >/dev/null; then groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid"; fi', ' if ! getent passwd "$fixed_uid" >/dev/null; then useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; fi', "done", + ...(resolveDaimonGrokRegistrations(runtimePlans).length === 0 ? [] : renderDaimonGrokHostPreflight()), ...renderDaimonBrokerProvisioning(runtimePlans), 'if ! getent passwd "$uid" >/dev/null; then', ' runtime_identity="daimon-$uid"', diff --git a/src/compiler/runProjectDaimonCodexDocker.ts b/src/compiler/runProjectDaimonCodexDocker.ts index 9738e84b..b3a11ff3 100644 --- a/src/compiler/runProjectDaimonCodexDocker.ts +++ b/src/compiler/runProjectDaimonCodexDocker.ts @@ -3,7 +3,8 @@ import path from "node:path"; import { readUtf8File } from "../filesystem/index.js"; import type { ContainerRuntimeInstanceReport } from "../report/index.js"; import { - codexNativeSandboxDockerSecurityArgsForConfigs, + daimonEngineDockerSecurityArgsForConfigs, + materializeDaimonGrokSeccompProfile, SpawnfileError } from "../shared/index.js"; @@ -55,5 +56,8 @@ export const resolveDaimonCodexNativeSandboxDockerSecurityOptions = async ( ); } } - return codexNativeSandboxDockerSecurityArgsForConfigs(sources); + return daimonEngineDockerSecurityArgsForConfigs( + sources, + () => materializeDaimonGrokSeccompProfile(path.join(compileResult.outputDirectory, "container", "security")) + ); }; diff --git a/src/distribution/consumeImage.ts b/src/distribution/consumeImage.ts index dc66155d..bd94988a 100644 --- a/src/distribution/consumeImage.ts +++ b/src/distribution/consumeImage.ts @@ -238,7 +238,7 @@ const consumeImageUpLocked = async ( }) ).mountArgs; const daimonDockerSecurityArgs = await resolveDaimonDockerSecurityArgsForImage( - imageRef, report, runDocker + imageRef, report, runDocker, workDir ); volumeReservation = await acquireExclusiveVolumeReservations( diff --git a/src/distribution/consumeImageDaimonDocker.ts b/src/distribution/consumeImageDaimonDocker.ts index 698ddcd9..eece4681 100644 --- a/src/distribution/consumeImageDaimonDocker.ts +++ b/src/distribution/consumeImageDaimonDocker.ts @@ -2,8 +2,9 @@ import { randomUUID } from "node:crypto"; import { SpawnfileError } from "../shared/errors.js"; import { - codexNativeSandboxDockerSecurityArgsForConfigs, - DAIMON_DOCKER_RUNTIME_SECURITY_ARGS + DAIMON_DOCKER_RUNTIME_SECURITY_ARGS, + daimonEngineDockerSecurityArgsForConfigs, + materializeDaimonGrokSeccompProfile } from "../shared/daimonCodexDocker.js"; import type { DockerCommandRunner } from "./dockerRunner.js"; @@ -56,13 +57,14 @@ const readDaimonConfigSourcesFromImage = async ( export const resolveDaimonDockerSecurityArgsForImage = async ( imageRef: string, report: DistributionReport, - runDocker: DockerCommandRunner + runDocker: DockerCommandRunner, + securityDirectory: string ): Promise => { const daimonInstances = daimonInstancesFrom(report); if (daimonInstances.length === 0) return []; const sources = await readDaimonConfigSourcesFromImage(imageRef, daimonInstances, runDocker); return [ ...DAIMON_DOCKER_RUNTIME_SECURITY_ARGS, - ...codexNativeSandboxDockerSecurityArgsForConfigs(sources) + ...(await daimonEngineDockerSecurityArgsForConfigs(sources, () => materializeDaimonGrokSeccompProfile(securityDirectory))) ]; }; diff --git a/src/shared/daimonCodexDocker.test.ts b/src/shared/daimonCodexDocker.test.ts new file mode 100644 index 00000000..704650c9 --- /dev/null +++ b/src/shared/daimonCodexDocker.test.ts @@ -0,0 +1,57 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + daimonEngineDockerSecurityArgsForConfigs, + materializeDaimonGrokSeccompProfile, + resolveDaimonEngineDockerInterop +} from "./daimonCodexDocker.js"; +import { DAIMON_GROK_SECCOMP_PROFILE_BYTES, DAIMON_GROK_SECCOMP_PROFILE_SHA256 } from "./daimonGrokSeccompProfile.js"; + +const config = (...engines: Array>): string => JSON.stringify({ + agents: engines.map((engine, index) => ({ engine, id: `agent:${index}` })), + version: "noopolis.daimon.organization-runtime.v1" +}); +const strictCodex = { codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, kind: "codex" }; +const grok = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }; +const profilePath = "/tmp/spawnfile-security/seccomp.json"; + +describe("Daimon engine Docker security options", () => { + it("runs Grok under the pinned default-plus-userns seccomp profile and AppArmor unconfined, never fully unconfined", async () => { + expect(resolveDaimonEngineDockerInterop(config(grok))).toEqual({ codex: false, grok: true }); + const args = await daimonEngineDockerSecurityArgsForConfigs([config(grok, { kind: "codex" }, { kind: "agy" })], async () => profilePath); + expect(args).toEqual([`--security-opt=seccomp=${profilePath}`, "--security-opt=apparmor=unconfined"]); + expect(args).not.toContain("--security-opt=seccomp=unconfined"); + }); + + it("keeps Codex's fully unconfined options only when a strict Codex agent exists, and nothing for neither", async () => { + let materialized = 0; + const materialize = async () => { materialized += 1; return profilePath; }; + await expect(daimonEngineDockerSecurityArgsForConfigs([config(grok), config(strictCodex)], materialize)) + .resolves.toEqual(["--security-opt=seccomp=unconfined", "--security-opt=apparmor=unconfined"]); + await expect(daimonEngineDockerSecurityArgsForConfigs([config({ kind: "codex" }, { kind: "agy" })], materialize)).resolves.toEqual([]); + expect(materialized).toBe(0); + await expect(daimonEngineDockerSecurityArgsForConfigs([config(grok)], async () => "relative.json")).rejects.toThrow(/absolute/u); + }); + + it("materializes the pinned seccomp profile byte-for-byte", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-grok-seccomp-")); + try { + const written = await materializeDaimonGrokSeccompProfile(path.join(directory, "security")); + expect(path.isAbsolute(written)).toBe(true); + const bytes = await readFile(written); + expect(createHash("sha256").update(bytes).digest("hex")).toBe(DAIMON_GROK_SECCOMP_PROFILE_SHA256); + expect(createHash("sha256").update(DAIMON_GROK_SECCOMP_PROFILE_BYTES).digest("hex")).toBe(DAIMON_GROK_SECCOMP_PROFILE_SHA256); + const profile = JSON.parse(bytes.toString("utf8")) as { defaultAction: string; syscalls: Array<{ action: string; names: string[] }> }; + expect(profile.defaultAction).toBe("SCMP_ACT_ERRNO"); + expect(profile.syscalls.some((rule) => rule.action === "SCMP_ACT_ALLOW" + && ["clone", "clone3", "unshare", "mount", "umount2", "pivot_root", "setns"].every((name) => rule.names.includes(name)))).toBe(true); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/src/shared/daimonCodexDocker.ts b/src/shared/daimonCodexDocker.ts index 778059f0..5004807a 100644 --- a/src/shared/daimonCodexDocker.ts +++ b/src/shared/daimonCodexDocker.ts @@ -1,3 +1,12 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { + DAIMON_GROK_SECCOMP_PROFILE_BYTES, + DAIMON_GROK_SECCOMP_PROFILE_FILE_NAME, + DAIMON_GROK_SECCOMP_PROFILE_SHA256 +} from "./daimonGrokSeccompProfile.js"; import { SpawnfileError } from "./errors.js"; export const DAIMON_DOCKER_RUNTIME_SECURITY_ARGS = [ @@ -56,7 +65,14 @@ const hasExactCodexStrictSandbox = (engine: Record): boolean => return true; }; -export const configRequiresCodexNativeSandboxDockerInterop = (source: string): boolean => { +export interface DaimonEngineDockerInterop { + /** A strict Codex agent needs seccomp and AppArmor fully unconfined for its own bubblewrap sandbox. */ + codex: boolean; + /** A brokered Grok worker needs bubblewrap's namespace syscalls: the pinned seccomp profile plus AppArmor unconfined. */ + grok: boolean; +} + +export const resolveDaimonEngineDockerInterop = (source: string): DaimonEngineDockerInterop => { let parsed: unknown; try { parsed = JSON.parse(source); @@ -78,7 +94,7 @@ export const configRequiresCodexNativeSandboxDockerInterop = (source: string): b "Daimon generated organization config is not a supported v1/v2 contract" ); } - let requiresInterop = false; + const interop = { codex: false, grok: false }; for (const agent of parsed.agents) { if (!isRecord(agent) || !isRecord(agent.engine)) { throw new SpawnfileError( @@ -93,19 +109,59 @@ export const configRequiresCodexNativeSandboxDockerInterop = (source: string): b "Daimon generated organization config has an unsupported agent engine" ); } - if (kind === "codex" && hasExactCodexStrictSandbox(agent.engine)) { - requiresInterop = true; - } + if (kind === "codex" && hasExactCodexStrictSandbox(agent.engine)) interop.codex = true; + if (kind === "grok") interop.grok = true; } - return requiresInterop; + return interop; }; -export const codexNativeSandboxDockerSecurityArgsForConfigs = (sources: string[]): string[] => { - let requiresInterop = false; - for (const source of sources) { - if (configRequiresCodexNativeSandboxDockerInterop(source)) { - requiresInterop = true; - } +export const configRequiresCodexNativeSandboxDockerInterop = (source: string): boolean => + resolveDaimonEngineDockerInterop(source).codex; + +export const DAIMON_GROK_HOST_USERNS_PREREQUISITE = + "kernel.apparmor_restrict_unprivileged_userns=0" as const; + +/** + * Writes the pinned Grok seccomp profile into `directory` (content-addressed, + * verified after writing) and returns its absolute path. Docker reads a + * `seccomp=` option on the client at `docker run` and stores the profile + * in the container config, so the file only has to exist for that call. + */ +export const materializeDaimonGrokSeccompProfile = async (directory: string): Promise => { + const target = path.resolve(directory, `${DAIMON_GROK_SECCOMP_PROFILE_SHA256.slice(0, 16)}-${DAIMON_GROK_SECCOMP_PROFILE_FILE_NAME}`); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, DAIMON_GROK_SECCOMP_PROFILE_BYTES, { mode: 0o644 }); + const written = await readFile(target); + if (createHash("sha256").update(written).digest("hex") !== DAIMON_GROK_SECCOMP_PROFILE_SHA256) { + throw new SpawnfileError("runtime_error", "Pinned Grok seccomp profile did not materialize byte-for-byte"); } - return requiresInterop ? [...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS] : []; + return target; +}; + +/** + * Docker security options for a Daimon container, by the engines its configs run. + * + * Codex's strict native sandbox needs seccomp and AppArmor fully unconfined, and + * that superset also lets Grok's bubblewrap run, so it wins whenever a strict + * Codex agent is present. Otherwise a Grok agent gets Docker's default seccomp + * profile plus bubblewrap's seven namespace syscalls (pinned bytes) and AppArmor + * unconfined — the narrowest combination under which Grok 1.0.34 starts. Grok + * additionally needs the Docker host to allow unprivileged user namespaces + * (`kernel.apparmor_restrict_unprivileged_userns=0`); the container entrypoint + * checks that before the broker starts. + */ +export const daimonEngineDockerSecurityArgsForConfigs = ( + sources: string[], + grokSeccompProfilePath: () => Promise +): Promise => { + const interop = sources.map(resolveDaimonEngineDockerInterop); + if (interop.some((entry) => entry.codex)) return Promise.resolve([...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS]); + if (!interop.some((entry) => entry.grok)) return Promise.resolve([]); + return grokSeccompProfilePath().then((profilePath) => { + if (!path.isAbsolute(profilePath)) throw new SpawnfileError("runtime_error", "Grok seccomp profile path must be absolute"); + return [`--security-opt=seccomp=${profilePath}`, "--security-opt=apparmor=unconfined"]; + }); }; + +export const codexNativeSandboxDockerSecurityArgsForConfigs = (sources: string[]): string[] => + sources.some(configRequiresCodexNativeSandboxDockerInterop) ? [...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS] : []; diff --git a/src/shared/daimonGrokSeccompProfile.ts b/src/shared/daimonGrokSeccompProfile.ts new file mode 100644 index 00000000..7fdf5a84 --- /dev/null +++ b/src/shared/daimonGrokSeccompProfile.ts @@ -0,0 +1,11 @@ +/* v8 ignore file -- pinned data module */ +/** + * Docker's default seccomp profile plus the seven namespace syscalls bubblewrap + * needs without CAP_SYS_ADMIN (clone, clone3, unshare, mount, umount2, pivot_root, + * setns): the narrowest profile under which Grok 1.0.34's sandbox starts + * (Noopolis P0 container evidence, 2026-09-17). Bytes are pinned by sha256; Daimon's + * Grok broker projection and slot preflight receipt bind the same digest. + */ +export const DAIMON_GROK_SECCOMP_PROFILE_FILE_NAME = "seccomp-default-plus-userns.json" as const; +export const DAIMON_GROK_SECCOMP_PROFILE_SHA256 = "9666074d2d6b1a261a410598b9ce1714d9060a6ef4c9c1d1cfb6f8915d7e842a" as const; +export const DAIMON_GROK_SECCOMP_PROFILE_BYTES = "{\n \"defaultAction\": \"SCMP_ACT_ERRNO\",\n \"defaultErrnoRet\": 1,\n \"archMap\": [\n {\n \"architecture\": \"SCMP_ARCH_X86_64\",\n \"subArchitectures\": [\n \"SCMP_ARCH_X86\",\n \"SCMP_ARCH_X32\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_AARCH64\",\n \"subArchitectures\": [\n \"SCMP_ARCH_ARM\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_MIPS64\",\n \"subArchitectures\": [\n \"SCMP_ARCH_MIPS\",\n \"SCMP_ARCH_MIPS64N32\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_MIPS64N32\",\n \"subArchitectures\": [\n \"SCMP_ARCH_MIPS\",\n \"SCMP_ARCH_MIPS64\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_MIPSEL64\",\n \"subArchitectures\": [\n \"SCMP_ARCH_MIPSEL\",\n \"SCMP_ARCH_MIPSEL64N32\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_MIPSEL64N32\",\n \"subArchitectures\": [\n \"SCMP_ARCH_MIPSEL\",\n \"SCMP_ARCH_MIPSEL64\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_S390X\",\n \"subArchitectures\": [\n \"SCMP_ARCH_S390\"\n ]\n },\n {\n \"architecture\": \"SCMP_ARCH_RISCV64\",\n \"subArchitectures\": null\n },\n {\n \"architecture\": \"SCMP_ARCH_LOONGARCH64\",\n \"subArchitectures\": null\n }\n ],\n \"syscalls\": [\n {\n \"names\": [\n \"clone\",\n \"clone3\",\n \"unshare\",\n \"mount\",\n \"umount2\",\n \"pivot_root\",\n \"setns\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"comment\": \"P0 spike: userns/mount syscalls bubblewrap needs, without CAP_SYS_ADMIN\"\n },\n {\n \"names\": [\n \"accept\",\n \"accept4\",\n \"access\",\n \"adjtimex\",\n \"alarm\",\n \"bind\",\n \"brk\",\n \"cachestat\",\n \"capget\",\n \"capset\",\n \"chdir\",\n \"chmod\",\n \"chown\",\n \"chown32\",\n \"clock_adjtime\",\n \"clock_adjtime64\",\n \"clock_getres\",\n \"clock_getres_time64\",\n \"clock_gettime\",\n \"clock_gettime64\",\n \"clock_nanosleep\",\n \"clock_nanosleep_time64\",\n \"close\",\n \"close_range\",\n \"connect\",\n \"copy_file_range\",\n \"creat\",\n \"dup\",\n \"dup2\",\n \"dup3\",\n \"epoll_create\",\n \"epoll_create1\",\n \"epoll_ctl\",\n \"epoll_ctl_old\",\n \"epoll_pwait\",\n \"epoll_pwait2\",\n \"epoll_wait\",\n \"epoll_wait_old\",\n \"eventfd\",\n \"eventfd2\",\n \"execve\",\n \"execveat\",\n \"exit\",\n \"exit_group\",\n \"faccessat\",\n \"faccessat2\",\n \"fadvise64\",\n \"fadvise64_64\",\n \"fallocate\",\n \"fanotify_mark\",\n \"fchdir\",\n \"fchmod\",\n \"fchmodat\",\n \"fchmodat2\",\n \"fchown\",\n \"fchown32\",\n \"fchownat\",\n \"fcntl\",\n \"fcntl64\",\n \"fdatasync\",\n \"fgetxattr\",\n \"flistxattr\",\n \"flock\",\n \"fork\",\n \"fremovexattr\",\n \"fsetxattr\",\n \"fstat\",\n \"fstat64\",\n \"fstatat64\",\n \"fstatfs\",\n \"fstatfs64\",\n \"fsync\",\n \"ftruncate\",\n \"ftruncate64\",\n \"futex\",\n \"futex_requeue\",\n \"futex_time64\",\n \"futex_wait\",\n \"futex_waitv\",\n \"futex_wake\",\n \"futimesat\",\n \"getcpu\",\n \"getcwd\",\n \"getdents\",\n \"getdents64\",\n \"getegid\",\n \"getegid32\",\n \"geteuid\",\n \"geteuid32\",\n \"getgid\",\n \"getgid32\",\n \"getgroups\",\n \"getgroups32\",\n \"getitimer\",\n \"getpeername\",\n \"getpgid\",\n \"getpgrp\",\n \"getpid\",\n \"getppid\",\n \"getpriority\",\n \"getrandom\",\n \"getresgid\",\n \"getresgid32\",\n \"getresuid\",\n \"getresuid32\",\n \"getrlimit\",\n \"get_robust_list\",\n \"getrusage\",\n \"getsid\",\n \"getsockname\",\n \"getsockopt\",\n \"get_thread_area\",\n \"gettid\",\n \"gettimeofday\",\n \"getuid\",\n \"getuid32\",\n \"getxattr\",\n \"getxattrat\",\n \"inotify_add_watch\",\n \"inotify_init\",\n \"inotify_init1\",\n \"inotify_rm_watch\",\n \"io_cancel\",\n \"ioctl\",\n \"io_destroy\",\n \"io_getevents\",\n \"io_pgetevents\",\n \"io_pgetevents_time64\",\n \"ioprio_get\",\n \"ioprio_set\",\n \"io_setup\",\n \"io_submit\",\n \"ipc\",\n \"kill\",\n \"landlock_add_rule\",\n \"landlock_create_ruleset\",\n \"landlock_restrict_self\",\n \"lchown\",\n \"lchown32\",\n \"lgetxattr\",\n \"link\",\n \"linkat\",\n \"listen\",\n \"listmount\",\n \"listxattr\",\n \"listxattrat\",\n \"llistxattr\",\n \"_llseek\",\n \"lremovexattr\",\n \"lseek\",\n \"lsetxattr\",\n \"lstat\",\n \"lstat64\",\n \"madvise\",\n \"map_shadow_stack\",\n \"membarrier\",\n \"memfd_create\",\n \"memfd_secret\",\n \"mincore\",\n \"mkdir\",\n \"mkdirat\",\n \"mknod\",\n \"mknodat\",\n \"mlock\",\n \"mlock2\",\n \"mlockall\",\n \"mmap\",\n \"mmap2\",\n \"mprotect\",\n \"mq_getsetattr\",\n \"mq_notify\",\n \"mq_open\",\n \"mq_timedreceive\",\n \"mq_timedreceive_time64\",\n \"mq_timedsend\",\n \"mq_timedsend_time64\",\n \"mq_unlink\",\n \"mremap\",\n \"mseal\",\n \"msgctl\",\n \"msgget\",\n \"msgrcv\",\n \"msgsnd\",\n \"msync\",\n \"munlock\",\n \"munlockall\",\n \"munmap\",\n \"name_to_handle_at\",\n \"nanosleep\",\n \"newfstatat\",\n \"_newselect\",\n \"open\",\n \"openat\",\n \"openat2\",\n \"pause\",\n \"pidfd_open\",\n \"pidfd_send_signal\",\n \"pipe\",\n \"pipe2\",\n \"pkey_alloc\",\n \"pkey_free\",\n \"pkey_mprotect\",\n \"poll\",\n \"ppoll\",\n \"ppoll_time64\",\n \"prctl\",\n \"pread64\",\n \"preadv\",\n \"preadv2\",\n \"prlimit64\",\n \"process_mrelease\",\n \"pselect6\",\n \"pselect6_time64\",\n \"pwrite64\",\n \"pwritev\",\n \"pwritev2\",\n \"read\",\n \"readahead\",\n \"readlink\",\n \"readlinkat\",\n \"readv\",\n \"recv\",\n \"recvfrom\",\n \"recvmmsg\",\n \"recvmmsg_time64\",\n \"recvmsg\",\n \"remap_file_pages\",\n \"removexattr\",\n \"removexattrat\",\n \"rename\",\n \"renameat\",\n \"renameat2\",\n \"restart_syscall\",\n \"riscv_hwprobe\",\n \"rmdir\",\n \"rseq\",\n \"rt_sigaction\",\n \"rt_sigpending\",\n \"rt_sigprocmask\",\n \"rt_sigqueueinfo\",\n \"rt_sigreturn\",\n \"rt_sigsuspend\",\n \"rt_sigtimedwait\",\n \"rt_sigtimedwait_time64\",\n \"rt_tgsigqueueinfo\",\n \"sched_getaffinity\",\n \"sched_getattr\",\n \"sched_getparam\",\n \"sched_get_priority_max\",\n \"sched_get_priority_min\",\n \"sched_getscheduler\",\n \"sched_rr_get_interval\",\n \"sched_rr_get_interval_time64\",\n \"sched_setaffinity\",\n \"sched_setattr\",\n \"sched_setparam\",\n \"sched_setscheduler\",\n \"sched_yield\",\n \"seccomp\",\n \"select\",\n \"semctl\",\n \"semget\",\n \"semop\",\n \"semtimedop\",\n \"semtimedop_time64\",\n \"send\",\n \"sendfile\",\n \"sendfile64\",\n \"sendmmsg\",\n \"sendmsg\",\n \"sendto\",\n \"setfsgid\",\n \"setfsgid32\",\n \"setfsuid\",\n \"setfsuid32\",\n \"setgid\",\n \"setgid32\",\n \"setgroups\",\n \"setgroups32\",\n \"setitimer\",\n \"setpgid\",\n \"setpriority\",\n \"setregid\",\n \"setregid32\",\n \"setresgid\",\n \"setresgid32\",\n \"setresuid\",\n \"setresuid32\",\n \"setreuid\",\n \"setreuid32\",\n \"setrlimit\",\n \"set_robust_list\",\n \"setsid\",\n \"setsockopt\",\n \"set_thread_area\",\n \"set_tid_address\",\n \"setuid\",\n \"setuid32\",\n \"setxattr\",\n \"setxattrat\",\n \"shmat\",\n \"shmctl\",\n \"shmdt\",\n \"shmget\",\n \"shutdown\",\n \"sigaltstack\",\n \"signalfd\",\n \"signalfd4\",\n \"sigprocmask\",\n \"sigreturn\",\n \"socketcall\",\n \"socketpair\",\n \"splice\",\n \"stat\",\n \"stat64\",\n \"statfs\",\n \"statfs64\",\n \"statmount\",\n \"statx\",\n \"symlink\",\n \"symlinkat\",\n \"sync\",\n \"sync_file_range\",\n \"syncfs\",\n \"sysinfo\",\n \"tee\",\n \"tgkill\",\n \"time\",\n \"timer_create\",\n \"timer_delete\",\n \"timer_getoverrun\",\n \"timer_gettime\",\n \"timer_gettime64\",\n \"timer_settime\",\n \"timer_settime64\",\n \"timerfd_create\",\n \"timerfd_gettime\",\n \"timerfd_gettime64\",\n \"timerfd_settime\",\n \"timerfd_settime64\",\n \"times\",\n \"tkill\",\n \"truncate\",\n \"truncate64\",\n \"ugetrlimit\",\n \"umask\",\n \"uname\",\n \"unlink\",\n \"unlinkat\",\n \"uretprobe\",\n \"utime\",\n \"utimensat\",\n \"utimensat_time64\",\n \"utimes\",\n \"vfork\",\n \"vmsplice\",\n \"wait4\",\n \"waitid\",\n \"waitpid\",\n \"write\",\n \"writev\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\"\n },\n {\n \"names\": [\n \"process_vm_readv\",\n \"process_vm_writev\",\n \"ptrace\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"minKernel\": \"4.8\"\n }\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 38,\n \"op\": \"SCMP_CMP_LT\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 39,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 41,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 42,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 43,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 44,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"socket\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 45,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"personality\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 0,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"personality\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 8,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"personality\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 131072,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"personality\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 131080,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"personality\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 4294967295,\n \"op\": \"SCMP_CMP_EQ\"\n }\n ]\n },\n {\n \"names\": [\n \"sync_file_range2\",\n \"swapcontext\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"ppc64le\"\n ]\n }\n },\n {\n \"names\": [\n \"arm_fadvise64_64\",\n \"arm_sync_file_range\",\n \"sync_file_range2\",\n \"breakpoint\",\n \"cacheflush\",\n \"set_tls\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"arm\",\n \"arm64\"\n ]\n }\n },\n {\n \"names\": [\n \"arch_prctl\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"amd64\",\n \"x32\"\n ]\n }\n },\n {\n \"names\": [\n \"modify_ldt\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"amd64\",\n \"x32\",\n \"x86\"\n ]\n }\n },\n {\n \"names\": [\n \"s390_pci_mmio_read\",\n \"s390_pci_mmio_write\",\n \"s390_runtime_instr\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"s390\",\n \"s390x\"\n ]\n }\n },\n {\n \"names\": [\n \"riscv_flush_icache\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"arches\": [\n \"riscv64\"\n ]\n }\n },\n {\n \"names\": [\n \"open_by_handle_at\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_DAC_READ_SEARCH\"\n ]\n }\n },\n {\n \"names\": [\n \"bpf\",\n \"clone\",\n \"clone3\",\n \"fanotify_init\",\n \"fsconfig\",\n \"fsmount\",\n \"fsopen\",\n \"fspick\",\n \"lookup_dcookie\",\n \"lsm_get_self_attr\",\n \"lsm_list_modules\",\n \"lsm_set_self_attr\",\n \"mount\",\n \"mount_setattr\",\n \"move_mount\",\n \"open_tree\",\n \"perf_event_open\",\n \"quotactl\",\n \"quotactl_fd\",\n \"setdomainname\",\n \"sethostname\",\n \"setns\",\n \"syslog\",\n \"umount\",\n \"umount2\",\n \"unshare\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_ADMIN\"\n ]\n }\n },\n {\n \"names\": [\n \"clone\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 0,\n \"value\": 2114060288,\n \"op\": \"SCMP_CMP_MASKED_EQ\"\n }\n ],\n \"excludes\": {\n \"caps\": [\n \"CAP_SYS_ADMIN\"\n ],\n \"arches\": [\n \"s390\",\n \"s390x\"\n ]\n }\n },\n {\n \"names\": [\n \"clone\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"args\": [\n {\n \"index\": 1,\n \"value\": 2114060288,\n \"op\": \"SCMP_CMP_MASKED_EQ\"\n }\n ],\n \"comment\": \"s390 parameter ordering for clone is different\",\n \"includes\": {\n \"arches\": [\n \"s390\",\n \"s390x\"\n ]\n },\n \"excludes\": {\n \"caps\": [\n \"CAP_SYS_ADMIN\"\n ]\n }\n },\n {\n \"names\": [\n \"reboot\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_BOOT\"\n ]\n }\n },\n {\n \"names\": [\n \"chroot\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_CHROOT\"\n ]\n }\n },\n {\n \"names\": [\n \"delete_module\",\n \"init_module\",\n \"finit_module\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_MODULE\"\n ]\n }\n },\n {\n \"names\": [\n \"acct\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_PACCT\"\n ]\n }\n },\n {\n \"names\": [\n \"kcmp\",\n \"pidfd_getfd\",\n \"process_madvise\",\n \"process_vm_readv\",\n \"process_vm_writev\",\n \"ptrace\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_PTRACE\"\n ]\n }\n },\n {\n \"names\": [\n \"iopl\",\n \"ioperm\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_RAWIO\"\n ]\n }\n },\n {\n \"names\": [\n \"settimeofday\",\n \"stime\",\n \"clock_settime\",\n \"clock_settime64\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_TIME\"\n ]\n }\n },\n {\n \"names\": [\n \"vhangup\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_TTY_CONFIG\"\n ]\n }\n },\n {\n \"names\": [\n \"get_mempolicy\",\n \"mbind\",\n \"set_mempolicy\",\n \"set_mempolicy_home_node\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYS_NICE\"\n ]\n }\n },\n {\n \"names\": [\n \"syslog\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_SYSLOG\"\n ]\n }\n },\n {\n \"names\": [\n \"bpf\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_BPF\"\n ]\n }\n },\n {\n \"names\": [\n \"perf_event_open\"\n ],\n \"action\": \"SCMP_ACT_ALLOW\",\n \"includes\": {\n \"caps\": [\n \"CAP_PERFMON\"\n ]\n }\n }\n ]\n}"; From 3d1d3cd3a446682b5aff2a0b2f2c64132d043167 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:19:03 +0200 Subject: [PATCH 25/84] feat: dedupe Daimon usage rows by turn and surface broker limit, model, and estimated usage fields --- src/cli/usageCommand.test.ts | 17 +++++ src/cli/usageCommand.ts | 9 ++- src/runtime/AGENTS.md | 12 ++++ src/runtime/index.ts | 1 + src/runtime/usageLedger.test.ts | 35 +++++++++++ src/runtime/usageLedger.ts | 76 +++++++++++++++++++--- src/runtime/usageLedgerRead.test.ts | 8 +++ src/runtime/usageLedgerRead.ts | 6 +- src/runtime/usageRequestLedger.test.ts | 40 ++++++++++++ src/runtime/usageRequestLedger.ts | 87 ++++++++++++++++++++++++++ 10 files changed, 280 insertions(+), 11 deletions(-) create mode 100644 src/runtime/usageRequestLedger.test.ts create mode 100644 src/runtime/usageRequestLedger.ts diff --git a/src/cli/usageCommand.test.ts b/src/cli/usageCommand.test.ts index d4354b8f..8ced1fc6 100644 --- a/src/cli/usageCommand.test.ts +++ b/src/cli/usageCommand.test.ts @@ -170,6 +170,23 @@ describe("spawnfile usage", () => { expect(result.output).toMatch(/^agy\s+\S*\s*1\s+45\.4k\s+—/mu); }); + it("counts a replayed broker turn once and marks estimated usage distinctly", async () => { + const turn = "a".repeat(64); + const sealed = line({ turn, limit_reason: "none", model: "grok-4.6", total: 100_000, notional_usd: 1, estimated_requests: 2, outcome: "completed" }); + const result = await executeUsageCommand("/tmp/project", { json: true }, handlersFor({ + [DAIMON_GROK_TURN_USAGE_LEDGER.rotatedFilePath]: `${sealed}\n`, + [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${sealed}\n${sealed}\n${line({ agent: "foreman", wake: "w2", total: 5_000, turn: "b".repeat(64) })}\n` + })); + const rendered = JSON.parse(result.output!); + expect(rendered.byEngine).toEqual([expect.objectContaining({ engine: "grok", estimatedRequests: 2, estimatedTurns: 1, tokens: 105_000, turns: 2 })]); + expect(rendered.coverage).toMatchObject({ estimatedTurnCount: 1 }); + const table = await executeUsageCommand("/tmp/project", {}, handlersFor({ + [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${sealed}\n${sealed}\n` + })); + expect(table.output).toMatch(/^cogsworth\s+grok\s+1\s+~100\.0k/mu); + expect(table.output).toContain("~ 1 turn(s) include ESTIMATED usage: 2 request(s) returned no provider-reported usage"); + }); + it("counts an all-zero turn as unknown rather than free", async () => { const result = await executeUsageCommand("/tmp/project", {}, handlersFor({ [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${line({ complete: false, input: 0, output: 0, cache_read: 0, cache_write: 0, total: 0 })}\n` diff --git a/src/cli/usageCommand.ts b/src/cli/usageCommand.ts index 18193321..6a77d41a 100644 --- a/src/cli/usageCommand.ts +++ b/src/cli/usageCommand.ts @@ -10,6 +10,7 @@ import { readUsageLedgerViaExec, type UsageLedgerExec } from "../runtime/usageLe import { DEFAULT_OUTPUT_DIRECTORY, errorExitCode } from "../shared/index.js"; import { computeUsageCoverage, + dedupeUsageRecordsByTurn, DEFAULT_USAGE_SINCE, filterUsageRecordsSince, groupUsageByAgent, @@ -109,7 +110,7 @@ const renderTable = ( ): string => { const groupBy = options.by ?? "agent"; const coverage = computeUsageCoverage(windowed, usage.roster.length, usage.unreadableUnits.length); - const totalTokens = windowed.reduce((sum, record) => sum + record.total, 0); + const totalTokens = dedupeUsageRecordsByTurn(windowed).reduce((sum, record) => sum + record.total, 0); const lines: string[] = []; const coverageLabel = coverage.partial @@ -128,7 +129,7 @@ const renderTable = ( const width = Math.max(8, ...rows.map((row) => row.agent.length)); lines.push(`${pad("agent", width)} ${pad("engine", 8)}${padStart("turns", 7)}${padStart("tokens", 9)}${padStart("notional", 11)}${padStart("share", 7)}`); for (const row of rows) { - lines.push(`${pad(row.agent, width)} ${pad(row.engine ?? "—", 8)}${padStart(row.turns === 0 ? "—" : String(row.turns), 7)}${padStart(formatTokens(row.tokens), 9)}${padStart(formatUsd(row.notionalUsd, row.turns), 11)}${padStart(formatShare(row.tokens, totalTokens), 7)}`); + lines.push(`${pad(row.agent, width)} ${pad(row.engine ?? "—", 8)}${padStart(row.turns === 0 ? "—" : String(row.turns), 7)}${padStart(`${row.estimatedTurns > 0 ? "~" : ""}${formatTokens(row.tokens)}`, 9)}${padStart(formatUsd(row.notionalUsd, row.turns), 11)}${padStart(formatShare(row.tokens, totalTokens), 7)}`); } lines.push("─".repeat(width + 44)); } @@ -143,6 +144,10 @@ const renderTable = ( lines.push(""); lines.push("Counts are a lower bound: the engine stream carries no completeness marker."); + if (coverage.estimatedTurnCount > 0) { + const estimatedRequests = engineRows.reduce((sum, row) => sum + row.estimatedRequests, 0); + lines.push(`~ ${coverage.estimatedTurnCount} turn(s) include ESTIMATED usage: ${estimatedRequests} request(s) returned no provider-reported usage and were charged a conservative estimate, not a measurement.`); + } if (coverage.incompleteRecordCount > 0) { lines.push(`${coverage.incompleteRecordCount} turn(s) reported all-zero usage and are counted as unknown, not free.`); } diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index d94cba70..c8db663b 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -17,6 +17,7 @@ src/runtime/ ├── registry.ts # Bundled adapter registration and lookup ├── usageLedger.ts # Pure parser/aggregator for Daimon's per-turn usage ledger ├── usageLedgerRead.ts # Ledger read transport: `cat`s both generations through a caller-supplied exec and separates "absent" from "unreadable" +├── usageRequestLedger.ts # Pure parser for Daimon's per-request stream (`requests.jsonl`): timing, model, usage source, broker-turn dedupe ├── scheduleUtils.ts # Shared duration schedule helpers for runtime lowering ├── daimon/ # Public Daimon organization-host adapter ├── openclaw/ # OpenClaw adapter implementation @@ -63,3 +64,14 @@ or receipt environment overrides fail closed. With no identity path, - Adapters receive resolved nodes, not raw manifests. - Keep runtime-specific behavior isolated here. - Share only the adapter contract, not runtime-specific implementation details. + +Daimon's Grok broker is the single sealed usage writer, and a replayed turn may +re-append its sealed rows. `usageLedger.ts` therefore dedupes every +`turn-usage.v1` row by its `turn` key (`dedupeUsageRecordsByTurn`) in parsing, +across both ledger generations (`usageLedgerRead.ts`), and inside every +aggregate; rows without a key are kept. It reads the broker's additive fields +only from Daimon's closed vocabularies — `limit_reason`, `model`, `outcome`, +`estimated_requests` — dropping a malformed optional field rather than the +row's spend. Rows with `estimated_requests` carry a conservative charge for +requests whose provider response had no valid usage; `spawnfile usage` marks +them `~` and says so, never presenting them as measured. diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 32fa4b88..90b8e20a 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -8,3 +8,4 @@ export * from "./statusProbes.js"; export * from "./types.js"; export * from "./usageLedger.js"; export * from "./usageLedgerRead.js"; +export * from "./usageRequestLedger.js"; diff --git a/src/runtime/usageLedger.test.ts b/src/runtime/usageLedger.test.ts index 83cdcbef..a0bc128d 100644 --- a/src/runtime/usageLedger.test.ts +++ b/src/runtime/usageLedger.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { computeUsageCoverage, + dedupeUsageRecordsByTurn, DEFAULT_USAGE_SINCE, filterUsageRecordsSince, groupUsageByAgent, @@ -32,6 +33,29 @@ const record = (overrides: Partial = {}): UsageRecord => ({ const line = (overrides: Partial = {}): string => JSON.stringify(record(overrides)); +describe("broker usage rows", () => { + const turn = "c".repeat(64); + + it("accepts turn, limit_reason, model, estimated_requests, and outcome from Daimon's closed vocabularies", () => { + expect(parseUsageLedgerLine(line({ estimated_requests: 3, limit_reason: "tokens", model: "grok-4.6", outcome: "failed", turn }))) + .toEqual(record({ estimated_requests: 3, limit_reason: "tokens", model: "grok-4.6", outcome: "failed", turn })); + const malformed = JSON.stringify({ ...record(), estimated_requests: 0, limit_reason: "budget", model: "gpt-5", outcome: "maybe", turn: "not-hex" }); + // A malformed optional field is dropped, never the row's spend. + expect(parseUsageLedgerLine(malformed)).toEqual(record()); + }); + + it("dedupes a replayed turn in parsing and in every aggregate, keeping unkeyed rows", () => { + const text = [line({ total: 10, turn }), line({ total: 10, turn }), line({ total: 5 }), line({ total: 5 })].join("\n"); + const parsed = parseUsageLedger(text); + expect(parsed.map((entry) => entry.total)).toEqual([10, 5, 5]); + const duplicated = [record({ total: 10, turn }), record({ total: 10, turn })]; + expect(dedupeUsageRecordsByTurn(duplicated)).toHaveLength(1); + expect(groupUsageByAgent(duplicated)[0]).toMatchObject({ tokens: 10, turns: 1 }); + expect(groupUsageByEngine(duplicated)[0]).toMatchObject({ tokens: 10, turns: 1 }); + expect(computeUsageCoverage([record({ complete: false, turn }), record({ complete: false, turn })], 1).incompleteRecordCount).toBe(1); + }); +}); + describe("parseUsageLedgerLine", () => { it("parses a well-formed line", () => { expect(parseUsageLedgerLine(line())).toEqual(record()); @@ -164,6 +188,8 @@ describe("groupUsageByAgent", () => { expect(cogsworth).toEqual({ agent: "cogsworth", engine: "grok", + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 1, notionalUsd: 3, tokens: 300, @@ -172,6 +198,8 @@ describe("groupUsageByAgent", () => { expect(groups.find((g) => g.agent === "foreman")).toEqual({ agent: "foreman", engine: "grok", + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 3, tokens: 300, @@ -189,6 +217,8 @@ describe("groupUsageByAgent", () => { expect(groups.find((g) => g.agent === "brass")).toEqual({ agent: "brass", engine: "codex", + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 0, tokens: 0, @@ -208,6 +238,8 @@ describe("groupUsageByEngine", () => { expect(groups.find((g) => g.engine === "grok")).toEqual({ engine: "grok", + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 11.1, tokens: 3_500_000, @@ -215,6 +247,8 @@ describe("groupUsageByEngine", () => { }); expect(groups.find((g) => g.engine === "codex")).toEqual({ engine: "codex", + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 0, tokens: 0, @@ -231,6 +265,7 @@ describe("computeUsageCoverage", () => { expect(coverage).toEqual({ agentsReporting: 2, agentsTotal: 16, + estimatedTurnCount: 0, incompleteRecordCount: 0, partial: true, unreadableUnitCount: 0 diff --git a/src/runtime/usageLedger.ts b/src/runtime/usageLedger.ts index 079cef35..00df4442 100644 --- a/src/runtime/usageLedger.ts +++ b/src/runtime/usageLedger.ts @@ -34,8 +34,52 @@ export interface UsageRecord { total: number; v: typeof USAGE_TURN_RECORD_VERSION; wake: string; + /** Broker rows: the sha256 turn id every reader dedupes on (a replay may re-append a sealed row). */ + turn?: string; + /** Broker rows: which per-turn limit ended the turn (`none` when none did). */ + limit_reason?: UsageLimitReason; + /** Broker rows: the declared model the broker verified the turn ran. */ + model?: string; + /** Broker rows: requests charged a conservative estimate because the provider response carried no valid usage. */ + estimated_requests?: number; + outcome?: "completed" | "failed"; } +export const USAGE_LIMIT_REASONS = ["tokens", "requests", "timeout", "none"] as const; +export type UsageLimitReason = typeof USAGE_LIMIT_REASONS[number]; +const USAGE_MODELS = ["grok-4.6", "grok-4.5", "grok-build"] as const; + +/** + * The additive broker fields of a `turn-usage.v1` row. Each is copied only when + * it is a member of Daimon's closed vocabulary; a malformed optional field is + * dropped rather than discarding the row's spend. + */ +const brokerFields = (record: Record): Partial => ({ + ...(typeof record.turn === "string" && /^[a-f0-9]{64}$/u.test(record.turn) ? { turn: record.turn } : {}), + ...((USAGE_LIMIT_REASONS as readonly unknown[]).includes(record.limit_reason) ? { limit_reason: record.limit_reason as UsageLimitReason } : {}), + ...((USAGE_MODELS as readonly unknown[]).includes(record.model) ? { model: record.model as string } : {}), + ...(Number.isSafeInteger(record.estimated_requests) && (record.estimated_requests as number) > 0 ? { estimated_requests: record.estimated_requests as number } : {}), + ...(record.outcome === "completed" || record.outcome === "failed" ? { outcome: record.outcome } : {}) +}); + +/** + * Keeps the first row of every `turn` and every row without one. + * + * Daimon's broker seals a turn's ledger bytes into the turn record and appends + * them again on replay when it cannot see the row, so two replays of one sealed + * turn can both append identical bytes. The key exists precisely so such a turn + * is never counted twice; every aggregate here must see deduplicated rows. + */ +export const dedupeUsageRecordsByTurn = (records: UsageRecord[]): UsageRecord[] => { + const seen = new Set(); + return records.filter((record) => { + if (record.turn === undefined) return true; + if (seen.has(record.turn)) return false; + seen.add(record.turn); + return true; + }); +}; + const NUMERIC_FIELDS = [ "input", "output", @@ -115,7 +159,8 @@ export const parseUsageLedgerLine = (line: string): UsageRecord | null => { output: record.output as number, total: record.total as number, v: USAGE_TURN_RECORD_VERSION, - wake: record.wake + wake: record.wake, + ...brokerFields(record) }; }; @@ -125,10 +170,10 @@ export const parseUsageLedgerLine = (line: string): UsageRecord | null => { * skipped, and the rest of the file still parses. */ export const parseUsageLedger = (text: string): UsageRecord[] => - text + dedupeUsageRecordsByTurn(text .split("\n") .map(parseUsageLedgerLine) - .filter((record): record is UsageRecord => record !== null); + .filter((record): record is UsageRecord => record !== null)); const DURATION_UNIT_MS: Record = { d: 24 * 60 * 60 * 1000, @@ -180,6 +225,9 @@ export interface UsageRosterEntry { export interface UsageAgentGroup { agent: string; engine: string | null; + /** Requests charged an estimate (no provider-reported usage); their tokens are a conservative charge, not a measurement. */ + estimatedRequests: number; + estimatedTurns: number; incompleteTurns: number; notionalUsd: number; tokens: number; @@ -189,6 +237,8 @@ export interface UsageAgentGroup { const emptyAgentGroup = (agent: string, engine: string | null): UsageAgentGroup => ({ agent, engine, + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 0, tokens: 0, @@ -206,9 +256,11 @@ export const groupUsageByAgent = ( for (const entry of roster) { byAgent.set(entry.agent, emptyAgentGroup(entry.agent, entry.engine)); } - for (const record of records) { + for (const record of dedupeUsageRecordsByTurn(records)) { const existing = byAgent.get(record.agent) ?? emptyAgentGroup(record.agent, record.engine); existing.turns += 1; + existing.estimatedRequests += record.estimated_requests ?? 0; + if (record.estimated_requests !== undefined) existing.estimatedTurns += 1; existing.tokens += record.total; existing.notionalUsd += record.notional_usd; if (!record.complete) { @@ -224,6 +276,8 @@ export const groupUsageByAgent = ( export interface UsageEngineGroup { engine: string; + estimatedRequests: number; + estimatedTurns: number; incompleteTurns: number; notionalUsd: number; tokens: number; @@ -232,6 +286,8 @@ export interface UsageEngineGroup { const emptyEngineGroup = (engine: string): UsageEngineGroup => ({ engine, + estimatedRequests: 0, + estimatedTurns: 0, incompleteTurns: 0, notionalUsd: 0, tokens: 0, @@ -249,9 +305,11 @@ export const groupUsageByEngine = ( for (const engine of engines) { byEngine.set(engine, emptyEngineGroup(engine)); } - for (const record of records) { + for (const record of dedupeUsageRecordsByTurn(records)) { const existing = byEngine.get(record.engine) ?? emptyEngineGroup(record.engine); existing.turns += 1; + existing.estimatedRequests += record.estimated_requests ?? 0; + if (record.estimated_requests !== undefined) existing.estimatedTurns += 1; existing.tokens += record.total; existing.notionalUsd += record.notional_usd; if (!record.complete) { @@ -265,6 +323,8 @@ export const groupUsageByEngine = ( export interface UsageCoverage { agentsReporting: number; agentsTotal: number; + /** Turns whose total includes an estimated charge for at least one request. */ + estimatedTurnCount: number; /** Count of `complete:false` records in the window — every count here is a * lower bound regardless of this number (see module doc / design * "Verification" — grok's `streaming-messages-json` carries no @@ -291,11 +351,13 @@ export const computeUsageCoverage = ( totalAgents: number, unreadableUnitCount = 0 ): UsageCoverage => { - const reporting = new Set(records.map((record) => record.agent)).size; + const unique = dedupeUsageRecordsByTurn(records); + const reporting = new Set(unique.map((record) => record.agent)).size; return { agentsReporting: reporting, agentsTotal: totalAgents, - incompleteRecordCount: records.filter((record) => !record.complete).length, + estimatedTurnCount: unique.filter((record) => record.estimated_requests !== undefined).length, + incompleteRecordCount: unique.filter((record) => !record.complete).length, partial: reporting < totalAgents || unreadableUnitCount > 0, unreadableUnitCount }; diff --git a/src/runtime/usageLedgerRead.test.ts b/src/runtime/usageLedgerRead.test.ts index a97600f3..1562abd4 100644 --- a/src/runtime/usageLedgerRead.test.ts +++ b/src/runtime/usageLedgerRead.test.ts @@ -28,6 +28,14 @@ describe("readUsageLedgerViaExec", () => { rotatedFilePath: "/var/lib/spawnfile/daimon/usage/usage.jsonl.1" }; + it("counts a broker turn re-appended across a rotation exactly once", async () => { + const sealed = line({ turn: "d".repeat(64) }); + const exec = async (command: string[]) => ({ stderr: "", stdout: `${sealed}\n` }); + const read = await readUsageLedgerViaExec(exec, paths); + expect(read.records).toHaveLength(1); + expect(read.unreadable).toEqual([]); + }); + it("merges both generations, rotated (older) first", async () => { const exec = async (command: string[]) => { const target = command[1]; diff --git a/src/runtime/usageLedgerRead.ts b/src/runtime/usageLedgerRead.ts index fbe067bb..d084cb0e 100644 --- a/src/runtime/usageLedgerRead.ts +++ b/src/runtime/usageLedgerRead.ts @@ -12,7 +12,7 @@ * real exports from this folder, and so must not be imported back. */ -import { parseUsageLedger, type UsageRecord } from "./usageLedger.js"; +import { dedupeUsageRecordsByTurn, parseUsageLedger, type UsageRecord } from "./usageLedger.js"; /** The exec shape this module needs to read a ledger — structurally the same * as `RuntimeProbeGateway.exec` (`./types.ts`), duck-typed here rather than @@ -131,7 +131,9 @@ export const readUsageLedgerViaExec = async ( readLedgerGeneration(exec, paths.filePath) ]); return { - records: [...parseUsageLedger(rotated.text), ...parseUsageLedger(primary.text)], + // A replayed broker turn can re-append its sealed row after a rotation, so + // dedupe across both generations, not only within each. + records: dedupeUsageRecordsByTurn([...parseUsageLedger(rotated.text), ...parseUsageLedger(primary.text)]), unreadable: [rotated.failure, primary.failure].filter( (failure): failure is UsageLedgerReadFailure => failure !== undefined ) diff --git a/src/runtime/usageRequestLedger.test.ts b/src/runtime/usageRequestLedger.test.ts new file mode 100644 index 00000000..c5f0522c --- /dev/null +++ b/src/runtime/usageRequestLedger.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; + +import { parseUsageRequestLedger, parseUsageRequestLedgerLine } from "./usageRequestLedger.js"; + +const turn = "e".repeat(64); +const grokRow = (overrides: Record = {}) => JSON.stringify({ + v: "noopolis.daimon.turn-requests.v1", agent: "foreman", wake: "wake-1", engine: "grok", at: "2026-09-17T10:00:03.000Z", + turn, model: "grok-4.6", request: 0, requests: 2, input: 2_800, cached_input: 2_400, fresh_input: 400, cache_write: 0, + output: 60, total: 2_860, usage_source: "stream", started_at: "2026-09-17T10:00:01.120Z", ended_at: "2026-09-17T10:00:02.004Z", + ...overrides +}); + +describe("per-request usage stream", () => { + it("accepts Grok broker rows with timing, model, and usage source, and Codex rows without them", () => { + expect(parseUsageRequestLedgerLine(grokRow())).toMatchObject({ + ended_at: "2026-09-17T10:00:02.004Z", model: "grok-4.6", started_at: "2026-09-17T10:00:01.120Z", turn, usage_source: "stream" + }); + expect(parseUsageRequestLedgerLine(grokRow({ usage_source: "estimated" }))).toMatchObject({ usage_source: "estimated" }); + const codex = JSON.stringify({ + v: "noopolis.daimon.turn-requests.v1", agent: "desk", wake: "w", engine: "codex", at: "2026-09-17T10:00:00.000Z", thread: "t-1", + request: 1, requests: 3, input: 30_000, cached_input: 28_000, fresh_input: 2_000, cache_write: 0, output: 200, reasoning: 50, total: 30_200 + }); + const parsed = parseUsageRequestLedgerLine(codex)!; + expect(parsed).toMatchObject({ engine: "codex", reasoning: 50, thread: "t-1" }); + expect(parsed.started_at).toBeUndefined(); + expect(parsed.usage_source).toBeUndefined(); + }); + + it("rejects malformed rows rather than inventing values", () => { + for (const bad of [ + grokRow({ usage_source: "guessed" }), grokRow({ started_at: "yesterday" }), grokRow({ turn: "short" }), + grokRow({ input: -1 }), grokRow({ requests: 0 }), grokRow({ v: "noopolis.daimon.turn-usage.v1" }), "{torn" + ]) expect(parseUsageRequestLedgerLine(bad)).toBeNull(); + }); + + it("counts a replayed broker turn's request rows once", () => { + const content = [grokRow(), grokRow(), grokRow({ request: 1 }), grokRow({ request: 1 }), ""].join("\n"); + expect(parseUsageRequestLedger(content).map((row) => row.request)).toEqual([0, 1]); + }); +}); diff --git a/src/runtime/usageRequestLedger.ts b/src/runtime/usageRequestLedger.ts new file mode 100644 index 00000000..d5f4b4e7 --- /dev/null +++ b/src/runtime/usageRequestLedger.ts @@ -0,0 +1,87 @@ +/** + * Pure parser for Daimon's per-model-request stream + * (`noopolis.daimon.turn-requests.v1`, `requests.jsonl` beside the usage + * ledger). Codex rows carry a `thread`; Grok broker rows carry the broker + * `turn`, the declared `model`, and `usage_source` (`stream`, `upstream`, or + * `estimated` — a conservative charge for a response without valid usage). + * Either may carry proxy- or rollout-measured `started_at`/`ended_at`, absent + * rather than substituted when not measured. No I/O happens here. + */ + +export const USAGE_REQUEST_RECORD_VERSION = "noopolis.daimon.turn-requests.v1" as const; +export const USAGE_REQUEST_SOURCES = ["stream", "upstream", "estimated"] as const; +export type UsageRequestSource = typeof USAGE_REQUEST_SOURCES[number]; + +export interface UsageRequestRecord { + agent: string; + at: string; + cache_write: number; + cached_input: number; + ended_at?: string; + engine: string; + fresh_input: number; + input: number; + model?: string; + output: number; + reasoning?: number; + request: number; + requests: number; + started_at?: string; + thread?: string; + total: number; + turn?: string; + usage_source?: UsageRequestSource; + v: typeof USAGE_REQUEST_RECORD_VERSION; + wake: string; +} + +const TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u; +const COUNTS = ["input", "cached_input", "fresh_input", "cache_write", "output", "total"] as const; +const nonNegative = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0; +const text = (value: unknown): value is string => typeof value === "string" && value.length > 0; + +/** One row, or `null` for a blank, torn, foreign-version, or malformed line. Never throws. */ +export const parseUsageRequestLedgerLine = (line: string): UsageRequestRecord | null => { + let parsed: unknown; + try { parsed = JSON.parse(line.trim()); } catch { return null; } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + const row = parsed as Record; + if (row.v !== USAGE_REQUEST_RECORD_VERSION || !text(row.agent) || !text(row.wake) || !text(row.engine) || !text(row.at) || Number.isNaN(Date.parse(row.at))) return null; + if (!Number.isSafeInteger(row.request) || !Number.isSafeInteger(row.requests) || (row.request as number) < 0 || (row.requests as number) < 1) return null; + if (COUNTS.some((field) => !nonNegative(row[field]))) return null; + if ((row.started_at !== undefined && !(text(row.started_at) && TIMESTAMP.test(row.started_at))) + || (row.ended_at !== undefined && !(text(row.ended_at) && TIMESTAMP.test(row.ended_at))) + || (row.usage_source !== undefined && !(USAGE_REQUEST_SOURCES as readonly unknown[]).includes(row.usage_source)) + || (row.turn !== undefined && !(typeof row.turn === "string" && /^[a-f0-9]{64}$/u.test(row.turn))) + || (row.reasoning !== undefined && !nonNegative(row.reasoning))) return null; + return { + agent: row.agent, at: row.at, cache_write: row.cache_write as number, cached_input: row.cached_input as number, + engine: row.engine, fresh_input: row.fresh_input as number, input: row.input as number, output: row.output as number, + request: row.request as number, requests: row.requests as number, total: row.total as number, + v: USAGE_REQUEST_RECORD_VERSION, wake: row.wake, + ...(row.started_at === undefined ? {} : { started_at: row.started_at as string }), + ...(row.ended_at === undefined ? {} : { ended_at: row.ended_at as string }), + ...(text(row.model) ? { model: row.model } : {}), + ...(row.reasoning === undefined ? {} : { reasoning: row.reasoning as number }), + ...(text(row.thread) ? { thread: row.thread } : {}), + ...(row.turn === undefined ? {} : { turn: row.turn as string }), + ...(row.usage_source === undefined ? {} : { usage_source: row.usage_source as UsageRequestSource }) + }; +}; + +/** + * Parses a whole stream, keeping the first row of each broker `(turn, request)` + * pair: a replayed broker turn re-appends its sealed request rows exactly as it + * re-appends its usage row. + */ +export const parseUsageRequestLedger = (content: string): UsageRequestRecord[] => { + const seen = new Set(); + return content.split("\n").map(parseUsageRequestLedgerLine).filter((row): row is UsageRequestRecord => { + if (row === null) return false; + if (row.turn === undefined) return true; + const key = `${row.turn}:${row.request}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +}; From 211b75479f9739ad32bbcb2f47e283b77988dd59 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:19:44 +0200 Subject: [PATCH 26/84] docs: describe Grok 1.0.34 brokered workers, container security, and deduped usage accounting --- specs/CONTAINERS.md | 26 ++++++++++-- specs/RUNTIMES.md | 70 +++++++++++++++++++++++++++++++- specs/USAGE_ACCOUNTING_DESIGN.md | 31 ++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/specs/CONTAINERS.md b/specs/CONTAINERS.md index 1097aa55..df9dca74 100644 --- a/specs/CONTAINERS.md +++ b/specs/CONTAINERS.md @@ -161,10 +161,28 @@ official AGY artifact exists for another architecture, this seam accepts only `linux/amd64` and fails closed elsewhere. The helper requires explicit `AGY_CLI_VERSION`, `AGY_CLI_URL`, -`AGY_CLI_SHA512`, and extracted `AGY_CLI_SHA256` pins. It preserves the Grok -artifact's `GROK_CLI_URL`/`GROK_CLI_SHA256` pin and additionally requires -`GROK_CLI_VERSION` for provenance. `CODEX_CLI_SHA256` remains required. All -artifact URLs must be credential-free HTTPS URLs without query or fragment. +`AGY_CLI_SHA512`, and extracted `AGY_CLI_SHA256` pins. `GROK_CLI_VERSION`, +`GROK_CLI_URL`, and `GROK_CLI_SHA256` must equal the Grok CLI build the vendored +Daimon contract manifest pins for the target architecture (1.0.34: +`grok-1.0.34-linux-aarch64` `39ab8766…c4a94`, `grok-1.0.34-linux-x86_64` +`be5905e1…c80d`); the image build re-checks them against the packaged manifest, +derives the native broker digest from it rather than a literal, and generated +organization images re-verify `/usr/local/bin/grok` against the same pin. +`CODEX_CLI_SHA256` remains required. All artifact URLs must be credential-free +HTTPS URLs without query or fragment. + +`npm run vendor:daimon-contract` (optionally `-- --check`) is the only way the +Daimon contract enters Spawnfile: it reads a Daimon checkout +(`SPAWNFILE_DAIMON_SOURCE_DIR`) and writes the canonical contract manifest and +digest plus Daimon's worker `config.toml` renderer bytes. + +Daimon organizations containing a Grok agent run with +`--security-opt=seccomp=` and +`--security-opt=apparmor=unconfined`; the profile ships with Spawnfile and is +materialized for each `docker run`. Codex's fully unconfined options are used +instead only when a strict Codex agent is present. The Docker host must have +`kernel.apparmor_restrict_unprivileged_userns=0`; the container entrypoint +fails with that instruction otherwise. OpenClaw and PicoClaw have equivalent overrides: diff --git a/specs/RUNTIMES.md b/specs/RUNTIMES.md index 550028f3..767b23ad 100644 --- a/specs/RUNTIMES.md +++ b/specs/RUNTIMES.md @@ -200,12 +200,78 @@ The consumed Daimon manifest declares one per-agent opaque slot for Codex. For Grok it declares one durable rotating-credential realm and one read-only operator bootstrap slot; Daimon serializes Grok turns through that authority, atomically reconciles provider rotation, and leaves sessions/cache per agent. -The generated Linux container installs `bubblewrap`, which Grok requires to -fail closed while applying the realm and peer-home deny set. +The generated Linux container installs `bubblewrap` and `ca-certificates`: +Grok CLI 1.0.34 runs every sandbox profile inside bubblewrap (even an empty +deny list), and its HTTP MCP client refuses to build without CA certificates +even for Daimon's loopback `http://` endpoint. The realm mount is `exclusive-reattach`: its host-stable volume survives run and deployment identities, cannot be attached by two live deployments, and is never copied by product-state migration. Standard concurrent canary is rejected; stop the old deployment and reattach the same realm for replacement. +### Daimon brokered Grok workers (Grok CLI 1.0.34) + +Every Daimon Grok agent runs through Daimon's engine broker as a lean worker +pinned to Grok CLI 1.0.34. Its model is declared, never inherited: + +```yaml +runtime: + name: daimon + options: + engine: grok +execution: + model: + primary: + provider: xai + name: grok-4.6 # grok-4.6 | grok-4.5 | grok-build + auth: + method: grok + reasoning_effort: low # low | medium | high +``` + +Both fields are required together, target-level, with no fallback; they lower +to Daimon's `engine.model`/`engine.reasoningEffort`. `grok` auth is only valid +for provider `xai` and is Daimon-owned (never a host import or an api-key +secret); `reasoning_effort` is only valid with it. Codex and AGY are unchanged. + +The worker `config.toml` is Daimon's own renderer output for that model x +effort, vendored and refused unless it hashes to the contract manifest pin the +broker attests every turn. Each worker's `GROK_HOME` is root-owned `1771` with +root-owned `0444` `config.toml`, `sandbox.toml`, `trusted_folders.toml`, +`managed_config.toml`, and `requirements.toml`, so the worker cannot change +model, trust, sandbox, or managed layers between turns; sandbox events are read +from `$GROK_HOME/sessions/sandbox-events.jsonl`. `service.json` is v2 with a +per-registration usage ledger (the container ledger in production), limits +(the manifest's v1 defaults), and model. + +On 1.0.13 a non-empty sandbox `deny` list made Grok refuse to start, so the +list was empty and unix modes were the only boundary. On 1.0.34 the deny list +is enforced inside bubblewrap for both shell and `read_file`, and it is +mandatory: Grok's strict base reads all of `/run`, `/var`, `/tmp`, and `/etc`, +and macOS bind mounts ignore unix modes. Each worker denies the Grok bootstrap +and realm, AGY realm and unlock secret when present, the wake-acceptance store, +every peer agent's runtime home and workspace, every other worker's home, the +broker's `/etc` and `/run` directories, the usage-ledger and wake-fuse volumes, +and `/run/secrets`, `/run/spawnfile`, `/run/spawnfile-secrets`. Every deny +entry and registration path must be canonical (present, not a symlink, its own +realpath) at provisioning or the container refuses to start. + +Workspace skills are not emitted for Grok agents: the worker workspace stays +untrusted and Daimon overrides the system prompt, so Grok loads neither +`.agents/skills` nor `.codex/skills`; a declared skill produces a compile +warning. + +Grok containers run with Docker's default seccomp profile plus bubblewrap's +namespace syscalls (a pinned profile) and `apparmor=unconfined` unless a strict +Codex agent already requires both fully unconfined. The Docker host must allow +unprivileged user namespaces: set `kernel.apparmor_restrict_unprivileged_userns=0` +(Ubuntu 24.04+ and Colima default to `1`). The entrypoint refuses to start a +Grok organization, naming the sysctl, when it is not. + +Not yet provided: a private per-worker `/tmp` (the strict profile grants +read-write shared `/tmp` and `/var/tmp`, and the broker launcher's fixed +environment sets no `TMPDIR`), and worker-readable tool-result spill files +(Daimon writes them `0600` as the organization uid). + For AGY it declares one host-realm durable mount plus one independent opaque unlock source slot. Spawnfile emits the stable RW volume, metadata-authorizes the caller-owned `0600` unlock source, and mounts it read-only; it never reads diff --git a/specs/USAGE_ACCOUNTING_DESIGN.md b/specs/USAGE_ACCOUNTING_DESIGN.md index 68923447..e36d574d 100644 --- a/specs/USAGE_ACCOUNTING_DESIGN.md +++ b/specs/USAGE_ACCOUNTING_DESIGN.md @@ -269,6 +269,37 @@ usage-adjacent command is `du`. not rewritten as failed. Each must turn a test red. 7. **Regression** — targeted `src/pi` and `src/runtime`, then the full suite. +## Revision 4 — Grok 1.0.34 broker accounting (2026-09-17) + +The broker remains the single sealed writer. What changed for readers: + +- **Dedupe by `turn`.** A broker row carries `turn` (64-hex turn id). The broker + seals a turn's ledger bytes into its turn record and re-appends them on replay + when it cannot find the row, so two replays can append identical bytes. + `spawnfile usage` keeps the first row per `turn` — within each generation, + across `usage.jsonl.1` and `usage.jsonl`, and inside every aggregate. Rows + without a key are kept. +- **Additive fields.** `limit_reason` (`tokens`/`requests`/`timeout`/`none`), + `model` (the declared model the broker verified), `outcome`, and + `estimated_requests` stay inside the unchanged `turn-usage.v1` record. A + malformed optional field is dropped, never the row. +- **Estimated usage is shown as estimated.** A request whose response carried no + valid usage is charged `ceil(bodyBytes/2) + 4096` tokens; the row counts those + in `estimated_requests`. The table prefixes such tokens with `~` and states + how many turns and requests are estimates; JSON exposes `estimatedTurns`, + `estimatedRequests`, and `coverage.estimatedTurnCount`. +- **Per-request stream.** `requests.jsonl` Grok rows carry `turn`, `model`, + proxy-measured `started_at`/`ended_at`, and `usage_source` + (`stream`/`upstream`/`estimated`); `src/runtime/usageRequestLedger.ts` parses + them and dedupes by `(turn, request)`. +- **Ledger location.** `service.json` v2 names a `usageLedgerPath` per + registration. Production points every registration at the container ledger, + because both `spawnfile usage` and Daimon's wake fuse read only that file. +- **Deny list.** Grok 1.0.13 could not start with a non-empty sandbox deny list, + so the ledger directory's protection was its unix mode alone. On 1.0.34 the + ledger directory is an enforced deny entry for every worker, as originally + designed above. + ## Findings folded in (revision 2 → 3) Found independently by two reviewers, by tracing container machinery: From 4bbd016677cdbeee545bbcae7ee865c1b70d91c8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:21:23 +0200 Subject: [PATCH 27/84] test: add the opt-in live check for a one-agent brokered Grok 1.0.34 organization --- fixtures/grok-lean-worker/AGENTS.md | 3 + fixtures/grok-lean-worker/Spawnfile | 22 ++++++ package.json | 3 +- scripts/README.md | 1 + scripts/grok-lean-worker-live-check.test.ts | 14 ++++ scripts/grok-lean-worker-live-check.ts | 79 +++++++++++++++++++++ 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 fixtures/grok-lean-worker/AGENTS.md create mode 100644 fixtures/grok-lean-worker/Spawnfile create mode 100644 scripts/grok-lean-worker-live-check.test.ts create mode 100644 scripts/grok-lean-worker-live-check.ts diff --git a/fixtures/grok-lean-worker/AGENTS.md b/fixtures/grok-lean-worker/AGENTS.md new file mode 100644 index 00000000..253460a8 --- /dev/null +++ b/fixtures/grok-lean-worker/AGENTS.md @@ -0,0 +1,3 @@ +# Grok lean worker live check + +You are a test agent. When woken, answer with the single word OK and call no tools. diff --git a/fixtures/grok-lean-worker/Spawnfile b/fixtures/grok-lean-worker/Spawnfile new file mode 100644 index 00000000..3e83acbc --- /dev/null +++ b/fixtures/grok-lean-worker/Spawnfile @@ -0,0 +1,22 @@ +spawnfile_version: "0.1" +kind: agent +name: grok-lean-worker +description: "One brokered Daimon Grok agent for the Grok 1.0.34 lean-worker live check" + +runtime: + name: daimon + options: + engine: grok + +execution: + model: + primary: + provider: xai + name: grok-4.6 + auth: + method: grok + reasoning_effort: low + +workspace: + docs: + system: AGENTS.md diff --git a/package.json b/package.json index 791937c4..cc05d83e 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,8 @@ "test:e2e:distribution-image": "tsx src/e2e/cli.ts distribution-image", "test:e2e:distribution-roundtrip": "tsx src/e2e/cli.ts distribution-roundtrip", "test:e2e:daimon-org": "tsx src/e2e/cli.ts daimon-org", - "test:scripts": "node --experimental-strip-types --test scripts/build-local-daimon-runtime.test.ts scripts/build-local-moltnet.test.ts scripts/compile-explicit-test-mcp.test.ts scripts/native-helper-workflows.test.ts scripts/typescript-policy.test.ts", + "test:live:grok-lean-worker": "node --experimental-strip-types ./scripts/grok-lean-worker-live-check.ts", + "test:scripts": "node --experimental-strip-types --test scripts/build-local-daimon-runtime.test.ts scripts/grok-lean-worker-live-check.test.ts scripts/build-local-moltnet.test.ts scripts/compile-explicit-test-mcp.test.ts scripts/native-helper-workflows.test.ts scripts/typescript-policy.test.ts", "test:unit": "npm run test:vitest; vitest_status=$?; npm run test:verdict; verdict_status=$?; npm run test:coverage-verdict; coverage_status=$?; npm run test:node; node_status=$?; failed_lanes=''; if [ \"$verdict_status\" -eq 2 ]; then failed_lanes='vitest test failures'; fi; if [ \"$node_status\" -eq 2 ]; then if [ -n \"$failed_lanes\" ]; then failed_lanes=\"$failed_lanes; node:test suite\"; else failed_lanes='node:test suite'; fi; fi; result_status=0; if [ -n \"$failed_lanes\" ]; then echo \"FAIL(tests): $failed_lanes\"; result_status=1; fi; if [ \"$verdict_status\" -ne 0 ] && [ \"$verdict_status\" -ne 2 ]; then echo 'FAIL(vitest-verdict): could not determine the vitest result'; result_status=1; fi; if [ \"$node_status\" -ne 0 ] && [ \"$node_status\" -ne 2 ]; then echo 'FAIL(node-lane): could not determine the node:test result'; result_status=1; fi; if [ \"$result_status\" -ne 0 ]; then exit 1; elif [ \"$vitest_status\" -ne 0 ]; then echo \"FAIL(vitest): exited $vitest_status with no test failure\"; exit 1; elif [ \"$coverage_status\" -eq 2 ]; then echo 'FAIL(coverage): thresholds not met (tests all passed)'; exit 2; elif [ \"$coverage_status\" -ne 0 ]; then echo 'FAIL(coverage-verdict): could not evaluate coverage (tests all passed)'; exit 1; else echo 'PASS'; fi", "test": "npm run test:unit && npm run test:scripts" }, diff --git a/scripts/README.md b/scripts/README.md index 2d97ef33..a9c7df70 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,6 +8,7 @@ in the npm package. Scripts run as native TypeScript on Node 22.19+. | --- | --- | --- | | `build-local-daimon-runtime.ts` | `npm run build:local-daimon` | Build a locally sourced runtime; explicit artifact pins, Docker, and loopback registry required | | `vendor-daimon-grok-contract.ts` | `npm run vendor:daimon-contract [-- --check]` | Vendor Daimon's contract manifest and Grok worker renderer bytes from a Daimon checkout (`SPAWNFILE_DAIMON_SOURCE_DIR`) | +| `grok-lean-worker-live-check.ts` | `SPAWNFILE_GROK_LIVE_CHECK=1 npm run test:live:grok-lean-worker` | Opt-in live check: one brokered Grok 1.0.34 agent, worker-home/deny/service attestation dump, one cheap wake, one deduped broker usage row; Docker, a local Daimon identity, and a dedicated Grok login required | | `build-local-moltnet.ts` | `npm run build:local-moltnet` | Build and stamp release binaries from an explicit source checkout | | `create-source-provenance-bundle.ts` | `npm run bundle:source-provenance` | Create deterministic archives with manifests and credential exclusions | | `create-linux-amd64-dependency-closure.ts` | `npm run prepare:linux-amd64-closure` | Prepare reviewed npm dependencies/cache in the pinned build container | diff --git a/scripts/grok-lean-worker-live-check.test.ts b/scripts/grok-lean-worker-live-check.test.ts new file mode 100644 index 00000000..7e51300e --- /dev/null +++ b/scripts/grok-lean-worker-live-check.test.ts @@ -0,0 +1,14 @@ +import assert from "node:assert/strict"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { requireLiveCheckEnvironment } from "./grok-lean-worker-live-check.ts"; + +test("the Grok lean-worker live check is explicit opt-in and refuses the desktop Grok login", () => { + const env = { SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: "/tmp/identity.json", SPAWNFILE_DAIMON_SOURCE_GROK_AUTH: "/tmp/grok-training/auth.json", SPAWNFILE_GROK_LIVE_CHECK: "1" }; + assert.deepEqual(requireLiveCheckEnvironment(env), { grokAuth: "/tmp/grok-training/auth.json", identity: "/tmp/identity.json" }); + assert.throws(() => requireLiveCheckEnvironment({ ...env, SPAWNFILE_GROK_LIVE_CHECK: undefined }), /SPAWNFILE_GROK_LIVE_CHECK=1/u); + assert.throws(() => requireLiveCheckEnvironment({ ...env, SPAWNFILE_DAIMON_SOURCE_GROK_AUTH: path.join(os.homedir(), ".grok", "auth.json") }), /desktop/u); + assert.throws(() => requireLiveCheckEnvironment({ ...env, SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: "relative.json" }), /identity/u); +}); diff --git a/scripts/grok-lean-worker-live-check.ts b/scripts/grok-lean-worker-live-check.ts new file mode 100644 index 00000000..c19ec19f --- /dev/null +++ b/scripts/grok-lean-worker-live-check.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// Explicit opt-in live check for a one-agent brokered Grok 1.0.34 organization: +// deploy fixtures/grok-lean-worker against a locally built Daimon runtime image, +// dump the worker home/sandbox/service attestation inputs, run one cheap wake, +// and require exactly one deduplicated usage row written by the broker. +// Needs Docker, a built CLI (`npm run build`), a local Daimon runtime identity +// built from the Daimon Grok accounting contract, and a dedicated Grok login +// file (never the desktop ~/.grok/auth.json). Makes one real model call. + +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const deployment = "grok-lean-live"; +const workerHome = "/var/lib/daimon-workers/2200"; +const usageLedger = "/var/lib/spawnfile/daimon/usage/usage.jsonl"; + +export const requireLiveCheckEnvironment = (env: Record): { grokAuth: string; identity: string } => { + if (env.SPAWNFILE_GROK_LIVE_CHECK !== "1") throw new Error("Set SPAWNFILE_GROK_LIVE_CHECK=1 to run the live Grok lean-worker check (one real model call)"); + const identity = env.SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY?.trim(); + const grokAuth = env.SPAWNFILE_DAIMON_SOURCE_GROK_AUTH?.trim(); + if (!identity || !path.isAbsolute(identity)) throw new Error("SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY must name the local Daimon runtime identity file"); + if (!grokAuth || !path.isAbsolute(grokAuth)) throw new Error("SPAWNFILE_DAIMON_SOURCE_GROK_AUTH must name a dedicated 0600 Grok login file"); + if (path.resolve(grokAuth) === path.join(os.homedir(), ".grok", "auth.json")) throw new Error("Refusing the desktop ~/.grok/auth.json; use a dedicated Grok login"); + return { grokAuth, identity }; +}; + +const run = (command: string, args: string[], input?: string): string => + execFileSync(command, args, { cwd: repoRoot, encoding: "utf8", input, stdio: [input === undefined ? "ignore" : "pipe", "pipe", "inherit"] }); + +const containerFor = (): string => { + const ids = run("docker", ["ps", "--filter", `label=com.spawnfile.deployment=${deployment}`, "--format", "{{.ID}}"]).trim().split("\n").filter(Boolean); + if (ids.length !== 1) throw new Error(`expected one running container for ${deployment}, found ${ids.length}`); + return ids[0]!; +}; + +const exec = (container: string, script: string): string => run("docker", ["exec", container, "bash", "-ceu", script]); + +const main = (): void => { + requireLiveCheckEnvironment(process.env); + const out = mkdtempSync(path.join(os.tmpdir(), "spawnfile-grok-lean-live-")); + try { + run(process.execPath, ["dist/cli/index.js", "up", "fixtures/grok-lean-worker", "--detach", "--deployment", deployment, "--out", out]); + const container = containerFor(); + const sysctl = exec(container, "cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null || echo absent").trim(); + process.stdout.write(`host apparmor_restrict_unprivileged_userns=${sysctl}\n`); + const layout = exec(container, `stat -c '%u:%g %a %F %n' ${workerHome} ${workerHome}/.grok ${workerHome}/.grok/sessions ${workerHome}/.grok/*.toml ${workerHome}/.grok/sessions/sandbox-events.jsonl`); + process.stdout.write(layout); + for (const expected of [ + `2200:2100 710 directory ${workerHome}\n`, `0:2200 1771 directory ${workerHome}/.grok\n`, `0:2200 1771 directory ${workerHome}/.grok/sessions\n`, + `0:0 444 regular file ${workerHome}/.grok/config.toml\n`, `0:0 444 regular file ${workerHome}/.grok/trusted_folders.toml\n`, + `2200:2100 640 regular` + ]) if (!layout.includes(expected)) throw new Error(`worker home layout is missing: ${expected.trim()}`); + const profile = exec(container, `cat ${workerHome}/.grok/sandbox.toml`); + if (!/deny = \["\//u.test(profile)) throw new Error("worker sandbox profile has an empty deny list"); + const service = JSON.parse(exec(container, "cat /etc/daimon-engine-broker/service.json")) as { version: string; registrations: Array<{ model: { id: string } }> }; + if (service.version !== "noopolis.daimon.engine-broker-service.v2" || service.registrations[0]?.model.id !== "grok-4.6") throw new Error("service.json is not the declared v2 registration"); + exec(container, `setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! printf x >> ${workerHome}/.grok/trusted_folders.toml' && setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! test -r /var/lib/spawnfile/daimon/grok-subscription-realm'`); + const wakeId = `live-${Date.now()}`; + const wake = JSON.stringify({ agentId: "agent:grok-lean-worker", event: { version: "noopolis.daimon.wake.v1", id: wakeId, kind: "manual", text: "Reply OK.", occurredAt: new Date().toISOString() } }); + const result = exec(container, `curl -fsS -X POST -H 'content-type: application/json' -H "authorization: Bearer $SPAWNFILE_DAIMON_CONTROL_TOKEN" --data-binary @- http://127.0.0.1:19700/v1/wake <<'WAKE'\n${wake}\nWAKE`); + process.stdout.write(`wake: ${result}\n`); + const events = exec(container, `tail -n 5 ${workerHome}/.grok/sessions/sandbox-events.jsonl`); + if (!events.includes("\"ProfileApplied\"") || !events.includes("\"deny_paths\"")) throw new Error("no enforced ProfileApplied event with deny_paths for the wake"); + // The ledger volume is exclusive-reattach and survives earlier runs: count only this wake's rows. + const rows = exec(container, `cat ${usageLedger}`).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line) as { engine: string; model?: string; turn?: string; wake: string }); + const grokRows = rows.filter((row) => row.engine === "grok" && row.wake === wakeId && typeof row.turn === "string"); + if (grokRows.length !== 1 || grokRows[0]!.model !== "grok-4.6") throw new Error(`expected exactly one broker usage row for grok-4.6, found ${grokRows.length}`); + process.stdout.write(`${run(process.execPath, ["dist/cli/index.js", "usage", "fixtures/grok-lean-worker", "--out", out, "--deployment", deployment])}\nGROK LEAN WORKER LIVE CHECK: PASS\n`); + } finally { + try { run(process.execPath, ["dist/cli/index.js", "down", "fixtures/grok-lean-worker", "--compiled", out, "--deployment", deployment, "--force"]); } catch { /* reported by Docker */ } + rmSync(out, { force: true, recursive: true }); + } +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) main(); From c86c5f790352817cb4d3cae0ae6ac778ba7faa8a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:22:59 +0200 Subject: [PATCH 28/84] docs: record the vendored Daimon Grok contract files in the Daimon adapter guide --- src/runtime/daimon/AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/runtime/daimon/AGENTS.md b/src/runtime/daimon/AGENTS.md index 9e575d9f..519d4464 100644 --- a/src/runtime/daimon/AGENTS.md +++ b/src/runtime/daimon/AGENTS.md @@ -54,6 +54,15 @@ excluded until Daimon learned to register its per-wake MCP endpoint through `agy mcp add`; the compiler-side MCP validations (explicit tools allowlist, absolute stdio command) are engine-independent and still apply. +`contract-manifest.json`/`.sha256` and `grokWorkerConfigBytes.ts` are vendored +from a Daimon checkout by `scripts/vendor-daimon-grok-contract.ts` and never +edited by hand. `grokWorkerContract.ts` serves Daimon's worker `config.toml` +bytes only when they hash to the manifest's per model x effort pin, and mirrors +Daimon's sandbox-profile renderer byte for byte (pinned by Daimon-rendered +samples). The manifest also pins Grok CLI 1.0.34 (URL + sha256 per +architecture), which the runtime image, the local builder, and generated +organization images all verify. + The consumed manifest also pins the native Grok broker source/x64/arm64 digests, fixed root/org/broker/worker identities, root-only registrations, and loopback-only provider/MCP endpoints. Container provisioning must match From 4ee37c94b90cb574b683ae639ab97a6ca000ae40 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:35:46 +0200 Subject: [PATCH 29/84] fix: deny Grok workers every Moltnet, memory, mount, instance, and peer resource path the container provisions --- specs/RUNTIMES.md | 11 +- src/compiler/AGENTS.md | 14 +- .../containerDaimonBrokerRender.test.ts | 2 +- ...tainerDaimonGrokWorkerDenyCoverage.test.ts | 132 ++++++++++++++++++ ...tainerDaimonGrokWorkerProvisioning.test.ts | 15 +- .../containerDaimonGrokWorkerProvisioning.ts | 6 +- .../containerDaimonGrokWorkerRender.ts | 83 +++++++++-- 7 files changed, 241 insertions(+), 22 deletions(-) create mode 100644 src/compiler/containerDaimonGrokWorkerDenyCoverage.test.ts diff --git a/specs/RUNTIMES.md b/specs/RUNTIMES.md index 767b23ad..1475ab6c 100644 --- a/specs/RUNTIMES.md +++ b/specs/RUNTIMES.md @@ -249,9 +249,14 @@ is enforced inside bubblewrap for both shell and `read_file`, and it is mandatory: Grok's strict base reads all of `/run`, `/var`, `/tmp`, and `/etc`, and macOS bind mounts ignore unix modes. Each worker denies the Grok bootstrap and realm, AGY realm and unlock secret when present, the wake-acceptance store, -every peer agent's runtime home and workspace, every other worker's home, the -broker's `/etc` and `/run` directories, the usage-ledger and wake-fuse volumes, -and `/run/secrets`, `/run/spawnfile`, `/run/spawnfile-secrets`. Every deny +every peer agent's runtime home and workspace, the organization config +directory, every persistent mount (its own tool state, credential home and +memory included), other runtime instance roots, the shared Moltnet, agent-token +and memory roots under `/var/lib/spawnfile`, every workspace resource backing +path not linked into its own workspace, every other worker's home, the broker's +`/etc` and `/run` directories, the usage-ledger and wake-fuse volumes, and +`/run/secrets`, `/run/spawnfile`, `/run/spawnfile-secrets`, `/run/world`. It +reaches Moltnet and memory only through Daimon's MCP tools. Every deny entry and registration path must be canonical (present, not a symlink, its own realpath) at provisioning or the container refuses to start. diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 17eb5a2f..96d68a69 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -177,9 +177,17 @@ src/compiler/ `src/runtime/daimon/grokWorkerConfigBytes.ts`, refused unless they hash to the manifest pin for the agent's declared model x effort. The sandbox profile's `deny` list is never empty: Daimon's protected set (realms, bootstrap, peers, - acceptance store) plus the broker's `/etc` and `/run` directories, the usage - ledger and wake fuse, every other worker's home, and the container secret - roots. Entries never nest. Provisioning + acceptance store, kept verbatim) plus the organization config directory, + every persistent mount of every runtime plan (the worker's own tool state, + credential home, and memory banks included), other runtime instance roots, + `/var/lib/spawnfile/{moltnet,agents,memory}`, every workspace resource backing + path not linked from the agent's own workspace, the broker's `/etc` and `/run` + directories, the usage ledger and wake fuse, every other worker's home, and + `/run/{secrets,spawnfile,spawnfile-secrets,world}`. Allowed on purpose: own + workspace, own worker home, own runtime home directory, own resource + backings. Masks never nest; `containerDaimonGrokWorkerDenyCoverage.test.ts` + enumerates everything the container provisions and fails on any uncovered, + unjustified path. Provisioning (`containerDaimonGrokWorkerProvisioning.ts`) writes Daimon's attested `GROK_HOME` layout — `root: 1771` home and `sessions/`, `root:root 0444` config/sandbox/trust/managed/requirements files, events under diff --git a/src/compiler/containerDaimonBrokerRender.test.ts b/src/compiler/containerDaimonBrokerRender.test.ts index 27d9b99c..857af10d 100644 --- a/src/compiler/containerDaimonBrokerRender.test.ts +++ b/src/compiler/containerDaimonBrokerRender.test.ts @@ -21,7 +21,7 @@ const grokPlan = (agentIds: string[], extra: Record = {}, model runtimeName: "daimon", engineByNodeId: { ...Object.fromEntries(agentIds.map((agentId) => [agentId, "grok"])), ...extra }, grokModelByNodeId: Object.fromEntries(agentIds.map((agentId) => [agentId, model])), - instancePaths: { instanceRoot: "/var/lib/spawnfile/instances/daimon/daimon-organization", workspacePath: "/var/lib/spawnfile/instances/daimon/daimon-organization/workspace" } + instancePaths: { configPath: "/var/lib/spawnfile/instances/daimon/daimon-organization/daimon/config.json", instanceRoot: "/var/lib/spawnfile/instances/daimon/daimon-organization", workspacePath: "/var/lib/spawnfile/instances/daimon/daimon-organization/workspace" } }) as unknown as Plan; const uid = process.getuid?.() ?? 501; const gid = process.getgid?.() ?? 20; diff --git a/src/compiler/containerDaimonGrokWorkerDenyCoverage.test.ts b/src/compiler/containerDaimonGrokWorkerDenyCoverage.test.ts new file mode 100644 index 00000000..0c12aa53 --- /dev/null +++ b/src/compiler/containerDaimonGrokWorkerDenyCoverage.test.ts @@ -0,0 +1,132 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { daimonAdapter } from "../runtime/daimon/adapter.js"; +import { DAIMON_CONTRACT_MANIFEST_SHA256, DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_SUBSCRIPTION_REALM, DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifest.js"; +import { DAIMON_WAKE_FUSE_DIRECTORY } from "../runtime/daimon/config.js"; +import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; +import { createRuntimeTargetPlans } from "./containerArtifactsPlans.js"; +import { resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js"; +import { resolveDaimonUidEntrypointOwnershipPlan } from "./containerDaimonUidEntrypointRender.js"; +import { MOLTNET_READINESS_DIRECTORY } from "./containerReadinessPaths.js"; +import type { EntrypointOptions } from "./containerEntrypointRender.js"; +import { createMoltnetDaimonReceiptStorePath, createMoltnetNetworkStateDirectory, createMoltnetOpenTokenPath } from "./moltnetConfigLowering.js"; +import type { CompilePlan, ResolvedAgentNode } from "./types.js"; + +const INSTANCE = "/var/lib/spawnfile/instances/daimon/daimon-organization"; +const temporary: string[] = []; +afterEach(async () => { + delete process.env.SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY; + await Promise.all(temporary.splice(0).map((directory) => rm(directory, { force: true, recursive: true }))); +}); + +const useCompatibleDaimonRuntime = async (): Promise => { + const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-deny-coverage-")); + temporary.push(directory); + const identity = path.join(directory, "identity.json"), digest = `sha256:${"a".repeat(64)}`; + await writeFile(identity, `${JSON.stringify({ + capability_receipt_sha256: digest, development: { mode: "local-development", non_production: true, unpublished: true, unsigned: true }, + image_architecture: "amd64", image_config_digest: digest, image_manifest_digest: digest, + image_reference: `127.0.0.1:54321/noopolis/spawnfile-runtime-daimon@${digest}`, manifest_sha256: DAIMON_CONTRACT_MANIFEST_SHA256, + registry_authority: "127.0.0.1:54321", version: "spawnfile.local-daimon-runtime-identity.v3" + })}\n`, { mode: 0o600 }); + process.env.SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY = identity; +}; + +const agent = (name: string, engine: string): ResolvedAgentNode => ({ + description: "", docs: [], env: {}, kind: "agent", mcpServers: [], name, policyMode: null, policyOnDegrade: null, + runtime: { name: "daimon", options: { engine } }, secrets: [], skills: [], source: `/tmp/${name}/Spawnfile`, subagents: [], + execution: engine === "grok" ? { model: { primary: { auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" } } } + : engine === "codex" ? { model: { primary: { auth: { method: "codex" }, name: "gpt-5.4-mini", provider: "openai" } } } : undefined +} as ResolvedAgentNode); + +const resource = (id: string, linkPath: string, backingPath: string) => ({ backingPath, id, kind: "git" as const, linkPath, mode: "readonly" as const, mount: `./${id}`, sharing: "agent" as const }); + +const plans = async (): Promise => { + await useCompatibleDaimonRuntime(); + const nodes = [["alpha", "grok"], ["beta", "grok"], ["gamma", "codex"], ["delta", "agy"]] as const; + const compiled = await Promise.all(nodes.map(async ([slug, engine]) => { + const value = agent(slug, engine); + return { emittedFiles: (await daimonAdapter.compileAgent(value)).files, id: `agent:${slug}`, kind: "agent" as const, runtimeName: "daimon", slug, value }; + })); + const [daimon] = await createRuntimeTargetPlans({ edges: [], nodes: [], root: "/tmp/Spawnfile", runtimes: { daimon: { nodeIds: [] } } } as unknown as CompilePlan, compiled); + const withResources: RuntimeTargetPlan = { + ...daimon!, + persistentMounts: [...(daimon!.persistentMounts ?? []), { id: "memory-bank", mount_path: "/var/lib/spawnfile/memory/alpha/notes", reason: "memory", volume_name: "memory" }], + resources: [ + resource("own", `${INSTANCE}/workspace/agents/alpha/own`, "/var/lib/spawnfile/resources/instances/daimon-organization/own"), + resource("peer", `${INSTANCE}/workspace/agents/beta/peer`, "/var/lib/spawnfile/resources/instances/daimon-organization/peer"), + resource("team", `${INSTANCE}/workspace/agents/alpha/team`, "/var/lib/spawnfile/resources/teams/crew/team"), + resource("team", `${INSTANCE}/workspace/agents/gamma/team`, "/var/lib/spawnfile/resources/teams/crew/team") + ] as unknown as RuntimeTargetPlan["resources"] + }; + const sibling = { + envFiles: [{ envName: "PICO_KEY", filePath: "/var/lib/spawnfile/instances/picoclaw/pico/picoclaw/.env" }], + instancePaths: { configPath: "/var/lib/spawnfile/instances/picoclaw/pico/picoclaw/config.json", instanceRoot: "/var/lib/spawnfile/instances/picoclaw/pico", workspacePath: "/var/lib/spawnfile/instances/picoclaw/pico/workspace" }, + persistentMounts: [{ id: "pico-state", mount_path: "/var/lib/spawnfile/instances/picoclaw/pico/home", reason: "state", volume_name: "pico" }], + runtimeName: "picoclaw" + } as unknown as RuntimeTargetPlan; + return [withResources, sibling]; +}; + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +describe("Grok worker deny coverage over everything the Daimon container provisions", () => { + it("denies every provisioned state path except the agent's own, explicitly justified ones", async () => { + const runtimePlans = await plans(); + const moltnet = { + nodePlans: [{ configPath: "/var/lib/spawnfile/moltnet/nodes/crew-net-alpha.json", receiptStorePath: createMoltnetDaimonReceiptStorePath("net", "alpha") }], + serverPlans: [{ configPath: "/var/lib/spawnfile/moltnet/servers/local/Moltnet.json", mode: "managed" }] + } as unknown as EntrypointOptions["moltnet"]; + const persistentMountPaths = runtimePlans.flatMap((plan) => (plan.persistentMounts ?? []).map((mount) => mount.mount_path)); + const ownership = resolveDaimonUidEntrypointOwnershipPlan(runtimePlans, persistentMountPaths, moltnet); + const registrations = resolveDaimonGrokRegistrations(runtimePlans); + const provisioned = [...new Set([ + ...ownership.stateRoots, ...ownership.privateDirectories, ...ownership.privateFiles, ...ownership.privateModeDirectories, + ...ownership.opaqueDescendantRoots, ...ownership.creatablePrivateDirectories.map((entry) => entry.target), + ...persistentMountPaths, ...runtimePlans.flatMap((plan) => plan.opaqueMountTargets ?? []), + ...runtimePlans.flatMap((plan) => [plan.instancePaths.configPath, ...(plan.envFiles ?? []).map((file) => file.filePath)]), + ...runtimePlans.flatMap((plan) => plan.resources ?? []).flatMap((entry) => [entry.backingPath, entry.linkPath]), + MOLTNET_READINESS_DIRECTORY, createMoltnetNetworkStateDirectory("net"), createMoltnetOpenTokenPath("net", "alpha"), "/var/lib/spawnfile/moltnet/servers", + DAIMON_GROK_ENGINE_BROKER.registrationPath, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.controlSocketPath, + DAIMON_GROK_ENGINE_BROKER.backendSocketPath, DAIMON_GROK_ENGINE_BROKER.launcherSocketPath, DAIMON_GROK_ENGINE_BROKER.turnStorePath, + DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath, DAIMON_GROK_TURN_USAGE_LEDGER.filePath, DAIMON_WAKE_FUSE_DIRECTORY, + ...registrations.map((entry) => entry.home), "/run/secrets", "/run/spawnfile-secrets/token", "/run/world/evidence" + ])].filter((entry) => entry.startsWith("/")); + + for (const registration of registrations) { + const slug = registration.agentId.replace(/^agent:/u, ""); + const ownRuntimeHome = `${INSTANCE}/runtime-homes/${slug}`; + const own = [registration.workspace, registration.home, ...runtimePlans.flatMap((plan) => plan.resources ?? []) + .filter((entry) => within(entry.linkPath, registration.workspace)).map((entry) => entry.backingPath)]; + /** Directories whose only provisioned children are separately denied; listing them reveals names, not content. */ + const justified = new Map([ + [`${INSTANCE}/state`, "sole child is the denied wake-acceptance store"], + ["/var/lib/spawnfile/daimon", "shared parent; every realm, ledger, fuse and bootstrap child is denied"], + [ownRuntimeHome, "Daimon names tool-result spill paths under it; every mount inside it is denied"], + ["/var/lib/spawnfile/instances/picoclaw", "runtime-kind parent of another runtime's denied instance root"] + ]); + const uncovered = provisioned.filter((entry) => + !registration.denyPaths.some((denied) => within(entry, denied)) + && !own.some((allowed) => within(entry, allowed)) + && ![...own, ownRuntimeHome].some((allowed) => allowed.startsWith(`${entry}/`)) + && !justified.has(entry)); + expect(uncovered, registration.agentId).toEqual([]); + for (const moltnetPath of ["/var/lib/spawnfile/moltnet", "/var/lib/spawnfile/agents", "/var/lib/spawnfile/memory"]) expect(registration.denyPaths).toContain(moltnetPath); + } + const [alpha, beta] = registrations; + expect(alpha!.denyPaths).toContain("/var/lib/spawnfile/resources/instances/daimon-organization/peer"); + expect(alpha!.denyPaths).not.toContain("/var/lib/spawnfile/resources/instances/daimon-organization/own"); + expect(alpha!.denyPaths).not.toContain("/var/lib/spawnfile/resources/teams/crew/team"); + expect(beta!.denyPaths).toEqual(expect.arrayContaining([ + "/var/lib/spawnfile/resources/instances/daimon-organization/own", "/var/lib/spawnfile/resources/teams/crew/team" + ])); + expect(beta!.denyPaths).not.toContain("/var/lib/spawnfile/resources/instances/daimon-organization/peer"); + expect(alpha!.deferredDenyPaths).toEqual(["/var/lib/spawnfile/resources/instances/daimon-organization/peer"]); + expect(alpha!.denyPaths).toEqual(expect.arrayContaining([`${INSTANCE}/runtime-homes/alpha/tool-state`, `${INSTANCE}/runtime-homes/alpha/.grok`, `${INSTANCE}/daimon`, "/var/lib/spawnfile/instances/picoclaw/pico"])); + for (const entry of alpha!.denyPaths) expect(alpha!.denyPaths.some((other) => other !== entry && within(entry, other))).toBe(false); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index a81c01b8..4d2aeecf 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -69,7 +69,7 @@ const seedFor = (registrations: readonly DaimonGrokRegistration[], omit: string[ ...registrations.map((entry) => [entry.workspace, { kind: "dir" as const }] as const), ["/etc/daimon-engine-broker", { kind: "dir" as const, mode: 0o700 }] as const, ...registrations.flatMap((entry) => entry.denyPaths) - .filter((denied) => !denied.startsWith("/var/lib/daimon-workers/") && !(["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"] as string[]).includes(denied)) + .filter((denied) => !denied.startsWith("/var/lib/daimon-workers/") && !(["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets", "/run/world"] as string[]).includes(denied)) .map((denied) => [denied, { kind: denied.endsWith("grok-bootstrap-auth") ? "file" as const : "dir" as const, mode: 0o700 }] as const) ].filter(([target]) => !omit.includes(target))); @@ -109,6 +109,19 @@ describe("Grok worker home provisioning", () => { for (const optional of ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"]) expect(nodes.get(optional)).toMatchObject({ kind: "dir", mode: 0o700, uid: 0 }); }); + it("allows a peer resource backing to be absent at provisioning (the entrypoint prepares it later) but never a symlink", () => { + const withResource = resolveDaimonGrokRegistrations([{ + ...plan({ "agent:a": "grok", "agent:b": "grok" }), + resources: [{ backingPath: "/var/lib/spawnfile/resources/instances/daimon-organization/b-repo", id: "b-repo", kind: "git", linkPath: `${INSTANCE}/workspace/agents/b/b-repo`, mode: "readonly", mount: "./b-repo", sharing: "agent" }] + } as unknown as RuntimeTargetPlan]); + const backing = "/var/lib/spawnfile/resources/instances/daimon-organization/b-repo"; + expect(withResource[0]!.deferredDenyPaths).toEqual([backing]); + expect(() => run(withResource, seedFor(withResource, [backing]))).not.toThrow(); + expect(() => run(withResource, { ...seedFor(withResource, [backing]), [backing]: { kind: "link", target: "/tmp/elsewhere" }, "/tmp/elsewhere": { kind: "dir" } })) + .toThrow(/canonical non-symlink/u); + expect(() => run(withResource, seedFor(withResource, ["/var/lib/spawnfile/daimon/usage"]))).toThrow(/deny path is missing/u); + }); + it("is restart-idempotent over an already provisioned home", () => { const first = run(registrations, seedFor(registrations)); const reseeded = Object.fromEntries([...first.entries()].filter(([target]) => target !== "/etc/daimon-engine-broker/service.json")); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 4082e695..9365ecc9 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -1,6 +1,7 @@ import { DAIMON_GROK_ENGINE_BROKER } from "../runtime/daimon/contractManifest.js"; import { DAIMON_BROKER_UID } from "../runtime/daimon/runtimeIdentity.js"; import { + DAIMON_GROK_DENIED_STATE_ROOTS, DAIMON_GROK_OPTIONAL_DENY_PATHS, DAIMON_GROK_WORKER_READ_ONLY_FILES, DAIMON_WORKER_ROOT, @@ -13,6 +14,7 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ agentId: entry.agentId, config: entry.config, configSha256: entry.configSha256, + deferredDenyPaths: entry.deferredDenyPaths, denyPaths: entry.denyPaths, eventsPath: entry.eventsPath, grokHome: entry.grokHome, @@ -46,7 +48,7 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ */ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly DaimonGrokRegistration[]): string[] => [ `const grokWorkers = ${JSON.stringify(registrations.map(programRegistration))};`, - `const optionalDenyPaths = new Set(${JSON.stringify(DAIMON_GROK_OPTIONAL_DENY_PATHS)});`, + `const optionalDenyPaths = new Set(${JSON.stringify([...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_STATE_ROOTS])});`, `const pinnedConfigSha256 = new Set(${JSON.stringify(Object.values(DAIMON_GROK_ENGINE_BROKER.worker.configSha256).flatMap((efforts) => Object.values(efforts)))});`, "const sha256Hex = (value) => crypto.createHash('sha256').update(value).digest('hex');", "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !pinnedConfigSha256.has(entry.configSha256) || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", @@ -58,7 +60,7 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo // Pass 1: every workspace and home exists, root-held, before any deny list is checked. `for (const entry of grokWorkers) { for (let ancestor = require('node:path').dirname(entry.workspace); ancestor.startsWith('/var/lib/spawnfile/') && ancestor.length > '/var/lib/spawnfile'.length; ancestor = require('node:path').dirname(ancestor)) fs.chmodSync(ancestor, fs.statSync(ancestor).mode & 0o7777 | 0o011); secureWorkspace(entry.workspace, entry.uid); assertCanonical(entry.workspace, 'workspace'); ensureDirectory(entry.home, 0, 0, 0o700); ensureDirectory(entry.grokHome, 0, 0, 0o700); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, 0, 0o700); for (const target of [entry.home, entry.grokHome]) assertCanonical(target, 'home'); }`, "for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }", - "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", + "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT' && entry.deferredDenyPaths.includes(denied)) continue; if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", // Pass 2: exact root-owned read-only files, the events file, then the final sticky modes. `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 3caaae2d..1fff49bc 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -30,7 +30,7 @@ import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; export const DAIMON_WORKER_ROOT = "/var/lib/daimon-workers"; export const DAIMON_GROK_WORKER_READ_ONLY_FILES = DAIMON_GROK_ENGINE_BROKER.worker.home.readOnlyFiles.names; /** Container paths no Grok worker may read that nothing else guarantees to exist; provisioning creates them root-owned 0700 when absent. */ -export const DAIMON_GROK_OPTIONAL_DENY_PATHS = ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"] as const; +export const DAIMON_GROK_OPTIONAL_DENY_PATHS = ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets", "/run/world"] as const; /** * The organization runtime state directory — the parent of the durable wake @@ -49,6 +49,8 @@ export interface DaimonGrokRegistration { config: string; configSha256: string; denyPaths: string[]; + /** Deny entries the main entrypoint materializes after root provisioning (workspace resource backings); absent is allowed there, a symlink never. */ + deferredDenyPaths: string[]; eventsPath: string; grokHome: string; home: string; @@ -79,20 +81,48 @@ const declaredModel = (plan: RuntimeTargetPlan, agentId: string): { model: Daimo return declared as { model: DaimonGrokBrokerModel; reasoningEffort: DaimonGrokBrokerReasoningEffort }; }; +/** + * Spawnfile-managed state roots no Grok worker needs: the shared Moltnet store + * (server data, node/bridge configs, receipt stores, network state), per-agent + * Moltnet open-mode token directories, and declared Mneme memory banks. A + * worker reaches Moltnet and memory only through Daimon's MCP tools, which run + * in the organization process, never by reading these files. Provisioning + * creates each root-owned `0700` when absent so the mask always has a target. + */ +export const DAIMON_GROK_DENIED_STATE_ROOTS = ["/var/lib/spawnfile/agents", "/var/lib/spawnfile/memory", "/var/lib/spawnfile/moltnet"] as const; + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + /** * One brokered worker's sandbox deny list. * * Daimon's own protected set for the agent (`grokSandboxProtectedPaths`: the * Grok bootstrap and realm, the AGY realm and unlock secret when any agent is * AGY, the wake-acceptance store, and every peer's runtime home and workspace) - * plus what this deployment adds: the broker's registration/service directory - * and control/relay socket directory, the usage-ledger and wake-fuse volumes, - * every other worker's home, and the container secret roots. Grok 1.0.34's - * strict base reads all of `/run`, `/var`, `/tmp` and `/etc`, and bind mounts - * from a macOS host ignore unix modes, so this list — not file modes — is the - * boundary. Entries must never nest: one mask inside another cannot be applied. + * is always kept verbatim, so a Daimon projection over the same inputs renders + * the same profile. This deployment adds everything else the container + * provisions that the worker does not need: the organization config directory + * (all agents' instructions and any env files), every persistent mount of + * every runtime plan (the worker's own tool state, credential home and memory + * included — it reaches them only through Daimon), every other runtime + * instance root, the shared Moltnet/agent/memory state roots, every workspace + * resource backing path not linked from this agent's own workspace, the broker's + * `/etc` and `/run` directories, the usage ledger and wake fuse, every other + * worker's home, and the container secret roots. + * + * Allowed on purpose: the agent's own workspace (the worker's cwd), its own + * runtime home directory itself (Daimon's tool-result spill contract names paths + * under it; its contents are persistent mounts and denied), its own worker home, + * and backing paths of resources linked into its own workspace (a mask over the + * backing inode would also hide the agent's own resource through its link). + * + * Grok 1.0.34's strict base reads all of `/run`, `/var`, `/tmp` and `/etc`, and + * macOS bind mounts ignore unix modes, so this list — not file modes — is the + * boundary. Masks cannot nest: an added entry already covered by another entry + * is dropped, and an added entry that would cover a Daimon entry is refused. */ export const resolveDaimonGrokWorkerDenyPaths = ( + plans: readonly RuntimeTargetPlan[], plan: RuntimeTargetPlan, agentId: string, workerHomes: readonly string[], @@ -100,8 +130,10 @@ export const resolveDaimonGrokWorkerDenyPaths = ( ): string[] => { const instanceRoot = plan.instancePaths.instanceRoot ?? fail("Daimon Grok registrations require an instance root"); const agents = Object.entries(plan.engineByNodeId ?? {}); + const ownWorkspace = path.posix.join(plan.instancePaths.workspacePath, "agents", nodeSlug(agentId)); + const ownRuntimeHome = path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, nodeSlug(agentId)); const peers = agents.filter(([id]) => id !== agentId).map(([id]) => nodeSlug(id)); - const denied = [...new Set([ + const daimonOwn = [ DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath, DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath, ...(agents.some(([, engine]) => engine === "agy") ? [DAIMON_AGY_SUBSCRIPTION_REALM.unlockMountPath, DAIMON_AGY_SUBSCRIPTION_REALM.durableMountPath] : []), @@ -109,17 +141,42 @@ export const resolveDaimonGrokWorkerDenyPaths = ( ...peers.flatMap((slug) => [ path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, slug), path.posix.join(plan.instancePaths.workspacePath, "agents", slug) + ]) + ]; + const ownResourceBackings = new Set((plan.resources ?? []) + .filter((resource) => within(resource.linkPath, ownWorkspace)) + .map((resource) => resource.backingPath)); + const added = [ + path.posix.dirname(plan.instancePaths.configPath), + ...plans.flatMap((candidate) => (candidate.persistentMounts ?? []).map((mount) => mount.mount_path)), + ...plans.filter((candidate) => candidate !== plan).flatMap((candidate) => [ + candidate.instancePaths.instanceRoot ?? path.posix.dirname(candidate.instancePaths.configPath), + ...(candidate.envFiles ?? []).map((binding) => binding.filePath) ]), + ...(plan.envFiles ?? []).map((binding) => binding.filePath), + ...plans.flatMap((candidate) => candidate.resources ?? []).map((resource) => resource.backingPath) + .filter((backing) => !ownResourceBackings.has(backing)), + ...DAIMON_GROK_DENIED_STATE_ROOTS, path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, DAIMON_WAKE_FUSE_DIRECTORY, ...workerHomes.filter((home) => home !== ownHome), ...DAIMON_GROK_OPTIONAL_DENY_PATHS - ])].sort(); + ].filter((entry) => entry.startsWith("/")); + for (const entry of added) { + if (within(ownWorkspace, entry) || within(ownHome, entry) || entry === ownRuntimeHome || within(ownRuntimeHome, entry) + || [...ownResourceBackings].some((backing) => within(backing, entry))) { + fail(`Grok worker deny path ${entry} would hide ${agentId}'s own workspace, home, or resources`); + } + } + const candidates = [...new Set([...daimonOwn, ...added])]; + const daimonSet = new Set(daimonOwn); + const denied = candidates.filter((entry) => daimonSet.has(entry) + || !candidates.some((other) => other !== entry && within(entry, other))).sort(); for (const entry of denied) { - const ancestor = denied.find((other) => other !== entry && entry.startsWith(`${other}/`)); - if (ancestor) fail(`Grok worker deny path ${entry} nests inside ${ancestor}`); + const ancestor = denied.find((other) => other !== entry && within(entry, other)); + if (ancestor) fail(`Grok worker deny path ${ancestor} would cover ${entry}; masks cannot nest`); } return denied; }; @@ -137,9 +194,11 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim const grokHome = path.posix.join(home, DAIMON_GROK_WORKER_HOME_DIRECTORY); const { model, reasoningEffort } = declaredModel(plan, agentId); const config = resolveDaimonGrokWorkerConfig(model, reasoningEffort); - const denyPaths = resolveDaimonGrokWorkerDenyPaths(plan, agentId, homes, home); + const denyPaths = resolveDaimonGrokWorkerDenyPaths(plans, plan, agentId, homes, home); const profile = renderDaimonGrokWorkerSandboxProfile(denyPaths); + const backings = new Set(plans.flatMap((candidate) => candidate.resources ?? []).map((resource) => resource.backingPath)); return { + deferredDenyPaths: denyPaths.filter((entry) => backings.has(entry)), agentId, config: config.bytes, configSha256: config.sha256, From fc3cc4d7d002a100b6f9b5d17670ff6528b4dd79 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:36:09 +0200 Subject: [PATCH 30/84] fix: require the declared model and effort pin for provisioned Grok worker config bytes --- src/compiler/containerDaimonGrokWorkerProvisioning.test.ts | 5 +++++ src/compiler/containerDaimonGrokWorkerProvisioning.ts | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index 4d2aeecf..d050f1fa 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -138,6 +138,11 @@ describe("Grok worker home provisioning", () => { .toThrow(/canonical non-symlink/u); const tampered = registrations.map((entry, index) => index === 0 ? { ...entry, config: entry.config.replace("grok-4.6", "grok-4.5") } : entry); expect(() => run(tampered, seedFor(tampered))).toThrow(/do not match their pins/u); + // Another model's pinned bytes are still refused: the pin is per declared model x effort, not any pin. + const otherPair = registrations.map((entry, index) => index === 0 ? { ...entry, config: DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.5"].high, configSha256: crypto.createHash("sha256").update(DAIMON_GROK_WORKER_CONFIG_BYTES["grok-4.5"].high).digest("hex") } : entry); + expect(() => run(otherPair, seedFor(otherPair))).toThrow(/do not match their pins/u); + const otherEffort = registrations.map((entry, index) => index === 0 ? { ...entry, reasoningEffort: "medium" as const } : entry); + expect(() => run(otherEffort, seedFor(otherEffort))).toThrow(/do not match their pins/u); const unpinned = registrations.map((entry, index) => index === 0 ? { ...entry, denyPaths: [] } : entry); expect(() => run(unpinned, seedFor(unpinned))).toThrow(/do not match their pins/u); const replaced = run(registrations, seedFor(registrations)); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 9365ecc9..757b3972 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -19,9 +19,11 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ eventsPath: entry.eventsPath, grokHome: entry.grokHome, home: entry.home, + model: entry.model, profile: entry.profile, profilePath: entry.profilePath, profileSha256: entry.profileSha256, + reasoningEffort: entry.reasoningEffort, slot: entry.slot, uid: entry.uid, workspace: entry.workspace @@ -49,9 +51,9 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly DaimonGrokRegistration[]): string[] => [ `const grokWorkers = ${JSON.stringify(registrations.map(programRegistration))};`, `const optionalDenyPaths = new Set(${JSON.stringify([...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_STATE_ROOTS])});`, - `const pinnedConfigSha256 = new Set(${JSON.stringify(Object.values(DAIMON_GROK_ENGINE_BROKER.worker.configSha256).flatMap((efforts) => Object.values(efforts)))});`, + `const pinnedConfigSha256 = ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.configSha256)};`, "const sha256Hex = (value) => crypto.createHash('sha256').update(value).digest('hex');", - "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !pinnedConfigSha256.has(entry.configSha256) || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", + "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !Object.hasOwn(pinnedConfigSha256, entry.model) || !Object.hasOwn(pinnedConfigSha256[entry.model], entry.reasoningEffort) || pinnedConfigSha256[entry.model][entry.reasoningEffort] !== entry.configSha256 || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", "const assertCanonical = (target, label) => { const info = fs.lstatSync(target); if (info.isSymbolicLink() || fs.realpathSync(target) !== target) throw new Error(`Grok worker ${label} is not a canonical non-symlink path: ${target}`); return info; };", "const ensureDirectory = (target, uid, gid, mode) => { fs.mkdirSync(target, { recursive: true, mode: 0o700 }); const info = fs.lstatSync(target); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('unsafe worker runtime directory'); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", "const ensureExactFile = (target, content, mode) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, content, { mode, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new Error('unsafe worker runtime file'); if (fs.readFileSync(target, 'utf8') !== content) throw new Error(`worker runtime file identity mismatch: ${target}`); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); };", From 151e416f4a64db7bd51e0b902a7486ca756678ff Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:36:23 +0200 Subject: [PATCH 31/84] fix: mark estimated tokens in the usage engine rollup --- src/cli/usageCommand.test.ts | 1 + src/cli/usageCommand.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli/usageCommand.test.ts b/src/cli/usageCommand.test.ts index 8ced1fc6..eba1328d 100644 --- a/src/cli/usageCommand.test.ts +++ b/src/cli/usageCommand.test.ts @@ -184,6 +184,7 @@ describe("spawnfile usage", () => { [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${sealed}\n${sealed}\n` })); expect(table.output).toMatch(/^cogsworth\s+grok\s+1\s+~100\.0k/mu); + expect(table.output).toMatch(/^grok\s+1\s+~100\.0k/mu); expect(table.output).toContain("~ 1 turn(s) include ESTIMATED usage: 2 request(s) returned no provider-reported usage"); }); diff --git a/src/cli/usageCommand.ts b/src/cli/usageCommand.ts index 6a77d41a..c0f8b21e 100644 --- a/src/cli/usageCommand.ts +++ b/src/cli/usageCommand.ts @@ -138,7 +138,7 @@ const renderTable = ( .sort((left, right) => right.tokens - left.tokens || left.engine.localeCompare(right.engine)); const engineWidth = Math.max(8, ...engineRows.map((row) => row.engine.length)); for (const row of engineRows) { - lines.push(`${pad(row.engine, engineWidth)} ${pad("", 8)}${padStart(String(row.turns), 7)}${padStart(formatTokens(row.tokens), 9)}${padStart(formatUsd(row.notionalUsd, row.turns), 11)}`); + lines.push(`${pad(row.engine, engineWidth)} ${pad("", 8)}${padStart(String(row.turns), 7)}${padStart(`${row.estimatedTurns > 0 ? "~" : ""}${formatTokens(row.tokens)}`, 9)}${padStart(formatUsd(row.notionalUsd, row.turns), 11)}`); } if (engineRows.length === 0) lines.push("no metered turns in this window"); From cd6e88d27d1e4d5132312fe07acd04875cc44c78 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:37:17 +0200 Subject: [PATCH 32/84] fix: report differing usage rows under one turn key as a partial-coverage conflict --- specs/USAGE_ACCOUNTING_DESIGN.md | 4 +++- src/cli/usageCommand.test.ts | 9 ++++++++ src/cli/usageCommand.ts | 5 +++++ src/runtime/usageLedger.test.ts | 16 +++++++++++++- src/runtime/usageLedger.ts | 34 ++++++++++++++++++++++++++--- src/runtime/usageLedgerRead.test.ts | 6 +++-- src/runtime/usageLedgerRead.ts | 9 ++++---- 7 files changed, 72 insertions(+), 11 deletions(-) diff --git a/specs/USAGE_ACCOUNTING_DESIGN.md b/specs/USAGE_ACCOUNTING_DESIGN.md index e36d574d..da9124fc 100644 --- a/specs/USAGE_ACCOUNTING_DESIGN.md +++ b/specs/USAGE_ACCOUNTING_DESIGN.md @@ -278,7 +278,9 @@ The broker remains the single sealed writer. What changed for readers: when it cannot find the row, so two replays can append identical bytes. `spawnfile usage` keeps the first row per `turn` — within each generation, across `usage.jsonl.1` and `usage.jsonl`, and inside every aggregate. Rows - without a key are kept. + without a key are kept. Rows sharing a key but differing are a conflict: the + first is counted, the keys are named in a warning, and coverage is PARTIAL + (`coverage.conflictingTurnCount`). - **Additive fields.** `limit_reason` (`tokens`/`requests`/`timeout`/`none`), `model` (the declared model the broker verified), `outcome`, and `estimated_requests` stay inside the unchanged `turn-usage.v1` record. A diff --git a/src/cli/usageCommand.test.ts b/src/cli/usageCommand.test.ts index eba1328d..52468baf 100644 --- a/src/cli/usageCommand.test.ts +++ b/src/cli/usageCommand.test.ts @@ -188,6 +188,15 @@ describe("spawnfile usage", () => { expect(table.output).toContain("~ 1 turn(s) include ESTIMATED usage: 2 request(s) returned no provider-reported usage"); }); + it("warns and reports PARTIAL when differing rows share one turn key", async () => { + const turn = "c".repeat(64); + const table = await executeUsageCommand("/tmp/project", {}, handlersFor({ + [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${line({ turn, total: 10 })}\n${line({ turn, total: 99 })}\n` + })); + expect(table.output).toContain("WARNING 1 turn(s) carry differing ledger rows under one turn key"); + expect(table.output).toContain("coverage PARTIAL"); + }); + it("counts an all-zero turn as unknown rather than free", async () => { const result = await executeUsageCommand("/tmp/project", {}, handlersFor({ [DAIMON_GROK_TURN_USAGE_LEDGER.filePath]: `${line({ complete: false, input: 0, output: 0, cache_read: 0, cache_write: 0, total: 0 })}\n` diff --git a/src/cli/usageCommand.ts b/src/cli/usageCommand.ts index c0f8b21e..2fdbeeea 100644 --- a/src/cli/usageCommand.ts +++ b/src/cli/usageCommand.ts @@ -12,6 +12,7 @@ import { computeUsageCoverage, dedupeUsageRecordsByTurn, DEFAULT_USAGE_SINCE, + findConflictingUsageTurns, filterUsageRecordsSince, groupUsageByAgent, groupUsageByEngine, @@ -144,6 +145,10 @@ const renderTable = ( lines.push(""); lines.push("Counts are a lower bound: the engine stream carries no completeness marker."); + const conflicts = findConflictingUsageTurns(windowed); + if (conflicts.length > 0) { + lines.push(`WARNING ${conflicts.length} turn(s) carry differing ledger rows under one turn key; the first row is counted and coverage is PARTIAL: ${conflicts.map((turn) => turn.slice(0, 12)).join(", ")}`); + } if (coverage.estimatedTurnCount > 0) { const estimatedRequests = engineRows.reduce((sum, row) => sum + row.estimatedRequests, 0); lines.push(`~ ${coverage.estimatedTurnCount} turn(s) include ESTIMATED usage: ${estimatedRequests} request(s) returned no provider-reported usage and were charged a conservative estimate, not a measurement.`); diff --git a/src/runtime/usageLedger.test.ts b/src/runtime/usageLedger.test.ts index a0bc128d..1eb51464 100644 --- a/src/runtime/usageLedger.test.ts +++ b/src/runtime/usageLedger.test.ts @@ -4,6 +4,7 @@ import { computeUsageCoverage, dedupeUsageRecordsByTurn, DEFAULT_USAGE_SINCE, + findConflictingUsageTurns, filterUsageRecordsSince, groupUsageByAgent, groupUsageByEngine, @@ -47,7 +48,9 @@ describe("broker usage rows", () => { it("dedupes a replayed turn in parsing and in every aggregate, keeping unkeyed rows", () => { const text = [line({ total: 10, turn }), line({ total: 10, turn }), line({ total: 5 }), line({ total: 5 })].join("\n"); const parsed = parseUsageLedger(text); - expect(parsed.map((entry) => entry.total)).toEqual([10, 5, 5]); + expect(dedupeUsageRecordsByTurn(parsed).map((entry) => entry.total)).toEqual([10, 5, 5]); + expect(groupUsageByEngine(parsed)[0]).toMatchObject({ tokens: 20, turns: 3 }); + expect(computeUsageCoverage(parsed, 1)).toMatchObject({ conflictingTurnCount: 0, partial: false }); const duplicated = [record({ total: 10, turn }), record({ total: 10, turn })]; expect(dedupeUsageRecordsByTurn(duplicated)).toHaveLength(1); expect(groupUsageByAgent(duplicated)[0]).toMatchObject({ tokens: 10, turns: 1 }); @@ -56,6 +59,16 @@ describe("broker usage rows", () => { }); }); +describe("conflicting rows under one turn key", () => { + it("counts the first row, names the conflict, and marks coverage partial", () => { + const turn = "f".repeat(64); + const records = [record({ total: 10, turn }), record({ total: 99, turn }), record({ total: 10, turn: "0".repeat(64) }), record({ total: 10, turn: "0".repeat(64) })]; + expect(findConflictingUsageTurns(records)).toEqual([turn]); + expect(groupUsageByAgent(records)[0]).toMatchObject({ tokens: 20, turns: 2 }); + expect(computeUsageCoverage(records, 1)).toMatchObject({ agentsReporting: 1, conflictingTurnCount: 1, partial: true }); + }); +}); + describe("parseUsageLedgerLine", () => { it("parses a well-formed line", () => { expect(parseUsageLedgerLine(line())).toEqual(record()); @@ -265,6 +278,7 @@ describe("computeUsageCoverage", () => { expect(coverage).toEqual({ agentsReporting: 2, agentsTotal: 16, + conflictingTurnCount: 0, estimatedTurnCount: 0, incompleteRecordCount: 0, partial: true, diff --git a/src/runtime/usageLedger.ts b/src/runtime/usageLedger.ts index 00df4442..f9abb213 100644 --- a/src/runtime/usageLedger.ts +++ b/src/runtime/usageLedger.ts @@ -69,6 +69,8 @@ const brokerFields = (record: Record): Partial => * them again on replay when it cannot see the row, so two replays of one sealed * turn can both append identical bytes. The key exists precisely so such a turn * is never counted twice; every aggregate here must see deduplicated rows. + * Readers keep every parsed row so {@link findConflictingUsageTurns} can see + * rows that share a key but differ. */ export const dedupeUsageRecordsByTurn = (records: UsageRecord[]): UsageRecord[] => { const seen = new Set(); @@ -80,6 +82,28 @@ export const dedupeUsageRecordsByTurn = (records: UsageRecord[]): UsageRecord[] }); }; +const canonicalRecord = (record: UsageRecord): string => + JSON.stringify(Object.keys(record).sort().map((key) => [key, record[key as keyof UsageRecord]])); + +/** + * `turn` keys whose rows are not byte-for-byte the same record. A replay + * re-appends identical sealed bytes, so differing rows under one key mean the + * ledger is not what the broker sealed; the first row is still the one counted, + * but the window is reported PARTIAL and the keys are named. + */ +export const findConflictingUsageTurns = (records: UsageRecord[]): string[] => { + const first = new Map(); + const conflicts = new Set(); + for (const record of records) { + if (record.turn === undefined) continue; + const canonical = canonicalRecord(record); + const seen = first.get(record.turn); + if (seen === undefined) first.set(record.turn, canonical); + else if (seen !== canonical) conflicts.add(record.turn); + } + return [...conflicts].sort(); +}; + const NUMERIC_FIELDS = [ "input", "output", @@ -170,10 +194,10 @@ export const parseUsageLedgerLine = (line: string): UsageRecord | null => { * skipped, and the rest of the file still parses. */ export const parseUsageLedger = (text: string): UsageRecord[] => - dedupeUsageRecordsByTurn(text + text .split("\n") .map(parseUsageLedgerLine) - .filter((record): record is UsageRecord => record !== null)); + .filter((record): record is UsageRecord => record !== null); const DURATION_UNIT_MS: Record = { d: 24 * 60 * 60 * 1000, @@ -323,6 +347,8 @@ export const groupUsageByEngine = ( export interface UsageCoverage { agentsReporting: number; agentsTotal: number; + /** `turn` keys carrying differing rows (first row counted); nonzero forces `partial`. */ + conflictingTurnCount: number; /** Turns whose total includes an estimated charge for at least one request. */ estimatedTurnCount: number; /** Count of `complete:false` records in the window — every count here is a @@ -353,12 +379,14 @@ export const computeUsageCoverage = ( ): UsageCoverage => { const unique = dedupeUsageRecordsByTurn(records); const reporting = new Set(unique.map((record) => record.agent)).size; + const conflictingTurnCount = findConflictingUsageTurns(records).length; return { agentsReporting: reporting, agentsTotal: totalAgents, + conflictingTurnCount, estimatedTurnCount: unique.filter((record) => record.estimated_requests !== undefined).length, incompleteRecordCount: unique.filter((record) => !record.complete).length, - partial: reporting < totalAgents || unreadableUnitCount > 0, + partial: reporting < totalAgents || unreadableUnitCount > 0 || conflictingTurnCount > 0, unreadableUnitCount }; }; diff --git a/src/runtime/usageLedgerRead.test.ts b/src/runtime/usageLedgerRead.test.ts index 1562abd4..b9fe9dbb 100644 --- a/src/runtime/usageLedgerRead.test.ts +++ b/src/runtime/usageLedgerRead.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { USAGE_TURN_RECORD_VERSION, type UsageRecord } from "./usageLedger.js"; +import { computeUsageCoverage, groupUsageByEngine, USAGE_TURN_RECORD_VERSION, type UsageRecord } from "./usageLedger.js"; import { readUsageLedgerViaExec } from "./usageLedgerRead.js"; const record = (overrides: Partial = {}): UsageRecord => ({ @@ -32,7 +32,9 @@ describe("readUsageLedgerViaExec", () => { const sealed = line({ turn: "d".repeat(64) }); const exec = async (command: string[]) => ({ stderr: "", stdout: `${sealed}\n` }); const read = await readUsageLedgerViaExec(exec, paths); - expect(read.records).toHaveLength(1); + expect(read.records).toHaveLength(2); + expect(groupUsageByEngine(read.records)[0]).toMatchObject({ tokens: 14_535, turns: 1 }); + expect(computeUsageCoverage(read.records, 1)).toMatchObject({ conflictingTurnCount: 0, partial: false }); expect(read.unreadable).toEqual([]); }); diff --git a/src/runtime/usageLedgerRead.ts b/src/runtime/usageLedgerRead.ts index d084cb0e..b08709e3 100644 --- a/src/runtime/usageLedgerRead.ts +++ b/src/runtime/usageLedgerRead.ts @@ -12,7 +12,7 @@ * real exports from this folder, and so must not be imported back. */ -import { dedupeUsageRecordsByTurn, parseUsageLedger, type UsageRecord } from "./usageLedger.js"; +import { parseUsageLedger, type UsageRecord } from "./usageLedger.js"; /** The exec shape this module needs to read a ledger — structurally the same * as `RuntimeProbeGateway.exec` (`./types.ts`), duck-typed here rather than @@ -131,9 +131,10 @@ export const readUsageLedgerViaExec = async ( readLedgerGeneration(exec, paths.filePath) ]); return { - // A replayed broker turn can re-append its sealed row after a rotation, so - // dedupe across both generations, not only within each. - records: dedupeUsageRecordsByTurn([...parseUsageLedger(rotated.text), ...parseUsageLedger(primary.text)]), + // Every row is kept, across both generations: aggregates dedupe by `turn` + // (a replay can re-append after a rotation) and coverage reports rows that + // share a key but differ. + records: [...parseUsageLedger(rotated.text), ...parseUsageLedger(primary.text)], unreadable: [rotated.failure, primary.failure].filter( (failure): failure is UsageLedgerReadFailure => failure !== undefined ) From 8ef468a91746e8b71ba3c4026bd5a073e78df904 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:37:33 +0200 Subject: [PATCH 33/84] test: probe Moltnet and memory denies and the /var/run alias in the Grok live check --- scripts/grok-lean-worker-live-check.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/grok-lean-worker-live-check.ts b/scripts/grok-lean-worker-live-check.ts index c19ec19f..8f4fabf0 100644 --- a/scripts/grok-lean-worker-live-check.ts +++ b/scripts/grok-lean-worker-live-check.ts @@ -56,6 +56,9 @@ const main = (): void => { ]) if (!layout.includes(expected)) throw new Error(`worker home layout is missing: ${expected.trim()}`); const profile = exec(container, `cat ${workerHome}/.grok/sandbox.toml`); if (!/deny = \["\//u.test(profile)) throw new Error("worker sandbox profile has an empty deny list"); + for (const required of ["/var/lib/spawnfile/moltnet", "/var/lib/spawnfile/memory", "/run/daimon-engine-broker"]) if (!profile.includes(JSON.stringify(required))) throw new Error(`worker sandbox profile does not deny ${required}`); + // /run denies rely on /var/run being the /run symlink (a bind mask covers both spellings); report what the image has. + process.stdout.write(`/var/run -> ${exec(container, "readlink /var/run || echo not-a-symlink").trim()}\n`); const service = JSON.parse(exec(container, "cat /etc/daimon-engine-broker/service.json")) as { version: string; registrations: Array<{ model: { id: string } }> }; if (service.version !== "noopolis.daimon.engine-broker-service.v2" || service.registrations[0]?.model.id !== "grok-4.6") throw new Error("service.json is not the declared v2 registration"); exec(container, `setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! printf x >> ${workerHome}/.grok/trusted_folders.toml' && setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! test -r /var/lib/spawnfile/daimon/grok-subscription-realm'`); From aa1e9a6fc1f7c5c8dea5c7db0b590a4711d2a668 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:12:07 +0200 Subject: [PATCH 34/84] feat: re-vendor the Daimon Grok contract with worker temp, spill, inference grant, and slot receipt v2 terms --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 38 +++++++++++++++++---- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 8144e93a..0d92caf1 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9","sourceSha256":"36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","x64Sha256":"36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v1","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382","sourceSha256":"c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75","x64Sha256":"69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index bec82dd2..1ed539ba 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:c9ea30c600c4844621c64a015e3d42e97f410c9942eb41dd675b5c04add34270 +sha256:401da56de1182a4c1834bc872ab3121d0b45d63486d729a02ff98d5627f30829 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 8860bd2d..117dbbd9 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:c9ea30c600c4844621c64a015e3d42e97f410c9942eb41dd675b5c04add34270" as const; + "sha256:401da56de1182a4c1834bc872ab3121d0b45d63486d729a02ff98d5627f30829" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -79,7 +79,10 @@ export const DAIMON_GROK_ENGINE_BROKER = { directory: { uid: 0, group: "worker", mode: 0o1771 }, sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, - sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, + privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, + sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, @@ -99,11 +102,34 @@ export const DAIMON_GROK_ENGINE_BROKER = { }, wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, projectionVersion: "noopolis.daimon.grok-broker-projection.v1", - slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", + inferenceGrants: { + requestKinds: ["request_inference_grant", "release_inference_grant"], + purposes: ["judge", "optimizer"], + tokenPrefix: "inference_", + ttlMs: 600_000, + limits: { maxRequests: 64, maxTokens: 2_000_000 }, + maxLiveGrants: 8, + maxInFlightRequestsPerGrant: 1, + bodyMembers: ["messages", "model", "reasoning_effort", "response_format", "stream", "stream_options"], + messageRoles: ["system", "user", "assistant"], + failureCodes: ["auth_stale", "grant_limit", "invalid_request", "unavailable"], + ledgerVersion: "noopolis.daimon.inference-usage.v1", + ledgerDedupeKey: ["grant", "request"], + client: { + modelId: "daimon-inference-grok", + envKey: "DAIMON_INFERENCE_GRANT", + configSha256: { + "grok-4.6": { low: "79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f", medium: "6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8", high: "5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac" }, + "grok-4.5": { low: "a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74", medium: "1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9", high: "ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2" }, + "grok-build": { low: "ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1", medium: "01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d", high: "98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c" } + } + } + }, artifacts: { - sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", - x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", - arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" + sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", + x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", + arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" } } as const; /** From e6af844c642951d7f30fe1ea8d62d53e601f3b95 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:16:05 +0200 Subject: [PATCH 35/84] feat: provision Grok worker private temp, closed shared temp, setgid spill directories, and a private broker TMPDIR --- src/compiler/AGENTS.md | 11 ++++++++++- src/compiler/containerDaimonBrokerRender.ts | 8 ++++++++ ...containerDaimonGrokWorkerProvisioning.test.ts | 16 ++++++++++++++++ .../containerDaimonGrokWorkerProvisioning.ts | 16 ++++++++++++++-- src/compiler/containerDaimonGrokWorkerRender.ts | 11 +++++++++++ .../containerDaimonUidEntrypointRender.test.ts | 6 +++++- .../containerDaimonUidEntrypointRender.ts | 6 +++--- src/runtime/daimon/config.test.ts | 11 +++++++++++ src/runtime/daimon/config.ts | 13 +++++++++---- 9 files changed, 87 insertions(+), 11 deletions(-) diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 96d68a69..93342f35 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -192,7 +192,16 @@ src/compiler/ `GROK_HOME` layout — `root: 1771` home and `sessions/`, `root:root 0444` config/sandbox/trust/managed/requirements files, events under `sessions/` — and refuses any registration or deny path that is missing, a - symlink, or not its own realpath. Production registrations all point + symlink, or not its own realpath. It also provisions Daimon's temp and spill + contract: `/tmp` `: 0700` (the launcher's + `TMPDIR`); `/tmp` and `/var/tmp` `root:2000 1774` (Grok refuses to start if + they are denied, so modes close them); `/tool-output` + `2000: 2750` under a runtime home `2000: 0710` whose + `/var/lib/spawnfile` ancestors are made traversable by reclaim-mode-restore. + The broker and relay (uid 2100, outside group 2000) run with + `TMPDIR=/run/daimon-engine-broker/tmp`; every other entrypoint process runs as + root or uid 2000, and workers get their `TMPDIR` from the launcher. The start + script never restates a Grok agent's runtime home mode. Production registrations all point `usageLedgerPath` at the one container ledger, because `spawnfile usage` and Daimon's wake fuse read only that file. - Declared names are checked for uniqueness across EVERY mount source diff --git a/src/compiler/containerDaimonBrokerRender.ts b/src/compiler/containerDaimonBrokerRender.ts index 34504095..6574e5f4 100644 --- a/src/compiler/containerDaimonBrokerRender.ts +++ b/src/compiler/containerDaimonBrokerRender.ts @@ -19,6 +19,13 @@ export const DAIMON_BROKER_BACKEND_SOCKET = "/run/daimon-engine-broker/backend.s export const DAIMON_BROKER_LAUNCHER_SOCKET = "/run/daimon-engine-broker/launcher.sock"; export const DAIMON_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; export const DAIMON_BROKER_REALM = "/var/lib/spawnfile/daimon/grok-subscription-realm"; +/** + * Private temp for the broker and its relay (uid 2100, outside the organization + * group): shared `/tmp` and `/var/tmp` are `root:2000 1774` in a Grok + * organization, so any non-root process outside group 2000 needs its own + * `TMPDIR`. It lives in the broker's `/run` directory, which every worker denies. + */ +export const DAIMON_BROKER_TMPDIR = "/run/daimon-engine-broker/tmp"; export { DAIMON_ORGANIZATION_STATE_DIRECTORY, DAIMON_WORKER_ROOT, @@ -128,6 +135,7 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri "if [ -d /run/daimon-engine-broker ]; then chmod u+rwx /run/daimon-engine-broker; fi", "rm -rf /etc/daimon-engine-broker /run/daimon-engine-broker", `install -d -o root -g ${DAIMON_BROKER_UID} -m 0731 /run/daimon-engine-broker`, + `install -d -o ${DAIMON_BROKER_UID} -g ${DAIMON_BROKER_UID} -m 0700 ${DAIMON_BROKER_TMPDIR}`, "node <<'SPAWNFILE_DAIMON_BROKER_PROVISION'", program, "SPAWNFILE_DAIMON_BROKER_PROVISION" diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index d050f1fa..3d3e0ef7 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -109,6 +109,22 @@ describe("Grok worker home provisioning", () => { for (const optional of ["/run/secrets", "/run/spawnfile", "/run/spawnfile-secrets"]) expect(nodes.get(optional)).toMatchObject({ kind: "dir", mode: 0o700, uid: 0 }); }); + it("provisions the P1b temp and spill contract: private TMPDIR, closed shared temp, setgid spills under a traversable runtime home", () => { + const runtimeHomes = `${INSTANCE}/runtime-homes`; + const nodes = run(registrations, { ...seedFor(registrations), [runtimeHomes]: { gid: 2000, kind: "dir", mode: 0o700, uid: 2000 }, "/tmp": { kind: "dir", mode: 0o1777 } }); + for (const shared of ["/tmp", "/var/tmp"]) expect(nodes.get(shared), shared).toMatchObject({ gid: 2000, kind: "dir", mode: 0o1774, uid: 0 }); + // Owner and group survive the traversal fix: root reclaims, modes, and restores (no CAP_FOWNER). + expect(nodes.get(runtimeHomes)).toMatchObject({ gid: 2000, mode: 0o711, uid: 2000 }); + for (const entry of registrations) { + expect(entry.privateTmp).toBe(`${entry.home}/tmp`); + expect(nodes.get(entry.privateTmp)).toMatchObject({ gid: entry.uid, kind: "dir", mode: 0o700, uid: entry.uid }); + expect(nodes.get(entry.runtimeHome)).toMatchObject({ gid: entry.uid, kind: "dir", mode: 0o710, uid: 2000 }); + expect(entry.spillDirectory).toBe(`${entry.runtimeHome}/tool-output`); + expect(nodes.get(entry.spillDirectory)).toMatchObject({ gid: entry.uid, kind: "dir", mode: 0o2750, uid: 2000 }); + expect(entry.profilePath).toBe(`${entry.home}/.grok/sandbox.toml`); + } + }); + it("allows a peer resource backing to be absent at provisioning (the entrypoint prepares it later) but never a symlink", () => { const withResource = resolveDaimonGrokRegistrations([{ ...plan({ "agent:a": "grok", "agent:b": "grok" }), diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 757b3972..1e160c16 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -1,5 +1,5 @@ import { DAIMON_GROK_ENGINE_BROKER } from "../runtime/daimon/contractManifest.js"; -import { DAIMON_BROKER_UID } from "../runtime/daimon/runtimeIdentity.js"; +import { DAIMON_BROKER_UID, DAIMON_ORGANIZATION_UID } from "../runtime/daimon/runtimeIdentity.js"; import { DAIMON_GROK_DENIED_STATE_ROOTS, DAIMON_GROK_OPTIONAL_DENY_PATHS, @@ -22,8 +22,11 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ model: entry.model, profile: entry.profile, profilePath: entry.profilePath, + privateTmp: entry.privateTmp, profileSha256: entry.profileSha256, reasoningEffort: entry.reasoningEffort, + runtimeHome: entry.runtimeHome, + spillDirectory: entry.spillDirectory, slot: entry.slot, uid: entry.uid, workspace: entry.workspace @@ -54,17 +57,26 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo `const pinnedConfigSha256 = ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.configSha256)};`, "const sha256Hex = (value) => crypto.createHash('sha256').update(value).digest('hex');", "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !Object.hasOwn(pinnedConfigSha256, entry.model) || !Object.hasOwn(pinnedConfigSha256[entry.model], entry.reasoningEffort) || pinnedConfigSha256[entry.model][entry.reasoningEffort] !== entry.configSha256 || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", + // Root holds no CAP_FOWNER: every mode change reclaims the inode, sets the mode, then restores or hands over ownership. + "const withMode = (target, mode, uid, gid) => { const info = fs.lstatSync(target); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`unsafe Grok worker directory: ${target}`); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", + "const traversable = (target) => { for (let ancestor = require('node:path').dirname(target); ancestor.startsWith('/var/lib/spawnfile/') && ancestor.length > '/var/lib/spawnfile'.length; ancestor = require('node:path').dirname(ancestor)) { const info = fs.lstatSync(ancestor); if ((info.mode & 0o011) !== 0o011) withMode(ancestor, (info.mode & 0o7777) | 0o011, info.uid, info.gid); } };", "const assertCanonical = (target, label) => { const info = fs.lstatSync(target); if (info.isSymbolicLink() || fs.realpathSync(target) !== target) throw new Error(`Grok worker ${label} is not a canonical non-symlink path: ${target}`); return info; };", "const ensureDirectory = (target, uid, gid, mode) => { fs.mkdirSync(target, { recursive: true, mode: 0o700 }); const info = fs.lstatSync(target); if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('unsafe worker runtime directory'); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); fs.chownSync(target, uid, gid); };", "const ensureExactFile = (target, content, mode) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, content, { mode, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new Error('unsafe worker runtime file'); if (fs.readFileSync(target, 'utf8') !== content) throw new Error(`worker runtime file identity mismatch: ${target}`); fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); };", `const ensureEventsFile = (target, uid) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.writeFileSync(target, '', { mode: 0o640, flag: 'wx' }); info = fs.lstatSync(target); } if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.uid !== uid && info.uid !== 0) || ![0, uid, ${DAIMON_BROKER_UID}].includes(info.gid) || ![0o600, 0o640, 0o644].includes(info.mode & 0o777)) throw new Error('unsafe worker attestation events'); fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o640); fs.chownSync(target, uid, ${DAIMON_BROKER_UID}); };`, `fs.mkdirSync('${DAIMON_WORKER_ROOT}', { recursive: true, mode: 0o711 }); fs.chownSync('${DAIMON_WORKER_ROOT}', 0, 0); fs.chmodSync('${DAIMON_WORKER_ROOT}', 0o711); assertCanonical('${DAIMON_WORKER_ROOT}', 'worker root');`, // Pass 1: every workspace and home exists, root-held, before any deny list is checked. - `for (const entry of grokWorkers) { for (let ancestor = require('node:path').dirname(entry.workspace); ancestor.startsWith('/var/lib/spawnfile/') && ancestor.length > '/var/lib/spawnfile'.length; ancestor = require('node:path').dirname(ancestor)) fs.chmodSync(ancestor, fs.statSync(ancestor).mode & 0o7777 | 0o011); secureWorkspace(entry.workspace, entry.uid); assertCanonical(entry.workspace, 'workspace'); ensureDirectory(entry.home, 0, 0, 0o700); ensureDirectory(entry.grokHome, 0, 0, 0o700); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, 0, 0o700); for (const target of [entry.home, entry.grokHome]) assertCanonical(target, 'home'); }`, + `for (const entry of grokWorkers) { traversable(entry.workspace); secureWorkspace(entry.workspace, entry.uid); assertCanonical(entry.workspace, 'workspace'); ensureDirectory(entry.home, 0, 0, 0o700); ensureDirectory(entry.grokHome, 0, 0, 0o700); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, 0, 0o700); for (const target of [entry.home, entry.grokHome]) assertCanonical(target, 'home'); }`, "for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }", "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT' && entry.deferredDenyPaths.includes(denied)) continue; if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", // Pass 2: exact root-owned read-only files, the events file, then the final sticky modes. `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, + // Worker-private temp: the launcher compiles TMPDIR=/tmp, the only temp the worker may write. + "for (const entry of grokWorkers) { ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); assertCanonical(entry.privateTmp, 'private temp'); }", + // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can traverse. + `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, + // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. + `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 1fff49bc..2e1f2fa2 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -58,7 +58,13 @@ export interface DaimonGrokRegistration { profile: string; profilePath: string; profileSha256: string; + /** `/tmp`: the launcher's compiled `TMPDIR` for this worker. */ + privateTmp: string; reasoningEffort: DaimonGrokBrokerReasoningEffort; + /** The organization runtime home whose `tool-output/` this worker reads. */ + runtimeHome: string; + /** `/tool-output`: setgid spill directory in the worker's group. */ + spillDirectory: string; slot: number; uid: number; usageLedgerPath: string; @@ -192,6 +198,8 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim return entries.map(({ agentId, plan }, slot) => { const home = homes[slot]!; const grokHome = path.posix.join(home, DAIMON_GROK_WORKER_HOME_DIRECTORY); + const instanceRoot = plan.instancePaths.instanceRoot ?? fail("Daimon Grok registrations require an instance root"); + const runtimeHome = path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, nodeSlug(agentId)); const { model, reasoningEffort } = declaredModel(plan, agentId); const config = resolveDaimonGrokWorkerConfig(model, reasoningEffort); const denyPaths = resolveDaimonGrokWorkerDenyPaths(plans, plan, agentId, homes, home); @@ -210,7 +218,10 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim profile, profilePath: path.posix.join(grokHome, "sandbox.toml"), profileSha256: daimonGrokWorkerSandboxProfileSha256(denyPaths), + privateTmp: path.posix.join(home, DAIMON_GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome), reasoningEffort, + runtimeHome, + spillDirectory: path.posix.join(runtimeHome, DAIMON_GROK_ENGINE_BROKER.worker.home.spillDirectory.relativeToRuntimeHome), slot, uid: DAIMON_FIRST_WORKER_UID + slot, // Production keeps one container ledger: `spawnfile usage` and Daimon's diff --git a/src/compiler/containerDaimonUidEntrypointRender.test.ts b/src/compiler/containerDaimonUidEntrypointRender.test.ts index cc99c401..dd91e2ab 100644 --- a/src/compiler/containerDaimonUidEntrypointRender.test.ts +++ b/src/compiler/containerDaimonUidEntrypointRender.test.ts @@ -247,7 +247,11 @@ describe("renderDaimonUidEntrypoint", () => { expect(rendered).toContain("bootstrapBytes.fill(0)"); expect(rendered).toContain("ensureExactFile(entry.profilePath, entry.profile, 0o444)"); expect(rendered).toContain("--bounding-set=-all,+chown,+setuid,+setgid -- '/opt/daimon/bin/daimon-engine-broker' &"); - expect(rendered).toContain("--bounding-set=-all,+chown,+setuid,+setgid,+setpcap -- '/opt/daimon/bin/daimon-engine-broker' --relay &"); + expect(rendered).toContain("--bounding-set=-all,+chown,+setuid,+setgid,+setpcap -- env TMPDIR='/run/daimon-engine-broker/tmp' '/opt/daimon/bin/daimon-engine-broker' --relay &"); + // Shared /tmp is root:2000 1774 in a Grok organization: every non-root process outside group 2000 needs its own TMPDIR. + expect(rendered).toContain("--bounding-set=-all -- env TMPDIR='/run/daimon-engine-broker/tmp' '/opt/daimon/bin/daimon-runtime' engine-broker serve &"); + expect(rendered).toContain("install -d -o 2100 -g 2100 -m 0700 /run/daimon-engine-broker/tmp"); + expect(rendered.match(/--reuid 2100 --regid 2100 [^\n]*daimon-runtime' engine-broker serve/gu)?.every((line) => line.includes("env TMPDIR="))).toBe(true); expect(rendered).toContain('"$relay_pid:2100:0000000000000000"'); expect(rendered).toContain('expected_caps=${rest##*:}'); expect(rendered).toContain("--reuid 2100 --regid 2100"); diff --git a/src/compiler/containerDaimonUidEntrypointRender.ts b/src/compiler/containerDaimonUidEntrypointRender.ts index 1dd5a8ca..665d7da7 100644 --- a/src/compiler/containerDaimonUidEntrypointRender.ts +++ b/src/compiler/containerDaimonUidEntrypointRender.ts @@ -23,7 +23,7 @@ import { DAIMON_BROKER_BACKEND_SOCKET, DAIMON_BROKER_LAUNCHER_SOCKET, DAIMON_BROKER_REALM, - DAIMON_BROKER_SOCKET, + DAIMON_BROKER_SOCKET, DAIMON_BROKER_TMPDIR, DAIMON_BROKER_UID, DAIMON_ORGANIZATION_UID, renderDaimonBrokerProvisioning, @@ -364,12 +364,12 @@ export const renderDaimonUidEntrypoint = ( "startup_children+=(\"$launcher_pid\")", `wait_for_broker_socket ${quote(DAIMON_BROKER_LAUNCHER_SOCKET)} "$launcher_pid" "engine broker launcher"`, `wait_for_broker_identity "$launcher_pid" 0 00000000000000c1 "engine broker launcher"`, - `setpriv --clear-groups --reuid ${DAIMON_BROKER_UID} --regid ${DAIMON_BROKER_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- ${quote(path.posix.join(daimonPlan?.runtimeRoot ?? "", "bin/daimon-runtime"))} engine-broker serve &`, + `setpriv --clear-groups --reuid ${DAIMON_BROKER_UID} --regid ${DAIMON_BROKER_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- env TMPDIR=${quote(DAIMON_BROKER_TMPDIR)} ${quote(path.posix.join(daimonPlan?.runtimeRoot ?? "", "bin/daimon-runtime"))} engine-broker serve &`, "broker_pid=$!", "startup_children+=(\"$broker_pid\")", `wait_for_broker_socket ${quote(DAIMON_BROKER_BACKEND_SOCKET)} "$broker_pid" "engine broker backend"`, `wait_for_broker_identity "$broker_pid" ${DAIMON_BROKER_UID} 0000000000000000 "engine broker backend"`, - `setpriv --inh-caps=-all --ambient-caps=-all --bounding-set=-all,+chown,+setuid,+setgid,+setpcap -- ${quote(DAIMON_BROKER_EXECUTABLE)} --relay &`, + `setpriv --inh-caps=-all --ambient-caps=-all --bounding-set=-all,+chown,+setuid,+setgid,+setpcap -- env TMPDIR=${quote(DAIMON_BROKER_TMPDIR)} ${quote(DAIMON_BROKER_EXECUTABLE)} --relay &`, "relay_pid=$!", "startup_children+=(\"$relay_pid\")", `wait_for_broker_socket ${quote(DAIMON_BROKER_SOCKET)} "$relay_pid" "engine broker control relay"`, diff --git a/src/runtime/daimon/config.test.ts b/src/runtime/daimon/config.test.ts index 9dea841b..d70719c8 100644 --- a/src/runtime/daimon/config.test.ts +++ b/src/runtime/daimon/config.test.ts @@ -170,6 +170,17 @@ describe("Daimon memory lowering", () => { expect(config.agents[0]!.engine).toEqual({ kind: "grok", model: "grok-4.5", reasoningEffort: "medium" }); }); + it("never restates a Grok agent's runtime home mode, which its worker's group must traverse for spills", async () => { + const [target] = await createDaimonContainerTargets([{ emittedFiles: [], id: "agent:grok", kind: "agent", slug: "grok", value: createPiTestNode({ + execution: { model: { primary: { auth: { method: "grok" }, name: "grok-4.6", provider: "xai", reasoning_effort: "low" } } }, + runtime: { name: "daimon", options: { engine: "grok" } } + }) }, { emittedFiles: [], id: "agent:codex", kind: "agent", slug: "codex", value: createDaimonNode() }]); + const start = target!.files.find((file) => file.path === "runtime/daimon-start.sh")!.content as string; + expect(start).toContain('[ -d "/runtime-homes/grok" ] || install -d -m 700 "/runtime-homes/grok"'); + expect(start).not.toMatch(/^install -d -m 700 "\/runtime-homes\/grok"/mu); + expect(start).toContain('install -d -m 700 "/runtime-homes/codex"'); + }); + it("omits Daimon's engine model selector when no execution model is declared", async () => { const config = await emitConfig(createDaimonNode({ execution: undefined })); diff --git a/src/runtime/daimon/config.ts b/src/runtime/daimon/config.ts index 4dba813a..2ba1ea54 100644 --- a/src/runtime/daimon/config.ts +++ b/src/runtime/daimon/config.ts @@ -140,10 +140,15 @@ const renderStartScript = (agents: Array<{ // group access. Forcing 0700 here would lock that worker out of its own workspace, // so create it only when absent and never restate the mode of an existing directory. `[ -d ${JSON.stringify(agent.workspacePath)} ] || install -d -m 700 ${JSON.stringify(agent.workspacePath)}`, - `install -d -m 700 ${[ - agent.runtimeHomePath, - ...(credential === undefined ? [] : [inbound]) - ].map((entry) => JSON.stringify(entry)).join(" ")}`, + // A Grok agent's runtime home is traversable by its worker's group (the + // broker provisioning grants `0710` so the worker can read its setgid + // `tool-output/` spills); restating 0700 here would revoke that. + agent.engine.kind === "grok" + ? `[ -d ${JSON.stringify(agent.runtimeHomePath)} ] || install -d -m 700 ${JSON.stringify(agent.runtimeHomePath)}` + : `install -d -m 700 ${[ + agent.runtimeHomePath, + ...(credential === undefined ? [] : [inbound]) + ].map((entry) => JSON.stringify(entry)).join(" ")}`, ...(credential === undefined ? [] : [ `if [ -e ${JSON.stringify(path.posix.join(agent.runtimeHomePath, credential.sourceRelativePath))} ]; then test "$(stat -c %a ${JSON.stringify(path.posix.join(agent.runtimeHomePath, credential.sourceRelativePath))})" = 600; fi` ]) From 9c9e0264234e10a24c68925a612ef80522c33a42 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:16:58 +0200 Subject: [PATCH 36/84] fix: refuse Grok worker deny entries equal to or above a base sandbox profile grant --- .../containerDaimonGrokWorkerRender.test.ts | 19 +++++++++++++++++++ .../containerDaimonGrokWorkerRender.ts | 13 +++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/compiler/containerDaimonGrokWorkerRender.test.ts b/src/compiler/containerDaimonGrokWorkerRender.test.ts index 8516ef37..d1cb05bd 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.test.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.test.ts @@ -103,3 +103,22 @@ describe("Daimon Grok worker registrations", () => { }]); }); }); + +describe("Grok base-profile grant guard", () => { + const plan = (extra: Partial): RuntimeTargetPlan => ({ ...grokWorkerPlan({ "agent:a": "grok", "agent:b": "grok" }), ...extra }) as RuntimeTargetPlan; + const mount = (mount_path: string) => ({ id: mount_path, mount_path, reason: "test", volume_name: "v" }); + + it("refuses a deny entry equal to or above a base-profile grant, and keeps entries below grants", () => { + for (const grant of ["/run", "/var", "/tmp", "/var/tmp", "/etc"]) { + expect(() => resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount(grant)] })]), grant).toThrow(/base profile grant/u); + } + expect(() => resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount("/var/lib/daimon-workers/2200/.grok/sessions")] })])).toThrow(/own workspace, home|base profile grant/u); + const [a] = resolveDaimonGrokRegistrations([plan({})]); + const grants = ["/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp", a!.workspace, a!.grokHome, `${a!.grokHome}/sessions`, `${a!.home}/tmp`]; + for (const entry of a!.denyPaths) { + for (const grant of grants) expect(grant === entry || grant.startsWith(`${entry}/`), `${entry} vs ${grant}`).toBe(false); + } + expect(a!.denyPaths).toEqual(expect.arrayContaining(["/run/secrets", "/run/daimon-engine-broker"])); + }); + +}); diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 2e1f2fa2..748b7b4f 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -44,6 +44,13 @@ export const DAIMON_ORGANIZATION_STATE_DIRECTORY = path.posix.join( "state" ); +/** + * Paths Grok 1.0.34's strict base profile grants (read or read-write). Grok + * refuses to start when a deny entry equals or contains one of them (verified + * for `/tmp`, `/var/tmp`, `/run`, `/etc` and `sessions`; `/tmp/sub` works), so a + * deny entry must always sit strictly below every grant it touches. + */ +export const DAIMON_GROK_BASE_PROFILE_GRANTS = ["/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp"] as const; export interface DaimonGrokRegistration { agentId: string; config: string; @@ -170,6 +177,12 @@ export const resolveDaimonGrokWorkerDenyPaths = ( ...workerHomes.filter((home) => home !== ownHome), ...DAIMON_GROK_OPTIONAL_DENY_PATHS ].filter((entry) => entry.startsWith("/")); + const grokHome = path.posix.join(ownHome, DAIMON_GROK_WORKER_HOME_DIRECTORY); + const grants = [...DAIMON_GROK_BASE_PROFILE_GRANTS, ownWorkspace, grokHome, path.posix.join(grokHome, "sessions"), path.posix.join(ownHome, "tmp")]; + for (const entry of [...daimonOwn, ...added]) { + const grant = grants.find((candidate) => within(candidate, entry)); + if (grant) fail(`Grok worker deny path ${entry} equals or contains the base profile grant ${grant}; Grok refuses such a profile`); + } for (const entry of added) { if (within(ownWorkspace, entry) || within(ownHome, entry) || entry === ownRuntimeHome || within(ownRuntimeHome, entry) || [...ownResourceBackings].some((backing) => within(backing, entry))) { From 38887486bf4d27bb61da2a55a8a7351470945567 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:16:59 +0200 Subject: [PATCH 37/84] test: prove an added ancestor covering a Daimon deny entry is refused --- src/compiler/containerDaimonGrokWorkerRender.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/compiler/containerDaimonGrokWorkerRender.test.ts b/src/compiler/containerDaimonGrokWorkerRender.test.ts index d1cb05bd..69e47cbc 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.test.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.test.ts @@ -104,7 +104,7 @@ describe("Daimon Grok worker registrations", () => { }); }); -describe("Grok base-profile grant guard", () => { +describe("Grok base-profile grant guard and nested-mask refusal", () => { const plan = (extra: Partial): RuntimeTargetPlan => ({ ...grokWorkerPlan({ "agent:a": "grok", "agent:b": "grok" }), ...extra }) as RuntimeTargetPlan; const mount = (mount_path: string) => ({ id: mount_path, mount_path, reason: "test", volume_name: "v" }); @@ -121,4 +121,8 @@ describe("Grok base-profile grant guard", () => { expect(a!.denyPaths).toEqual(expect.arrayContaining(["/run/secrets", "/run/daimon-engine-broker"])); }); + it("refuses an added ancestor that would cover a Daimon deny entry, since masks cannot nest", () => { + expect(() => resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount("/var/lib/spawnfile/instances/daimon/daimon-organization/state")] })])) + .toThrow(/would cover .*state\/wake-acceptance; masks cannot nest/u); + }); }); From eefa9fb41389edc80ef7ad1c7152ffe402b47165 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:17:44 +0200 Subject: [PATCH 38/84] fix: write and validate canonical Grok registration paths within the 255-byte registration bound --- .../containerDaimonGrokWorkerRender.test.ts | 22 ++++++++++++++++ .../containerDaimonGrokWorkerRender.ts | 26 ++++++++++++++++--- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/compiler/containerDaimonGrokWorkerRender.test.ts b/src/compiler/containerDaimonGrokWorkerRender.test.ts index 69e47cbc..c38cb3c5 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.test.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.test.ts @@ -6,6 +6,7 @@ import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_TURN_USAGE_LEDGER } from "../run import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; import { + assertCanonicalRegisteredPath, DAIMON_GROK_OPTIONAL_DENY_PATHS, renderDaimonGrokServiceConfig, resolveDaimonGrokRegistrations @@ -126,3 +127,24 @@ describe("Grok base-profile grant guard and nested-mask refusal", () => { .toThrow(/would cover .*state\/wake-acceptance; masks cannot nest/u); }); }); + +describe("canonical registered paths", () => { + it("accepts only absolute canonical paths of at most 255 bytes", () => { + expect(assertCanonicalRegisteredPath("home", "/var/lib/daimon-workers/2200")).toBe("/var/lib/daimon-workers/2200"); + for (const bad of ["relative/home", "/", "/var/lib/", "/var//lib", "/var/./lib", "/var/../lib", "/var/lib/.", `/${"a".repeat(255)}`]) { + expect(() => assertCanonicalRegisteredPath("home", bad), bad).toThrow(/canonical registered path/u); + } + expect(assertCanonicalRegisteredPath("home", `/${"a".repeat(254)}`)).toHaveLength(255); + }); + + it("writes only canonical workspace, home, and runtime paths, and refuses one that cannot fit the registration record", () => { + const messy = { ...grokWorkerPlan({ "agent:a": "grok" }), instancePaths: { configPath: `${INSTANCE}/daimon/config.json`, instanceRoot: `${INSTANCE}/`, workspacePath: `${INSTANCE}//workspace/` } } as RuntimeTargetPlan; + const [entry] = resolveDaimonGrokRegistrations([messy]); + for (const value of [entry!.workspace, entry!.home, entry!.grokHome, entry!.profilePath, entry!.eventsPath, entry!.privateTmp, entry!.runtimeHome, entry!.spillDirectory]) { + expect(() => assertCanonicalRegisteredPath("path", value), value).not.toThrow(); + } + const longId = `agent:${"x".repeat(240)}`; + expect(() => resolveDaimonGrokRegistrations([grokWorkerPlan({ [longId]: "grok" })])).toThrow(/canonical registered path/u); + expect(() => resolveDaimonGrokRegistrations([grokWorkerPlan({ "agent:!!!": "grok" })])).toThrow(/no path-safe slug/u); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 748b7b4f..5bab54c0 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -51,6 +51,22 @@ export const DAIMON_ORGANIZATION_STATE_DIRECTORY = path.posix.join( * deny entry must always sit strictly below every grant it touches. */ export const DAIMON_GROK_BASE_PROFILE_GRANTS = ["/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp"] as const; +export const DAIMON_GROK_MAX_REGISTERED_PATH_BYTES = 255; + +/** + * The canonical path rule Daimon's service parser and native launcher enforce: + * absolute, no empty, `.` or `..` component, no trailing slash, and at most 255 + * bytes (the registration record's NUL-terminated 256-byte fields). + */ +export const assertCanonicalRegisteredPath = (label: string, value: string): string => { + const components = value.split("/").slice(1); + if (!value.startsWith("/") || value.length < 2 || value.endsWith("/") || components.some((part) => part === "" || part === "." || part === "..") + || path.posix.normalize(value) !== value || value.includes("\0") || Buffer.byteLength(value) > DAIMON_GROK_MAX_REGISTERED_PATH_BYTES) { + fail(`Grok worker ${label} is not a canonical registered path: ${JSON.stringify(value)}`); + } + return value; +}; + export interface DaimonGrokRegistration { agentId: string; config: string; @@ -209,14 +225,18 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim .sort((left, right) => left.agentId.localeCompare(right.agentId)); const homes = entries.map((_, slot) => path.posix.join(DAIMON_WORKER_ROOT, String(DAIMON_FIRST_WORKER_UID + slot))); return entries.map(({ agentId, plan }, slot) => { - const home = homes[slot]!; + if (nodeSlug(agentId) === "") fail(`Daimon Grok agent ${agentId} has no path-safe slug`); + const home = assertCanonicalRegisteredPath("home", homes[slot]!); const grokHome = path.posix.join(home, DAIMON_GROK_WORKER_HOME_DIRECTORY); const instanceRoot = plan.instancePaths.instanceRoot ?? fail("Daimon Grok registrations require an instance root"); - const runtimeHome = path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, nodeSlug(agentId)); + const runtimeHome = assertCanonicalRegisteredPath("runtime home", path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, nodeSlug(agentId))); const { model, reasoningEffort } = declaredModel(plan, agentId); const config = resolveDaimonGrokWorkerConfig(model, reasoningEffort); const denyPaths = resolveDaimonGrokWorkerDenyPaths(plans, plan, agentId, homes, home); const profile = renderDaimonGrokWorkerSandboxProfile(denyPaths); + for (const [label, value] of [["GROK_HOME", grokHome], ["profile", path.posix.join(grokHome, "sandbox.toml")], ["events", path.posix.join(grokHome, DAIMON_GROK_ENGINE_BROKER.worker.home.sandboxEvents.relativePath)], ["private temp", path.posix.join(home, DAIMON_GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome)]] as const) assertCanonicalRegisteredPath(label, value); + // The launcher derives HOME, GROK_HOME=/.grok and TMPDIR=/tmp from the registered home, and the broker reads the profile from GROK_HOME. + if (path.posix.dirname(path.posix.dirname(path.posix.join(grokHome, "sandbox.toml"))) !== home) fail(`Grok worker ${agentId} profile is not under its registered home`); const backings = new Set(plans.flatMap((candidate) => candidate.resources ?? []).map((resource) => resource.backingPath)); return { deferredDenyPaths: denyPaths.filter((entry) => backings.has(entry)), @@ -240,7 +260,7 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim // Production keeps one container ledger: `spawnfile usage` and Daimon's // wake fuse both read it, so a per-slot file would hide Grok spend from both. usageLedgerPath: DAIMON_GROK_TURN_USAGE_LEDGER.filePath, - workspace: path.posix.join(plan.instancePaths.workspacePath, "agents", nodeSlug(agentId)) + workspace: assertCanonicalRegisteredPath("workspace", path.posix.join(plan.instancePaths.workspacePath, "agents", nodeSlug(agentId))) }; }); }; From 408848bd5000ea66d2e5465d4adc82d03ae8b37f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:18:10 +0200 Subject: [PATCH 39/84] test: assert Grok temp, spill, and broker TMPDIR provisioning in the live check --- scripts/grok-lean-worker-live-check.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/grok-lean-worker-live-check.ts b/scripts/grok-lean-worker-live-check.ts index 8f4fabf0..4166adcd 100644 --- a/scripts/grok-lean-worker-live-check.ts +++ b/scripts/grok-lean-worker-live-check.ts @@ -2,7 +2,9 @@ // Explicit opt-in live check for a one-agent brokered Grok 1.0.34 organization: // deploy fixtures/grok-lean-worker against a locally built Daimon runtime image, // dump the worker home/sandbox/service attestation inputs, run one cheap wake, -// and require exactly one deduplicated usage row written by the broker. +// assert the P1b temp/spill layout and broker TMPDIR, and require exactly one +// deduplicated usage row written by the broker. The spill read is a unix-level +// read as the worker uid; a `read_file` spill read needs a >16 KiB tool result. // Needs Docker, a built CLI (`npm run build`), a local Daimon runtime identity // built from the Daimon Grok accounting contract, and a dedicated Grok login // file (never the desktop ~/.grok/auth.json). Makes one real model call. @@ -59,6 +61,18 @@ const main = (): void => { for (const required of ["/var/lib/spawnfile/moltnet", "/var/lib/spawnfile/memory", "/run/daimon-engine-broker"]) if (!profile.includes(JSON.stringify(required))) throw new Error(`worker sandbox profile does not deny ${required}`); // /run denies rely on /var/run being the /run symlink (a bind mask covers both spellings); report what the image has. process.stdout.write(`/var/run -> ${exec(container, "readlink /var/run || echo not-a-symlink").trim()}\n`); + const runtimeHome = "/var/lib/spawnfile/instances/daimon/daimon-organization/runtime-homes/grok-lean-worker"; + const tempLayout = exec(container, `stat -c '%u:%g %a %n' /tmp /var/tmp ${workerHome}/tmp ${runtimeHome} ${runtimeHome}/tool-output`); + process.stdout.write(tempLayout); + for (const expected of [ + "0:2000 1774 /tmp\n", "0:2000 1774 /var/tmp\n", `2200:2200 700 ${workerHome}/tmp\n`, `2000:2200 710 ${runtimeHome}\n`, `2000:2200 2750 ${runtimeHome}/tool-output\n` + ]) if (!tempLayout.includes(expected)) throw new Error(`temp/spill layout is missing: ${expected.trim()}`); + for (const pattern of ["engine-broker serve", "daimon-engine-broker --relay"]) { + const environ = exec(container, `pid=$(pgrep -f -o ${JSON.stringify(pattern)}); tr '\\0' '\\n' < /proc/$pid/environ | grep '^TMPDIR=' || true`).trim(); + if (environ !== "TMPDIR=/run/daimon-engine-broker/tmp") throw new Error(`${pattern} runs without its private TMPDIR (${environ || "unset"})`); + } + // A spill as Daimon writes it (uid 2000, 0640, setgid group) must be readable by the worker, and shared /tmp must not be. + exec(container, `setpriv --reuid 2000 --regid 2000 --clear-groups bash -c 'umask 027; printf spill-ok > ${runtimeHome}/tool-output/live-check.log' && setpriv --reuid 2200 --regid 2200 --clear-groups bash -c 'test "$(cat ${runtimeHome}/tool-output/live-check.log)" = spill-ok && ! printf x > /tmp/worker-probe' && rm -f ${runtimeHome}/tool-output/live-check.log`); const service = JSON.parse(exec(container, "cat /etc/daimon-engine-broker/service.json")) as { version: string; registrations: Array<{ model: { id: string } }> }; if (service.version !== "noopolis.daimon.engine-broker-service.v2" || service.registrations[0]?.model.id !== "grok-4.6") throw new Error("service.json is not the declared v2 registration"); exec(container, `setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! printf x >> ${workerHome}/.grok/trusted_folders.toml' && setpriv --reuid 2200 --regid 2200 --clear-groups bash -c '! test -r /var/lib/spawnfile/daimon/grok-subscription-realm'`); From 051a5ce83903e151cf6ff5897b9ff2344a692268 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:18:10 +0200 Subject: [PATCH 40/84] docs: describe Grok worker temp, spill, and deny-grant rules --- specs/RUNTIMES.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/specs/RUNTIMES.md b/specs/RUNTIMES.md index 1475ab6c..3340b159 100644 --- a/specs/RUNTIMES.md +++ b/specs/RUNTIMES.md @@ -272,10 +272,18 @@ unprivileged user namespaces: set `kernel.apparmor_restrict_unprivileged_userns= (Ubuntu 24.04+ and Colima default to `1`). The entrypoint refuses to start a Grok organization, naming the sysctl, when it is not. -Not yet provided: a private per-worker `/tmp` (the strict profile grants -read-write shared `/tmp` and `/var/tmp`, and the broker launcher's fixed -environment sets no `TMPDIR`), and worker-readable tool-result spill files -(Daimon writes them `0600` as the organization uid). +Grok refuses a profile whose deny list contains a path equal to or above a +base-profile grant (`/tmp`, `/var/tmp`, `/run`, `/etc`, `/var`, the workspace, +`GROK_HOME`, `sessions`, the worker's `TMPDIR`); Spawnfile refuses such an entry +at compile time. Shared temp is therefore closed by modes: `/tmp` and +`/var/tmp` are `root:2000 1774` (a worker can list names, not read or create), +each worker writes only its launcher-compiled `TMPDIR` `/tmp` +(`: 0700`), and the broker and relay, the only non-root +processes outside group 2000, run with `TMPDIR=/run/daimon-engine-broker/tmp`. +Tool-result spills go to `/tool-output` `2000: 2750` +under a runtime home `2000: 0710`, so only the agent's own worker can +read them. Registered workspace and home paths are canonical and at most 255 +bytes. For AGY it declares one host-realm durable mount plus one independent opaque unlock source slot. Spawnfile emits the stable RW volume, metadata-authorizes From a5bf084ecdf28c2c29242795e0816bbc8015a8b3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:21:55 +0200 Subject: [PATCH 41/84] refactor: share the Daimon broker root provisioning program and service.json v2 options --- src/compiler/containerDaimonBrokerRender.ts | 27 +++- .../containerDaimonGrokWorkerProvisioning.ts | 10 +- .../containerDaimonGrokWorkerRender.ts | 38 +++++- src/compiler/training/broker/paths.ts | 128 +++++++++++++++++ src/compiler/training/broker/registration.ts | 129 ++++++++++++++++++ 5 files changed, 325 insertions(+), 7 deletions(-) create mode 100644 src/compiler/training/broker/paths.ts create mode 100644 src/compiler/training/broker/registration.ts diff --git a/src/compiler/containerDaimonBrokerRender.ts b/src/compiler/containerDaimonBrokerRender.ts index 6574e5f4..8357a80e 100644 --- a/src/compiler/containerDaimonBrokerRender.ts +++ b/src/compiler/containerDaimonBrokerRender.ts @@ -3,7 +3,7 @@ import { DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifest.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; -import { resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js"; +import { resolveDaimonGrokRegistrations, type DaimonGrokRegistration, type DaimonGrokServiceConfigOptions } from "./containerDaimonGrokWorkerRender.js"; import { renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; import { DAIMON_BROKER_UID, @@ -101,6 +101,29 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri : null })) .sort((left, right) => left.linkPath.localeCompare(right.linkPath)); + return renderDaimonBrokerProvisioningProgram(registrations, workspaceResources); +}; + +/** + * The root broker provisioning program itself, over already-resolved + * registrations and workspace resources. + * + * Production reaches it through {@link renderDaimonBrokerProvisioning}; the + * broker-capable training container reaches it directly for its single fixed + * slot, with `serviceOptions` pointing the turn store at per-slot tmpfs and + * declaring the evaluator inference ledger. Both callers get byte-identical + * credential, registration, worker-home, temp and spill provisioning — the + * whole point of sharing it rather than writing a second root program. + * + * It is re-runnable: the shell wrapper removes `/etc/daimon-engine-broker` and + * `/run/daimon-engine-broker` first, so a slot recycle replays exactly the + * audited start-up path, credential-journal recovery included. + */ +export const renderDaimonBrokerProvisioningProgram = ( + registrations: readonly DaimonGrokRegistration[], + workspaceResources: WorkspaceSecurityResource[], + serviceOptions: DaimonGrokServiceConfigOptions = {} +): string[] => { const program = [ "const crypto = require('node:crypto'); const fs = require('node:fs');", `const registrations = ${JSON.stringify(registrations.map(({ agentId, home, slot, uid, workspace }) => ({ agentId, home, slot, uid, workspace })))};`, @@ -126,7 +149,7 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri `try { const journalPath = '${DAIMON_BROKER_REALM}/.daimon-broker/credential-journal.json'; let journal; try { const raw = readSecure(journalPath, 2100, 'recovery journal'); journal = JSON.parse(raw.toString('utf8')); raw.fill(0); } catch (error) { if (error.code !== 'ENOENT') throw error; } const stale = journal?.version === 'noopolis.daimon.broker-credential-journal.v1' && journal.state === 'stale'; const recover = () => { if (!stale || !Number.isSafeInteger(journal.generation) || journal.generation < 0 || !/^[a-f0-9]{64}$/.test(journal.sourceDigest) || journal.sourceDigest !== journal.promotedDigest || bootstrapDigest === journal.sourceDigest) throw new Error('unsafe broker credential recovery'); atomicOwned(authority, bootstrapBytes); const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } }; if (!existing) { if (stale) recover(); else atomicOwned(authority, bootstrapBytes); } else { const authorityBytes = readSecure(authority, 2100, 'authority'); try { const authorityDigest = crypto.createHash('sha256').update(authorityBytes).digest('hex'); if (stale) { if (authorityDigest !== journal.sourceDigest && authorityDigest !== bootstrapDigest) throw new Error('unsafe broker credential recovery'); if (authorityDigest === bootstrapDigest) { const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } } else recover(); } } finally { authorityBytes.fill(0); } } } finally { bootstrapBytes.fill(0); }`, "if (journalRootExists) { fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); fs.chownSync(journalRoot, 2100, 2100); }", ...renderDaimonWorkspaceResourceSecurity(workspaceResources), - ...renderDaimonGrokWorkerProvisioning(registrations), + ...renderDaimonGrokWorkerProvisioning(registrations, serviceOptions), `fs.chownSync('${DAIMON_BROKER_REALM}', 0, 0); fs.chmodSync('${DAIMON_BROKER_REALM}', 0o700); fs.chownSync('${DAIMON_BROKER_REALM}', 2100, 2100);`, "fs.chmodSync('/etc/daimon-engine-broker', 0o555);" ].join("\n"); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 1e160c16..e6abdbe4 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -6,7 +6,8 @@ import { DAIMON_GROK_WORKER_READ_ONLY_FILES, DAIMON_WORKER_ROOT, renderDaimonGrokServiceConfig, - type DaimonGrokRegistration + type DaimonGrokRegistration, + type DaimonGrokServiceConfigOptions } from "./containerDaimonGrokWorkerRender.js"; /** Only the bytes the program writes; everything else it needs is recomputed or checked in place. */ @@ -51,7 +52,10 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ * the wrong inode. Root chmods only paths it owns at that moment (no * `CAP_FOWNER`): reclaim, mode, then hand over. */ -export const renderDaimonGrokWorkerProvisioning = (registrations: readonly DaimonGrokRegistration[]): string[] => [ +export const renderDaimonGrokWorkerProvisioning = ( + registrations: readonly DaimonGrokRegistration[], + serviceOptions: DaimonGrokServiceConfigOptions = {} +): string[] => [ `const grokWorkers = ${JSON.stringify(registrations.map(programRegistration))};`, `const optionalDenyPaths = new Set(${JSON.stringify([...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_STATE_ROOTS])});`, `const pinnedConfigSha256 = ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.configSha256)};`, @@ -77,7 +81,7 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, - `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, + `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations, serviceOptions))};`, "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` ]; diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 5bab54c0..c6e16b54 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -265,11 +265,45 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim }); }; +export interface DaimonGrokServiceConfigOptions { + /** + * Where the broker keeps its durable turn registry. Production keeps it on + * the realm volume beside the credential; training overrides it to per-slot + * tmpfs so a recycled slot can never replay trial N's sealed turn in trial + * N+1 (R2), and refuses any path on the realm volume for that reason. + */ + turnStore?: string; + /** `service.json` v2's optional evaluator inference ledger. Without it the broker refuses every grant. */ + inferenceLedgerPath?: string; +} + +const resolveDaimonGrokTurnStore = (turnStore?: string): string => { + if (turnStore === undefined) return DAIMON_GROK_ENGINE_BROKER.turnStorePath; + assertCanonicalRegisteredPath("turn store", turnStore); + if (turnStore === DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath || turnStore.startsWith(`${DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath}/`)) { + fail(`Grok broker turn store ${turnStore} is on the durable credential realm; a recycled slot would replay the previous trial's sealed turns`); + } + return turnStore; +}; + +const resolveDaimonGrokInferenceLedger = (inferenceLedgerPath: string, registrations: readonly DaimonGrokRegistration[]): string => { + assertCanonicalRegisteredPath("inference ledger", inferenceLedgerPath); + if (!inferenceLedgerPath.endsWith(".jsonl")) fail(`Grok inference ledger must be a .jsonl path: ${inferenceLedgerPath}`); + const subject = new Set([DAIMON_GROK_TURN_USAGE_LEDGER.filePath, path.posix.join(DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, "requests.jsonl"), + ...registrations.flatMap((entry) => [entry.usageLedgerPath, path.posix.join(path.posix.dirname(entry.usageLedgerPath), "requests.jsonl")])]); + if (subject.has(inferenceLedgerPath)) fail(`Grok inference ledger ${inferenceLedgerPath} is a subject usage ledger; judge spend must never reach the wake fuse`); + return inferenceLedgerPath; +}; + /** `service.json` v2, exactly the shape Daimon's strict `parseEngineBrokerServiceConfig` accepts. */ -export const renderDaimonGrokServiceConfig = (registrations: readonly DaimonGrokRegistration[]) => ({ +export const renderDaimonGrokServiceConfig = ( + registrations: readonly DaimonGrokRegistration[], + options: DaimonGrokServiceConfigOptions = {} +) => ({ version: DAIMON_GROK_ENGINE_BROKER.serviceConfigVersions[1], credentialHome: DAIMON_GROK_ENGINE_BROKER.credentialHomePath, - turnStore: DAIMON_GROK_ENGINE_BROKER.turnStorePath, + turnStore: resolveDaimonGrokTurnStore(options.turnStore), + ...(options.inferenceLedgerPath === undefined ? {} : { inferenceLedgerPath: resolveDaimonGrokInferenceLedger(options.inferenceLedgerPath, registrations) }), registrations: registrations.map((entry) => ({ agentId: entry.agentId, slot: entry.slot, diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts new file mode 100644 index 00000000..94da73a9 --- /dev/null +++ b/src/compiler/training/broker/paths.ts @@ -0,0 +1,128 @@ +import path from "node:path"; + +import { + DAIMON_GROK_ENGINE_BROKER, + DAIMON_GROK_SUBSCRIPTION_REALM, + DAIMON_GROK_TURN_USAGE_LEDGER +} from "../../../runtime/daimon/contractManifest.js"; +import { + DAIMON_BROKER_UID, + DAIMON_FIRST_WORKER_UID, + DAIMON_ORGANIZATION_UID +} from "../../../runtime/daimon/runtimeIdentity.js"; +import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js"; +import { DAIMON_WORKER_ROOT } from "../../containerDaimonGrokWorkerRender.js"; + +/** + * Every container path the broker-capable training container fixes. + * + * They are fixed rather than derived because three parties must agree on them + * without talking to each other: the root entrypoint that provisions them, the + * root slot supervisor that wipes and re-provisions them on recycle, and + * Paideia's native adapter, which writes them into `paideia.daimon-native.launch.v2` + * and has no way to ask the container what it chose. Daimon's projection never + * resolves a path, so each one must also be canonical and never a symlink; the + * provisioning program asserts that before a slot is used. + * + * Everything under `/run/training/slot` is per-trial state on tmpfs — never the + * Grok realm volume (R2). The realm volume holds only `auth.json` and the + * broker credential journal, so a recycle can wipe the slot without touching + * the one durable rotating credential. + */ +export const TRAINING_RUN_ROOT = "/run/training/output"; +export const TRAINING_SEALED_INPUTS_ROOT = "/run/training/inputs"; +export const TRAINING_PAIDEIA_ROOT = "/run/paideia"; +export const TRAINING_CONTEXT_FILE = `${TRAINING_PAIDEIA_ROOT}/context.json`; +/** Host-written, read-only `spawnfile.training-broker.v1` declaration the root entrypoint reads. */ +export const TRAINING_BROKER_DECLARATION_FILE = `${TRAINING_PAIDEIA_ROOT}/training-broker.json`; + +export const TRAINING_SLOT_ROOT = "/run/training/slot"; +export const TRAINING_SLOT_WORKSPACE = `${TRAINING_SLOT_ROOT}/workspace`; +export const TRAINING_SLOT_RUNTIME_HOME = `${TRAINING_SLOT_ROOT}/runtime-home`; +export const TRAINING_SLOT_STATE_ROOT = `${TRAINING_SLOT_ROOT}/state`; +export const TRAINING_SLOT_ACCEPTANCE_STORE = `${TRAINING_SLOT_STATE_ROOT}/wake-acceptance`; +/** Per-slot turn store. On tmpfs on purpose: a replayed turn from trial N must never satisfy trial N+1 (R2). */ +export const TRAINING_SLOT_TURN_STORE = `${TRAINING_SLOT_ROOT}/turns`; +export const TRAINING_SLOT_USAGE_DIRECTORY = `${TRAINING_SLOT_ROOT}/usage`; +export const TRAINING_SLOT_USAGE_LEDGER = `${TRAINING_SLOT_USAGE_DIRECTORY}/usage.jsonl`; +export const TRAINING_SLOT_PREFLIGHT_RECEIPT = `${TRAINING_SLOT_ROOT}/preflight.json`; +/** Supervisor-owned monotonic generation counter; survives a recycle, never a container restart. */ +export const TRAINING_SLOT_GENERATION_FILE = `${TRAINING_SLOT_ROOT}/generation.json`; + +export const TRAINING_INFERENCE_DIRECTORY = "/run/training/inference"; +export const TRAINING_INFERENCE_LEDGER = `${TRAINING_INFERENCE_DIRECTORY}/inference.jsonl`; +/** Judge grant homes: `2000:2000 0700`, denied to every worker uid. Paideia reads it as `PAIDEIA_GROK_GRANT_HOME_ROOT`. */ +export const TRAINING_GRANT_HOME_ROOT = "/run/training/grants"; + +export const TRAINING_SUPERVISOR_DIRECTORY = "/run/training/supervisor"; +export const TRAINING_SUPERVISOR_SOCKET = `${TRAINING_SUPERVISOR_DIRECTORY}/control.sock`; +export const TRAINING_SUPERVISOR_LOG = `${TRAINING_SUPERVISOR_DIRECTORY}/supervisor.log`; + +export const TRAINING_WORKER_ROOT = DAIMON_WORKER_ROOT; +export const TRAINING_SLOT_INDEX = 0; +export const TRAINING_WORKER_UID = DAIMON_FIRST_WORKER_UID + TRAINING_SLOT_INDEX; +export const TRAINING_WORKER_HOME = path.posix.join(TRAINING_WORKER_ROOT, String(TRAINING_WORKER_UID)); + +/** Paideia's own fixed container paths (`containerPaths` in its native launch schema); all protected from the worker. */ +export const TRAINING_CALLER_PROTECTED_PATHS = [ + `${TRAINING_PAIDEIA_ROOT}/config.json`, + `${TRAINING_PAIDEIA_ROOT}/control`, + `${TRAINING_PAIDEIA_ROOT}/launch.json`, + `${TRAINING_PAIDEIA_ROOT}/token`, + `${TRAINING_PAIDEIA_ROOT}/env`, + `${TRAINING_PAIDEIA_ROOT}/preparation.json`, + `${TRAINING_PAIDEIA_ROOT}/repair.json` +] as const; + +/** `paideia.daimon-native.launch.v2`'s five `broker.evaluatorPaths` roles, at this container's real paths. */ +export const TRAINING_EVALUATOR_ROOTS = [ + { role: "run-root", path: TRAINING_RUN_ROOT }, + { role: "context", path: TRAINING_CONTEXT_FILE }, + { role: "sealed-inputs", path: TRAINING_SEALED_INPUTS_ROOT }, + { role: "judge-home", path: TRAINING_GRANT_HOME_ROOT }, + { role: "slot-ledger", path: TRAINING_SLOT_USAGE_DIRECTORY } +] as const; + +/** + * Everything this container provisions that the one training worker must not + * read, beyond Daimon's own protected set (realm, bootstrap, acceptance store). + * Each entry sits strictly below a Grok 1.0.34 base-profile grant — Grok + * refuses a profile whose deny entry equals or contains `/run`, `/var`, `/etc` + * or `/tmp` — and no entry covers another, because masks cannot nest. + */ +export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), + DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, + DAIMON_WAKE_FUSE_DIRECTORY, + TRAINING_BROKER_DECLARATION_FILE, + TRAINING_INFERENCE_DIRECTORY, + TRAINING_SLOT_STATE_ROOT, + TRAINING_SLOT_TURN_STORE, + TRAINING_SUPERVISOR_DIRECTORY, + ...TRAINING_CALLER_PROTECTED_PATHS, + ...TRAINING_EVALUATOR_ROOTS.map((entry) => entry.path) +]; + +/** Paths the provisioning program creates root-owned `0700` when absent, so every deny entry always has a target inode. */ +export const TRAINING_OPTIONAL_DENY_DIRECTORIES: readonly string[] = [ + TRAINING_INFERENCE_DIRECTORY, + TRAINING_SLOT_STATE_ROOT, + TRAINING_SLOT_TURN_STORE, + TRAINING_SUPERVISOR_DIRECTORY, + TRAINING_RUN_ROOT, + TRAINING_SEALED_INPUTS_ROOT, + TRAINING_GRANT_HOME_ROOT +]; + +/** + * Deny entries Paideia materializes per trial inside its own `/run/paideia` + * tmpfs. Absent is allowed at provisioning time (the profile still masks the + * path once it exists, because Grok applies the profile at every worker spawn); + * a symlink never is. + */ +export const TRAINING_DEFERRED_DENY_PATHS: readonly string[] = [...TRAINING_CALLER_PROTECTED_PATHS]; + +export const TRAINING_REALM_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath; +export const TRAINING_BOOTSTRAP_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath; +export { DAIMON_BROKER_UID, DAIMON_ORGANIZATION_UID }; diff --git a/src/compiler/training/broker/registration.ts b/src/compiler/training/broker/registration.ts new file mode 100644 index 00000000..881f0f8e --- /dev/null +++ b/src/compiler/training/broker/registration.ts @@ -0,0 +1,129 @@ +import path from "node:path"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { + DAIMON_GROK_BROKER_MODELS, + DAIMON_GROK_BROKER_REASONING_EFFORTS, + DAIMON_GROK_ENGINE_BROKER, + type DaimonGrokBrokerModel, + type DaimonGrokBrokerReasoningEffort +} from "../../../runtime/daimon/contractManifest.js"; +import { + DAIMON_GROK_WORKER_HOME_DIRECTORY, + daimonGrokWorkerSandboxProfileSha256, + renderDaimonGrokWorkerSandboxProfile, + resolveDaimonGrokWorkerConfig +} from "../../../runtime/daimon/grokWorkerContract.js"; +import { + DAIMON_GROK_BASE_PROFILE_GRANTS, + assertCanonicalRegisteredPath, + type DaimonGrokRegistration +} from "../../containerDaimonGrokWorkerRender.js"; +import { + TRAINING_ADDED_DENY_PATHS, + TRAINING_BOOTSTRAP_MOUNT, + TRAINING_DEFERRED_DENY_PATHS, + TRAINING_REALM_MOUNT, + TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SLOT_INDEX, + TRAINING_SLOT_RUNTIME_HOME, + TRAINING_SLOT_USAGE_LEDGER, + TRAINING_SLOT_WORKSPACE, + TRAINING_WORKER_HOME, + TRAINING_WORKER_UID +} from "./paths.js"; + +const fail = (message: string): never => { + throw new SpawnfileError("compile_error", message); +}; + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +/** + * The one training slot's deny list: Daimon's own protected set for a + * single-agent organization (Grok bootstrap, realm, wake-acceptance store) + * plus everything this container provisions that the subject must not read — + * the evaluator roots, the caller's protected `/run/paideia` paths, the broker + * control and registration roots, the grant home root, the inference ledger, + * the per-slot ledger, turn store and supervisor socket directory. + * + * The same two refusals the production render enforces apply here: an entry + * that equals or contains a Grok 1.0.34 base-profile grant makes Grok refuse + * the profile outright, and masks cannot nest, so an entry covered by another + * entry is a provisioning bug rather than something to silently drop. + */ +export const resolveTrainingGrokDenyPaths = (added: readonly string[] = TRAINING_ADDED_DENY_PATHS): string[] => { + const daimonOwn = [TRAINING_BOOTSTRAP_MOUNT, TRAINING_REALM_MOUNT, TRAINING_SLOT_ACCEPTANCE_STORE]; + const grokHome = path.posix.join(TRAINING_WORKER_HOME, DAIMON_GROK_WORKER_HOME_DIRECTORY); + const grants = [ + ...DAIMON_GROK_BASE_PROFILE_GRANTS, TRAINING_SLOT_WORKSPACE, grokHome, + path.posix.join(grokHome, "sessions"), path.posix.join(TRAINING_WORKER_HOME, "tmp") + ]; + const candidates = [...new Set([...daimonOwn, ...added])].sort(); + for (const entry of candidates) { + assertCanonicalRegisteredPath("deny path", entry); + const grant = grants.find((candidate) => within(candidate, entry)); + if (grant) fail(`Training Grok deny path ${entry} equals or contains the base profile grant ${grant}; Grok refuses such a profile`); + if (within(TRAINING_SLOT_WORKSPACE, entry) || within(TRAINING_WORKER_HOME, entry) || within(TRAINING_SLOT_RUNTIME_HOME, entry)) { + fail(`Training Grok deny path ${entry} would hide the subject's own workspace, home, or runtime home`); + } + const ancestor = candidates.find((other) => other !== entry && within(entry, other)); + if (ancestor) fail(`Training Grok deny path ${ancestor} would cover ${entry}; masks cannot nest`); + } + return candidates; +}; + +export interface TrainingGrokSlotInput { + agentId: string; + model: DaimonGrokBrokerModel; + reasoningEffort: DaimonGrokBrokerReasoningEffort; + /** Deny entries this container adds beyond Daimon's own protected set; defaults to the fixed training set. */ + denyPaths?: readonly string[]; +} + +/** + * The single brokered Grok registration a training container runs, at the + * fixed container paths in `paths.ts`. Unlike production it points + * `usageLedgerPath` at the per-slot ledger, because each trial's spend must be + * separable and a recycle wipes it; the container-wide ledger and wake fuse do + * not exist here. + */ +export const resolveTrainingGrokRegistration = (input: TrainingGrokSlotInput): DaimonGrokRegistration => { + if (!input.agentId.trim()) fail("Training Grok slot requires an agent id"); + if (!(DAIMON_GROK_BROKER_MODELS as readonly string[]).includes(input.model) + || !(DAIMON_GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(input.reasoningEffort)) { + fail(`Training Grok slot has no declared broker model and reasoning effort: ${input.agentId}`); + } + const home = assertCanonicalRegisteredPath("home", TRAINING_WORKER_HOME); + const grokHome = path.posix.join(home, DAIMON_GROK_WORKER_HOME_DIRECTORY); + const config = resolveDaimonGrokWorkerConfig(input.model, input.reasoningEffort); + const denyPaths = resolveTrainingGrokDenyPaths(input.denyPaths); + const profile = renderDaimonGrokWorkerSandboxProfile(denyPaths); + const eventsPath = path.posix.join(grokHome, DAIMON_GROK_ENGINE_BROKER.worker.home.sandboxEvents.relativePath); + const profilePath = path.posix.join(grokHome, "sandbox.toml"); + for (const [label, value] of [["GROK_HOME", grokHome], ["profile", profilePath], ["events", eventsPath], + ["workspace", TRAINING_SLOT_WORKSPACE], ["runtime home", TRAINING_SLOT_RUNTIME_HOME], + ["usage ledger", TRAINING_SLOT_USAGE_LEDGER]] as const) assertCanonicalRegisteredPath(label, value); + return { + agentId: input.agentId, + config: config.bytes, + configSha256: config.sha256, + deferredDenyPaths: denyPaths.filter((entry) => TRAINING_DEFERRED_DENY_PATHS.includes(entry)), + denyPaths, + eventsPath, + grokHome, + home, + model: input.model, + profile, + profilePath, + profileSha256: daimonGrokWorkerSandboxProfileSha256(denyPaths), + privateTmp: path.posix.join(home, DAIMON_GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome), + reasoningEffort: input.reasoningEffort, + runtimeHome: TRAINING_SLOT_RUNTIME_HOME, + slot: TRAINING_SLOT_INDEX, + spillDirectory: path.posix.join(TRAINING_SLOT_RUNTIME_HOME, DAIMON_GROK_ENGINE_BROKER.worker.home.spillDirectory.relativeToRuntimeHome), + uid: TRAINING_WORKER_UID, + usageLedgerPath: TRAINING_SLOT_USAGE_LEDGER, + workspace: TRAINING_SLOT_WORKSPACE + }; +}; From e2a0c3235cfcc4f55bc0ae0fd66ebb96c7f4b59c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:29:51 +0200 Subject: [PATCH 42/84] feat: provision, supervise and certify a broker-capable training Grok slot --- src/compiler/training/broker/AGENTS.md | 41 +++++ src/compiler/training/broker/CLAUDE.md | 1 + src/compiler/training/broker/declaration.ts | 62 +++++++ src/compiler/training/broker/entrypoint.ts | 129 ++++++++++++++ src/compiler/training/broker/index.ts | 12 ++ src/compiler/training/broker/main.ts | 16 ++ src/compiler/training/broker/processes.ts | 102 +++++++++++ src/compiler/training/broker/projection.ts | 96 +++++++++++ src/compiler/training/broker/provisioning.ts | 111 ++++++++++++ src/compiler/training/broker/receipt.ts | 116 +++++++++++++ src/compiler/training/broker/runtime.ts | 162 ++++++++++++++++++ src/compiler/training/broker/supervisor.ts | 167 +++++++++++++++++++ 12 files changed, 1015 insertions(+) create mode 100644 src/compiler/training/broker/AGENTS.md create mode 120000 src/compiler/training/broker/CLAUDE.md create mode 100644 src/compiler/training/broker/declaration.ts create mode 100644 src/compiler/training/broker/entrypoint.ts create mode 100644 src/compiler/training/broker/index.ts create mode 100644 src/compiler/training/broker/main.ts create mode 100644 src/compiler/training/broker/processes.ts create mode 100644 src/compiler/training/broker/projection.ts create mode 100644 src/compiler/training/broker/provisioning.ts create mode 100644 src/compiler/training/broker/receipt.ts create mode 100644 src/compiler/training/broker/runtime.ts create mode 100644 src/compiler/training/broker/supervisor.ts diff --git a/src/compiler/training/broker/AGENTS.md b/src/compiler/training/broker/AGENTS.md new file mode 100644 index 00000000..2c2586bd --- /dev/null +++ b/src/compiler/training/broker/AGENTS.md @@ -0,0 +1,41 @@ +# Broker-capable training container + +Owns the root privilege model, slot provisioning, slot recycle supervisor and +slot preflight receipt of `spawnfile.training-container.v3` — the training +container that runs a brokered Grok subject beside the evaluator. + +- `paths.ts` fixes every container path. They are constants because the root + entrypoint, the slot supervisor and Paideia's `launch.v2` must agree on them + without talking to each other, and Daimon's projection never resolves a path. +- `declaration.ts` is the host's read-only `spawnfile.training-broker.v1` input. + It is data, never code: no host executable or script is ever mounted, so the + image's own Spawnfile distribution renders the provisioning from it. +- `registration.ts` builds the one `DaimonGrokRegistration` and its deny list. +- `provisioning.ts` renders the root bash program, delegating everything below + the slot skeleton to the production renderers in `../../containerDaimonBrokerRender.ts`. + Never fork that program: a recycle must replay the audited start-up path, + credential-journal recovery included. +- `processes.ts` starts the launcher (root), broker and relay (2100) and proves + each one's socket and post-drop uid/`CapBnd` before the next starts. +- `runtime.ts` implements drain, wipe, provision, start, canaries and receipt. +- `supervisor.ts` is the one-verb root socket server. One verb, one argument, + no caller-supplied path or command. +- `receipt.ts` writes `noopolis.daimon.grok-slot-preflight.v2`. +- `entrypoint.ts`/`main.ts` are the image's root entrypoint. + +Local constraints: + +- Nothing per-trial may live on the Grok realm volume. The realm holds only + `auth.json` and the broker credential journal; the turn store, worker home, + workspace, wake-acceptance store, per-slot ledger and Grok session state are + per-slot tmpfs and are wiped on every recycle (R2). +- The two ledger directories are setgid to the organization group + (`2100:2000 2750`). The broker writes rows `0640` in its own group, so + without setgid uid 2000 cannot read a usage or inference row it paid for. +- The supervisor socket's uid gate is the socket node (`root:2000 0660` in a + root-owned `0711` directory on tmpfs), because Node exposes no `SO_PEERCRED`. + A `0600` root-owned socket would deny the one caller it exists for. +- A worker-uid canary over a host bind proves nothing (Docker Desktop and + Colima ignore `chown`), so `unenforcedBindPolicy` decides between refusing + the slot and accepting the bubblewrap deny list as that path's only boundary. +- Keep files under 400 lines and tests adjacent. diff --git a/src/compiler/training/broker/CLAUDE.md b/src/compiler/training/broker/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/src/compiler/training/broker/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/compiler/training/broker/declaration.ts b/src/compiler/training/broker/declaration.ts new file mode 100644 index 00000000..a1b8e996 --- /dev/null +++ b/src/compiler/training/broker/declaration.ts @@ -0,0 +1,62 @@ +import path from "node:path"; +import { z } from "zod"; + +import { + DAIMON_GROK_BROKER_MODELS, + DAIMON_GROK_BROKER_REASONING_EFFORTS, + DAIMON_GROK_ENGINE_BROKER +} from "../../../runtime/daimon/contractManifest.js"; + +const canonical = z.string().min(2).max(4_096).refine((value) => + path.posix.isAbsolute(value) && path.posix.normalize(value) === value && !value.endsWith("/") && !value.includes("\0"), + "canonical absolute path"); + +/** + * `spawnfile.training-broker.v1`: the only thing the host tells the + * broker-capable training container about its slot. It is data, never code — + * the image's own Spawnfile distribution renders the provisioning from it with + * the same renderers production uses, so no host-written executable is ever + * mounted (`../container/AGENTS.md`). + * + * It carries no credentials: the training Grok login reaches the container only + * as the read-only bootstrap leaf bind, and the rotating credential lives on + * the named realm volume. + */ +export const trainingBrokerDeclarationSchema = z.strictObject({ + version: z.literal("spawnfile.training-broker.v1"), + engine: z.literal("grok"), + agentId: z.string().min(1).max(128).regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u), + model: z.enum(DAIMON_GROK_BROKER_MODELS), + reasoningEffort: z.enum(DAIMON_GROK_BROKER_REASONING_EFFORTS), + architecture: z.enum(["arm64", "x64"]), + limits: z.strictObject({ + maxRequests: z.number().int().min(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.maxRequests[0]).max(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.maxRequests[1]), + maxTokens: z.number().int().min(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.maxTokens[0]).max(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.maxTokens[1]), + timeoutMs: z.number().int().min(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.timeoutMs[0]).max(DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds.timeoutMs[1]) + }), + /** The read-only bootstrap credential leaf, already refused if it is the desktop `~/.grok/auth.json`. */ + bootstrap: canonical, + /** The uid the trained evaluator (Paideia, DSPy, judges) runs as; the only uid the slot supervisor serves. */ + organizationUid: z.literal(DAIMON_GROK_ENGINE_BROKER.identities.organizationUid), + seccompProfileSha256: z.string().regex(/^[a-f0-9]{64}$/u), + /** + * What the slot supervisor does with a deny path whose backing filesystem + * does not enforce unix ownership — every host bind under Docker Desktop and + * Colima, where `chown` is silently ignored and uid 2200 reads a root `0600` + * file (P0 §5). + * + * `refuse` (the default) fails the recycle and writes no receipt: a + * worker-uid probe over such a path carries no information, so the + * supervisor will not certify it. `profile-only` is the operator's explicit + * acceptance that for those paths the boundary is the bubblewrap-enforced + * `deny` list alone — verified to block both shell `cat` and `read_file` on + * 1.0.34 (P0 §4) — and the supervisor records every path that used that + * weaker evidence in its log. + */ + unenforcedBindPolicy: z.enum(["refuse", "profile-only"]).default("refuse") +}).strict(); + +export type TrainingBrokerDeclaration = z.infer; + +export const parseTrainingBrokerDeclaration = (value: unknown): TrainingBrokerDeclaration => + trainingBrokerDeclarationSchema.parse(value); diff --git a/src/compiler/training/broker/entrypoint.ts b/src/compiler/training/broker/entrypoint.ts new file mode 100644 index 00000000..dcdb0cd0 --- /dev/null +++ b/src/compiler/training/broker/entrypoint.ts @@ -0,0 +1,129 @@ +import { spawn } from "node:child_process"; +import { lstat, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import { parseTrainingBrokerDeclaration, type TrainingBrokerDeclaration } from "./declaration.js"; +import { loadDaimonProjectionModule, resolveTrainingGrokProjection, type DaimonProjectionModule } from "./projection.js"; +import { resolveTrainingGrokRegistration } from "./registration.js"; +import { createTrainingSlotRuntime } from "./runtime.js"; +import { createTrainingSlotSupervisor, serveTrainingSlotSupervisor } from "./supervisor.js"; +import { + DAIMON_ORGANIZATION_UID, + TRAINING_BOOTSTRAP_MOUNT, + TRAINING_BROKER_DECLARATION_FILE, + TRAINING_GRANT_HOME_ROOT, + TRAINING_SUPERVISOR_SOCKET +} from "./paths.js"; + +export const TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV = "PAIDEIA_GROK_BROKER_CONTROL_SOCKET"; +export const TRAINING_GROK_GRANT_HOME_ROOT_ENV = "PAIDEIA_GROK_GRANT_HOME_ROOT"; +export const TRAINING_ENTRYPOINT_COMMAND = "/opt/training/bin/train"; + +/** + * The desktop Grok login, in every spelling a caller might reach it by. + * + * D2 is explicit: training uses one dedicated Grok login and never the + * developer's own. An in-container refresh rotates the credential, so mounting + * the desktop leaf would invalidate the developer's desktop session as a side + * effect of a training run — and the refreshed token would live on tmpfs, so + * the desktop login would be gone rather than moved. + */ +export const desktopGrokAuthPaths = (home = os.homedir()): string[] => + [path.resolve(home, ".grok/auth.json"), path.resolve(home, ".grok", "auth.json")]; + +export const assertNotDesktopGrokAuth = (source: string, home = os.homedir()): string => { + if (desktopGrokAuthPaths(home).includes(path.resolve(source))) { + throw new SpawnfileError("validation_error", + "Training refuses the desktop ~/.grok/auth.json as its Grok bootstrap; import a dedicated login with `spawnfile auth import grok --profile paideia-training --from `"); + } + return source; +}; + +/** The bootstrap leaf inside the container: the fixed read-only mount, a bounded regular file, never a symlink. */ +const assertBootstrap = async (declaration: TrainingBrokerDeclaration): Promise => { + if (declaration.bootstrap !== TRAINING_BOOTSTRAP_MOUNT) { + throw new SpawnfileError("validation_error", `The training Grok bootstrap must be mounted at ${TRAINING_BOOTSTRAP_MOUNT}`); + } + const info = await lstat(declaration.bootstrap); + if (!info.isFile() || info.isSymbolicLink() || info.size < 2 || info.size > DAIMON_GROK_SUBSCRIPTION_MAX_BYTES) { + throw new SpawnfileError("validation_error", "The training Grok bootstrap must be a bounded regular credential leaf"); + } +}; +const DAIMON_GROK_SUBSCRIPTION_MAX_BYTES = 64 * 1024; + +export interface TrainingEntrypointOptions { + argv: readonly string[]; + declarationPath?: string; + loadDaimon?: () => Promise; + log?: (line: string) => void; +} + +/** + * The broker-capable training container's root entrypoint. + * + * Order matters and is the whole privilege model: provision as root with the + * production capability set, start the launcher (root) and the broker and + * relay (2100) and prove each one's post-drop identity, publish the slot + * supervisor socket, and only then drop to uid 2000 with an empty capability + * bounding set to run `train`. Paideia, DSPy and the judges are that uid; the + * model's own tools are the worker uid the launcher alone can reach. + */ +export const runTrainingBrokerEntrypoint = async (options: TrainingEntrypointOptions): Promise => { + const log = options.log ?? ((line: string) => process.stderr.write(`[training-entrypoint] ${line}\n`)); + if (process.getuid?.() !== 0) throw new SpawnfileError("runtime_error", "The broker-capable training container must start as root"); + const declaration = parseTrainingBrokerDeclaration(JSON.parse(await readFile(options.declarationPath ?? TRAINING_BROKER_DECLARATION_FILE, "utf8"))); + await assertBootstrap(declaration); + const registration = resolveTrainingGrokRegistration(declaration); + const daimon = await (options.loadDaimon ?? loadDaimonProjectionModule)(); + // Daimon's projection is I/O-free, so the digest the receipt will bind is known before anything is provisioned. + const { projectionSha256 } = await resolveTrainingGrokProjection(declaration, registration, daimon); + const runtime = createTrainingSlotRuntime({ declaration, registration, projectionSha256 }); + await runtime.provision(); + await runtime.start(); + const supervisor = createTrainingSlotSupervisor({ runtime, organizationUid: declaration.organizationUid }); + const server = serveTrainingSlotSupervisor(supervisor, TRAINING_SUPERVISOR_SOCKET, log, declaration.organizationUid); + log(`slot ${registration.slot} provisioned for ${registration.agentId} (${registration.model}/${registration.reasoningEffort}), projection ${projectionSha256}`); + const status = await runTrainingChild(options.argv, declaration, log); + server.close(); + await runtime.stop(); + return status; +}; + +/** + * `train` as uid 2000 with an empty bounding set, proven inside the child + * before it execs: `setpriv --bounding-set=-all` silently does nothing without + * `CAP_SETPCAP`, so the guard has to read `/proc/self/status` rather than + * trust the flag. + */ +export const trainingChildArgv = (argv: readonly string[], uid = DAIMON_ORGANIZATION_UID): string[] => [ + "setpriv", "--clear-groups", `--reuid=${uid}`, `--regid=${uid}`, "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", + "--", "/bin/bash", "-ceu", + 'if [ "$EUID" -eq 0 ]; then echo "training must not run as root" >&2; exit 1; fi\n' + + 'test "$(sed -n "s/^CapBnd:[[:space:]]*//p" /proc/self/status)" = 0000000000000000\n' + + 'test "$(sed -n "s/^CapEff:[[:space:]]*//p" /proc/self/status)" = 0000000000000000\n' + + 'exec "$@"', + "bash", TRAINING_ENTRYPOINT_COMMAND, ...argv +]; + +const runTrainingChild = (argv: readonly string[], declaration: TrainingBrokerDeclaration, log: (line: string) => void): Promise => + new Promise((resolve) => { + const command = trainingChildArgv(argv, declaration.organizationUid); + const child = spawn(command[0]!, command.slice(1), { + stdio: ["ignore", "inherit", "inherit"], + env: { + ...process.env, + [TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV]: DAIMON_GROK_ENGINE_BROKER.controlSocketPath, + [TRAINING_GROK_GRANT_HOME_ROOT_ENV]: TRAINING_GRANT_HOME_ROOT, + HOME: "/home/training" + } + }); + const forward = (signal: NodeJS.Signals) => () => child.kill(signal); + for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(signal, forward(signal)); + child.once("exit", (code, signal) => { + log(`training exited code=${code ?? "null"} signal=${signal ?? "none"}`); + resolve(signal ? (signal === "SIGINT" ? 130 : 143) : code ?? 1); + }); + }); diff --git a/src/compiler/training/broker/index.ts b/src/compiler/training/broker/index.ts new file mode 100644 index 00000000..09ca24d9 --- /dev/null +++ b/src/compiler/training/broker/index.ts @@ -0,0 +1,12 @@ +export { parseTrainingBrokerDeclaration, trainingBrokerDeclarationSchema, type TrainingBrokerDeclaration } from "./declaration.js"; +export { assertNotDesktopGrokAuth, desktopGrokAuthPaths, runTrainingBrokerEntrypoint, trainingChildArgv, + TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV, TRAINING_GROK_GRANT_HOME_ROOT_ENV } from "./entrypoint.js"; +export { brokerProcessPlan, startBrokerProcesses, stopBrokerProcesses } from "./processes.js"; +export { loadDaimonProjectionModule, resolveTrainingGrokProjection, trainingOrganizationRuntimeConfig } from "./projection.js"; +export { renderTrainingBrokerProvisioning, renderTrainingIdentities, trainingSlotDirectories } from "./provisioning.js"; +export { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveBackingFilesystem, resolveTrainingCanaries } from "./receipt.js"; +export { resolveTrainingGrokDenyPaths, resolveTrainingGrokRegistration } from "./registration.js"; +export { createTrainingSlotRuntime } from "./runtime.js"; +export { assertTrainingSupervisorSocketIdentity, createTrainingSlotSupervisor, serveTrainingSlotSupervisor, + TRAINING_SUPERVISOR_PROTOCOL, type TrainingSlotRuntime } from "./supervisor.js"; +export * from "./paths.js"; diff --git a/src/compiler/training/broker/main.ts b/src/compiler/training/broker/main.ts new file mode 100644 index 00000000..a7e84452 --- /dev/null +++ b/src/compiler/training/broker/main.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import { runTrainingBrokerEntrypoint } from "./entrypoint.js"; + +/** + * The broker-capable training image's root entrypoint process. The image ships + * it; the host supplies only the read-only `spawnfile.training-broker.v1` + * declaration and never a script, so no host executable enters the container. + */ +const main = async (): Promise => { + process.exitCode = await runTrainingBrokerEntrypoint({ argv: process.argv.slice(2) }); +}; + +main().catch((error: unknown) => { + process.stderr.write(`[training-entrypoint] ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/src/compiler/training/broker/processes.ts b/src/compiler/training/broker/processes.ts new file mode 100644 index 00000000..d9dfbd63 --- /dev/null +++ b/src/compiler/training/broker/processes.ts @@ -0,0 +1,102 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { readFile, stat } from "node:fs/promises"; +import { setTimeout as pause } from "node:timers/promises"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import { DAIMON_BROKER_TMPDIR } from "../../containerDaimonBrokerRender.js"; +import { DAIMON_BROKER_UID } from "./paths.js"; + +export const DAIMON_RUNTIME_ROOT = "/opt/spawnfile/runtime-installs/daimon"; +/** `00000000000000c1` = CHOWN|SETGID|SETUID, the launcher's bounding set after `setpriv`. */ +export const LAUNCHER_CAPABILITY_BOUND = "00000000000000c1"; +export const DROPPED_CAPABILITY_BOUND = "0000000000000000"; + +export interface BrokerChild { name: string; pid: number; child: ChildProcess; uid: number; capBnd: string } + +const setpriv = (args: string[]): string[] => ["setpriv", ...args]; + +/** + * The three broker processes, in the order and with the identities production + * starts them: the root launcher (which alone may `setuid` to a worker uid), + * the broker backend as uid 2100, and the native control relay as uid 2100. + * Each one runs with the same bounding set the production entrypoint verifies, + * and the broker and relay carry their own `TMPDIR` because shared `/tmp` is + * closed to every process outside the organization group. + */ +export const brokerProcessPlan = (): { name: string; argv: string[]; socket: string; uid: number; capBnd: string }[] => [ + { + name: "engine broker launcher", socket: DAIMON_GROK_ENGINE_BROKER.launcherSocketPath, uid: 0, capBnd: LAUNCHER_CAPABILITY_BOUND, + argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid", "--", DAIMON_GROK_ENGINE_BROKER.nativeExecutablePath]) + }, + { + name: "engine broker backend", socket: DAIMON_GROK_ENGINE_BROKER.backendSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND, + argv: setpriv(["--clear-groups", `--reuid=${DAIMON_BROKER_UID}`, `--regid=${DAIMON_BROKER_UID}`, "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", + "--", "env", `TMPDIR=${DAIMON_BROKER_TMPDIR}`, `${DAIMON_RUNTIME_ROOT}/bin/daimon-runtime`, "engine-broker", "serve"]) + }, + { + name: "engine broker control relay", socket: DAIMON_GROK_ENGINE_BROKER.controlSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND, + argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid,+setpcap", "--", "env", `TMPDIR=${DAIMON_BROKER_TMPDIR}`, + DAIMON_GROK_ENGINE_BROKER.nativeExecutablePath, "--relay"]) + } +]; + +const processIdentity = async (pid: number, procRoot: string): Promise<{ uid: string; capBnd: string }> => { + const status = await readFile(`${procRoot}/${pid}/status`, "utf8"); + return { + uid: /^Uid:\s+(\d+)/mu.exec(status)?.[1] ?? "", + capBnd: /^CapBnd:\s+([0-9a-f]+)/mu.exec(status)?.[1] ?? "" + }; +}; + +export interface StartBrokerOptions { + timeoutMs?: number; + pollMs?: number; + procRoot?: string; + log(line: string): void; +} + +/** + * Starts all three, then proves each one reached its socket *and* its expected + * post-drop identity before the next starts. A broker that is alive but still + * root, or a relay whose socket never appeared, fails the slot here rather + * than at the first trial wake. + */ +export const startBrokerProcesses = async (options: StartBrokerOptions): Promise => { + const timeoutMs = options.timeoutMs ?? 60_000, pollMs = options.pollMs ?? 100, procRoot = options.procRoot ?? "/proc"; + const started: BrokerChild[] = []; + const deadline = Date.now() + timeoutMs; + try { + for (const entry of brokerProcessPlan()) { + const child = spawn(entry.argv[0]!, entry.argv.slice(1), { stdio: ["ignore", "inherit", "inherit"] }); + if (child.pid === undefined) throw new SpawnfileError("runtime_error", `${entry.name} did not start`); + let exited = false; + child.once("exit", (code) => { exited = true; options.log(`${entry.name} exited with status ${code ?? "signal"}`); }); + started.push({ name: entry.name, pid: child.pid, child, uid: entry.uid, capBnd: entry.capBnd }); + for (;;) { + if (exited) throw new SpawnfileError("runtime_error", `${entry.name} exited before readiness`); + if (Date.now() > deadline) throw new SpawnfileError("runtime_error", `${entry.name} readiness timed out`); + const socket = await stat(entry.socket).catch(() => undefined); + if (socket?.isSocket()) { + const identity = await processIdentity(child.pid, procRoot); + if (identity.uid === String(entry.uid) && identity.capBnd === entry.capBnd) break; + } + await pause(pollMs); + } + options.log(`${entry.name} ready pid=${child.pid} uid=${entry.uid} capbnd=${entry.capBnd}`); + } + return started; + } catch (error) { await stopBrokerProcesses(started, options.log); throw error; } +}; + +/** Graceful stop in reverse start order: relay, backend, launcher. `SIGKILL` only after the grace window. */ +export const stopBrokerProcesses = async (children: readonly BrokerChild[], log: (line: string) => void, graceMs = 5_000): Promise => { + for (const entry of [...children].reverse()) { + if (entry.child.exitCode !== null || entry.child.signalCode !== null) continue; + entry.child.kill("SIGTERM"); + const deadline = Date.now() + graceMs; + while (entry.child.exitCode === null && entry.child.signalCode === null && Date.now() < deadline) await pause(25); + if (entry.child.exitCode === null && entry.child.signalCode === null) { log(`${entry.name} did not stop gracefully; killing`); entry.child.kill("SIGKILL"); } + while (entry.child.exitCode === null && entry.child.signalCode === null) await pause(25); + } +}; diff --git a/src/compiler/training/broker/projection.ts b/src/compiler/training/broker/projection.ts new file mode 100644 index 00000000..c4cd5391 --- /dev/null +++ b/src/compiler/training/broker/projection.ts @@ -0,0 +1,96 @@ +import { SpawnfileError } from "../../../shared/index.js"; +import { + DAIMON_GROK_ENGINE_BROKER, + type DaimonGrokBrokerModel, + type DaimonGrokBrokerReasoningEffort +} from "../../../runtime/daimon/contractManifest.js"; +import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; +import type { TrainingBrokerDeclaration } from "./declaration.js"; +import { TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME } from "./paths.js"; + +/** + * The slice of Daimon's public `@noopolis/daimon/runtime` export the slot + * supervisor needs. Daimon stays the resolver: the slot preflight receipt binds + * `projection_sha256`, and a second implementation of that digest would fork + * the contract the moment either side changed a member. + */ +export interface DaimonProjectionModule { + resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: Record): { version: string; profileSha256: string; denyPaths: readonly string[] }; + grokBrokerProjectionSha256(projection: unknown): string; + GROK_ENGINE_BROKER: { projectionVersion: string; slotPreflightVersion: string; nativeAbiVersion: number }; +} + +/** Indirect specifier: Daimon is an in-image runtime peer of the training container, never a Spawnfile source dependency. */ +const loadRuntime = async (): Promise => { + const packageName = "@noopolis/daimon/runtime"; + return import(packageName); +}; + +export const loadDaimonProjectionModule = async (load: () => Promise = loadRuntime): Promise => { + let daimon: DaimonProjectionModule; + try { daimon = await load() as DaimonProjectionModule; } + catch { throw new SpawnfileError("runtime_error", "The training image must install @noopolis/daimon with its public /runtime export"); } + if (typeof daimon.resolveOrganizationGrokBrokerProjection !== "function" || typeof daimon.grokBrokerProjectionSha256 !== "function" + || daimon.GROK_ENGINE_BROKER?.projectionVersion !== DAIMON_GROK_ENGINE_BROKER.projectionVersion + || daimon.GROK_ENGINE_BROKER.slotPreflightVersion !== DAIMON_GROK_ENGINE_BROKER.slotPreflightVersion + || daimon.GROK_ENGINE_BROKER.nativeAbiVersion !== DAIMON_GROK_ENGINE_BROKER.nativeAbiVersion) { + throw new SpawnfileError("runtime_error", "The installed Daimon Grok broker contract differs from this compiler's vendored manifest"); + } + return daimon; +}; + +/** + * The one-agent organization runtime config the training slot's projection is + * resolved from. It is a projection *input*, never a runtime config Daimon + * hosts: the trial's real config is Paideia's, and both describe the same + * single agent at the same fixed container paths, so both compute the same + * projection digest. A peer agent would change Daimon's protected set and the + * digest with it, which is exactly why training runs one slot. + */ +export const trainingOrganizationRuntimeConfig = (declaration: { + agentId: string; model: DaimonGrokBrokerModel; reasoningEffort: DaimonGrokBrokerReasoningEffort; +}): Record => ({ + version: "noopolis.daimon.organization-runtime.v2", + id: "daimon-organization", + agents: [{ + id: declaration.agentId, + name: declaration.agentId, + instructions: "", + workspacePath: TRAINING_SLOT_WORKSPACE, + runtimeHomePath: TRAINING_SLOT_RUNTIME_HOME, + engine: { kind: "grok", model: declaration.model, reasoningEffort: declaration.reasoningEffort } + }] +}); + +export interface TrainingGrokProjection { projection: Record; projectionSha256: string } + +/** + * Resolve the slot's public Grok broker projection through Daimon, over exactly + * the provisioned registration. A profile digest that differs from Daimon's own + * render is refused there, so the receipt can never certify a slot whose deny + * list drifted from the bytes provisioning wrote. + */ +export const resolveTrainingGrokProjection = async ( + declaration: TrainingBrokerDeclaration, + registration: DaimonGrokRegistration, + daimon: DaimonProjectionModule +): Promise => { + const projection = daimon.resolveOrganizationGrokBrokerProjection( + trainingOrganizationRuntimeConfig(declaration), declaration.agentId, { + slot: registration.slot, + workerUid: registration.uid, + workerHomePath: TRAINING_WORKER_HOME, + architecture: declaration.architecture, + usageLedgerPath: registration.usageLedgerPath, + limits: declaration.limits, + acceptanceStorePath: TRAINING_SLOT_ACCEPTANCE_STORE, + denyPaths: registration.denyPaths, + seccompProfileSha256: declaration.seccompProfileSha256, + profileSha256: registration.profileSha256 + }) as unknown as Record; + const denied = [...(projection.denyPaths as string[])]; + if (denied.length !== registration.denyPaths.length || denied.some((entry, index) => entry !== registration.denyPaths[index])) { + throw new SpawnfileError("runtime_error", "Daimon's Grok broker projection deny list differs from the provisioned sandbox profile"); + } + return { projection, projectionSha256: daimon.grokBrokerProjectionSha256(projection) }; +}; diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts new file mode 100644 index 00000000..b2b6d2a1 --- /dev/null +++ b/src/compiler/training/broker/provisioning.ts @@ -0,0 +1,111 @@ +import path from "node:path"; + +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; +import { renderDaimonBrokerProvisioningProgram } from "../../containerDaimonBrokerRender.js"; +import { renderDaimonGrokHostPreflight } from "../../containerDaimonGrokWorkerProvisioning.js"; +import { + DAIMON_BROKER_UID, + DAIMON_ORGANIZATION_UID, + TRAINING_GRANT_HOME_ROOT, + TRAINING_INFERENCE_DIRECTORY, + TRAINING_INFERENCE_LEDGER, + TRAINING_OPTIONAL_DENY_DIRECTORIES, + TRAINING_PAIDEIA_ROOT, + TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SLOT_ROOT, + TRAINING_SLOT_RUNTIME_HOME, + TRAINING_SLOT_STATE_ROOT, + TRAINING_SLOT_TURN_STORE, + TRAINING_SLOT_USAGE_DIRECTORY, + TRAINING_SLOT_USAGE_LEDGER, + TRAINING_SLOT_WORKSPACE, + TRAINING_WORKER_ROOT, + TRAINING_WORKER_UID +} from "./paths.js"; + +const quote = (value: string): string => `'${value.replace(/'/g, `'"'"'`)}'`; + +/** + * Root only ever chmods an inode it currently owns: the container's capability + * set grants `CAP_CHOWN` but not `CAP_FOWNER` (P0 §2), so `install -d -o u -g g + * -m m` — which chowns before it chmods — fails with `EPERM` for every + * non-root owner. Reclaim, set the mode, then hand over. + */ +const PROVISION_DIRECTORY_HELPER = [ + "provision_dir() {", + " target=$1; mode=$2; owner=$3; group=$4", + " mkdir -p \"$target\"", + " test -d \"$target\" && test ! -L \"$target\"", + " chown 0:0 \"$target\"; chmod \"$mode\" \"$target\"; chown \"$owner:$group\" \"$target\"", + "}" +]; + +/** + * Every directory the training slot owns, with the identity that must hold it. + * + * The two ledger directories are setgid to the organization group on purpose: + * the broker writes its rows `0640` owned by the broker group, so without + * `2100:2000 2750` uid 2000 — Paideia, DSPy and every judge — cannot read a + * single subject usage row or inference row it paid for. + */ +export const trainingSlotDirectories = (): readonly { path: string; mode: string; uid: number; gid: number }[] => [ + { path: "/run/training", mode: "0755", uid: 0, gid: 0 }, + { path: TRAINING_SLOT_ROOT, mode: "0755", uid: 0, gid: 0 }, + { path: TRAINING_PAIDEIA_ROOT, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_SLOT_WORKSPACE, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, + { path: TRAINING_SLOT_RUNTIME_HOME, mode: "0710", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, + { path: TRAINING_SLOT_STATE_ROOT, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_SLOT_ACCEPTANCE_STORE, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_SLOT_TURN_STORE, mode: "0700", uid: DAIMON_BROKER_UID, gid: DAIMON_BROKER_UID }, + { path: TRAINING_SLOT_USAGE_DIRECTORY, mode: "2750", uid: DAIMON_BROKER_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_INFERENCE_DIRECTORY, mode: "2750", uid: DAIMON_BROKER_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_GRANT_HOME_ROOT, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: TRAINING_WORKER_ROOT, mode: "0711", uid: 0, gid: 0 } +]; + +/** Fixed uid/gid identities the slot needs before anything is provisioned. */ +export const renderTrainingIdentities = (): string[] => [ + `for fixed_uid in ${DAIMON_ORGANIZATION_UID} ${DAIMON_BROKER_UID} ${TRAINING_WORKER_UID}; do`, + ' if ! getent group "$fixed_uid" >/dev/null; then groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid"; fi', + ' if ! getent passwd "$fixed_uid" >/dev/null; then useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; fi', + "done" +]; + +/** + * Root provisioning for the training container's one broker slot, as bash + * lines run by the image's own root entrypoint and replayed verbatim by the + * slot supervisor on every recycle. + * + * Everything below the slot skeleton is Daimon's audited production + * provisioning: the same credential bootstrap and journal recovery, the same + * `registrations.bin`, the same attested worker `GROK_HOME` layout, the same + * P1b private worker temp, closed shared temp and setgid spill directory, and + * the same `service.json` — here v2 with the per-slot usage ledger, the + * per-slot tmpfs turn store and the evaluator `inferenceLedgerPath`. + */ +export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistration): string[] => [ + ...PROVISION_DIRECTORY_HELPER, + ...trainingSlotDirectories().map((entry) => `provision_dir ${quote(entry.path)} ${entry.mode} ${entry.uid} ${entry.gid}`), + // Every deny entry needs an inode to mask; a bind the host did not supply is created root-owned and unreadable. + ...TRAINING_OPTIONAL_DENY_DIRECTORIES.map((target) => `if [ ! -e ${quote(target)} ]; then provision_dir ${quote(target)} 0700 0 0; fi`), + ...renderDaimonGrokHostPreflight(), + ...renderDaimonBrokerProvisioningProgram([registration], [], { + turnStore: TRAINING_SLOT_TURN_STORE, + inferenceLedgerPath: TRAINING_INFERENCE_LEDGER + }), + // The broker creates both ledgers 0640 in its own group; the setgid directories above give the group to uid 2000. + ...[TRAINING_SLOT_USAGE_LEDGER, TRAINING_INFERENCE_LEDGER].map((ledger) => + `if [ ! -e ${quote(ledger)} ]; then : > ${quote(ledger)}; chown 0:0 ${quote(ledger)}; chmod 0640 ${quote(ledger)}; chown ${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} ${quote(ledger)}; fi`), + `test "$(stat -c '%u:%g %a' ${quote(TRAINING_SLOT_USAGE_DIRECTORY)})" = "${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} 2750"`, + `test "$(stat -c '%u:%g %a' ${quote(TRAINING_INFERENCE_DIRECTORY)})" = "${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} 2750"`, + `test "$(stat -c '%u:%g %a' ${quote(TRAINING_GRANT_HOME_ROOT)})" = "${DAIMON_ORGANIZATION_UID}:${DAIMON_ORGANIZATION_UID} 700"`, + // The judge grant home root is the evaluator's alone: no worker uid may even traverse it. + `setpriv --clear-groups --reuid ${TRAINING_WORKER_UID} --regid ${TRAINING_WORKER_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu '! test -r ${quote(TRAINING_GRANT_HOME_ROOT)}'`, + `setpriv --clear-groups --reuid ${DAIMON_ORGANIZATION_UID} --regid ${DAIMON_ORGANIZATION_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu 'test -r ${quote(TRAINING_SLOT_USAGE_LEDGER)} && test -r ${quote(TRAINING_INFERENCE_LEDGER)}'` +]; + +/** Where the broker's own `service.json` lands, so the supervisor can prove the slot it restarted is the slot it provisioned. */ +export const TRAINING_SERVICE_CONFIG_PATH = DAIMON_GROK_ENGINE_BROKER.serviceConfigPath; +export const TRAINING_REGISTRATION_PATH = DAIMON_GROK_ENGINE_BROKER.registrationPath; +export const TRAINING_BROKER_CONTROL_ROOT = path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath); diff --git a/src/compiler/training/broker/receipt.ts b/src/compiler/training/broker/receipt.ts new file mode 100644 index 00000000..505be6c1 --- /dev/null +++ b/src/compiler/training/broker/receipt.ts @@ -0,0 +1,116 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; + +/** Filesystems that silently ignore `chown`/`chmod`, so a worker-uid probe over them proves nothing. */ +export const UNENFORCED_OWNERSHIP_FILESYSTEMS = ["virtiofs", "9p", "nfs", "nfs4", "cifs", "smb3", "vboxsf", "grpcfuse"] as const; + +const isUnenforced = (fstype: string): boolean => + (UNENFORCED_OWNERSHIP_FILESYSTEMS as readonly string[]).includes(fstype) || fstype.startsWith("fuse"); + +/** + * The filesystem type backing `target`, from the longest mount point that is a + * prefix of it. `/proc/self/mountinfo` octal-escapes space, tab, newline and + * backslash in the mount point field, exactly as the durable-mount guard + * already handles elsewhere in this compiler. + */ +export const resolveBackingFilesystem = (target: string, mountinfo: string): string => { + let best = "", fstype = ""; + for (const line of mountinfo.split("\n")) { + const fields = line.split(" "); + const separator = fields.indexOf("-"); + if (separator < 0 || fields.length < separator + 2) continue; + const mountPoint = (fields[4] ?? "").replace(/\\(04[0011]|134)/gu, (match) => ({ "\\040": " ", "\\011": "\t", "\\012": "\n", "\\134": "\\" }[match] ?? match)); + if (mountPoint !== "/" && target !== mountPoint && !target.startsWith(`${mountPoint}/`)) continue; + if (mountPoint.length >= best.length) { best = mountPoint; fstype = fields[separator + 1] ?? ""; } + } + return fstype; +}; + +export interface TrainingCanaryProbe { + /** Runs one worker-uid `open()` attempt; resolves true when the read was denied. */ + (target: string): Promise; +} + +export interface TrainingCanaryOptions { + denyPaths: readonly string[]; + probe: TrainingCanaryProbe; + mountinfo: string; + unenforcedBindPolicy: "refuse" | "profile-only"; + log(line: string): void; +} + +/** + * One denied canary per deny path, or a refusal naming the first path that is + * still reachable. + * + * A path on a filesystem that enforces unix ownership must be observed denied + * to the worker uid. A path on a host bind that ignores ownership cannot be: + * the declaration's `unenforcedBindPolicy` decides whether that refuses the + * recycle or is accepted on the attested profile's deny entry alone, and every + * path taking the weaker route is named in the supervisor log. + */ +export const resolveTrainingCanaries = async (options: TrainingCanaryOptions): Promise<{ path: string; method: "sandboxed-read"; result: "denied" }[]> => { + const canaries: { path: string; method: "sandboxed-read"; result: "denied" }[] = []; + for (const target of options.denyPaths) { + const fstype = resolveBackingFilesystem(target, options.mountinfo); + if (isUnenforced(fstype)) { + if (options.unenforcedBindPolicy === "refuse") { + throw new SpawnfileError("runtime_error", + `Grok slot canary ${target} is backed by ${fstype || "an unknown filesystem"}, which ignores unix ownership; declare unenforcedBindPolicy "profile-only" to accept the bubblewrap deny list as its only boundary`); + } + options.log(`canary ${target} certified by the enforced sandbox profile only (${fstype} ignores unix ownership)`); + } else if (!await options.probe(target)) { + throw new SpawnfileError("runtime_error", `Grok slot canary ${target} is still readable by the worker uid`); + } + canaries.push({ path: target, method: "sandboxed-read", result: "denied" }); + } + return canaries; +}; + +export interface TrainingSlotReceiptInput { + slot: number; + workerUid: number; + generation: number; + nonce: string; + projectionSha256: string; + sandboxProfileSha256: string; + seccompProfileSha256: string; + grokExecutableSha256: string; + canaries: readonly { path: string; method: "sandboxed-read"; result: "denied" }[]; + createdAt: Date; +} + +/** `noopolis.daimon.grok-slot-preflight.v2`, in the exact member order Daimon's strict schema accepts. */ +export const buildTrainingSlotReceipt = (input: TrainingSlotReceiptInput): Record => { + if (!/^[a-f0-9]{64}$/u.test(input.nonce)) throw new SpawnfileError("runtime_error", "Grok slot recycle nonce must be 32 random bytes in lowercase hex"); + if (!Number.isSafeInteger(input.generation) || input.generation < 1) throw new SpawnfileError("runtime_error", "Grok slot generation must be a positive integer"); + if (input.canaries.length === 0) throw new SpawnfileError("runtime_error", "Grok slot receipt requires at least one denied canary"); + return { + version: DAIMON_GROK_ENGINE_BROKER.slotPreflightVersion, + slot: input.slot, + worker_uid: input.workerUid, + generation: input.generation, + nonce: input.nonce, + projection_sha256: input.projectionSha256, + sandbox_profile_sha256: input.sandboxProfileSha256, + seccomp_profile_sha256: input.seccompProfileSha256, + sandbox_runtime: "bubblewrap", + grok_executable_sha256: input.grokExecutableSha256, + canaries: [...input.canaries], + created_at: `${input.createdAt.toISOString().slice(0, 23)}Z` + }; +}; + +/** sha256 of the pinned Grok executable the slot's workers exec, refused unless it is a manifest-pinned 1.0.34 build. */ +export const readGrokExecutableSha256 = (executable = DAIMON_GROK_ENGINE_BROKER.grokExecutablePath): string => { + const digest = createHash("sha256").update(readFileSync(executable)).digest("hex"); + const pinned: readonly string[] = [DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.x64.sha256]; + if (!pinned.includes(digest)) { + throw new SpawnfileError("runtime_error", `${path.basename(executable)} is not the manifest-pinned Grok ${DAIMON_GROK_ENGINE_BROKER.grokCliVersion} build`); + } + return digest; +}; diff --git a/src/compiler/training/broker/runtime.ts b/src/compiler/training/broker/runtime.ts new file mode 100644 index 00000000..7b153529 --- /dev/null +++ b/src/compiler/training/broker/runtime.ts @@ -0,0 +1,162 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { appendFileSync } from "node:fs"; +import { chmod, chown, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as pause } from "node:timers/promises"; +import { promisify } from "node:util"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; +import type { TrainingBrokerDeclaration } from "./declaration.js"; +import { renderTrainingBrokerProvisioning, renderTrainingIdentities } from "./provisioning.js"; +import { startBrokerProcesses, stopBrokerProcesses, type BrokerChild } from "./processes.js"; +import { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveTrainingCanaries } from "./receipt.js"; +import type { TrainingSlotRuntime } from "./supervisor.js"; +import { + DAIMON_ORGANIZATION_UID, + TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SLOT_GENERATION_FILE, + TRAINING_SLOT_PREFLIGHT_RECEIPT, + TRAINING_SLOT_TURN_STORE, + TRAINING_SLOT_USAGE_DIRECTORY, + TRAINING_SLOT_WORKSPACE, + TRAINING_SUPERVISOR_LOG +} from "./paths.js"; + +const run = promisify(execFile); + +export interface TrainingSlotRuntimeOptions { + declaration: TrainingBrokerDeclaration; + registration: DaimonGrokRegistration; + projectionSha256: string; + /** Test seam: the shell the provisioning script runs in and the `/proc` it reads identities from. */ + shell?: string; + procRoot?: string; + logPath?: string; +} + +const CREDENTIAL_JOURNAL = `${DAIMON_GROK_ENGINE_BROKER.credentialHomePath}/.daimon-broker/credential-journal.json`; + +/** State the broker's credential authority may be left in; only `promoted` (or no journal at all) is settled. */ +const settledJournal = async (): Promise<"settled" | "refreshing" | "stale"> => { + const raw = await readFile(CREDENTIAL_JOURNAL, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); + if (raw === undefined) return "settled"; + const journal = JSON.parse(raw) as { version?: unknown; state?: unknown }; + if (journal.version !== "noopolis.daimon.broker-credential-journal.v1") throw new SpawnfileError("runtime_error", "The broker credential journal is not a recognised version"); + return journal.state === "promoted" ? "settled" : journal.state === "refreshing" ? "refreshing" : "stale"; +}; + +/** A turn is active while its registry record says so; a terminal record is finished work and never blocks a recycle. */ +const activeTurns = async (): Promise => { + const names = await readdir(TRAINING_SLOT_TURN_STORE).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; + throw error; + }); + let active = 0; + for (const name of names) { + if (!name.endsWith(".json")) continue; + const raw = await readFile(path.join(TRAINING_SLOT_TURN_STORE, name), "utf8").catch(() => ""); + try { if ((JSON.parse(raw) as { state?: unknown }).state === "active") active += 1; } catch { /* a half-written record is not an active turn */ } + } + return active; +}; + +/** + * The real slot runtime: what `recycle` actually does to this container. + * + * Every mutation goes through the same rendered provisioning the root + * entrypoint ran at startup, so a recycled slot is byte-identically + * provisioned — including the credential journal recovery, which is how a + * recycle that lands on a `refreshing` or `stale` realm either recovers or + * fails closed with a named error instead of leaving a silently stale realm. + */ +export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): TrainingSlotRuntime & { children(): readonly BrokerChild[] } => { + const shell = options.shell ?? "/bin/bash"; + const logPath = options.logPath ?? TRAINING_SUPERVISOR_LOG; + const script = [...renderTrainingIdentities(), ...renderTrainingBrokerProvisioning(options.registration)].join("\n"); + let children: BrokerChild[] = []; + const log = (line: string): void => { + const entry = `${new Date().toISOString()} ${line}\n`; + process.stderr.write(`[slot-supervisor] ${entry}`); + try { appendFileSync(logPath, entry, { mode: 0o640 }); } catch { /* the log is diagnostics, never the gate */ } + }; + const probe = async (target: string): Promise => { + try { + await run("setpriv", ["--clear-groups", `--reuid=${options.registration.uid}`, `--regid=${options.registration.uid}`, + "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", "--", "/bin/sh", "-c", `exec test -r ${JSON.stringify(target)}`], { timeout: 10_000 }); + return false; + } catch { return true; } + }; + return { + log, + children: () => children, + drain: async (signal) => { + for (;;) { + signal.throwIfAborted(); + const state = await settledJournal(); + if (state === "stale") { + // The audited provisioning below owns stale-realm recovery; it promotes the bootstrap or refuses outright. + log("credential journal is stale; the recycle will replay the root credential recovery"); + return; + } + if (state === "settled" && await activeTurns() === 0) return; + await pause(100); + } + }, + stop: async () => { await stopBrokerProcesses(children, log); children = []; }, + wipe: async () => { + for (const target of [options.registration.home, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_ACCEPTANCE_STORE, + options.registration.spillDirectory, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.registrationPath]) { + await rm(target, { recursive: true, force: true }); + } + // The workspace keeps its root; only the trial's content goes, so the registered path stays canonical. + for (const name of await readdir(TRAINING_SLOT_WORKSPACE).catch(() => [] as string[])) { + await rm(path.join(TRAINING_SLOT_WORKSPACE, name), { recursive: true, force: true }); + } + for (const name of await readdir(TRAINING_SLOT_USAGE_DIRECTORY).catch(() => [] as string[])) { + await rm(path.join(TRAINING_SLOT_USAGE_DIRECTORY, name), { recursive: true, force: true }); + } + }, + provision: async () => { + const result = await run(shell, ["-ceu", script], { maxBuffer: 8 * 1024 * 1024 }).catch((error: unknown) => { + const detail = error as { stderr?: string; stdout?: string }; + throw new SpawnfileError("runtime_error", `Training slot provisioning failed: ${(detail.stderr ?? detail.stdout ?? "").slice(-2048).trim()}`); + }); + if (result.stdout.trim()) log(`provisioning: ${result.stdout.trim().slice(-2048)}`); + }, + start: async () => { children = await startBrokerProcesses({ log, procRoot: options.procRoot }); }, + canaries: async () => resolveTrainingCanaries({ + denyPaths: options.registration.denyPaths, probe, log, + mountinfo: await readFile(`${options.procRoot ?? "/proc"}/self/mountinfo`, "utf8"), + unenforcedBindPolicy: options.declaration.unenforcedBindPolicy + }), + nextGeneration: async () => { + const previous = await readFile(TRAINING_SLOT_GENERATION_FILE, "utf8").catch(() => ""); + const parsed = Number.parseInt((JSON.parse(previous || "{}") as { generation?: unknown }).generation as string ?? "0", 10); + // Seeded from the wall clock so a container restart cannot hand out a generation an evaluator already accepted. + const generation = Math.max(Number.isSafeInteger(parsed) && parsed > 0 ? parsed + 1 : 1, Math.floor(Date.now() / 1000)); + await writeFile(TRAINING_SLOT_GENERATION_FILE, `${JSON.stringify({ generation })}\n`, { mode: 0o600 }); + return generation; + }, + publishReceipt: async ({ generation, nonce, canaries }) => { + const receipt = buildTrainingSlotReceipt({ + slot: options.registration.slot, workerUid: options.registration.uid, generation, nonce, + projectionSha256: options.projectionSha256, sandboxProfileSha256: options.registration.profileSha256, + seccompProfileSha256: options.declaration.seccompProfileSha256, + grokExecutableSha256: readGrokExecutableSha256(), canaries, createdAt: new Date() + }); + const temporary = `${TRAINING_SLOT_PREFLIGHT_RECEIPT}.${randomUUID()}.tmp`; + await mkdir(path.dirname(TRAINING_SLOT_PREFLIGHT_RECEIPT), { recursive: true }); + await writeFile(temporary, `${JSON.stringify(receipt)}\n`, { mode: 0o600, flag: "wx" }); + await chown(temporary, 0, DAIMON_ORGANIZATION_UID); + await chmod(temporary, 0o640); + await rename(temporary, TRAINING_SLOT_PREFLIGHT_RECEIPT); + return TRAINING_SLOT_PREFLIGHT_RECEIPT; + } + }; +}; diff --git a/src/compiler/training/broker/supervisor.ts b/src/compiler/training/broker/supervisor.ts new file mode 100644 index 00000000..343d2327 --- /dev/null +++ b/src/compiler/training/broker/supervisor.ts @@ -0,0 +1,167 @@ +import { chmodSync, chownSync, lstatSync, rmSync } from "node:fs"; +import { createServer, type Socket } from "node:net"; +import path from "node:path"; + +import { SpawnfileError } from "../../../shared/index.js"; +import { DAIMON_ORGANIZATION_UID } from "./paths.js"; + +export const TRAINING_SUPERVISOR_PROTOCOL = "spawnfile.training-slot-supervisor.v1" as const; +export const TRAINING_SUPERVISOR_MAX_REQUEST_BYTES = 4_096; +const NONCE = /^[a-f0-9]{64}$/u; + +/** + * Everything the recycle verb does to the running slot. It is an interface so + * the state machine below can be tested without Docker, a broker, or root: + * the order of these calls *is* the contract, and getting it wrong (wiping + * before draining, writing a receipt before the canaries) is exactly the class + * of bug a live check finds far too late. + */ +export interface TrainingSlotRuntime { + /** Waits for no active turn and a settled credential journal; throws a named error on an unrecoverable realm. */ + drain(signal: AbortSignal): Promise; + stop(): Promise; + /** Removes worker home, slot workspace, wake-acceptance, turn store, per-slot ledger and Grok session state. */ + wipe(): Promise; + /** Replays the audited root provisioning: registrations, service.json, worker home, temp, spill, ledgers. */ + provision(): Promise; + start(): Promise; + /** Worker-uid denial canaries over every deny path; throws when one is still reachable. */ + canaries(): Promise<{ path: string; method: "sandboxed-read"; result: "denied" }[]>; + /** Writes the slot preflight receipt v2 atomically, readable by the organization uid. */ + publishReceipt(input: { generation: number; nonce: string; canaries: readonly { path: string; method: "sandboxed-read"; result: "denied" }[] }): Promise; + /** Monotonic per-slot generation; strictly greater than every generation this slot has ever published. */ + nextGeneration(): Promise; + log(line: string): void; +} + +export interface TrainingSlotSupervisorOptions { + runtime: TrainingSlotRuntime; + /** Only this uid may recycle. Paideia, DSPy and the judges all run as it; every worker uid is refused. */ + organizationUid?: number; + drainTimeoutMs?: number; + now?: () => number; +} + +export interface TrainingRecycleResult { + ok: true; verb: "recycle"; generation: number; nonce: string; receipt: string; durationMs: number; +} + +/** + * The single-verb root slot supervisor. + * + * One verb, one argument, no path or command ever supplied by the caller: the + * whole point of D4 is that recycling replays the container's own audited + * provisioning rather than exposing a root file-system API to the evaluator. + * Recycles are serialized — a second caller waits rather than interleaving a + * wipe with a provision. + */ +export const createTrainingSlotSupervisor = (options: TrainingSlotSupervisorOptions) => { + const organizationUid = options.organizationUid ?? DAIMON_ORGANIZATION_UID; + const drainTimeoutMs = options.drainTimeoutMs ?? 120_000; + const now = options.now ?? Date.now; + let queue: Promise = Promise.resolve(); + const recycle = async (nonce: string, peerUid: number): Promise => { + if (peerUid !== organizationUid) throw new SpawnfileError("validation_error", `Grok slot recycle refused for uid ${peerUid}`); + if (!NONCE.test(nonce)) throw new SpawnfileError("validation_error", "Grok slot recycle requires a 32-byte hex nonce"); + const started = now(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), drainTimeoutMs); + try { + await options.runtime.drain(controller.signal); + } finally { clearTimeout(timer); } + await options.runtime.stop(); + await options.runtime.wipe(); + await options.runtime.provision(); + await options.runtime.start(); + const canaries = await options.runtime.canaries(); + const generation = await options.runtime.nextGeneration(); + const receipt = await options.runtime.publishReceipt({ generation, nonce, canaries }); + const durationMs = now() - started; + options.runtime.log(`recycle generation=${generation} canaries=${canaries.length} durationMs=${durationMs}`); + return { ok: true, verb: "recycle", generation, nonce, receipt, durationMs }; + }; + return { + /** Serialized: a recycle already in flight completes before the next one starts. */ + recycle: (nonce: string, peerUid: number): Promise => { + const next = queue.then(() => recycle(nonce, peerUid)); + queue = next.catch(() => undefined); + return next; + } + }; +}; + +export type TrainingSlotSupervisor = ReturnType; + +const failure = (message: string): string => `${JSON.stringify({ v: TRAINING_SUPERVISOR_PROTOCOL, ok: false, error: message })}\n`; + +/** + * Re-establishes, and re-checks on every connection, the only uid gate this + * socket has. + * + * The scope asked for `SO_PEERCRED`. Node exposes no ancillary-data or + * peer-credential API on a unix socket and no native addon ships in this + * image, so the gate is the socket node itself, which the kernel enforces on + * `connect()` exactly as it does on `open()`: the node is + * `root: 0660` inside a root-owned `0711` directory on + * tmpfs, where modes *are* enforced. Only uid 2000 (and root) can connect; + * uid 2200 gets `EACCES` before a byte is written. A `0600` root-owned socket + * — the literal reading of the scope — would deny the one caller it exists + * for, so this is the same restriction stated in the only mechanism available. + * The directory holds no write permission for anyone but root, so the node + * cannot be replaced by a laxer one. + */ +export const assertTrainingSupervisorSocketIdentity = (socketPath: string, organizationGid: number): void => { + const directory = lstatSync(path.posix.dirname(socketPath)); + if (!directory.isDirectory() || directory.uid !== 0 || directory.gid !== 0 || (directory.mode & 0o777) !== 0o711) { + throw new SpawnfileError("runtime_error", "The slot supervisor socket directory must be root-owned 0711"); + } + const node = lstatSync(socketPath); + if (!node.isSocket() || node.uid !== 0 || node.gid !== organizationGid || (node.mode & 0o777) !== 0o660) { + throw new SpawnfileError("runtime_error", "The slot supervisor socket must be root-owned, organization-group 0660"); + } +}; + +/** + * Line-delimited JSON over a unix socket. Every request is bounded, + * single-line, and must name the protocol, the one verb and nothing else. + */ +export const serveTrainingSlotSupervisor = (supervisor: TrainingSlotSupervisor, socketPath: string, log: (line: string) => void, organizationUid = DAIMON_ORGANIZATION_UID) => { + const server = createServer((socket: Socket) => { + let peerUid: number | undefined; + try { assertTrainingSupervisorSocketIdentity(socketPath, organizationUid); peerUid = organizationUid; } + catch (error) { log(`refusing a connection on an unverified supervisor socket: ${error instanceof Error ? error.message : "unknown"}`); } + let buffer = "", settled = false; + socket.setTimeout(600_000, () => socket.destroy()); + const answer = (line: string): void => { settled = true; socket.end(line); }; + socket.on("data", (chunk) => { + if (settled) return; + buffer += chunk.toString("utf8"); + if (Buffer.byteLength(buffer) > TRAINING_SUPERVISOR_MAX_REQUEST_BYTES) { answer(failure("request too large")); return; } + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + const line = buffer.slice(0, newline); + let request: { v?: unknown; verb?: unknown; nonce?: unknown }; + try { request = JSON.parse(line) as typeof request; } catch { answer(failure("request is not JSON")); return; } + if (request.v !== TRAINING_SUPERVISOR_PROTOCOL || request.verb !== "recycle" || typeof request.nonce !== "string" + || Object.keys(request).length !== 3) { answer(failure("unsupported request")); return; } + if (peerUid === undefined) { answer(failure("peer identity is unavailable")); return; } + supervisor.recycle(request.nonce, peerUid) + .then((result) => answer(`${JSON.stringify({ v: TRAINING_SUPERVISOR_PROTOCOL, ...result })}\n`)) + .catch((error: unknown) => { + const message = error instanceof Error ? error.message : "recycle failed"; + log(`recycle failed: ${message}`); + answer(failure(message)); + }); + }); + socket.on("error", () => socket.destroy()); + }); + rmSync(socketPath, { force: true }); + server.listen(socketPath, () => { + chownSync(socketPath, 0, 0); + chmodSync(socketPath, 0o660); + chownSync(socketPath, 0, organizationUid); + assertTrainingSupervisorSocketIdentity(socketPath, organizationUid); + log(`slot supervisor listening on ${socketPath}`); + }); + return server; +}; From 46ad2b5a1439db36dc15c93a284c89c2a07ddc5c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:37:01 +0200 Subject: [PATCH 43/84] feat: launch spawnfile.training-container.v3 as root with a brokered Grok slot --- runtime-images/training/Dockerfile | 4 +- src/auth/trainingAuth.test.ts | 2 +- src/auth/trainingAuth.ts | 10 ++- src/cli/paideiaDelegation.ts | 5 +- src/compiler/training/broker/provisioning.ts | 2 + src/compiler/training/container/contract.ts | 19 ++++- .../container/fixtures.test-helper.ts | 2 +- .../training/container/launch.test.ts | 2 +- src/compiler/training/container/launch.ts | 22 ++++-- .../training/container/prepare.test.ts | 4 +- src/compiler/training/container/security.ts | 73 +++++++++++++++++++ src/compiler/training/preparation/broker.ts | 53 ++++++++++++++ .../training/preparation/contract.test.ts | 2 +- src/compiler/training/preparation/contract.ts | 48 +++++++++++- .../preparation/fixtures.test-helper.ts | 2 +- .../training/preparation/image.test.ts | 4 +- src/compiler/training/preparation/image.ts | 13 ++-- src/compiler/training/preparation/prepare.ts | 16 +++- src/compiler/training/repair/contract.ts | 2 +- src/compiler/training/repair/witness.ts | 5 +- 20 files changed, 251 insertions(+), 39 deletions(-) create mode 100644 src/compiler/training/container/security.ts create mode 100644 src/compiler/training/preparation/broker.ts diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index b41e4a00..77db1b0f 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -32,7 +32,6 @@ RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \ COPY claude/package.json claude/package-lock.json /opt/training/claude/ RUN cd /opt/training/claude && npm ci --omit=dev -COPY grok /opt/training/bin/grok COPY paideia/dist /opt/training/paideia/dist COPY spawnfile/dist /opt/training/spawnfile/dist @@ -42,7 +41,8 @@ COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawn COPY integration /opt/training/integration COPY bootstrap /opt/training/bootstrap COPY train /opt/training/bin/train -RUN chmod 0555 /opt/training/bin/train /opt/training/bin/grok \ +COPY train-broker /opt/training/bin/train-broker +RUN chmod 0555 /opt/training/bin/train /opt/training/bin/train-broker \ && chmod -R a+rX /opt/training \ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \ diff --git a/src/auth/trainingAuth.test.ts b/src/auth/trainingAuth.test.ts index c72893c3..fcd8ed54 100644 --- a/src/auth/trainingAuth.test.ts +++ b/src/auth/trainingAuth.test.ts @@ -6,7 +6,7 @@ import { stageTrainingAuth, type TrainingAuthProvider } from "./trainingAuth.js" const roots: string[] = []; afterEach(async () => { for (const root of roots.splice(0)) await rm(root,{recursive:true,force:true}); }); const setup = async () => { const root = await realpath(await mkdtemp(path.join(os.tmpdir(),"spawnfile-training-auth-")));roots.push(root);const home=path.join(root,"home"),source=path.join(root,"auth");await mkdir(home,{mode:0o700});await writeFile(source,"opaque-fixture-only");return {root,home,source}; }; -it.each(["codex","grok","claude"] as const)("stages only %s auth to its native private leaf without overwriting renewal",async(provider)=>{ +it.each(["codex","claude"] as const)("stages only %s auth to its native private leaf without overwriting renewal",async(provider)=>{ const f=await setup();const receipt=await stageTrainingAuth({...f,provider}); expect(receipt.version).toBe("spawnfile.training-auth-stage.v1");expect(await readFile(receipt.destination,"utf8")).toBe("opaque-fixture-only");expect((await stat(receipt.destination)).mode&0o777).toBe(0o600); await writeFile(receipt.destination,"renewed");await expect(stageTrainingAuth({...f,provider})).rejects.toThrow("renewed credential preserved");expect(await readFile(receipt.destination,"utf8")).toBe("renewed"); diff --git a/src/auth/trainingAuth.ts b/src/auth/trainingAuth.ts index cef43962..f62dcdc4 100644 --- a/src/auth/trainingAuth.ts +++ b/src/auth/trainingAuth.ts @@ -3,7 +3,14 @@ import { constants } from "node:fs"; import { link, lstat, mkdir, open, realpath, unlink } from "node:fs/promises"; import path from "node:path"; -export type TrainingAuthProvider = "codex" | "grok" | "claude"; +/** + * Grok is deliberately absent. `spawnfile.training-container.v3` holds the one + * dedicated training Grok login in a broker-owned realm volume seeded from a + * read-only bootstrap leaf, and judges spend it through short inference grants + * (D2). Staging a Grok credential into a shared runtime home would put a + * rotating login where the evaluator and the image both reach it. + */ +export type TrainingAuthProvider = "codex" | "claude"; export interface TrainingAuthStageOptions { /** Caller-provisioned, existing private runtime home; never a host home mount. */ home: string; @@ -13,7 +20,6 @@ export interface TrainingAuthStageOptions { } const targets: Record = { codex: [".daimon-inbound", "codex-auth"], - grok: [".grok", "auth.json"], claude: [".claude", ".credentials.json"] }; /** Stages opaque credential bytes once. Never imports configuration, logs bytes, or overwrites renewed auth. */ diff --git a/src/cli/paideiaDelegation.ts b/src/cli/paideiaDelegation.ts index de58a3c3..bd424f19 100644 --- a/src/cli/paideiaDelegation.ts +++ b/src/cli/paideiaDelegation.ts @@ -136,8 +136,9 @@ export const delegatePaideiaTraining = async (options: DelegatePaideiaTrainingOp if (options.signal?.aborted) return 130; if (options.repairWitness && !options.repairMeasurements) throw failure("--repair-witness requires --repair-measurements"); if (options.trainingConfig) { - const config = await readBoundedJson(options.trainingConfig) as { version?: unknown }; - if (config.version === "spawnfile.training-container.v2") { + const config = await readBoundedJson(options.trainingConfig) as { version?: unknown; image?: unknown }; + // v2 and v3 are both declarative preparations; a v3 *launch* config (already lowered, with `inputs`) goes straight to Docker. + if (config.version === "spawnfile.training-container.v2" || (config.version === "spawnfile.training-container.v3" && config.image !== undefined)) { if (options.trainingImage) throw failure("V2 owns its image declaration; --training-image is only for v1"); const prepared = await prepareTraining({ configPath: options.trainingConfig, context: options.context, args: options.args, repairMeasurements: options.repairMeasurements, repairWitness: options.repairWitness, diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts index b2b6d2a1..0e731d58 100644 --- a/src/compiler/training/broker/provisioning.ts +++ b/src/compiler/training/broker/provisioning.ts @@ -53,6 +53,8 @@ export const trainingSlotDirectories = (): readonly { path: string; mode: string { path: "/run/training", mode: "0755", uid: 0, gid: 0 }, { path: TRAINING_SLOT_ROOT, mode: "0755", uid: 0, gid: 0 }, { path: TRAINING_PAIDEIA_ROOT, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: "/home/training", mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: "/work", mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_SLOT_WORKSPACE, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, { path: TRAINING_SLOT_RUNTIME_HOME, mode: "0710", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, { path: TRAINING_SLOT_STATE_ROOT, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, diff --git a/src/compiler/training/container/contract.ts b/src/compiler/training/container/contract.ts index fdebbf68..29860540 100644 --- a/src/compiler/training/container/contract.ts +++ b/src/compiler/training/container/contract.ts @@ -4,17 +4,32 @@ import { z } from "zod"; const hostPath = z.string().min(1).refine((value) => path.isAbsolute(value) && !/[,\r\n\0]/u.test(value), "Expected an absolute bind path"); const inputPath = z.string().regex(/^\/run\/training\/inputs\/[A-Za-z0-9._/-]+$/u).refine((value) => path.posix.normalize(value) === value && !value.endsWith("/")); export const trainingImageSchema = z.string().regex(/^(?:sha256:[a-f0-9]{64}|[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[a-f0-9]{64})$/u); +const brokerLaunchSchema = z.object({ + engine: z.literal("grok"), + /** The named Docker volume holding the training Grok realm: `auth.json` and the broker credential journal, nothing else. */ + realmVolume: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), + /** Host leaf of the dedicated training Grok login. Never the desktop `~/.grok/auth.json`. */ + bootstrap: hostPath, + /** `spawnfile.training-broker.v1`, written beside the launch config and bound read-only into the container. */ + declaration: hostPath +}).strict(); + export const trainingContainerConfigSchema = z.object({ - version: z.literal("spawnfile.training-container.v1"), + version: z.enum(["spawnfile.training-container.v1", "spawnfile.training-container.v3"]), dockerContext: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), inputs: z.array(z.object({ source: hostPath, destination: inputPath }).strict()).min(1).max(64), output: z.object({ source: hostPath, destination: z.literal("/run/training/output") }).strict(), - auth: z.array(z.object({ source: hostPath, provider: z.enum(["codex", "grok", "claude"]) }).strict()).max(3) + auth: z.array(z.object({ source: hostPath, provider: z.enum(["codex", "claude"]) }).strict()).max(2), + /** Present only on `spawnfile.training-container.v3`: the brokered Grok slot this container runs as root. */ + broker: brokerLaunchSchema.optional() }).strict().superRefine((value, context) => { const destinations = value.inputs.map((entry) => entry.destination); if (destinations.some((entry, index) => destinations.some((other, otherIndex) => otherIndex !== index && (entry === other || entry.startsWith(`${other}/`))))) { context.addIssue({ code: "custom", message: "Input destinations must not overlap" }); } if (new Set(value.auth.map((entry) => entry.provider)).size !== value.auth.length) context.addIssue({ code: "custom", message: "Duplicate auth provider" }); + if ((value.version === "spawnfile.training-container.v3") !== (value.broker !== undefined)) { + context.addIssue({ code: "custom", message: "Only spawnfile.training-container.v3 declares a broker slot, and it always does" }); + } }); export type TrainingContainerConfig = z.infer; diff --git a/src/compiler/training/container/fixtures.test-helper.ts b/src/compiler/training/container/fixtures.test-helper.ts index 6a990382..233548bb 100644 --- a/src/compiler/training/container/fixtures.test-helper.ts +++ b/src/compiler/training/container/fixtures.test-helper.ts @@ -21,7 +21,7 @@ export const fixture = async () => { sources: [source], documents: [{ ...source, role: "system" }], skills: [{ ...source, name: "skill", ref: "skill", requiresMcp: [] }], resources: [], requirements: { nativeCompilation: true, isolatedPreparation: true } }; const config = { version: "spawnfile.training-container.v1", dockerContext: "desktop-linux", inputs: [{ source: project, destination: "/run/training/inputs/project" }], - output: { source: output, destination: "/run/training/output" }, auth: [{ source: path.join(root, "auth-leaf"), provider: "grok" }] }; + output: { source: output, destination: "/run/training/output" }, auth: [{ source: path.join(root, "auth-leaf"), provider: "codex" }] }; const configPath = path.join(root, "launch.json"); await writeFile(configPath, JSON.stringify(config)); const args = ["--train", path.join(project, "train.yaml"), "--out", output]; return { root, project, output, context, config, configPath, args }; diff --git a/src/compiler/training/container/launch.test.ts b/src/compiler/training/container/launch.test.ts index 13e5dde6..e31b3918 100644 --- a/src/compiler/training/container/launch.test.ts +++ b/src/compiler/training/container/launch.test.ts @@ -26,7 +26,7 @@ it("launches one immutable container, streams a verified persisted completion an expect(create.some((value) => value.startsWith("/work:") && value.includes(`uid=${process.getuid!()}`))).toBe(true); expect(create).toContain(image); expect(create).not.toContain("--privileged"); expect(create.join(" ")).not.toContain("docker.sock"); - expect(create.join(" ")).toContain("dst=/run/paideia-auth/grok,readonly"); + expect(create.join(" ")).toContain("dst=/run/paideia-auth/codex,readonly"); expect(output.stdout).toHaveBeenLastCalledWith('{"status":"completed","index":"/run/training/output/index.json"}'); expect(docker.calls.some((args) => args[2] === "rm" && args.at(-1) === id)).toBe(true); }); diff --git a/src/compiler/training/container/launch.ts b/src/compiler/training/container/launch.ts index 6eb5984b..db3c1234 100644 --- a/src/compiler/training/container/launch.ts +++ b/src/compiler/training/container/launch.ts @@ -2,6 +2,8 @@ import { randomUUID } from "node:crypto"; import { chmod, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import { DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS } from "../../../shared/index.js"; +import { assertNotDesktopGrokAuth } from "../broker/entrypoint.js"; +import { TRAINING_BROKER_ENTRYPOINT, trainingBrokerMounts, trainingBrokerSecurityArgs, trainingBrokerTmpfsTargets } from "./security.js"; import { parseDetachedContainerInspect } from "../../runProjectDocker.js"; import type { TrainingContext } from "../contract.js"; import { trainingImageSchema } from "./contract.js"; @@ -28,6 +30,8 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp const configBytes = await readFile(options.configPath, "utf8"); if (Buffer.byteLength(configBytes) > 1024 * 1024) throw Error("Training launch config exceeds 1 MiB"); const prepared = await prepareTrainingContainer(JSON.parse(configBytes), options.context, options.args); + // D2: the training Grok login is a dedicated one. Refuse the developer's desktop leaf before Docker ever sees it. + if (prepared.config.broker) assertNotDesktopGrokAuth(prepared.config.broker.bootstrap); if (options.preparationPath) { if (await realpath(options.preparationPath) !== options.preparationPath) throw Error("Preparation receipt path must be canonical"); parseTrainingMappedPreparation(JSON.parse(await readFile(options.preparationPath, "utf8"))); @@ -74,12 +78,20 @@ export const launchTrainingContainer = async (options: LaunchTrainingContainerOp ...options.preparationPath ? [`type=bind,src=${options.preparationPath},dst=/run/paideia/preparation.json,readonly`] : [], ...options.repairPath ? [`type=bind,src=${options.repairPath},dst=/run/paideia/repair.json,readonly`] : [], ...prepared.config.auth.map((entry) => `type=bind,src=${entry.source},dst=/run/paideia-auth/${entry.provider},readonly`)]; + const broker = prepared.config.broker; + // A brokered Grok slot runs the launcher, the broker and the model's own worker uid inside this container, so it + // starts as root with the production Daimon capability set instead of the host user with no capabilities at all. + const privilege = broker + ? [...await trainingBrokerSecurityArgs(privateRoot), "--pids-limit", "2048", + ...trainingBrokerTmpfsTargets().flatMap((entry) => ["--tmpfs", `${entry.path}:rw,nosuid,nodev,size=${entry.size},mode=${entry.mode}`])] + : ["--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + ...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS, "--pids-limit", "512", + "--tmpfs", `/tmp:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=1777`, "--tmpfs", `/work:rw,nosuid,nodev,size=4g,uid=${uid},gid=${gid},mode=700`, + "--tmpfs", `/home/training:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=700`]; const args = ["create", "--name", name, "--label", `com.spawnfile.training.owner=${name}`, - "--init", "--read-only", "--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", - ...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS, "--pids-limit", "512", - "--tmpfs", `/tmp:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=1777`, "--tmpfs", `/work:rw,nosuid,nodev,size=4g,uid=${uid},gid=${gid},mode=700`, "--tmpfs", `/home/training:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=700`, - "--env", "HOME=/home/training", "--workdir", "/work", "--entrypoint", "/opt/training/bin/train", - ...mounts.flatMap((mount) => ["--mount", mount]), + "--init", "--read-only", ...privilege, + "--env", "HOME=/home/training", "--workdir", "/work", "--entrypoint", broker ? TRAINING_BROKER_ENTRYPOINT : "/opt/training/bin/train", + ...[...mounts, ...(broker ? trainingBrokerMounts(broker) : [])].flatMap((mount) => ["--mount", mount]), ...(prepared.viewerPort === undefined ? [] : ["--publish", `127.0.0.1:${prepared.viewerPort}:${prepared.viewerPort}`]), expectedImage, "train", "--spawnfile-context", "/run/paideia/context.json", ...prepared.args]; creationAttempted = true; diff --git a/src/compiler/training/container/prepare.test.ts b/src/compiler/training/container/prepare.test.ts index 78adac76..1023fc84 100644 --- a/src/compiler/training/container/prepare.test.ts +++ b/src/compiler/training/container/prepare.test.ts @@ -33,7 +33,7 @@ it("rejects unmapped paths, output under readonly mounts, malformed resources an it("rejects symlink aliases, whole auth directories and overlapping writable mounts", async () => { const f = await setup(); const alias = path.join(f.root, "alias"); await symlink(f.project, alias); for (const config of [{ ...f.config, inputs: [{ source: alias, destination: "/run/training/inputs/project" }] }, - { ...f.config, auth: [{ provider: "grok", source: f.project }] }, + { ...f.config, auth: [{ provider: "codex", source: f.project }] }, { ...f.config, output: { source: f.project, destination: "/run/training/output" } }, { ...f.config, output: { source: path.join(f.root, "auth-leaf"), destination: "/run/training/output" } }, { ...f.config, inputs: [{ source: f.root, destination: "/run/training/inputs/project" }] }]) await expect(prepareTrainingContainer(config, f.context, f.args)).rejects.toThrow(); @@ -44,7 +44,7 @@ it("rejects symlink aliases, whole auth directories and overlapping writable mou it("rejects noncanonical raw spellings and overlapping host input roots before remapping",async()=>{ const f=await setup();const inner=path.join(f.project,"inner");await mkdir(inner);const other=path.join(f.root,"other");await mkdir(other);const leaf=path.join(f.project,"credential");await writeFile(leaf,"fixture"); - await expect(prepareTrainingContainer({...f.config,auth:[{provider:"grok",source:other+"/../project/credential"}]},f.context,f.args)).rejects.toThrow("canonical"); + await expect(prepareTrainingContainer({...f.config,auth:[{provider:"codex",source:other+"/../project/credential"}]},f.context,f.args)).rejects.toThrow("canonical"); await expect(prepareTrainingContainer({...f.config,inputs:[...f.config.inputs,{source:inner,destination:"/run/training/inputs/inner"}]},f.context,f.args)).rejects.toThrow("source roots must not overlap"); await expect(prepareTrainingContainer({...f.config,output:{source:f.output+"/../output",destination:"/run/training/output"}},f.context,f.args)).rejects.toThrow("canonical"); }); diff --git a/src/compiler/training/container/security.ts b/src/compiler/training/container/security.ts new file mode 100644 index 00000000..7020bf32 --- /dev/null +++ b/src/compiler/training/container/security.ts @@ -0,0 +1,73 @@ +import { + DAIMON_DOCKER_RUNTIME_SECURITY_ARGS, + materializeDaimonGrokSeccompProfile +} from "../../../shared/index.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import { DAIMON_BROKER_TMPDIR } from "../../containerDaimonBrokerRender.js"; +import { DAIMON_GROK_TURN_USAGE_LEDGER } from "../../../runtime/daimon/contractManifest.js"; +import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js"; +import { + TRAINING_GRANT_HOME_ROOT, + TRAINING_INFERENCE_DIRECTORY, + TRAINING_PAIDEIA_ROOT, + TRAINING_SLOT_ROOT, + TRAINING_SUPERVISOR_DIRECTORY, + TRAINING_WORKER_ROOT +} from "../broker/paths.js"; + +export const TRAINING_BROKER_ENTRYPOINT = "/opt/training/bin/train-broker"; + +/** + * Writable state of a broker-capable training container, all on tmpfs. + * + * The image root stays read-only and nothing here is a host bind, so a trial's + * worker home, workspace, turn store, wake-acceptance store and ledgers are + * on a filesystem that enforces unix ownership — which is what lets the slot + * supervisor's worker-uid canaries mean anything (P0 §5: a host bind under + * Docker Desktop or Colima silently ignores `chown`). The Grok realm is the + * one durable mount, and it is a named volume, not a bind. + */ +export const trainingBrokerTmpfsTargets = (): readonly { path: string; size: string; mode: string }[] => [ + { path: "/tmp", size: "1g", mode: "1777" }, + { path: "/var/tmp", size: "256m", mode: "1777" }, + { path: "/work", size: "4g", mode: "0755" }, + { path: "/home/training", size: "1g", mode: "0755" }, + { path: TRAINING_PAIDEIA_ROOT, size: "64m", mode: "0755" }, + { path: TRAINING_SLOT_ROOT, size: "4g", mode: "0755" }, + { path: TRAINING_GRANT_HOME_ROOT, size: "64m", mode: "0755" }, + { path: TRAINING_INFERENCE_DIRECTORY, size: "64m", mode: "0755" }, + { path: TRAINING_SUPERVISOR_DIRECTORY, size: "16m", mode: "0755" }, + { path: TRAINING_WORKER_ROOT, size: "1g", mode: "0755" }, + { path: "/etc/daimon-engine-broker", size: "16m", mode: "0755" }, + { path: "/run/daimon-engine-broker", size: "64m", mode: "0755" }, + { path: DAIMON_BROKER_TMPDIR, size: "64m", mode: "0755" }, + { path: DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, size: "64m", mode: "0755" }, + { path: DAIMON_WAKE_FUSE_DIRECTORY, size: "16m", mode: "0755" } +]; + +/** + * `docker create` arguments for the broker-capable training container. + * + * It starts as root with the *production* Daimon capability set rather than + * the v1 path's `--cap-drop ALL` as the host user: the root entrypoint has to + * provision uid-owned directories (`CAP_CHOWN`), let the native launcher + * `setuid` to a worker (`CAP_SETUID`/`SETGID`), drop bounding sets + * (`CAP_SETPCAP`), supervise dropped children (`CAP_KILL`) and read the + * attested layout it does not own (`CAP_DAC_READ_SEARCH`). `CAP_FOWNER` is + * deliberately absent, which is why every provisioning chmod reclaims the + * inode first. Grok 1.0.34 runs every sandbox profile inside bubblewrap, so + * the container also needs the pinned default-plus-userns seccomp profile and + * AppArmor unconfined — the narrowest combination P0 found. + */ +export const trainingBrokerSecurityArgs = async (seccompProfileDirectory: string): Promise => [ + "--user", "0:0", + ...DAIMON_DOCKER_RUNTIME_SECURITY_ARGS, + `--security-opt=seccomp=${await materializeDaimonGrokSeccompProfile(seccompProfileDirectory)}`, + "--security-opt=apparmor=unconfined" +]; + +export const trainingBrokerMounts = (broker: { realmVolume: string; bootstrap: string; declaration: string }): string[] => [ + `type=volume,src=${broker.realmVolume},dst=${DAIMON_GROK_ENGINE_BROKER.credentialHomePath}`, + `type=bind,src=${broker.bootstrap},dst=/var/lib/spawnfile/daimon/grok-bootstrap-auth,readonly`, + `type=bind,src=${broker.declaration},dst=${TRAINING_PAIDEIA_ROOT}/training-broker.json,readonly` +]; diff --git a/src/compiler/training/preparation/broker.ts b/src/compiler/training/preparation/broker.ts new file mode 100644 index 00000000..7be88834 --- /dev/null +++ b/src/compiler/training/preparation/broker.ts @@ -0,0 +1,53 @@ +import { lstat, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { DAIMON_GROK_SECCOMP_PROFILE_SHA256 } from "../../../shared/daimonGrokSeccompProfile.js"; +import { assertNotDesktopGrokAuth } from "../broker/entrypoint.js"; +import { parseTrainingBrokerDeclaration } from "../broker/declaration.js"; +import { TRAINING_BOOTSTRAP_MOUNT } from "../broker/paths.js"; +import { DAIMON_ORGANIZATION_UID } from "../../../runtime/daimon/runtimeIdentity.js"; +import type { TrainingPreparationConfig } from "./contract.js"; + +export interface PreparedTrainingBroker { + declarationPath: string; + launch: { engine: "grok"; realmVolume: string; bootstrap: string; declaration: string }; +} + +/** + * Lowers `spawnfile.training-container.v3`'s broker block into the read-only + * `spawnfile.training-broker.v1` declaration the container's own root + * entrypoint reads, plus the launch bindings that carry the realm volume and + * bootstrap leaf. + * + * The bootstrap is checked here as well as at launch: a run that never reaches + * `docker create` — a dry run, a failed build — must still refuse the + * developer's desktop Grok login rather than report it later. + */ +export const prepareTrainingBroker = async ( + broker: NonNullable, + root: string, + staging: string, + write: boolean +): Promise => { + const bootstrap = path.resolve(root, broker.bootstrap); + assertNotDesktopGrokAuth(bootstrap); + const info = await lstat(bootstrap).catch(() => undefined); + if (!info?.isFile() || info.isSymbolicLink()) throw Error("The training Grok bootstrap must be an existing regular credential leaf"); + if ((info.mode & 0o077) !== 0) throw Error("The training Grok bootstrap must not be group- or world-accessible"); + const declaration = parseTrainingBrokerDeclaration({ + version: "spawnfile.training-broker.v1", + engine: broker.engine, + agentId: broker.agentId, + model: broker.model, + reasoningEffort: broker.reasoningEffort, + architecture: broker.architecture, + limits: broker.limits, + bootstrap: TRAINING_BOOTSTRAP_MOUNT, + organizationUid: DAIMON_ORGANIZATION_UID, + seccompProfileSha256: DAIMON_GROK_SECCOMP_PROFILE_SHA256, + ...(broker.unenforcedBindPolicy === undefined ? {} : { unenforcedBindPolicy: broker.unenforcedBindPolicy }) + }); + const declarationPath = path.join(staging, "training-broker.json"); + if (write) await writeFile(declarationPath, `${JSON.stringify(declaration)}\n`, { mode: 0o444, flag: "wx" }); + return { declarationPath, launch: { engine: "grok", realmVolume: broker.realmVolume, bootstrap, declaration: declarationPath } }; +}; diff --git a/src/compiler/training/preparation/contract.test.ts b/src/compiler/training/preparation/contract.test.ts index d2708be4..01766915 100644 --- a/src/compiler/training/preparation/contract.test.ts +++ b/src/compiler/training/preparation/contract.test.ts @@ -7,7 +7,7 @@ const config = () => ({ version: "spawnfile.training-container.v2", dockerContex it("keeps authoring strict and rejects ambiguous input and settings references", () => { const base = config(); expect(trainingPreparationSchema.parse(base)).toEqual(base); expect(() => trainingPreparationSchema.parse({ ...base, inputs: [...base.inputs, ...base.inputs] })).toThrow("unique"); - expect(() => trainingPreparationSchema.parse({ ...base, auth: [{ source: "one", provider: "grok" }, { source: "two", provider: "grok" }] })).toThrow("unique"); + expect(() => trainingPreparationSchema.parse({ ...base, auth: [{ source: "one", provider: "codex" }, { source: "two", provider: "codex" }] })).toThrow("unique"); expect(() => trainingPreparationSchema.parse({ ...base, integration: { settings: { input: "missing", path: "settings.json" } } })).toThrow("declared"); expect(() => trainingPreparationSchema.parse({ ...base, inputs: [{ ...base.inputs[0], include: ["x"], git: { revision: "a".repeat(40) } }] })).toThrow("separate"); for (const bad of ["../escape", "/absolute", "a\\b", "a//b"]) expect(() => trainingPreparationSchema.parse({ ...base, inputs: [{ ...base.inputs[0], include: [bad] }] })).toThrow(); diff --git a/src/compiler/training/preparation/contract.ts b/src/compiler/training/preparation/contract.ts index 6ed2a584..c7874b97 100644 --- a/src/compiler/training/preparation/contract.ts +++ b/src/compiler/training/preparation/contract.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { z } from "zod"; import { trainingImageSchema } from "../container/contract.js"; +import { DAIMON_GROK_BROKER_MODELS, DAIMON_GROK_BROKER_REASONING_EFFORTS, DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; const local = z.string().min(1).refine(value => !/[,\r\n\0]/u.test(value)); const relative = local.refine(value => !path.isAbsolute(value) && !value.includes("\\") && value.split("/").every(part => part !== "" && part !== "." && part !== ".." && part !== ".git")); @@ -11,11 +12,45 @@ export const trainingBuildSchema = z.object({ recipe: z.literal("daimon-dspy.v1"), nativeImage: trainingImageSchema, pythonImage: trainingImageSchema, platform: z.enum(["linux/arm64", "linux/amd64"]), paideia: local, bridge: local, claude: local, compiler: local.optional(), - grok: z.object({ source: local, sha256: sha }).strict(), + /** + * Deprecated with `spawnfile.training-container.v3`: the image no longer + * copies a Grok binary at all. Judges run the native parent's pinned + * `/usr/local/bin/grok` through a broker inference grant, so a second copy + * could only ever be a different, unattested build. + */ + grok: z.object({ source: local, sha256: sha }).strict().optional(), integration: z.object({ source: local, entry: relative.refine(value => /^[A-Za-z0-9._/-]+$/u.test(value)) }).strict(), bootstrap: local.optional() }).strict(); +const bounds = DAIMON_GROK_ENGINE_BROKER.turnLimits.bounds; +/** + * `spawnfile.training-container.v3`'s brokered Grok slot. + * + * It is the whole difference from v2: the container starts as root with the + * production capability set, runs one broker slot for the subject, holds the + * dedicated training Grok login in a named realm volume seeded from + * `bootstrap`, and serves judge inference grants out of the same credential. + * No Grok binary and no Grok auth staging enter the image. + */ +export const trainingBrokerSchema = z.object({ + engine: z.literal("grok"), + agentId: z.string().min(1).max(128).regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u), + model: z.enum(DAIMON_GROK_BROKER_MODELS), + reasoningEffort: z.enum(DAIMON_GROK_BROKER_REASONING_EFFORTS), + architecture: z.enum(["arm64", "x64"]), + limits: z.object({ + maxRequests: z.number().int().min(bounds.maxRequests[0]).max(bounds.maxRequests[1]), + maxTokens: z.number().int().min(bounds.maxTokens[0]).max(bounds.maxTokens[1]), + timeoutMs: z.number().int().min(bounds.timeoutMs[0]).max(bounds.timeoutMs[1]) + }).strict(), + /** Named Docker volume for the rotating training credential and its journal; never a host bind. */ + realmVolume: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), + /** The dedicated training Grok login leaf, relative to the declaration. Never the desktop `~/.grok/auth.json`. */ + bootstrap: local, + unenforcedBindPolicy: z.enum(["refuse", "profile-only"]).optional() +}).strict(); + export const trainingPreparationSchema = z.object({ - version: z.literal("spawnfile.training-container.v2"), + version: z.enum(["spawnfile.training-container.v2", "spawnfile.training-container.v3"]), dockerContext: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u), image: z.union([z.object({ ref: trainingImageSchema }).strict(), z.object({ build: trainingBuildSchema }).strict()]), integration: z.object({ settings: reference }).strict(), @@ -27,8 +62,15 @@ export const trainingPreparationSchema = z.object({ }).strict().optional() }).strict()).min(1).max(64), output: z.object({ source: local, destination: z.literal("/run/training/output") }).strict(), - auth: z.array(z.object({ source: local, provider: z.enum(["codex", "grok", "claude"]) }).strict()).max(3) + auth: z.array(z.object({ source: local, provider: z.enum(["codex", "claude"]) }).strict()).max(2), + broker: trainingBrokerSchema.optional() }).strict().superRefine((value, context) => { + if ((value.version === "spawnfile.training-container.v3") !== (value.broker !== undefined)) { + context.addIssue({ code: "custom", message: "Only spawnfile.training-container.v3 declares a broker slot, and it always does" }); + } + if (value.broker && "build" in value.image && value.image.build.grok) { + context.addIssue({ code: "custom", message: "A v3 training image never copies a Grok binary; judges use the native parent's pinned /usr/local/bin/grok" }); + } if (new Set(value.inputs.map(input => input.id)).size !== value.inputs.length) context.addIssue({ code: "custom", message: "Input IDs must be unique" }); if (new Set(value.auth.map(auth => auth.provider)).size !== value.auth.length) context.addIssue({ code: "custom", message: "Auth providers must be unique" }); if (!value.inputs.some(input => input.id === value.integration.settings.input)) context.addIssue({ code: "custom", message: "Integration settings require a declared input" }); diff --git a/src/compiler/training/preparation/fixtures.test-helper.ts b/src/compiler/training/preparation/fixtures.test-helper.ts index 608507eb..15ec9a5b 100644 --- a/src/compiler/training/preparation/fixtures.test-helper.ts +++ b/src/compiler/training/preparation/fixtures.test-helper.ts @@ -25,7 +25,7 @@ export async function preparationFixture() { const config: TrainingPreparationConfig = { version: "spawnfile.training-container.v2", dockerContext: "local", image: { build: { recipe: "daimon-dspy.v1", nativeImage: image, pythonImage: image, platform: "linux/arm64", - paideia: "paideia", bridge: "bridge", claude: "claude", grok: { source: "grok", sha256: sha("native-binary") }, + paideia: "paideia", bridge: "bridge", claude: "claude", integration: { source: "integration", entry: "entry.ts" }, bootstrap: "bootstrap" } }, integration: { settings: { input: "settings", path: "settings.json" } }, inputs: [{ id: "project", source: "project", destination: "/run/training/inputs/project" }, { id: "settings", source: "settings", destination: "/run/training/inputs/settings" }], diff --git a/src/compiler/training/preparation/image.test.ts b/src/compiler/training/preparation/image.test.ts index 2c09852d..7b28c302 100644 --- a/src/compiler/training/preparation/image.test.ts +++ b/src/compiler/training/preparation/image.test.ts @@ -24,10 +24,8 @@ it("binds actual runtime bytes, locks, recipe, native parent, platform and entry expect((await plan()).digest).toBe(initial.digest); }); -it("rejects dependency drift, executable pin mismatch, missing required entries and image symlink escapes", async () => { +it("rejects dependency drift, missing required entries and image symlink escapes", async () => { const f = await fixture(); const plan = () => planTrainingImage(f.build, f.root, [], f.own); - f.build.grok.sha256 = image; await expect(plan()).rejects.toThrow("digest mismatch"); - const { sha } = await import("./fixtures.test-helper.js"); f.build.grok.sha256 = sha("native-binary"); f.build.integration.entry = "missing.ts"; await expect(plan()).rejects.toThrow("missing integration"); f.build.integration.entry = "entry.ts"; await f.put("claude/package.json", JSON.stringify({ dependencies: { injected: "1" } })); await expect(plan()).rejects.toThrow("manifest/lock mismatch"); await f.put("claude/package.json", JSON.stringify({ dependencies: {} })); diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 5ff11252..6c26e3b8 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -8,7 +8,7 @@ import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); export const trainingAssets = path.extname(fileURLToPath(import.meta.url)) === ".ts" ? path.join(packageRoot, "runtime-images/training") : fileURLToPath(new URL("./assets/", import.meta.url)); -export interface TrainingImagePlan { digest: string; files: SealedFile[]; dockerfile: string; entry: string; build: TrainingImageBuild } +export interface TrainingImagePlan { digest: string; files: SealedFile[]; dockerfile: string; entry: string; brokerEntry: string; build: TrainingImageBuild } async function packageFiles(root: string, target: string, withDist: boolean, lockSource = path.join(root, "package-lock.json")): Promise { const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8")); @@ -26,7 +26,7 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { const resolve = (value: string) => path.resolve(root, value); - for (const source of [build.paideia, build.bridge, build.claude, build.grok.source, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); + for (const source of [build.paideia, build.bridge, build.claude, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); const dockerfile = await readFile(path.join(assets, "Dockerfile"), "utf8"); const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot @@ -40,20 +40,20 @@ export async function planTrainingImage(build: TrainingImageBuild, root: string, ...await sealTree(resolve(build.bridge), "bridge", { ignoreDevelopment: true }), ...await sealTree(resolve(build.integration.source), "integration", { ignoreDevelopment: true }), ...build.bootstrap ? await sealTree(resolve(build.bootstrap), "bootstrap", { ignoreDevelopment: true }) : [], - await sealFile(resolve(build.grok.source), "grok"), await sealFile(path.join(ownRoot, "runtimes.yaml"), "spawnfile/runtimes.yaml"), await sealFile(path.join(ownRoot, "moltnet-releases.json"), "spawnfile/moltnet-releases.json"), ...await Promise.all(["runtimes.yaml", "moltnet-releases.json"].map(name => sealFile(path.join(build.compiler ? resolve(build.compiler) : ownRoot, name), `compiler/${name}`))) ]; - if (files.find(file => file.destination === "grok")!.sha256 !== build.grok.sha256) throw Error("Grok executable digest mismatch"); for (const required of [`integration/${build.integration.entry}`, "bridge/pyproject.toml", "bridge/requirements.lock", "bridge/paideia_dspy/__init__.py", "paideia/dist/src/cli/main.js", "spawnfile/dist/cli/index.js", "compiler/dist/cli/index.js"]) { if (!files.some(file => file.destination === required)) throw Error(`Training distribution is missing ${required}`); } const entry = `#!/bin/sh\nexec /usr/local/bin/node --experimental-strip-types ${JSON.stringify(`/opt/training/integration/${build.integration.entry}`)} "$@"\n`; + // The broker-capable entrypoint is the image's own Spawnfile distribution, never a host-written script. + const brokerEntry = `#!/bin/sh\nexec /usr/local/bin/node /opt/training/spawnfile/dist/compiler/training/broker/main.js "$@"\n`; const digest = hashJson({ recipe: build.recipe, nativeImage: build.nativeImage, pythonImage: build.pythonImage, platform: build.platform, - files: fileIdentity(files), dockerfile, entry }); - return { digest, files, dockerfile, entry, build }; + files: fileIdentity(files), dockerfile, entry, brokerEntry }); + return { digest, files, dockerfile, entry, brokerEntry, build }; } /** Cache is content-addressed and still requires a matching immutable image and label. */ @@ -79,6 +79,7 @@ export async function buildTrainingImage(plan: TrainingImagePlan, options: { await mkdir(path.join(staging, "bootstrap"), { recursive: true, mode: 0o700 }); await writeFile(path.join(staging, "Dockerfile"), `${plan.dockerfile}\nLABEL com.spawnfile.training.recipe=${JSON.stringify(plan.digest)}\n`, { mode: 0o600 }); await writeFile(path.join(staging, "train"), plan.entry, { mode: 0o755 }); + await writeFile(path.join(staging, "train-broker"), plan.brokerEntry, { mode: 0o755 }); const result = await call(["build", "--platform", plan.build.platform, "--build-arg", `NATIVE_IMAGE=${plan.build.nativeImage}`, "--build-arg", `PYTHON_IMAGE=${plan.build.pythonImage}`, "--tag", tag, staging], true); if (result.code !== 0) throw Error("Training image build failed; inspect the streamed build diagnostic"); diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index 10e2e278..46aad01e 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -8,6 +8,7 @@ import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js"; import { planTrainingImage, buildTrainingImage } from "./image.js"; import { planMeasurementRepair, stageMeasurementRepair, writeTrainingWitness } from "../repair/index.js"; +import { prepareTrainingBroker } from "./broker.js"; function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { return parseTrainingMappedPreparation({ version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, @@ -32,6 +33,8 @@ export interface PreparedTraining { /** Reads only until the explicit dry-run boundary; preparation never executes project code. */ export async function prepareTraining(options: PrepareTrainingOptions): Promise { const config = trainingPreparationSchema.parse(await readBoundedJson(options.configPath)); + // A v2 declaration still lowers to the unchanged v1 launch config; only v3 carries a broker slot into Docker. + const launchVersion = config.version === "spawnfile.training-container.v3" ? config.version : "spawnfile.training-container.v1"; const root = path.dirname(path.resolve(options.configPath)); const auth = config.auth.map(entry => ({ ...entry, source: path.resolve(root, entry.source) })); const output = path.resolve(root, config.output.source), parent = path.dirname(output); @@ -59,6 +62,7 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< if (repair && hashJson(config.integration) !== hashJson(repair.witness.manifest.config.integration)) throw Error("Repair integration settings binding changed"); const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image, canonical: options.context.project.sourceDigest, ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); + if (config.broker) await prepareTrainingBroker(config.broker, root, path.dirname(path.resolve(options.configPath)), false); if (options.dryRun) return { digest, dryRun: true }; options.signal?.throwIfAborted(); for (const entry of auth) if (await exactPath(entry.source) !== entry.source || !(await lstat(entry.source)).isFile()) throw Error("Training auth must be a canonical regular leaf"); @@ -100,8 +104,10 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< if (inspected.code !== 0) throw Error("Training image is unavailable"); image = inspected.stdout.trim(); } const mapped = mappedReceipt(digest, image, config); - const launch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, - inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth }); + const preparedBroker = config.broker ? await prepareTrainingBroker(config.broker, root, staging, true) : undefined; + const launch = trainingContainerConfigSchema.parse({ version: launchVersion, dockerContext: config.dockerContext, + inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth, + ...(preparedBroker ? { broker: preparedBroker.launch } : {}) }); await writeFile(configPath, JSON.stringify(launch), { flag: "wx", mode: 0o600 }); await writeFile(preparationPath, JSON.stringify(parseTrainingMappedPreparation(mapped)), { flag: "wx", mode: 0o400 }); const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index]!, "input", { ignoreGit: true, internalSymlinks: true }))) : null)); @@ -111,8 +117,10 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) }); } catch (error) { if (owned) await rm(staging, { recursive: true, force: true }); throw error; } } - const expectedLaunch = trainingContainerConfigSchema.parse({ version: "spawnfile.training-container.v1", dockerContext: config.dockerContext, - inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth }); + const expectedBroker = config.broker ? await prepareTrainingBroker(config.broker, root, staging, false) : undefined; + const expectedLaunch = trainingContainerConfigSchema.parse({ version: launchVersion, dockerContext: config.dockerContext, + inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth, + ...(expectedBroker ? { broker: expectedBroker.launch } : {}) }); if (hashJson(await readBoundedJson(configPath)) !== hashJson(expectedLaunch) || hashJson(await readBoundedJson(preparationPath)) !== hashJson(mappedReceipt(digest, image, config))) throw Error("Saved training launch or mapped receipt changed"); const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]); diff --git a/src/compiler/training/repair/contract.ts b/src/compiler/training/repair/contract.ts index 58810a2b..5bb06411 100644 --- a/src/compiler/training/repair/contract.ts +++ b/src/compiler/training/repair/contract.ts @@ -16,7 +16,7 @@ export const witnessSchema = z.object({ inputs: z.array(z.object({ id: z.string(), source: z.string(), destination: z.string(), digest: sha, staged: z.string(), snapshotDigest: sha.nullable() }).strict()), repair: z.object({ witness: sha, parent: sha }).strict().optional(), - image: z.object({ build: trainingBuildSchema, dockerfile: z.string(), entry: z.string(), files: z.array(fileSchema).min(1).max(10000) }).strict() + image: z.object({ build: trainingBuildSchema, dockerfile: z.string(), entry: z.string(), brokerEntry: z.string(), files: z.array(fileSchema).min(1).max(10000) }).strict() }).strict(); export type TrainingWitness = z.infer; export const witnessEnvelopeSchema = z.object({ manifest: witnessSchema, digest: sha }).strict(); diff --git a/src/compiler/training/repair/witness.ts b/src/compiler/training/repair/witness.ts index e5559319..802fbabf 100644 --- a/src/compiler/training/repair/witness.ts +++ b/src/compiler/training/repair/witness.ts @@ -18,7 +18,7 @@ export async function readTrainingWitness(file: string) { if (hashJson(sorted(fileIdentity(files))) !== hashJson(sorted(witness.image.files))) throw Error("Training witness image bytes changed"); const { recipe, nativeImage, pythonImage, platform } = witness.image.build; if (hashJson({ recipe, nativeImage, pythonImage, platform, files: witness.image.files, - dockerfile: witness.image.dockerfile, entry: witness.image.entry }) !== witness.imagePlanDigest) throw Error("Training witness recipe identity mismatch"); + dockerfile: witness.image.dockerfile, entry: witness.image.entry, brokerEntry: witness.image.brokerEntry }) !== witness.imagePlanDigest) throw Error("Training witness recipe identity mismatch"); const expected = hashJson({ config: raw.manifest && (raw.manifest as TrainingWitness).config, sources: witness.inputs.map(input => ({ id: input.id, digest: input.digest })), image: witness.imagePlanDigest, canonical: witness.context.project.sourceDigest, ...(witness.repair ? { repair: witness.repair } : {}) }); @@ -39,6 +39,7 @@ export async function writeTrainingWitness(options: { directory: string; digest: context: options.context, command: { args: [...options.args] }, inputs: options.inputs.map((input, index) => ({ id: input.id, source: input.source, destination: input.destination, digest: input.digest, staged: options.staged[index]!, snapshotDigest: options.snapshots[index]! })), - image: { build: options.plan.build, files: fileIdentity(options.plan.files), dockerfile: options.plan.dockerfile, entry: options.plan.entry } }; + image: { build: options.plan.build, files: fileIdentity(options.plan.files), dockerfile: options.plan.dockerfile, + entry: options.plan.entry, brokerEntry: options.plan.brokerEntry } }; await writeFile(path.join(directory, "manifest.json"), JSON.stringify({ manifest, digest: hashJson(manifest) }), { flag: "wx", mode: 0o400 }); } From 846794d0ec59068203cde86a219ebc42ffdc5968 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:39:23 +0200 Subject: [PATCH 44/84] feat: import a dedicated training Grok login and refuse the desktop home --- src/auth/importers.ts | 36 ++++++++++++++++++++++++++++++++++++ src/auth/paths.ts | 2 +- src/auth/types.ts | 2 +- src/cli/authCommands.ts | 11 +++++++++++ src/cli/runCli.ts | 4 +++- 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/auth/importers.ts b/src/auth/importers.ts index 6fb8af7f..c03f6357 100644 --- a/src/auth/importers.ts +++ b/src/auth/importers.ts @@ -111,6 +111,42 @@ export const importCodexAuth = async ( return profile; }; +/** + * Imports a **dedicated** Grok login into a profile. + * + * Unlike Codex, the source is always explicit: training rotates the credential + * it is given, so defaulting to `~/.grok` would let a routine training run + * invalidate the developer's own desktop session, and the refreshed token + * would land on container tmpfs rather than back in `~/.grok` (D2). The + * desktop home is refused outright, by resolved path and through `GROK_HOME`. + */ +export const importGrokAuth = async ( + profileName: string, + sourceDirectory: string | undefined +) => { + if (!sourceDirectory) { + throw new SpawnfileError( + "validation_error", + "Grok import requires --from holding a dedicated training login; training never uses the desktop ~/.grok" + ); + } + const resolvedSource = path.resolve(sourceDirectory); + const desktop = path.resolve(process.env.GROK_HOME ?? path.join(os.homedir(), ".grok")); + if (resolvedSource === desktop || resolvedSource === path.join(os.homedir(), ".grok")) { + throw new SpawnfileError( + "validation_error", + `Refusing the desktop Grok home ${resolvedSource}; a training run rotates the credential it imports` + ); + } + const authFilePath = path.join(resolvedSource, "auth.json"); + if (!(await fileExists(authFilePath))) { + throw new SpawnfileError("validation_error", `Grok auth file does not exist: ${authFilePath}`); + } + const { directory, profile } = await registerImportedAuth(profileName, "grok"); + await writePrivateUtf8File(path.join(directory, "auth.json"), await readUtf8File(authFilePath)); + return profile; +}; + export const importClaudeCodeAuth = async ( profileName: string, sourceDirectory?: string, diff --git a/src/auth/paths.ts b/src/auth/paths.ts index 393da80d..88ff37f8 100644 --- a/src/auth/paths.ts +++ b/src/auth/paths.ts @@ -42,7 +42,7 @@ export const resolveProfilePath = (profileName: string): string => export const resolveImportedAuthDirectory = ( profileName: string, - kind: "claude-code" | "codex" + kind: "claude-code" | "codex" | "grok" ): string => path.join(resolveProfileDirectory(profileName), "imports", kind); export const resolveTargetSecretsRoot = (): string => diff --git a/src/auth/types.ts b/src/auth/types.ts index c58c9325..4ea3a474 100644 --- a/src/auth/types.ts +++ b/src/auth/types.ts @@ -1,4 +1,4 @@ -export type ImportedAuthKind = "claude-code" | "codex"; +export type ImportedAuthKind = "claude-code" | "codex" | "grok"; export interface ImportedAuthEntry { kind: ImportedAuthKind; diff --git a/src/cli/authCommands.ts b/src/cli/authCommands.ts index 6ebaa269..3c740a90 100644 --- a/src/cli/authCommands.ts +++ b/src/cli/authCommands.ts @@ -31,6 +31,7 @@ type AuthCommandHandlers = Pick< CliHandlers, | "importClaudeCodeAuth" | "importCodexAuth" + | "importGrokAuth" | "importEnvFile" | "initializeTargetSecretSourceLifecycle" | "provisionCredentials" @@ -122,6 +123,16 @@ export const registerAuthCommands = ( emitLines(streams, formatAuthProfileSummary(profile)); }); + authImportCommand + .command("grok") + .description("Import a dedicated training Grok login into a profile") + .option("-p, --profile ", "Auth profile name", "default") + .requiredOption("--from ", "Source Grok config directory holding the dedicated login") + .action(async (options: { from: string; profile: string }) => { + const profile = await handlers.importGrokAuth(options.profile, options.from); + emitLines(streams, formatAuthProfileSummary(profile)); + }); + authImportCommand .command("codex") .description("Import Codex subscription credentials into a profile") diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index 9da2c183..0cd3d604 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -5,6 +5,7 @@ import { Command } from "commander"; import { importClaudeCodeAuth, importCodexAuth, + importGrokAuth, importEnvFile, initializeTargetSecretSourceLifecycle, provisionCredentials, @@ -107,6 +108,7 @@ export interface CliHandlers { addProjectSurface: typeof addProjectSurface; addSubagentProject: typeof addSubagentProject; addTeamProject: typeof addTeamProject; clearProjectModelFallbacks: typeof clearProjectModelFallbacks; importClaudeCodeAuth: typeof importClaudeCodeAuth; importCodexAuth: typeof importCodexAuth; + importGrokAuth: typeof importGrokAuth; importEnvFile: typeof importEnvFile; initProject: typeof initProject; listInitTemplates: typeof listInitTemplates; initializeTargetSecretSourceLifecycle: typeof initializeTargetSecretSourceLifecycle; @@ -136,7 +138,7 @@ const createDefaultHandlers = (): CliHandlers => ({ buildCompilePlan, buildOrganizationView, buildProject, compileProject, publishProject, addAgentProject, addProjectModelFallback, addProjectSurface, addSubagentProject, addTeamProject, clearProjectModelFallbacks, - importClaudeCodeAuth, importCodexAuth, importEnvFile, initializeTargetSecretSourceLifecycle, + importClaudeCodeAuth, importCodexAuth, importGrokAuth, importEnvFile, initializeTargetSecretSourceLifecycle, provisionCredentials, createDockerProbeGateway, exportRunArtifacts, downDeployment, inspectDockerDeployment, listDeploymentRecords, listHomeDeploymentRecords, From a3177cd79229daa5158f1dfd4c31d72d30644412 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:42:28 +0200 Subject: [PATCH 45/84] test: pin the training slot privilege model, recycle order, canaries and receipt freshness --- .../training/broker/entrypoint.test.ts | 78 ++++++++++++++ src/compiler/training/broker/receipt.test.ts | 82 ++++++++++++++ .../training/broker/registration.test.ts | 101 ++++++++++++++++++ .../training/broker/supervisor.test.ts | 77 +++++++++++++ 4 files changed, 338 insertions(+) create mode 100644 src/compiler/training/broker/entrypoint.test.ts create mode 100644 src/compiler/training/broker/receipt.test.ts create mode 100644 src/compiler/training/broker/registration.test.ts create mode 100644 src/compiler/training/broker/supervisor.test.ts diff --git a/src/compiler/training/broker/entrypoint.test.ts b/src/compiler/training/broker/entrypoint.test.ts new file mode 100644 index 00000000..a07a3482 --- /dev/null +++ b/src/compiler/training/broker/entrypoint.test.ts @@ -0,0 +1,78 @@ +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { DAIMON_DOCKER_RUNTIME_SECURITY_ARGS } from "../../../shared/index.js"; +import { trainingBrokerMounts, trainingBrokerTmpfsTargets } from "../container/security.js"; +import { assertNotDesktopGrokAuth, trainingChildArgv, TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV, TRAINING_GROK_GRANT_HOME_ROOT_ENV } from "./entrypoint.js"; +import { parseTrainingBrokerDeclaration } from "./declaration.js"; +import { brokerProcessPlan } from "./processes.js"; +import { TRAINING_REALM_MOUNT, TRAINING_SLOT_ROOT } from "./paths.js"; + +const declaration = (overrides: Record = {}) => parseTrainingBrokerDeclaration({ + version: "spawnfile.training-broker.v1", engine: "grok", agentId: "agent:author", model: "grok-4.6", + reasoningEffort: "low", architecture: "arm64", limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + bootstrap: "/var/lib/spawnfile/daimon/grok-bootstrap-auth", organizationUid: 2000, seccompProfileSha256: "f".repeat(64), ...overrides +}); + +describe("training broker declaration", () => { + it("defaults to refusing a deny path on a filesystem that ignores unix ownership", () => { + expect(declaration().unenforcedBindPolicy).toBe("refuse"); + }); + + it("stays strict about engine, model, effort and unknown members", () => { + expect(() => declaration({ engine: "codex" })).toThrow(); + expect(() => declaration({ model: "gpt-5" })).toThrow(); + expect(() => declaration({ reasoningEffort: "xhigh" })).toThrow(); + expect(() => declaration({ organizationUid: 2200 })).toThrow(); + expect(() => declaration({ extra: 1 })).toThrow(); + }); +}); + +describe("training Grok credential authority", () => { + it("refuses the desktop Grok login as a training bootstrap", () => { + const home = path.join(os.tmpdir(), "spawnfile-desktop"); + expect(() => assertNotDesktopGrokAuth(path.join(home, ".grok/auth.json"), home)).toThrow(/desktop ~\/.grok\/auth.json/u); + expect(() => assertNotDesktopGrokAuth(path.join(home, ".grok", "..", ".grok", "auth.json"), home)).toThrow(/desktop/u); + expect(assertNotDesktopGrokAuth(path.join(home, "training-grok/auth.json"), home)).toContain("training-grok"); + }); +}); + +describe("training container privilege model", () => { + it("runs train as the organization uid and refuses to exec with any capability left", () => { + const argv = trainingChildArgv(["--spawnfile-context", "/run/paideia/context.json"]); + expect(argv.slice(0, 8)).toEqual(["setpriv", "--clear-groups", "--reuid=2000", "--regid=2000", "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", "--"]); + const guard = argv[10]!; + expect(guard).toContain('test "$(sed -n "s/^CapBnd:[[:space:]]*//p" /proc/self/status)" = 0000000000000000'); + expect(guard).toContain('test "$(sed -n "s/^CapEff:[[:space:]]*//p" /proc/self/status)" = 0000000000000000'); + expect(guard).toContain('if [ "$EUID" -eq 0 ]'); + expect(argv).toContain("/opt/training/bin/train"); + }); + + it("keeps the production capability set and never grants CAP_FOWNER or CAP_DAC_OVERRIDE", () => { + expect([...DAIMON_DOCKER_RUNTIME_SECURITY_ARGS]).toEqual(["--cap-drop=ALL", "--cap-add=CHOWN", "--cap-add=SETUID", + "--cap-add=SETGID", "--cap-add=DAC_READ_SEARCH", "--cap-add=SETPCAP", "--cap-add=KILL", "--security-opt=no-new-privileges:true"]); + }); + + it("starts the launcher as root and the broker and relay as uid 2100 with an empty bounding set", () => { + expect(brokerProcessPlan().map((entry) => [entry.uid, entry.capBnd])).toEqual([ + [0, "00000000000000c1"], [2100, "0000000000000000"], [2100, "0000000000000000"] + ]); + }); + + it("keeps every per-trial path on tmpfs and the realm on a named volume", () => { + const tmpfs = trainingBrokerTmpfsTargets().map((entry) => entry.path); + expect(tmpfs).toContain(TRAINING_SLOT_ROOT); + expect(tmpfs).toContain("/var/lib/daimon-workers"); + expect(tmpfs).not.toContain(TRAINING_REALM_MOUNT); + const mounts = trainingBrokerMounts({ realmVolume: "training-realm", bootstrap: "/host/auth.json", declaration: "/host/training-broker.json" }); + expect(mounts[0]).toBe(`type=volume,src=training-realm,dst=${TRAINING_REALM_MOUNT}`); + expect(mounts[1]).toContain("readonly"); + expect(mounts[2]).toContain("dst=/run/paideia/training-broker.json,readonly"); + }); + + it("exports the broker control socket and a private grant home root to the evaluator", () => { + expect(TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV).toBe("PAIDEIA_GROK_BROKER_CONTROL_SOCKET"); + expect(TRAINING_GROK_GRANT_HOME_ROOT_ENV).toBe("PAIDEIA_GROK_GRANT_HOME_ROOT"); + }); +}); diff --git a/src/compiler/training/broker/receipt.test.ts b/src/compiler/training/broker/receipt.test.ts new file mode 100644 index 00000000..80ea0f6e --- /dev/null +++ b/src/compiler/training/broker/receipt.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; + +import { buildTrainingSlotReceipt, resolveBackingFilesystem, resolveTrainingCanaries } from "./receipt.js"; + +const digest = (fill: string) => fill.repeat(64).slice(0, 64); +const nonce = digest("a"); +const canary = (target: string) => ({ path: target, method: "sandboxed-read" as const, result: "denied" as const }); + +const receipt = (overrides: Partial[0]> = {}) => buildTrainingSlotReceipt({ + slot: 0, workerUid: 2200, generation: 7, nonce, projectionSha256: digest("b"), sandboxProfileSha256: digest("c"), + seccompProfileSha256: digest("d"), grokExecutableSha256: digest("e"), canaries: [canary("/run/paideia/context.json")], + createdAt: new Date("2026-09-17T12:00:00.000Z"), ...overrides +}); + +const mountinfo = [ + "21 20 0:20 / / rw,relatime - overlay overlay rw", + "30 21 0:31 / /run/training/slot rw,relatime - tmpfs tmpfs rw", + "31 21 0:32 / /run/training/output rw,relatime - virtiofs docker rw" +].join("\n"); + +describe("training slot preflight receipt v2", () => { + it("is the exact v2 shape with the caller's nonce and the supervisor's generation", () => { + expect(receipt()).toEqual({ + version: "noopolis.daimon.grok-slot-preflight.v2", slot: 0, worker_uid: 2200, generation: 7, nonce, + projection_sha256: digest("b"), sandbox_profile_sha256: digest("c"), seccomp_profile_sha256: digest("d"), + sandbox_runtime: "bubblewrap", grok_executable_sha256: digest("e"), + canaries: [canary("/run/paideia/context.json")], created_at: "2026-09-17T12:00:00.000Z" + }); + }); + + it("refuses a generation below one and a nonce that is not 32 hex bytes", () => { + expect(() => receipt({ generation: 0 })).toThrow(/positive integer/u); + expect(() => receipt({ nonce: "not-a-nonce" })).toThrow(/lowercase hex/u); + expect(() => receipt({ nonce: nonce.toUpperCase() })).toThrow(/lowercase hex/u); + }); + + it("refuses a receipt with no canary at all", () => { + expect(() => receipt({ canaries: [] })).toThrow(/at least one denied canary/u); + }); +}); + +describe("training slot canaries", () => { + it("requires a worker-uid denial for every deny path on a filesystem that enforces ownership", async () => { + const probed: string[] = []; + const canaries = await resolveTrainingCanaries({ + denyPaths: ["/run/training/slot/turns", "/run/training/slot/usage"], mountinfo, unenforcedBindPolicy: "refuse", + probe: async (target) => { probed.push(target); return true; }, log: () => undefined + }); + expect(probed).toEqual(["/run/training/slot/turns", "/run/training/slot/usage"]); + expect(canaries).toHaveLength(2); + }); + + it("refuses to certify a path the worker uid can still read", async () => { + await expect(resolveTrainingCanaries({ + denyPaths: ["/run/training/slot/turns"], mountinfo, unenforcedBindPolicy: "refuse", + probe: async () => false, log: () => undefined + })).rejects.toThrow(/still readable by the worker uid/u); + }); + + it("refuses a host bind by default, because chown there is silently ignored", async () => { + await expect(resolveTrainingCanaries({ + denyPaths: ["/run/training/output"], mountinfo, unenforcedBindPolicy: "refuse", + probe: async () => { throw Error("the probe must never run on an unenforced filesystem"); }, log: () => undefined + })).rejects.toThrow(/virtiofs, which ignores unix ownership/u); + }); + + it("accepts a host bind on the profile's deny entry alone only when the operator declared that, and says so", async () => { + const lines: string[] = []; + const canaries = await resolveTrainingCanaries({ + denyPaths: ["/run/training/output"], mountinfo, unenforcedBindPolicy: "profile-only", + probe: async () => false, log: (line) => lines.push(line) + }); + expect(canaries).toEqual([canary("/run/training/output")]); + expect(lines[0]).toContain("certified by the enforced sandbox profile only"); + }); + + it("resolves the longest matching mount point", () => { + expect(resolveBackingFilesystem("/run/training/slot/usage/usage.jsonl", mountinfo)).toBe("tmpfs"); + expect(resolveBackingFilesystem("/run/training/output/runs", mountinfo)).toBe("virtiofs"); + expect(resolveBackingFilesystem("/etc/daimon-engine-broker", mountinfo)).toBe("overlay"); + }); +}); diff --git a/src/compiler/training/broker/registration.test.ts b/src/compiler/training/broker/registration.test.ts new file mode 100644 index 00000000..25c88be4 --- /dev/null +++ b/src/compiler/training/broker/registration.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; + +import { renderDaimonGrokServiceConfig } from "../../containerDaimonGrokWorkerRender.js"; +import { renderTrainingBrokerProvisioning, trainingSlotDirectories } from "./provisioning.js"; +import { resolveTrainingGrokDenyPaths, resolveTrainingGrokRegistration } from "./registration.js"; +import { + TRAINING_ADDED_DENY_PATHS, + TRAINING_CALLER_PROTECTED_PATHS, + TRAINING_EVALUATOR_ROOTS, + TRAINING_GRANT_HOME_ROOT, + TRAINING_INFERENCE_DIRECTORY, + TRAINING_INFERENCE_LEDGER, + TRAINING_REALM_MOUNT, + TRAINING_SLOT_TURN_STORE, + TRAINING_SLOT_USAGE_DIRECTORY, + TRAINING_SLOT_USAGE_LEDGER +} from "./paths.js"; + +const slot = () => resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-4.6", reasoningEffort: "low" }); + +describe("training Grok slot registration", () => { + it("denies every evaluator root, caller protected path, control root and the judge grant home", () => { + const denied = slot().denyPaths; + for (const entry of [...TRAINING_EVALUATOR_ROOTS.map((role) => role.path), ...TRAINING_CALLER_PROTECTED_PATHS, + "/run/daimon-engine-broker", "/etc/daimon-engine-broker", TRAINING_GRANT_HOME_ROOT, TRAINING_INFERENCE_DIRECTORY, + TRAINING_SLOT_TURN_STORE, TRAINING_REALM_MOUNT]) { + expect(denied, entry).toContain(entry); + } + expect(denied).toEqual([...denied].sort()); + expect(new Set(denied).size).toBe(denied.length); + }); + + it("keeps the subject's own workspace, worker home and runtime home reachable", () => { + const registration = slot(); + for (const own of [registration.workspace, registration.home, registration.grokHome, registration.runtimeHome, registration.privateTmp, registration.spillDirectory]) { + expect(registration.denyPaths.some((entry) => own === entry || own.startsWith(`${entry}/`))).toBe(false); + } + }); + + it("refuses a deny entry that equals or contains a Grok base-profile grant", () => { + expect(() => resolveTrainingGrokDenyPaths([...TRAINING_ADDED_DENY_PATHS, "/run"])).toThrow(/base profile grant/u); + expect(() => resolveTrainingGrokDenyPaths([...TRAINING_ADDED_DENY_PATHS, "/var/tmp"])).toThrow(/base profile grant/u); + }); + + it("refuses nested masks and a mask over the subject's own workspace", () => { + expect(() => resolveTrainingGrokDenyPaths([...TRAINING_ADDED_DENY_PATHS, "/run/training/output/runs"])).toThrow(/masks cannot nest/u); + expect(() => resolveTrainingGrokDenyPaths([...TRAINING_ADDED_DENY_PATHS, "/run/training/slot/runtime-home"])).toThrow(/own workspace, home, or runtime home/u); + }); + + it("points the registration at the per-slot ledger, never the container ledger", () => { + expect(slot().usageLedgerPath).toBe(TRAINING_SLOT_USAGE_LEDGER); + expect(slot().usageLedgerPath.startsWith("/var/lib/spawnfile/daimon/usage")).toBe(false); + }); + + it("refuses an undeclared model or reasoning effort", () => { + expect(() => resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-9" as never, reasoningEffort: "low" })).toThrow(/declared broker model/u); + }); +}); + +describe("training service.json v2", () => { + it("keeps the turn store on per-slot tmpfs and declares the inference ledger", () => { + const service = renderDaimonGrokServiceConfig([slot()], { turnStore: TRAINING_SLOT_TURN_STORE, inferenceLedgerPath: TRAINING_INFERENCE_LEDGER }); + expect(service.turnStore).toBe(TRAINING_SLOT_TURN_STORE); + expect(service).toHaveProperty("inferenceLedgerPath", TRAINING_INFERENCE_LEDGER); + expect(service.registrations[0]!.usageLedgerPath).toBe(TRAINING_SLOT_USAGE_LEDGER); + }); + + it("refuses a turn store on the durable credential realm", () => { + expect(() => renderDaimonGrokServiceConfig([slot()], { turnStore: `${TRAINING_REALM_MOUNT}/turns` })) + .toThrow(/durable credential realm/u); + }); + + it("refuses an inference ledger that is a subject usage ledger", () => { + expect(() => renderDaimonGrokServiceConfig([slot()], { turnStore: TRAINING_SLOT_TURN_STORE, inferenceLedgerPath: TRAINING_SLOT_USAGE_LEDGER })) + .toThrow(/subject usage ledger/u); + }); +}); + +describe("training slot provisioning", () => { + it("makes both ledger directories setgid to the organization group and the grant home private", () => { + const directories = trainingSlotDirectories(); + for (const target of [TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_INFERENCE_DIRECTORY]) { + expect(directories.find((entry) => entry.path === target)).toMatchObject({ mode: "2750", uid: 2100, gid: 2000 }); + } + expect(directories.find((entry) => entry.path === TRAINING_GRANT_HOME_ROOT)).toMatchObject({ mode: "0700", uid: 2000, gid: 2000 }); + }); + + it("asserts the provisioned ledger and grant modes, and proves both ledgers are readable by uid 2000", () => { + const script = renderTrainingBrokerProvisioning(slot()).join("\n"); + expect(script).toContain(`test "$(stat -c '%u:%g %a' '${TRAINING_SLOT_USAGE_DIRECTORY}')" = "2100:2000 2750"`); + expect(script).toContain(`test "$(stat -c '%u:%g %a' '${TRAINING_INFERENCE_DIRECTORY}')" = "2100:2000 2750"`); + expect(script).toContain(`test "$(stat -c '%u:%g %a' '${TRAINING_GRANT_HOME_ROOT}')" = "2000:2000 700"`); + expect(script).toContain(`! test -r '${TRAINING_GRANT_HOME_ROOT}'`); + expect(script).toContain(`test -r '${TRAINING_SLOT_USAGE_LEDGER}' && test -r '${TRAINING_INFERENCE_LEDGER}'`); + }); + + it("reclaims every directory before it chmods it, because root holds no CAP_FOWNER", () => { + const script = renderTrainingBrokerProvisioning(slot()).join("\n"); + expect(script).toContain('chown 0:0 "$target"; chmod "$mode" "$target"; chown "$owner:$group" "$target"'); + }); +}); diff --git a/src/compiler/training/broker/supervisor.test.ts b/src/compiler/training/broker/supervisor.test.ts new file mode 100644 index 00000000..c536af35 --- /dev/null +++ b/src/compiler/training/broker/supervisor.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createTrainingSlotSupervisor, type TrainingSlotRuntime } from "./supervisor.js"; + +const nonce = "a".repeat(64); + +const stubRuntime = (overrides: Partial = {}) => { + const order: string[] = []; + const state = { leftovers: ["trial-1.json"], generation: 0 }; + const runtime: TrainingSlotRuntime = { + drain: async () => { order.push("drain"); }, + stop: async () => { order.push("stop"); }, + wipe: async () => { order.push("wipe"); state.leftovers = []; }, + provision: async () => { order.push("provision"); }, + start: async () => { order.push("start"); }, + canaries: async () => { order.push("canaries"); return [{ path: "/run/paideia/context.json", method: "sandboxed-read" as const, result: "denied" as const }]; }, + nextGeneration: async () => { order.push("generation"); state.generation += 1; return state.generation; }, + publishReceipt: async () => { order.push("receipt"); return "/run/training/slot/preflight.json"; }, + log: () => undefined, + ...overrides + }; + return { runtime, order, state }; +}; + +describe("training slot recycle", () => { + it("drains, stops, wipes, re-provisions, restarts, runs canaries and only then publishes the receipt", async () => { + const { runtime, order, state } = stubRuntime(); + const result = await createTrainingSlotSupervisor({ runtime }).recycle(nonce, 2000); + expect(order).toEqual(["drain", "stop", "wipe", "provision", "start", "canaries", "generation", "receipt"]); + // Mutation guard: a recycle that left the previous trial's state behind must not reach a receipt. + expect(state.leftovers).toEqual([]); + expect(result).toMatchObject({ ok: true, verb: "recycle", generation: 1, nonce }); + }); + + it("refuses a caller that is not the organization uid", async () => { + const { runtime, order } = stubRuntime(); + await expect(createTrainingSlotSupervisor({ runtime }).recycle(nonce, 2200)).rejects.toThrow(/refused for uid 2200/u); + expect(order).toEqual([]); + }); + + it("refuses a nonce that is not 32 random bytes of hex", async () => { + const { runtime, order } = stubRuntime(); + for (const bad of ["", "not-hex", "A".repeat(64), "a".repeat(63)]) { + await expect(createTrainingSlotSupervisor({ runtime }).recycle(bad, 2000)).rejects.toThrow(/hex nonce/u); + } + expect(order).toEqual([]); + }); + + it("never publishes a receipt when a canary is still reachable", async () => { + const { runtime, order } = stubRuntime({ canaries: async () => { throw Error("Grok slot canary /run/training/output is still readable by the worker uid"); } }); + await expect(createTrainingSlotSupervisor({ runtime }).recycle(nonce, 2000)).rejects.toThrow(/still readable/u); + expect(order).not.toContain("receipt"); + }); + + it("aborts the drain at its deadline instead of recycling under an active turn", async () => { + const drain = vi.fn(async (signal: AbortSignal) => { await new Promise((resolve, reject) => { signal.addEventListener("abort", () => reject(Error("drain aborted"))); setTimeout(resolve, 5_000); }); }); + const { runtime, order } = stubRuntime({ drain }); + await expect(createTrainingSlotSupervisor({ runtime, drainTimeoutMs: 20 }).recycle(nonce, 2000)).rejects.toThrow(/drain aborted/u); + expect(order).not.toContain("wipe"); + }); + + it("serializes overlapping recycles instead of interleaving a wipe with a provision", async () => { + const { runtime, order } = stubRuntime(); + const supervisor = createTrainingSlotSupervisor({ runtime }); + await Promise.all([supervisor.recycle(nonce, 2000), supervisor.recycle("b".repeat(64), 2000)]); + expect(order.join(",")).toBe(["drain", "stop", "wipe", "provision", "start", "canaries", "generation", "receipt"].concat( + ["drain", "stop", "wipe", "provision", "start", "canaries", "generation", "receipt"]).join(",")); + }); + + it("keeps the queue usable after a failed recycle", async () => { + let fail = true; + const { runtime } = stubRuntime({ provision: async () => { if (fail) { fail = false; throw Error("provisioning failed"); } } }); + const supervisor = createTrainingSlotSupervisor({ runtime }); + await expect(supervisor.recycle(nonce, 2000)).rejects.toThrow(/provisioning failed/u); + await expect(supervisor.recycle("c".repeat(64), 2000)).resolves.toMatchObject({ ok: true }); + }); +}); From 3aee56f5b7aae73f9bffd42fd988bda1db9ebb29 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 14:42:44 +0200 Subject: [PATCH 46/84] fix: keep the training slot state root out of the deny list so masks do not nest --- src/compiler/training/broker/paths.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 94da73a9..3acb5f8b 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -97,7 +97,9 @@ export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ DAIMON_WAKE_FUSE_DIRECTORY, TRAINING_BROKER_DECLARATION_FILE, TRAINING_INFERENCE_DIRECTORY, - TRAINING_SLOT_STATE_ROOT, + // `TRAINING_SLOT_STATE_ROOT` is deliberately absent: Daimon's own protected set + // already denies the wake-acceptance store beneath it, and masks cannot nest, + // so denying the parent as well would make Grok refuse the whole profile. TRAINING_SLOT_TURN_STORE, TRAINING_SUPERVISOR_DIRECTORY, ...TRAINING_CALLER_PROTECTED_PATHS, From 6f6a314837bbad0460d7714aba2c7222950b74af Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:06:37 +0200 Subject: [PATCH 47/84] fix: order training slot provisioning so root without CAP_DAC_OVERRIDE can build every path --- runtime-images/training/Dockerfile | 8 +++ src/compiler/containerDaimonBrokerRender.ts | 18 +++++- .../containerDaimonGrokWorkerProvisioning.ts | 24 ++++++-- src/compiler/training/broker/paths.ts | 46 +++++++++++---- src/compiler/training/broker/projection.ts | 20 ++++--- src/compiler/training/broker/provisioning.ts | 56 ++++++++++++++----- src/compiler/training/broker/receipt.test.ts | 17 ++++-- src/compiler/training/broker/receipt.ts | 9 ++- .../training/broker/registration.test.ts | 9 ++- src/compiler/training/broker/runtime.ts | 37 +++++++----- 10 files changed, 177 insertions(+), 67 deletions(-) diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 77db1b0f..cd113789 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -10,6 +10,14 @@ COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/ # libraries in the image so Python extensions also load in that clean process. RUN echo /opt/python/lib > /etc/ld.so.conf.d/training-python.conf && ldconfig +# The broker-capable container runs on a read-only root, so the fixed Daimon identities are baked here +# instead of being created by the entrypoint the way the production organization image creates them. +RUN for fixed_uid in 2000 2100 2200; do \ + groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid" \ + && useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" \ + --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; \ + done + WORKDIR /opt/training/paideia COPY paideia/package.json paideia/package-lock.json ./ RUN npm ci --omit=dev --omit=peer --ignore-scripts \ diff --git a/src/compiler/containerDaimonBrokerRender.ts b/src/compiler/containerDaimonBrokerRender.ts index 8357a80e..691c735f 100644 --- a/src/compiler/containerDaimonBrokerRender.ts +++ b/src/compiler/containerDaimonBrokerRender.ts @@ -122,7 +122,17 @@ export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): stri export const renderDaimonBrokerProvisioningProgram = ( registrations: readonly DaimonGrokRegistration[], workspaceResources: WorkspaceSecurityResource[], - serviceOptions: DaimonGrokServiceConfigOptions = {} + serviceOptions: DaimonGrokServiceConfigOptions = {}, + /** + * How the broker's `/etc` and `/run` roots are reset before provisioning. + * A production organization owns both as ordinary image directories and + * removes them outright. The training container mounts each as its own + * tmpfs, which cannot be unlinked, so it empties them instead — the same + * end state, reached the only way a mount point allows. + */ + rootReset: "remove" | "clear" = "remove", + /** Deny targets the worker provisioning creates when absent; see `renderDaimonGrokWorkerProvisioning`. */ + optionalDenyPaths?: readonly string[] ): string[] => { const program = [ "const crypto = require('node:crypto'); const fs = require('node:fs');", @@ -149,14 +159,16 @@ export const renderDaimonBrokerProvisioningProgram = ( `try { const journalPath = '${DAIMON_BROKER_REALM}/.daimon-broker/credential-journal.json'; let journal; try { const raw = readSecure(journalPath, 2100, 'recovery journal'); journal = JSON.parse(raw.toString('utf8')); raw.fill(0); } catch (error) { if (error.code !== 'ENOENT') throw error; } const stale = journal?.version === 'noopolis.daimon.broker-credential-journal.v1' && journal.state === 'stale'; const recover = () => { if (!stale || !Number.isSafeInteger(journal.generation) || journal.generation < 0 || !/^[a-f0-9]{64}$/.test(journal.sourceDigest) || journal.sourceDigest !== journal.promotedDigest || bootstrapDigest === journal.sourceDigest) throw new Error('unsafe broker credential recovery'); atomicOwned(authority, bootstrapBytes); const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } }; if (!existing) { if (stale) recover(); else atomicOwned(authority, bootstrapBytes); } else { const authorityBytes = readSecure(authority, 2100, 'authority'); try { const authorityDigest = crypto.createHash('sha256').update(authorityBytes).digest('hex'); if (stale) { if (authorityDigest !== journal.sourceDigest && authorityDigest !== bootstrapDigest) throw new Error('unsafe broker credential recovery'); if (authorityDigest === bootstrapDigest) { const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } } else recover(); } } finally { authorityBytes.fill(0); } } } finally { bootstrapBytes.fill(0); }`, "if (journalRootExists) { fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); fs.chownSync(journalRoot, 2100, 2100); }", ...renderDaimonWorkspaceResourceSecurity(workspaceResources), - ...renderDaimonGrokWorkerProvisioning(registrations, serviceOptions), + ...renderDaimonGrokWorkerProvisioning(registrations, serviceOptions, ...(optionalDenyPaths ? [optionalDenyPaths] : [])), `fs.chownSync('${DAIMON_BROKER_REALM}', 0, 0); fs.chmodSync('${DAIMON_BROKER_REALM}', 0o700); fs.chownSync('${DAIMON_BROKER_REALM}', 2100, 2100);`, "fs.chmodSync('/etc/daimon-engine-broker', 0o555);" ].join("\n"); return [ "if [ -d /etc/daimon-engine-broker ]; then chmod u+rwx /etc/daimon-engine-broker; fi", "if [ -d /run/daimon-engine-broker ]; then chmod u+rwx /run/daimon-engine-broker; fi", - "rm -rf /etc/daimon-engine-broker /run/daimon-engine-broker", + ...(rootReset === "remove" + ? ["rm -rf /etc/daimon-engine-broker /run/daimon-engine-broker"] + : ["for broker_root in /etc/daimon-engine-broker /run/daimon-engine-broker; do if [ -d \"$broker_root\" ]; then find \"$broker_root\" -mindepth 1 -delete; fi; done"]), `install -d -o root -g ${DAIMON_BROKER_UID} -m 0731 /run/daimon-engine-broker`, `install -d -o ${DAIMON_BROKER_UID} -g ${DAIMON_BROKER_UID} -m 0700 ${DAIMON_BROKER_TMPDIR}`, "node <<'SPAWNFILE_DAIMON_BROKER_PROVISION'", diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index e6abdbe4..c6784d0b 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -54,10 +54,18 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ */ export const renderDaimonGrokWorkerProvisioning = ( registrations: readonly DaimonGrokRegistration[], - serviceOptions: DaimonGrokServiceConfigOptions = {} + serviceOptions: DaimonGrokServiceConfigOptions = {}, + /** + * Deny targets this program creates root-owned `0700` when they are absent, + * so every mask always has an inode. The default is the production + * container's set; the training container passes its own, because its roots + * are tmpfs mounts and `/run/secrets` and the shared state roots do not + * exist there at all. + */ + optionalDenyPaths: readonly string[] = [...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_STATE_ROOTS] ): string[] => [ `const grokWorkers = ${JSON.stringify(registrations.map(programRegistration))};`, - `const optionalDenyPaths = new Set(${JSON.stringify([...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_STATE_ROOTS])});`, + `const optionalDenyPaths = new Set(${JSON.stringify([...optionalDenyPaths])});`, `const pinnedConfigSha256 = ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.configSha256)};`, "const sha256Hex = (value) => crypto.createHash('sha256').update(value).digest('hex');", "for (const entry of grokWorkers) { if (sha256Hex(entry.config) !== entry.configSha256 || !Object.hasOwn(pinnedConfigSha256, entry.model) || !Object.hasOwn(pinnedConfigSha256[entry.model], entry.reasoningEffort) || pinnedConfigSha256[entry.model][entry.reasoningEffort] !== entry.configSha256 || sha256Hex(entry.profile) !== entry.profileSha256 || entry.denyPaths.length === 0) throw new Error(`Grok worker contract bytes for ${entry.agentId} do not match their pins`); }", @@ -74,11 +82,15 @@ export const renderDaimonGrokWorkerProvisioning = ( "for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }", "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT' && entry.deferredDenyPaths.includes(denied)) continue; if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", // Pass 2: exact root-owned read-only files, the events file, then the final sticky modes. - `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, - // Worker-private temp: the launcher compiles TMPDIR=/tmp, the only temp the worker may write. + `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, + // Worker-private temp: the launcher compiles TMPDIR=/tmp, the only temp the worker may write. It is + // created in the pass above, while the home is still root-owned: root here holds no CAP_DAC_OVERRIDE, so + // once the home is `: 0710` root can no longer create anything inside it. "for (const entry of grokWorkers) { ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); assertCanonical(entry.privateTmp, 'private temp'); }", - // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can traverse. - `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, + // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can + // traverse. The spill directory is created before the runtime home is narrowed to 0710: without + // CAP_DAC_OVERRIDE root cannot create inside a directory it does not own once the mode excludes it. + `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations, serviceOptions))};`, diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 3acb5f8b..97c8209f 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -63,7 +63,14 @@ export const TRAINING_SLOT_INDEX = 0; export const TRAINING_WORKER_UID = DAIMON_FIRST_WORKER_UID + TRAINING_SLOT_INDEX; export const TRAINING_WORKER_HOME = path.posix.join(TRAINING_WORKER_ROOT, String(TRAINING_WORKER_UID)); -/** Paideia's own fixed container paths (`containerPaths` in its native launch schema); all protected from the worker. */ +/** + * Paideia's own fixed container paths (`containerPaths` in its native launch + * schema). They are *not* individual deny entries: Grok 1.0.34 materializes + * every `deny` target inside bubblewrap as the worker uid, and it cannot + * create a file inside `/run/paideia`, which belongs to uid 2000 alone. The + * single `/run/paideia` mask below covers all of them, which is also stronger + * — a file Paideia adds later is covered without re-provisioning the slot. + */ export const TRAINING_CALLER_PROTECTED_PATHS = [ `${TRAINING_PAIDEIA_ROOT}/config.json`, `${TRAINING_PAIDEIA_ROOT}/control`, @@ -77,7 +84,7 @@ export const TRAINING_CALLER_PROTECTED_PATHS = [ /** `paideia.daimon-native.launch.v2`'s five `broker.evaluatorPaths` roles, at this container's real paths. */ export const TRAINING_EVALUATOR_ROOTS = [ { role: "run-root", path: TRAINING_RUN_ROOT }, - { role: "context", path: TRAINING_CONTEXT_FILE }, + { role: "context", path: TRAINING_PAIDEIA_ROOT }, { role: "sealed-inputs", path: TRAINING_SEALED_INPUTS_ROOT }, { role: "judge-home", path: TRAINING_GRANT_HOME_ROOT }, { role: "slot-ledger", path: TRAINING_SLOT_USAGE_DIRECTORY } @@ -95,14 +102,12 @@ export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, DAIMON_WAKE_FUSE_DIRECTORY, - TRAINING_BROKER_DECLARATION_FILE, TRAINING_INFERENCE_DIRECTORY, // `TRAINING_SLOT_STATE_ROOT` is deliberately absent: Daimon's own protected set // already denies the wake-acceptance store beneath it, and masks cannot nest, // so denying the parent as well would make Grok refuse the whole profile. TRAINING_SLOT_TURN_STORE, TRAINING_SUPERVISOR_DIRECTORY, - ...TRAINING_CALLER_PROTECTED_PATHS, ...TRAINING_EVALUATOR_ROOTS.map((entry) => entry.path) ]; @@ -112,18 +117,37 @@ export const TRAINING_OPTIONAL_DENY_DIRECTORIES: readonly string[] = [ TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, TRAINING_SUPERVISOR_DIRECTORY, + TRAINING_GRANT_HOME_ROOT, + // Denied and unused here — training meters per slot — but the mask still needs an inode. The launch + // mounts each as tmpfs, so this only creates them when a caller ran the entrypoint without them. + DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, + DAIMON_WAKE_FUSE_DIRECTORY +]; + +/** + * Deny entries a launch may legitimately not have mounted. Everything else must + * exist before a worker turn: Grok 1.0.34 creates an absent `deny` target as + * the worker uid inside bubblewrap and refuses the whole profile when it + * cannot, so provisioning materializes every other entry itself. + */ +export const TRAINING_DEFERRED_DENY_PATHS: readonly string[] = [ + // Docker materializes both host binds before the entrypoint runs. They sit on the read-only image + // root, which root cannot create into, so provisioning must tolerate a launch that declared neither. TRAINING_RUN_ROOT, - TRAINING_SEALED_INPUTS_ROOT, - TRAINING_GRANT_HOME_ROOT + TRAINING_SEALED_INPUTS_ROOT ]; /** - * Deny entries Paideia materializes per trial inside its own `/run/paideia` - * tmpfs. Absent is allowed at provisioning time (the profile still masks the - * path once it exists, because Grok applies the profile at every worker spawn); - * a symlink never is. + * Deny entries that are host bind mounts. + * + * A worker-uid read probe over one of these proves nothing: the host owns the + * inode, its mode is whatever the operator's project directory happens to be, + * and Docker Desktop and Colima ignore `chown` outright (P0 §5). Their boundary + * is the bubblewrap-enforced `deny` list, which P0 verified blocks both shell + * `cat` and `read_file` on 1.0.34. The declaration's `unenforcedBindPolicy` + * decides whether the slot supervisor accepts that or refuses the slot. */ -export const TRAINING_DEFERRED_DENY_PATHS: readonly string[] = [...TRAINING_CALLER_PROTECTED_PATHS]; +export const TRAINING_HOST_BIND_DENY_PATHS: readonly string[] = [TRAINING_RUN_ROOT, TRAINING_SEALED_INPUTS_ROOT]; export const TRAINING_REALM_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath; export const TRAINING_BOOTSTRAP_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath; diff --git a/src/compiler/training/broker/projection.ts b/src/compiler/training/broker/projection.ts index c4cd5391..b1616e78 100644 --- a/src/compiler/training/broker/projection.ts +++ b/src/compiler/training/broker/projection.ts @@ -41,21 +41,25 @@ export const loadDaimonProjectionModule = async (load: () => Promise = /** * The one-agent organization runtime config the training slot's projection is - * resolved from. It is a projection *input*, never a runtime config Daimon - * hosts: the trial's real config is Paideia's, and both describe the same - * single agent at the same fixed container paths, so both compute the same - * projection digest. A peer agent would change Daimon's protected set and the - * digest with it, which is exactly why training runs one slot. + * resolved from. + * + * It is a projection *input*, never a runtime config Daimon hosts: the trial's + * real config is Paideia's. Only `agents[].id`, `workspacePath`, + * `runtimeHomePath`, the engine's declared model and effort, and the *absence* + * of peers reach `resolveOrganizationGrokBrokerProjection`, so this minimal + * config and Paideia's fuller one compute the same projection digest. A peer + * agent would change Daimon's protected set and the digest with it, which is + * exactly why training runs one slot. */ export const trainingOrganizationRuntimeConfig = (declaration: { agentId: string; model: DaimonGrokBrokerModel; reasoningEffort: DaimonGrokBrokerReasoningEffort; }): Record => ({ - version: "noopolis.daimon.organization-runtime.v2", - id: "daimon-organization", + version: "noopolis.daimon.organization-runtime.v1", + host: { bindHost: "127.0.0.1", port: 19_700, controlTokenEnv: "SPAWNFILE_DAIMON_CONTROL_TOKEN" }, agents: [{ id: declaration.agentId, name: declaration.agentId, - instructions: "", + instructions: "training subject", workspacePath: TRAINING_SLOT_WORKSPACE, runtimeHomePath: TRAINING_SLOT_RUNTIME_HOME, engine: { kind: "grok", model: declaration.model, reasoningEffort: declaration.reasoningEffort } diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts index 0e731d58..fd5d2498 100644 --- a/src/compiler/training/broker/provisioning.ts +++ b/src/compiler/training/broker/provisioning.ts @@ -1,6 +1,7 @@ import path from "node:path"; -import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; +import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_TURN_USAGE_LEDGER } from "../../../runtime/daimon/contractManifest.js"; +import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js"; import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; import { renderDaimonBrokerProvisioningProgram } from "../../containerDaimonBrokerRender.js"; import { renderDaimonGrokHostPreflight } from "../../containerDaimonGrokWorkerProvisioning.js"; @@ -14,11 +15,11 @@ import { TRAINING_PAIDEIA_ROOT, TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_ROOT, - TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_SLOT_USAGE_LEDGER, + TRAINING_SUPERVISOR_DIRECTORY, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_ROOT, TRAINING_WORKER_UID @@ -35,7 +36,6 @@ const quote = (value: string): string => `'${value.replace(/'/g, `'"'"'`)}'`; const PROVISION_DIRECTORY_HELPER = [ "provision_dir() {", " target=$1; mode=$2; owner=$3; group=$4", - " mkdir -p \"$target\"", " test -d \"$target\" && test ! -L \"$target\"", " chown 0:0 \"$target\"; chmod \"$mode\" \"$target\"; chown \"$owner:$group\" \"$target\"", "}" @@ -50,27 +50,44 @@ const PROVISION_DIRECTORY_HELPER = [ * single subject usage row or inference row it paid for. */ export const trainingSlotDirectories = (): readonly { path: string; mode: string; uid: number; gid: number }[] => [ - { path: "/run/training", mode: "0755", uid: 0, gid: 0 }, + // `/run/training` itself is deliberately absent: it is a mount-point parent on the read-only + // image root, and every writable child below it is its own tmpfs or bind. { path: TRAINING_SLOT_ROOT, mode: "0755", uid: 0, gid: 0 }, { path: TRAINING_PAIDEIA_ROOT, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: "/home/training", mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: "/work", mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_SLOT_WORKSPACE, mode: "0750", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, - { path: TRAINING_SLOT_RUNTIME_HOME, mode: "0710", uid: DAIMON_ORGANIZATION_UID, gid: TRAINING_WORKER_UID }, + // The agent runtime home is deliberately absent: the shared broker provisioning creates it root-owned, + // fills its setgid `tool-output/`, and only then narrows it to `2000: 0710`. Handing it over here + // would leave root — which holds no `CAP_DAC_OVERRIDE` — unable to create the spill directory inside it. { path: TRAINING_SLOT_STATE_ROOT, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_SLOT_ACCEPTANCE_STORE, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_SLOT_TURN_STORE, mode: "0700", uid: DAIMON_BROKER_UID, gid: DAIMON_BROKER_UID }, { path: TRAINING_SLOT_USAGE_DIRECTORY, mode: "2750", uid: DAIMON_BROKER_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_INFERENCE_DIRECTORY, mode: "2750", uid: DAIMON_BROKER_UID, gid: DAIMON_ORGANIZATION_UID }, { path: TRAINING_GRANT_HOME_ROOT, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID }, - { path: TRAINING_WORKER_ROOT, mode: "0711", uid: 0, gid: 0 } + // Traverse only: the supervisor socket inside it is the uid gate, and a directory nobody but root may + // write is what keeps that socket from being replaced by a laxer one. + { path: TRAINING_SUPERVISOR_DIRECTORY, mode: "0711", uid: 0, gid: 0 }, + { path: TRAINING_WORKER_ROOT, mode: "0711", uid: 0, gid: 0 }, + // Denied and unused by training, which meters per slot — but a world-readable directory would make its + // worker-uid canary meaningless, so both get the modes the production organization gives them. + { path: DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, mode: "0750", uid: DAIMON_BROKER_UID, gid: DAIMON_ORGANIZATION_UID }, + { path: DAIMON_WAKE_FUSE_DIRECTORY, mode: "0700", uid: DAIMON_ORGANIZATION_UID, gid: DAIMON_ORGANIZATION_UID } ]; -/** Fixed uid/gid identities the slot needs before anything is provisioned. */ +/** + * The fixed identities the slot needs, checked rather than created. + * + * The broker-capable training container runs on a read-only root, so + * `/etc/passwd` and `/etc/group` cannot be written at start-up the way the + * production Daimon entrypoint writes them. The image bakes uid/gid 2000, 2100 + * and the worker uid instead, and this refuses to provision a slot in an image + * that did not. + */ export const renderTrainingIdentities = (): string[] => [ `for fixed_uid in ${DAIMON_ORGANIZATION_UID} ${DAIMON_BROKER_UID} ${TRAINING_WORKER_UID}; do`, - ' if ! getent group "$fixed_uid" >/dev/null; then groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid"; fi', - ' if ! getent passwd "$fixed_uid" >/dev/null; then useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; fi', + ' getent passwd "$fixed_uid" >/dev/null && getent group "$fixed_uid" >/dev/null || { echo "the training image must bake uid/gid $fixed_uid; the container root is read-only" >&2; exit 1; }', "done" ]; @@ -88,17 +105,26 @@ export const renderTrainingIdentities = (): string[] => [ */ export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistration): string[] => [ ...PROVISION_DIRECTORY_HELPER, - ...trainingSlotDirectories().map((entry) => `provision_dir ${quote(entry.path)} ${entry.mode} ${entry.uid} ${entry.gid}`), + // Two passes, because root here holds neither `CAP_DAC_OVERRIDE` nor `CAP_FOWNER`: create every + // directory while they are all still root-owned and traversable, then set ownership and mode from the + // deepest path up, so tightening a parent to `0700` never strands a child that still has to be created. + ...trainingSlotDirectories().map((entry) => `mkdir -p ${quote(entry.path)}`), // Every deny entry needs an inode to mask; a bind the host did not supply is created root-owned and unreadable. - ...TRAINING_OPTIONAL_DENY_DIRECTORIES.map((target) => `if [ ! -e ${quote(target)} ]; then provision_dir ${quote(target)} 0700 0 0; fi`), + ...TRAINING_OPTIONAL_DENY_DIRECTORIES.map((target) => `if [ ! -e ${quote(target)} ]; then mkdir -p ${quote(target)}; provision_dir ${quote(target)} 0700 0 0; fi`), + // Both ledgers exist before their directories are handed to the broker: the broker appends rows `0640` in + // its own group, and the setgid directory below is what lets uid 2000 read a row it paid for. + ...[TRAINING_SLOT_USAGE_LEDGER, TRAINING_INFERENCE_LEDGER].map((ledger) => + `if [ ! -e ${quote(ledger)} ]; then : > ${quote(ledger)}; fi; chown 0:0 ${quote(ledger)}; chmod 0640 ${quote(ledger)}; chown ${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} ${quote(ledger)}`), + ...[...trainingSlotDirectories()].sort((left, right) => right.path.split("/").length - left.path.split("/").length) + .map((entry) => `provision_dir ${quote(entry.path)} ${entry.mode} ${entry.uid} ${entry.gid}`), ...renderDaimonGrokHostPreflight(), ...renderDaimonBrokerProvisioningProgram([registration], [], { turnStore: TRAINING_SLOT_TURN_STORE, inferenceLedgerPath: TRAINING_INFERENCE_LEDGER - }), - // The broker creates both ledgers 0640 in its own group; the setgid directories above give the group to uid 2000. - ...[TRAINING_SLOT_USAGE_LEDGER, TRAINING_INFERENCE_LEDGER].map((ledger) => - `if [ ! -e ${quote(ledger)} ]; then : > ${quote(ledger)}; chown 0:0 ${quote(ledger)}; chmod 0640 ${quote(ledger)}; chown ${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} ${quote(ledger)}; fi`), + }, "clear", TRAINING_OPTIONAL_DENY_DIRECTORIES), + // The shared program leaves the broker's `/etc` root `0555 root:root`, which every uid can list. Training + // denies that directory to its worker, and a canary can only observe a denial the kernel actually enforces. + `chmod 0550 /etc/daimon-engine-broker; chown 0:${DAIMON_BROKER_UID} /etc/daimon-engine-broker`, `test "$(stat -c '%u:%g %a' ${quote(TRAINING_SLOT_USAGE_DIRECTORY)})" = "${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} 2750"`, `test "$(stat -c '%u:%g %a' ${quote(TRAINING_INFERENCE_DIRECTORY)})" = "${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} 2750"`, `test "$(stat -c '%u:%g %a' ${quote(TRAINING_GRANT_HOME_ROOT)})" = "${DAIMON_ORGANIZATION_UID}:${DAIMON_ORGANIZATION_UID} 700"`, diff --git a/src/compiler/training/broker/receipt.test.ts b/src/compiler/training/broker/receipt.test.ts index 80ea0f6e..ff833140 100644 --- a/src/compiler/training/broker/receipt.test.ts +++ b/src/compiler/training/broker/receipt.test.ts @@ -43,7 +43,7 @@ describe("training slot canaries", () => { it("requires a worker-uid denial for every deny path on a filesystem that enforces ownership", async () => { const probed: string[] = []; const canaries = await resolveTrainingCanaries({ - denyPaths: ["/run/training/slot/turns", "/run/training/slot/usage"], mountinfo, unenforcedBindPolicy: "refuse", + denyPaths: ["/run/training/slot/turns", "/run/training/slot/usage"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", probe: async (target) => { probed.push(target); return true; }, log: () => undefined }); expect(probed).toEqual(["/run/training/slot/turns", "/run/training/slot/usage"]); @@ -52,14 +52,21 @@ describe("training slot canaries", () => { it("refuses to certify a path the worker uid can still read", async () => { await expect(resolveTrainingCanaries({ - denyPaths: ["/run/training/slot/turns"], mountinfo, unenforcedBindPolicy: "refuse", + denyPaths: ["/run/training/slot/turns"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", probe: async () => false, log: () => undefined })).rejects.toThrow(/still readable by the worker uid/u); }); - it("refuses a host bind by default, because chown there is silently ignored", async () => { + it("refuses a declared host bind by default, because its mode is the operator's and not the container's", async () => { await expect(resolveTrainingCanaries({ - denyPaths: ["/run/training/output"], mountinfo, unenforcedBindPolicy: "refuse", + denyPaths: ["/run/training/inputs"], mountinfo, hostBindPaths: ["/run/training/inputs"], unenforcedBindPolicy: "refuse", + probe: async () => { throw Error("the probe must never run on a host bind"); }, log: () => undefined + })).rejects.toThrow(/is a host bind mount/u); + }); + + it("refuses a filesystem that ignores unix ownership by default, even when the launch did not declare it", async () => { + await expect(resolveTrainingCanaries({ + denyPaths: ["/run/training/output"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", probe: async () => { throw Error("the probe must never run on an unenforced filesystem"); }, log: () => undefined })).rejects.toThrow(/virtiofs, which ignores unix ownership/u); }); @@ -67,7 +74,7 @@ describe("training slot canaries", () => { it("accepts a host bind on the profile's deny entry alone only when the operator declared that, and says so", async () => { const lines: string[] = []; const canaries = await resolveTrainingCanaries({ - denyPaths: ["/run/training/output"], mountinfo, unenforcedBindPolicy: "profile-only", + denyPaths: ["/run/training/output"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "profile-only", probe: async () => false, log: (line) => lines.push(line) }); expect(canaries).toEqual([canary("/run/training/output")]); diff --git a/src/compiler/training/broker/receipt.ts b/src/compiler/training/broker/receipt.ts index 505be6c1..aaeed365 100644 --- a/src/compiler/training/broker/receipt.ts +++ b/src/compiler/training/broker/receipt.ts @@ -39,6 +39,8 @@ export interface TrainingCanaryOptions { denyPaths: readonly string[]; probe: TrainingCanaryProbe; mountinfo: string; + /** Deny entries the launch bound from the host; a worker-uid probe over one of them proves nothing. */ + hostBindPaths: readonly string[]; unenforcedBindPolicy: "refuse" | "profile-only"; log(line: string): void; } @@ -57,12 +59,13 @@ export const resolveTrainingCanaries = async (options: TrainingCanaryOptions): P const canaries: { path: string; method: "sandboxed-read"; result: "denied" }[] = []; for (const target of options.denyPaths) { const fstype = resolveBackingFilesystem(target, options.mountinfo); - if (isUnenforced(fstype)) { + const hostBind = options.hostBindPaths.includes(target); + if (hostBind || isUnenforced(fstype)) { if (options.unenforcedBindPolicy === "refuse") { throw new SpawnfileError("runtime_error", - `Grok slot canary ${target} is backed by ${fstype || "an unknown filesystem"}, which ignores unix ownership; declare unenforcedBindPolicy "profile-only" to accept the bubblewrap deny list as its only boundary`); + `Grok slot canary ${target} is ${hostBind ? "a host bind mount" : `backed by ${fstype || "an unknown filesystem"}, which ignores unix ownership`}; declare unenforcedBindPolicy "profile-only" to accept the bubblewrap deny list as its only boundary`); } - options.log(`canary ${target} certified by the enforced sandbox profile only (${fstype} ignores unix ownership)`); + options.log(`canary ${target} certified by the enforced sandbox profile only (${hostBind ? "host bind mount" : `${fstype} ignores unix ownership`})`); } else if (!await options.probe(target)) { throw new SpawnfileError("runtime_error", `Grok slot canary ${target} is still readable by the worker uid`); } diff --git a/src/compiler/training/broker/registration.test.ts b/src/compiler/training/broker/registration.test.ts index 25c88be4..56e9dcf2 100644 --- a/src/compiler/training/broker/registration.test.ts +++ b/src/compiler/training/broker/registration.test.ts @@ -19,9 +19,14 @@ import { const slot = () => resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-4.6", reasoningEffort: "low" }); describe("training Grok slot registration", () => { - it("denies every evaluator root, caller protected path, control root and the judge grant home", () => { + it("denies every evaluator root, the caller's whole /run/paideia, the control root and the judge grant home", () => { const denied = slot().denyPaths; - for (const entry of [...TRAINING_EVALUATOR_ROOTS.map((role) => role.path), ...TRAINING_CALLER_PROTECTED_PATHS, + // Every caller-protected path is covered by the single `/run/paideia` mask rather than listed itself: + // Grok materializes each deny target inside bubblewrap and cannot create one there as the worker uid. + for (const covered of TRAINING_CALLER_PROTECTED_PATHS) { + expect(denied.some((entry) => covered === entry || covered.startsWith(`${entry}/`)), covered).toBe(true); + } + for (const entry of [...TRAINING_EVALUATOR_ROOTS.map((role) => role.path), "/run/daimon-engine-broker", "/etc/daimon-engine-broker", TRAINING_GRANT_HOME_ROOT, TRAINING_INFERENCE_DIRECTORY, TRAINING_SLOT_TURN_STORE, TRAINING_REALM_MOUNT]) { expect(denied, entry).toContain(entry); diff --git a/src/compiler/training/broker/runtime.ts b/src/compiler/training/broker/runtime.ts index 7b153529..20026d0b 100644 --- a/src/compiler/training/broker/runtime.ts +++ b/src/compiler/training/broker/runtime.ts @@ -1,7 +1,7 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import { appendFileSync } from "node:fs"; -import { chmod, chown, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import { chmod, chown, mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as pause } from "node:timers/promises"; import { promisify } from "node:util"; @@ -16,9 +16,11 @@ import { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveTrainingCana import type { TrainingSlotRuntime } from "./supervisor.js"; import { DAIMON_ORGANIZATION_UID, + TRAINING_HOST_BIND_DENY_PATHS, TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_GENERATION_FILE, TRAINING_SLOT_PREFLIGHT_RECEIPT, + TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_SLOT_WORKSPACE, @@ -26,6 +28,7 @@ import { } from "./paths.js"; const run = promisify(execFile); +const quote = (value: string): string => `'${value.replace(/'/gu, `'"'"'`)}'`; export interface TrainingSlotRuntimeOptions { declaration: TrainingBrokerDeclaration; @@ -110,20 +113,26 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): }, stop: async () => { await stopBrokerProcesses(children, log); children = []; }, wipe: async () => { - for (const target of [options.registration.home, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_ACCEPTANCE_STORE, - options.registration.spillDirectory, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.registrationPath]) { - await rm(target, { recursive: true, force: true }); - } - // The workspace keeps its root; only the trial's content goes, so the registered path stays canonical. - for (const name of await readdir(TRAINING_SLOT_WORKSPACE).catch(() => [] as string[])) { - await rm(path.join(TRAINING_SLOT_WORKSPACE, name), { recursive: true, force: true }); - } - for (const name of await readdir(TRAINING_SLOT_USAGE_DIRECTORY).catch(() => [] as string[])) { - await rm(path.join(TRAINING_SLOT_USAGE_DIRECTORY, name), { recursive: true, force: true }); - } + // Root holds `CAP_CHOWN` and `CAP_DAC_READ_SEARCH` but not `CAP_DAC_OVERRIDE`, so it cannot delete + // inside a directory it handed to the worker or the broker. Reclaim each tree first, then remove it. + const targets = [options.registration.home, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_ACCEPTANCE_STORE, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.registrationPath]; + const contents = [TRAINING_SLOT_WORKSPACE, TRAINING_SLOT_USAGE_DIRECTORY]; + const script = [ + // The parent has to be reclaimed too: unlinking is a write to the *directory*, and the slot hands + // several of these parents to uid 2000 or the broker. Provisioning restores every mode right after. + 'reclaim() { parent=$(dirname "$1"); chown 0:0 "$parent"; chmod u+rwx "$parent"; if [ -e "$1" ]; then chown -R 0:0 "$1"; chmod -R u+rwX "$1"; fi; }', + ...targets.map((target) => `reclaim ${quote(target)}; rm -rf ${quote(target)}`), + // The workspace and the per-slot ledger directory keep their own root: the registered paths must + // stay canonical across a recycle, so only what a trial put inside them goes. + ...contents.map((target) => `reclaim ${quote(target)}; if [ -d ${quote(target)} ]; then find ${quote(target)} -mindepth 1 -delete; fi`) + ].join("\n"); + await run(shell, ["--noprofile", "--norc", "-ceu", script], { maxBuffer: 1024 * 1024 }).catch((error: unknown) => { + const detail = error as { stderr?: string }; + throw new SpawnfileError("runtime_error", `Training slot wipe failed: ${(detail.stderr ?? "").slice(-1024).trim()}`); + }); }, provision: async () => { - const result = await run(shell, ["-ceu", script], { maxBuffer: 8 * 1024 * 1024 }).catch((error: unknown) => { + const result = await run(shell, ["--noprofile", "--norc", "-ceu", script], { maxBuffer: 8 * 1024 * 1024 }).catch((error: unknown) => { const detail = error as { stderr?: string; stdout?: string }; throw new SpawnfileError("runtime_error", `Training slot provisioning failed: ${(detail.stderr ?? detail.stdout ?? "").slice(-2048).trim()}`); }); @@ -131,7 +140,7 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): }, start: async () => { children = await startBrokerProcesses({ log, procRoot: options.procRoot }); }, canaries: async () => resolveTrainingCanaries({ - denyPaths: options.registration.denyPaths, probe, log, + denyPaths: options.registration.denyPaths, probe, log, hostBindPaths: TRAINING_HOST_BIND_DENY_PATHS, mountinfo: await readFile(`${options.procRoot ?? "/proc"}/self/mountinfo`, "utf8"), unenforcedBindPolicy: options.declaration.unenforcedBindPolicy }), From 38221355d52709b7b6194c734beea2fc2617935f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:07:36 +0200 Subject: [PATCH 48/84] docs: specify the broker-capable training container, slot lifecycle and credential lineage --- specs/TRAINING.md | 9 +++ specs/TRAINING_CONTAINERS.md | 144 +++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/specs/TRAINING.md b/specs/TRAINING.md index e425c804..596f33ed 100644 --- a/specs/TRAINING.md +++ b/specs/TRAINING.md @@ -9,6 +9,15 @@ inside one immutable image. The v2 training configuration prepares its pinned image and inputs behind the same command; the v1 image/mount path remains available. Dry-run remains a host-only estimate. +`spawnfile.training-container.v3` adds a brokered Grok slot to that boundary: +the subject's model runs in the same container under a root-provisioned broker, +the evaluator recycles that slot between trials through a single-verb root +supervisor, and judges spend the one dedicated training Grok login through +bounded inference grants instead of holding it. Import that login with +`spawnfile auth import grok --profile paideia-training --from `; the +desktop `~/.grok` is refused, because a training run rotates the credential it +is given. + ```sh spawnfile train ./Spawnfile --agent agent:writer \ --train evals/train.paideia.yaml --test evals/test.paideia.yaml \ diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 030a013a..b3084e04 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -216,3 +216,147 @@ and checkpoint bytes are hashed. `/run/paideia/repair.json` is a protected read- Paideia owns error-only rescoring, retaining successful historical pass/fail results, paired optimizer import, cumulative accounting and the new experiment lineage. A repair receipt does not itself assert that any judgment or continuation succeeded. + +## Broker-capable training (v3) + +`spawnfile.training-container.v3` is v2 plus one brokered Grok slot, so the +subject's model runs beside the evaluator in the same container instead of +reaching a model the evaluator also holds. It is selected by adding a `broker` +block to the declarative preparation; everything else in v2 is unchanged. + +```json +{ + "version": "spawnfile.training-container.v3", + "broker": { + "engine": "grok", + "agentId": "agent:author", + "model": "grok-4.6", + "reasoningEffort": "low", + "architecture": "arm64", + "limits": { "maxRequests": 32, "maxTokens": 300000, "timeoutMs": 240000 }, + "realmVolume": "spawnfile-training-grok-realm", + "bootstrap": "./secrets/paideia-training-grok/auth.json", + "unenforcedBindPolicy": "refuse" + } +} +``` + +`image.build.grok` is refused under v3 and ignored under v2: the image copies no +Grok binary at all, and the `grok` auth provider is gone from `auth` and from +`stageTrainingAuth`. Judges run the native parent's pinned `/usr/local/bin/grok` +through a broker inference grant, so there is exactly one Grok build and exactly +one Grok credential in the container. + +### Privilege table + +| Process | uid | Capability bounding set | Where it comes from | +| --- | --- | --- | --- | +| container | 0 | `CHOWN,SETUID,SETGID,SETPCAP,KILL,DAC_READ_SEARCH` | `docker create` (`no-new-privileges`, read-only root, pids 2048) | +| root entrypoint | 0 | same | image `/opt/training/bin/train-broker` | +| engine broker launcher | 0 | `CHOWN,SETUID,SETGID` (`…c1`) | `setpriv --bounding-set` | +| engine broker backend | 2100 | empty | `setpriv --reuid 2100` | +| control relay | 2100 | empty | `setpriv --reuid 2100` | +| slot supervisor | 0 | same as the entrypoint | in-process with the entrypoint | +| `train` — Paideia, DSPy, judges | 2000 | empty, verified from `/proc/self/status` | `setpriv --bounding-set=-all` | +| model tools | 2200 | empty | the native launcher alone can `setuid` there | + +`CAP_FOWNER` and `CAP_DAC_OVERRIDE` are deliberately absent, so provisioning +always reclaims an inode before it chmods one, creates every directory while the +tree is still root-owned, and sets ownership from the deepest path upwards. + +Grok 1.0.34 runs every sandbox profile inside bubblewrap, so the container adds +the pinned `seccomp-default-plus-userns` profile and `apparmor=unconfined`, and +the Docker host must allow unprivileged user namespaces +(`kernel.apparmor_restrict_unprivileged_userns=0`); the entrypoint refuses to +provision a slot otherwise, naming that sysctl. + +### Slot lifecycle + +Every per-trial path is tmpfs, never the realm volume: the worker home, the slot +workspace, the agent runtime home and its setgid `tool-output/`, the +wake-acceptance store, the broker turn store, and the per-slot usage ledger. The +realm volume holds only `auth.json` and the broker credential journal, so an +identical `(agent, wake, prompt)` in trial N+1 can never replay trial N's sealed +turn. + +The root slot supervisor listens on `/run/training/supervisor/control.sock` with +one verb and one argument: + +```json +{"v": "spawnfile.training-slot-supervisor.v1", "verb": "recycle", "nonce": "<32 random bytes, hex>"} +``` + +It answers `{"ok": true, "generation": N, "receipt": "/run/training/slot/preflight.json", "durationMs": …}`. +`recycle` drains (no active turn in the registry and a settled credential +journal), stops the relay, backend and launcher in that order, wipes the slot, +replays the same audited provisioning the entrypoint ran — credential-journal +recovery included, so a crash during a refresh either recovers or fails closed +with a named error — restarts and re-verifies all three identities, runs the +worker-uid denial canaries, and only then publishes +`noopolis.daimon.grok-slot-preflight.v2` with a monotonic `generation` and the +caller's `nonce`. Recycles are serialized; the caller never names a path or a +command. Measured: three recycles at 611–620 ms each. + +The socket node is `root:2000 0660` inside a root-owned `0711` directory on +tmpfs, which is the uid gate: the kernel enforces it on `connect()` and uid 2200 +gets `EACCES`. Node exposes no `SO_PEERCRED`, and a `0600` root-owned socket +would deny the one caller it exists for. + +### Credential lineage + +``` +spawnfile auth import grok --profile paideia-training --from + │ refuses ~/.grok and $GROK_HOME outright; --from is required + ▼ + profile store ──► declaration `broker.bootstrap` ──► read-only bind + /var/lib/spawnfile/daimon/grok-bootstrap-auth + │ + root entrypoint promotes it into the named realm volume + ▼ + /var/lib/spawnfile/daimon/grok-subscription-realm/auth.json + (broker uid 2100, rotated in place, journalled) + │ │ + subject turns judge/optimizer grants +``` + +The launch refuses a bootstrap that resolves to the desktop `~/.grok/auth.json`, +both at preparation and again before `docker create`. Judges never hold the +credential: the container exports `PAIDEIA_GROK_BROKER_CONTROL_SOCKET` and a +private `PAIDEIA_GROK_GRANT_HOME_ROOT` (`2000:2000 0700`, denied to every worker +uid) to the `train` child, and each judge lane asks the broker for a bounded +inference grant. + +Both ledger directories are setgid to the organization group +(`2100:2000 2750`): the broker writes rows `0640` in its own group, so without +setgid uid 2000 could not read a single usage or inference row it paid for. + +### Evaluator roots and deny list + +`paideia.daimon-native.launch.v2`'s five evaluator roles carry these real +container paths, and every one of them is a deny entry in the worker's sandbox +profile and a canary in the receipt: + +| role | path | +| --- | --- | +| run-root | `/run/training/output` | +| context | `/run/paideia` | +| sealed-inputs | `/run/training/inputs` | +| judge-home | `/run/training/grants` | +| slot-ledger | `/run/training/slot/usage` | + +The rest of the deny list is `/etc/daimon-engine-broker`, +`/run/daimon-engine-broker`, `/run/training/inference`, +`/run/training/slot/turns`, `/run/training/supervisor`, +`/var/lib/spawnfile/daimon/{usage,wake-fuse}`, and Daimon's own protected set +(the Grok bootstrap, the realm and the wake-acceptance store). The caller's +`config.json`, `launch.json`, `token`, `env`, `control`, `preparation.json` and +`repair.json` are covered by the single `/run/paideia` mask rather than listed +individually — Grok materializes each deny target inside bubblewrap as the +worker uid and cannot create one inside a directory only uid 2000 may write. + +A canary is a worker-uid `open()` that must fail. For a deny entry that is a +host bind mount, or sits on a filesystem that ignores unix ownership (Docker +Desktop and Colima both do), that probe proves nothing, so +`unenforcedBindPolicy` decides: `refuse` (the default) fails the recycle and +writes no receipt; `profile-only` accepts the bubblewrap-enforced `deny` list as +that path's only boundary and names every such path in the supervisor log. From 907eb73dfa2c7523da55d14ef9d5a97bd1f4c426 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:07:47 +0200 Subject: [PATCH 49/84] docs: drop the staged Grok credential and copied binary from the training image contract --- specs/TRAINING_CONTAINERS.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index b3084e04..3f39c47b 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -76,10 +76,13 @@ an existing or refreshed credential, and returns a non-secret versioned receipt. | Provider | Destination relative to runtime home | | --- | --- | | codex | `.daimon-inbound/codex-auth` | -| grok | `.grok/auth.json` | | claude | `.claude/.credentials.json` | -The image startup can stage Grok/Claude into its clean shared home. A native trial +Grok is not stageable and `grok` is not an `auth` provider: the one training +Grok login lives in the broker-owned realm volume of a v3 container and is spent +through inference grants, never copied into a runtime home. + +The image startup can stage Claude into its clean shared home. A native trial preparation callback stages Codex into that trial's fresh home before Daimon starts. Credentials remain writable only inside the runtime home; renewed state is not silently copied back to the host bootstrap file. @@ -121,7 +124,6 @@ context. The command does not configure or start a machine-global VM. "paideia": "./packages/paideia", "bridge": "./packages/dspy", "claude": "./packages/claude", - "grok": { "source": "./bin/grok", "sha256": "sha256:<64 hex characters>" }, "integration": { "source": "./integration", "entry": "container/entry.ts" }, "bootstrap": "./bootstrap" } From 7c0b10c3fe4788dd8af48cacf0ab39526e2769c6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:20:01 +0200 Subject: [PATCH 50/84] test: give the cold, warm and resume training preparation path an explicit timeout --- src/cli/trainingPreparation.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli/trainingPreparation.test.ts b/src/cli/trainingPreparation.test.ts index a1e39f9c..5e2be6af 100644 --- a/src/cli/trainingPreparation.test.ts +++ b/src/cli/trainingPreparation.test.ts @@ -35,7 +35,9 @@ it("runs cold, warm and exact-resume through the actual public command and exist f.config.output.source = "warm"; await f.save(); expect(await run()).toBe(0); expect(calls.filter(args => args[2] === "build")).toHaveLength(1); expect(calls.filter(args => args[2] === "create")).toHaveLength(3); -}); +// Three whole cold/warm/resume preparations, each sealing and hashing the complete installed distribution: +// it sits just under the default 30s bound on an idle machine and just over it under full-suite load. +}, 180_000); it("keeps v2 public dry-run free of Docker, auth and preparation writes", async () => { const f = await preparationFixture(); roots.push(f.root); f.config.image = { ref: image }; f.config.auth[0]!.source = "missing"; await f.save(); From 430bf0a8629f715c49fead820b776ea663179065 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:23:26 +0200 Subject: [PATCH 51/84] docs: record the bubblewrap deny-target and root-capability constraints in the broker guide --- src/compiler/training/broker/AGENTS.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/compiler/training/broker/AGENTS.md b/src/compiler/training/broker/AGENTS.md index 2c2586bd..fd0c194a 100644 --- a/src/compiler/training/broker/AGENTS.md +++ b/src/compiler/training/broker/AGENTS.md @@ -35,7 +35,19 @@ Local constraints: - The supervisor socket's uid gate is the socket node (`root:2000 0660` in a root-owned `0711` directory on tmpfs), because Node exposes no `SO_PEERCRED`. A `0600` root-owned socket would deny the one caller it exists for. -- A worker-uid canary over a host bind proves nothing (Docker Desktop and - Colima ignore `chown`), so `unenforcedBindPolicy` decides between refusing - the slot and accepting the bubblewrap deny list as that path's only boundary. +- A worker-uid canary over a host bind proves nothing — the host owns the inode, + and Docker Desktop and Colima ignore `chown` outright — so + `unenforcedBindPolicy` decides between refusing the slot and accepting the + bubblewrap deny list as that path's only boundary. +- Grok 1.0.34 materializes every `deny` target inside bubblewrap as the worker + uid, so a deny entry it cannot create makes the whole profile fail. That is + why `/run/paideia` is masked as a directory rather than file by file, and why + provisioning creates every non-bind deny target itself. One entry still fails + this way (`.runtime/grok-p5/EVIDENCE.md`): the wake-acceptance store, which is + Daimon's own protected path under a `2000:2000 0700` parent. +- Root here holds `CAP_CHOWN`, `CAP_SETUID`, `CAP_SETGID`, `CAP_SETPCAP`, + `CAP_KILL` and `CAP_DAC_READ_SEARCH` — never `CAP_FOWNER` or + `CAP_DAC_OVERRIDE`. Create the whole tree while it is still root-owned, set + ownership from the deepest path up, and reclaim an inode (and its parent) + before chmod-ing or unlinking it. - Keep files under 400 lines and tests adjacent. From 40e81117094b18f9115e62b9cd203f68123f3094 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:56:46 +0200 Subject: [PATCH 52/84] fix: order Grok worker temp and spill provisioning for a root without CAP_DAC_OVERRIDE --- .../containerDaimonGrokWorkerProvisioning.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index 1e160c16..ab666b89 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -70,11 +70,15 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo "for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }", "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT' && entry.deferredDenyPaths.includes(denied)) continue; if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }", // Pass 2: exact root-owned read-only files, the events file, then the final sticky modes. - `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, - // Worker-private temp: the launcher compiles TMPDIR=/tmp, the only temp the worker may write. + `for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710); for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`, + // Worker-private temp: the launcher compiles TMPDIR=/tmp, the only temp the worker may write. It is + // created in the pass above, while the home is still root-owned: root here holds no CAP_DAC_OVERRIDE, so + // once the home is `: 0710` root can no longer create anything inside it. "for (const entry of grokWorkers) { ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); assertCanonical(entry.privateTmp, 'private temp'); }", - // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can traverse. - `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, + // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can + // traverse. The spill directory is created before the runtime home is narrowed to 0710: without + // CAP_DAC_OVERRIDE root cannot create inside a directory it does not own once the mode excludes it. + `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, From 21982cb62b765e20c54929a052b1f840622310b2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:56:46 +0200 Subject: [PATCH 53/84] fix: place Grok worker deny entries where bubblewrap can materialize them as the worker uid --- src/compiler/AGENTS.md | 20 +++++++++-- ...tainerDaimonGrokWorkerProvisioning.test.ts | 31 ++++++++++++++++ .../containerDaimonGrokWorkerProvisioning.ts | 8 +++++ .../containerDaimonGrokWorkerRender.test.ts | 20 +++++++++-- .../containerDaimonGrokWorkerRender.ts | 35 +++++++++++++++++-- 5 files changed, 105 insertions(+), 9 deletions(-) diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 93342f35..3ee7b90d 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -177,7 +177,8 @@ src/compiler/ `src/runtime/daimon/grokWorkerConfigBytes.ts`, refused unless they hash to the manifest pin for the agent's declared model x effort. The sandbox profile's `deny` list is never empty: Daimon's protected set (realms, bootstrap, peers, - acceptance store, kept verbatim) plus the organization config directory, + and the organization `state` directory holding the acceptance store, kept + verbatim) plus the organization config directory, every persistent mount of every runtime plan (the worker's own tool state, credential home, and memory banks included), other runtime instance roots, `/var/lib/spawnfile/{moltnet,agents,memory}`, every workspace resource backing @@ -192,12 +193,25 @@ src/compiler/ `GROK_HOME` layout — `root: 1771` home and `sessions/`, `root:root 0444` config/sandbox/trust/managed/requirements files, events under `sessions/` — and refuses any registration or deny path that is missing, a - symlink, or not its own realpath. It also provisions Daimon's temp and spill - contract: `/tmp` `: 0700` (the launcher's + symlink, or not its own realpath, and — once every mode below is final — any + deny entry bubblewrap could not *place*. Grok 1.0.34 materializes each deny + target inside bubblewrap as the worker uid, so the target must exist and the + worker must be able to search every ancestor directory; one unplaceable entry + makes Grok refuse the whole profile and every turn of that worker fails with a + bare `bwrap: Can't create file at …: Permission denied` (matrix: + `.runtime/grok-deny-placement/EVIDENCE.md`). That is why the wake-acceptance + store is masked through its `0700 2000:2000` parent + (`daimonGrokAcceptanceStoreDenyPath`) rather than directly: lifting a mask to a + private ancestor is strictly stronger and, unlike opening that ancestor with + `o+x`, adds the worker no reach at all. It also provisions Daimon's temp and + spill contract: `/tmp` `: 0700` (the launcher's `TMPDIR`); `/tmp` and `/var/tmp` `root:2000 1774` (Grok refuses to start if they are denied, so modes close them); `/tool-output` `2000: 2750` under a runtime home `2000: 0710` whose `/var/lib/spawnfile` ancestors are made traversable by reclaim-mode-restore. + Root here holds no `CAP_DAC_OVERRIDE`, so the private temp is created while the + worker home is still root-owned and the spill directory before the runtime home + is narrowed to `0710`; either done the other way round fails outright. The broker and relay (uid 2100, outside group 2000) run with `TMPDIR=/run/daimon-engine-broker/tmp`; every other entrypoint process runs as root or uid 2000, and workers get their `TMPDIR` from the launcher. The start diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index 3d3e0ef7..2f45909b 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { daimonGrokWorkerSandboxProfileSha256, renderDaimonGrokWorkerSandboxProfile } from "../runtime/daimon/grokWorkerContract.js"; import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; import { renderDaimonGrokHostPreflight, renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; @@ -194,3 +195,33 @@ describe("Grok host user-namespace preflight", () => { expect((await preflight(null, null)).stdout).toContain("preflight-ok"); }); }); + +describe("Grok deny-path placement", () => { + const registrations = resolveDaimonGrokRegistrations([plan({ "agent:a": "grok", "agent:b": "grok" })]); + + it("refuses a deny entry under a directory the worker uid cannot search", () => { + // Grok 1.0.34 materializes every deny target inside bubblewrap as the worker uid, so a private + // ancestor makes the whole profile unusable — every turn dies with `bwrap: Can't create file at …`. + const shared = "/var/lib/spawnfile/daimon"; + expect(() => run(registrations, { ...seedFor(registrations), [shared]: { gid: 0, kind: "dir", mode: 0o700, uid: 0 } })) + .toThrow(/is not placeable: worker uid 2200 cannot search \/var\/lib\/spawnfile\/daimon \(700 0:0\); deny that directory itself instead/u); + // 0711 — search without read — is exactly what the shared state ancestor is provisioned as, and is enough. + expect(() => run(registrations, { ...seedFor(registrations), [shared]: { gid: 0, kind: "dir", mode: 0o711, uid: 0 } })).not.toThrow(); + }); + + it("refuses the wake-acceptance store as a deny entry, and accepts the private state directory that covers it", () => { + const state = `${INSTANCE}/state`; + const store = `${state}/wake-acceptance`; + const asDenied = (denyPaths: readonly string[]): DaimonGrokRegistration[] => registrations.map((entry, index) => { + const denied = index === 0 ? [...denyPaths].sort() : entry.denyPaths; + const profile = renderDaimonGrokWorkerSandboxProfile(denied); + return { ...entry, denyPaths: denied, profile, profileSha256: daimonGrokWorkerSandboxProfileSha256(denied) }; + }); + const privateState = { [state]: { gid: 2000, kind: "dir" as const, mode: 0o700, uid: 2000 }, [store]: { gid: 2000, kind: "dir" as const, mode: 0o700, uid: 2000 } }; + const leaf = asDenied([...registrations[0]!.denyPaths.filter((entry) => entry !== state), store]); + expect(() => run(leaf, { ...seedFor(leaf), ...privateState })).toThrow(new RegExp(`is not placeable: worker uid 2200 cannot search ${state} \\(700 2000:2000\\)`, "u")); + // What the collector emits instead: the mask on the directory itself, which bubblewrap can place. + expect(registrations[0]!.denyPaths).toContain(state); + expect(() => run(registrations, { ...seedFor(registrations), ...privateState })).not.toThrow(); + }); +}); diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index ab666b89..dc7f1624 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -81,6 +81,14 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, + // Deny-path placement, asserted once every mode above is final. Grok 1.0.34 materializes each deny + // target inside bubblewrap AS THE WORKER UID, so the worker must be able to search every ancestor and + // the target must already exist; one unplaceable entry makes Grok refuse the whole profile and every + // turn of that worker fails with a bare `bwrap: Can't create file at ...: Permission denied`. Root here + // holds CAP_DAC_READ_SEARCH, so it can read every mode the worker cannot, and decides for it. + "const searches = (info, uid, gid) => info.uid === uid ? (info.mode & 0o100) !== 0 : info.gid === gid ? (info.mode & 0o010) !== 0 : (info.mode & 0o001) !== 0;", + "const assertPlaceable = (denied, uid, gid) => { const parts = denied.split('/').slice(1); let at = ''; for (const part of parts.slice(0, -1)) { at += `/${part}`; let info; try { info = fs.lstatSync(at); } catch (error) { throw new Error(`Grok worker deny path ${denied} is not placeable: ${at} could not be read (${error.code})`); } if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`Grok worker deny path ${denied} is not placeable: ${at} is not a directory`); if (!searches(info, uid, gid)) throw new Error(`Grok worker deny path ${denied} is not placeable: worker uid ${uid} cannot search ${at} (${(info.mode & 0o7777).toString(8)} ${info.uid}:${info.gid}); deny that directory itself instead`); } };", + "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { if (entry.deferredDenyPaths.includes(denied)) { try { fs.lstatSync(denied); } catch (error) { if (error.code === 'ENOENT') continue; throw error; } } assertPlaceable(denied, entry.uid, entry.uid); }", `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations))};`, "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` diff --git a/src/compiler/containerDaimonGrokWorkerRender.test.ts b/src/compiler/containerDaimonGrokWorkerRender.test.ts index c38cb3c5..edf798d9 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.test.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.test.ts @@ -7,6 +7,7 @@ import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerCon import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; import { assertCanonicalRegisteredPath, + daimonGrokAcceptanceStoreDenyPath, DAIMON_GROK_OPTIONAL_DENY_PATHS, renderDaimonGrokServiceConfig, resolveDaimonGrokRegistrations @@ -68,7 +69,9 @@ describe("Daimon Grok worker registrations", () => { "/run/daimon-engine-broker", DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, "/var/lib/spawnfile/daimon/wake-fuse", - `${INSTANCE}/state/wake-acceptance`, + // The wake-acceptance store is masked through its private `state` parent: bubblewrap cannot + // materialize a deny target under a directory the worker uid cannot search. + `${INSTANCE}/state`, `${INSTANCE}/runtime-homes/b`, `${INSTANCE}/workspace/agents/b`, `${INSTANCE}/runtime-homes/c`, `${INSTANCE}/workspace/agents/c`, `${INSTANCE}/runtime-homes/d`, `${INSTANCE}/workspace/agents/d`, @@ -123,8 +126,19 @@ describe("Grok base-profile grant guard and nested-mask refusal", () => { }); it("refuses an added ancestor that would cover a Daimon deny entry, since masks cannot nest", () => { - expect(() => resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount("/var/lib/spawnfile/instances/daimon/daimon-organization/state")] })])) - .toThrow(/would cover .*state\/wake-acceptance; masks cannot nest/u); + expect(() => resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount("/var/lib/spawnfile/daimon")] })])) + .toThrow(/would cover .*grok-bootstrap-auth; masks cannot nest/u); + }); + + it("masks the wake-acceptance store through its private parent, never the store itself", () => { + // The `0700 2000:2000` state directory is unplaceable as an ancestor of a deny entry, so the mask + // moves onto it. Declaring it again as a mount is the same path, not a nested one. + const [a] = resolveDaimonGrokRegistrations([plan({})]); + expect(a!.denyPaths).toContain(`${INSTANCE}/state`); + expect(a!.denyPaths).not.toContain(`${INSTANCE}/state/wake-acceptance`); + expect(daimonGrokAcceptanceStoreDenyPath(INSTANCE)).toBe(`${INSTANCE}/state`); + const again = resolveDaimonGrokRegistrations([plan({ persistentMounts: [mount(`${INSTANCE}/state`)] })]); + expect(again[0]!.denyPaths.filter((entry) => entry === `${INSTANCE}/state`)).toHaveLength(1); }); }); diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 5bab54c0..48f00ed3 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -44,6 +44,27 @@ export const DAIMON_ORGANIZATION_STATE_DIRECTORY = path.posix.join( "state" ); +/** + * The deny entry that protects the durable wake-acceptance store: the + * organization state directory the store lives in, not the store itself. + * + * Grok 1.0.34 materializes every `deny` target inside bubblewrap **as the + * worker uid** (it bind-mounts `$GROK_HOME/sandbox-blocked-{file,dir}` over + * the target), so an entry is placeable only when the worker can search every + * ancestor directory and the target already exists. The ownership guard secures + * this state directory to `0700 2000:2000`, so the store beneath it can never + * be a deny entry — and a single unplaceable entry makes Grok refuse the whole + * profile, failing *every* turn with `bwrap: Can't create file at …: + * Permission denied`, not just that path (matrix: + * `.runtime/grok-deny-placement/EVIDENCE.md`). + * + * Lifting the mask to the directory is strictly stronger than masking the store + * — nothing else lives there — and it adds no traversal right to the worker, + * which opening the directory with `o+x` would have done. + */ +export const daimonGrokAcceptanceStoreDenyPath = (instanceRoot: string): string => + path.posix.dirname(path.posix.join(instanceRoot, DAIMON_RUNTIME_ACCEPTANCE_STORE_DIRECTORY)); + /** * Paths Grok 1.0.34's strict base profile grants (read or read-write). Grok * refuses to start when a deny entry equals or contains one of them (verified @@ -127,8 +148,8 @@ const within = (candidate: string, root: string): boolean => candidate === root * * Daimon's own protected set for the agent (`grokSandboxProtectedPaths`: the * Grok bootstrap and realm, the AGY realm and unlock secret when any agent is - * AGY, the wake-acceptance store, and every peer's runtime home and workspace) - * is always kept verbatim, so a Daimon projection over the same inputs renders + * AGY, the organization state directory that holds the wake-acceptance store, + * and every peer's runtime home and workspace) is always kept verbatim, so a Daimon projection over the same inputs renders * the same profile. This deployment adds everything else the container * provisions that the worker does not need: the organization config directory * (all agents' instructions and any env files), every persistent mount of @@ -149,6 +170,14 @@ const within = (candidate: string, root: string): boolean => candidate === root * macOS bind mounts ignore unix modes, so this list — not file modes — is the * boundary. Masks cannot nest: an added entry already covered by another entry * is dropped, and an added entry that would cover a Daimon entry is refused. + * + * Every entry must also be *placeable*: Grok materializes each deny target + * inside bubblewrap as the worker uid, so the target must exist and the worker + * must be able to search every ancestor. That depends on modes, not paths, so + * the root provisioning program asserts it once every mode is final and refuses + * to start the container otherwise (`containerDaimonGrokWorkerProvisioning.ts`); + * here a protected path whose parent is private is lifted to that parent + * instead (`daimonGrokAcceptanceStoreDenyPath`). */ export const resolveDaimonGrokWorkerDenyPaths = ( plans: readonly RuntimeTargetPlan[], @@ -166,7 +195,7 @@ export const resolveDaimonGrokWorkerDenyPaths = ( DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath, DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath, ...(agents.some(([, engine]) => engine === "agy") ? [DAIMON_AGY_SUBSCRIPTION_REALM.unlockMountPath, DAIMON_AGY_SUBSCRIPTION_REALM.durableMountPath] : []), - path.posix.join(instanceRoot, DAIMON_RUNTIME_ACCEPTANCE_STORE_DIRECTORY), + daimonGrokAcceptanceStoreDenyPath(instanceRoot), ...peers.flatMap((slug) => [ path.posix.join(instanceRoot, DAIMON_RUNTIME_HOMES_DIRECTORY, slug), path.posix.join(plan.instancePaths.workspacePath, "agents", slug) From d940b3e95e3307301a46016ac4145c71e0f3a63d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:59:07 +0200 Subject: [PATCH 54/84] fix: mask the training wake-acceptance store through its slot state root so bubblewrap can place it --- specs/RUNTIMES.md | 12 ++++++++++-- specs/TRAINING_CONTAINERS.md | 16 +++++++++++++++- .../containerDaimonGrokWorkerProvisioning.ts | 8 ++++++++ src/compiler/training/broker/paths.ts | 6 +++--- src/compiler/training/broker/projection.ts | 5 ++++- .../training/broker/registration.test.ts | 16 +++++++++++++++- src/compiler/training/broker/registration.ts | 12 ++++++++---- 7 files changed, 63 insertions(+), 12 deletions(-) diff --git a/specs/RUNTIMES.md b/specs/RUNTIMES.md index 3340b159..d8980467 100644 --- a/specs/RUNTIMES.md +++ b/specs/RUNTIMES.md @@ -248,7 +248,8 @@ list was empty and unix modes were the only boundary. On 1.0.34 the deny list is enforced inside bubblewrap for both shell and `read_file`, and it is mandatory: Grok's strict base reads all of `/run`, `/var`, `/tmp`, and `/etc`, and macOS bind mounts ignore unix modes. Each worker denies the Grok bootstrap -and realm, AGY realm and unlock secret when present, the wake-acceptance store, +and realm, AGY realm and unlock secret when present, the organization `state` +directory that holds the wake-acceptance store, every peer agent's runtime home and workspace, the organization config directory, every persistent mount (its own tool state, credential home and memory included), other runtime instance roots, the shared Moltnet, agent-token @@ -258,7 +259,14 @@ path not linked into its own workspace, every other worker's home, the broker's `/run/secrets`, `/run/spawnfile`, `/run/spawnfile-secrets`, `/run/world`. It reaches Moltnet and memory only through Daimon's MCP tools. Every deny entry and registration path must be canonical (present, not a symlink, its own -realpath) at provisioning or the container refuses to start. +realpath) at provisioning or the container refuses to start, and must be +*placeable*: Grok materializes each deny target inside bubblewrap as the worker +uid, so the worker must be able to search every ancestor directory and the +target must already exist. A single unplaceable entry makes Grok refuse the +whole profile, failing every turn of that worker. Provisioning asserts this once +every mode is final; where a protected path sits under a private parent — the +wake-acceptance store under the `0700 2000:2000` organization state directory — +the mask is lifted to that parent rather than opening it with `o+x`. Workspace skills are not emitted for Grok agents: the worker workspace stays untrusted and Daimon overrides the system prompt, so Grok loads neither diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 3f39c47b..eaf05c85 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -350,12 +350,26 @@ The rest of the deny list is `/etc/daimon-engine-broker`, `/run/daimon-engine-broker`, `/run/training/inference`, `/run/training/slot/turns`, `/run/training/supervisor`, `/var/lib/spawnfile/daimon/{usage,wake-fuse}`, and Daimon's own protected set -(the Grok bootstrap, the realm and the wake-acceptance store). The caller's +(the Grok bootstrap, the realm and `/run/training/slot/state`, the slot state +root that holds the wake-acceptance store). The caller's `config.json`, `launch.json`, `token`, `env`, `control`, `preparation.json` and `repair.json` are covered by the single `/run/paideia` mask rather than listed individually — Grok materializes each deny target inside bubblewrap as the worker uid and cannot create one inside a directory only uid 2000 may write. +The same rule governs deny *placement* everywhere. A deny entry is placeable +only when it already exists, is not a symlink, and the worker uid can search +every ancestor directory; one unplaceable entry makes Grok refuse the whole +profile, so every turn of that slot fails with `bwrap: Can't create file at …: +Permission denied`, not just that path (matrix: +`.runtime/grok-deny-placement/EVIDENCE.md`). The wake-acceptance store is that +case: its parent `/run/training/slot/state` is `2000:2000 0700`, so the mask +moves onto the state root, which covers it and nothing else. Lifting a mask to a +private ancestor is strictly stronger than masking the leaf and, unlike opening +the ancestor with `o+x`, gives the worker no additional reach. Root provisioning +asserts placement for every entry once all modes are final — at container start +and on every recycle — and refuses the slot otherwise. + A canary is a worker-uid `open()` that must fail. For a deny entry that is a host bind mount, or sits on a filesystem that ignores unix ownership (Docker Desktop and Colima both do), that probe proves nothing, so diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index c6784d0b..29b889c6 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -93,6 +93,14 @@ export const renderDaimonGrokWorkerProvisioning = ( `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, + // Deny-path placement, asserted once every mode above is final. Grok 1.0.34 materializes each deny + // target inside bubblewrap AS THE WORKER UID, so the worker must be able to search every ancestor and + // the target must already exist; one unplaceable entry makes Grok refuse the whole profile and every + // turn of that worker fails with a bare `bwrap: Can't create file at ...: Permission denied`. Root here + // holds CAP_DAC_READ_SEARCH, so it can read every mode the worker cannot, and decides for it. + "const searches = (info, uid, gid) => info.uid === uid ? (info.mode & 0o100) !== 0 : info.gid === gid ? (info.mode & 0o010) !== 0 : (info.mode & 0o001) !== 0;", + "const assertPlaceable = (denied, uid, gid) => { const parts = denied.split('/').slice(1); let at = ''; for (const part of parts.slice(0, -1)) { at += `/${part}`; let info; try { info = fs.lstatSync(at); } catch (error) { throw new Error(`Grok worker deny path ${denied} is not placeable: ${at} could not be read (${error.code})`); } if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`Grok worker deny path ${denied} is not placeable: ${at} is not a directory`); if (!searches(info, uid, gid)) throw new Error(`Grok worker deny path ${denied} is not placeable: worker uid ${uid} cannot search ${at} (${(info.mode & 0o7777).toString(8)} ${info.uid}:${info.gid}); deny that directory itself instead`); } };", + "for (const entry of grokWorkers) for (const denied of entry.denyPaths) { if (entry.deferredDenyPaths.includes(denied)) { try { fs.lstatSync(denied); } catch (error) { if (error.code === 'ENOENT') continue; throw error; } } assertPlaceable(denied, entry.uid, entry.uid); }", `const service = ${JSON.stringify(renderDaimonGrokServiceConfig(registrations, serviceOptions))};`, "for (const registration of service.registrations) { const entry = grokWorkers.find((worker) => worker.agentId === registration.agentId); if (!entry || registration.profileSha256 !== sha256Hex(fs.readFileSync(entry.profilePath, 'utf8'))) throw new Error('Grok broker service registration does not match its provisioned profile'); }", `fs.writeFileSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', \`\${JSON.stringify(service)}\\n\`, { mode: 0o440, flag: 'wx' }); fs.chownSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0, ${DAIMON_BROKER_UID}); fs.chmodSync('${DAIMON_GROK_ENGINE_BROKER.serviceConfigPath}', 0o440);` diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 97c8209f..1bc53a00 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -103,9 +103,9 @@ export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, DAIMON_WAKE_FUSE_DIRECTORY, TRAINING_INFERENCE_DIRECTORY, - // `TRAINING_SLOT_STATE_ROOT` is deliberately absent: Daimon's own protected set - // already denies the wake-acceptance store beneath it, and masks cannot nest, - // so denying the parent as well would make Grok refuse the whole profile. + // `TRAINING_SLOT_STATE_ROOT` is Daimon's own protected entry here (it covers the + // wake-acceptance store, which is itself unplaceable under a `2000:2000 0700` + // parent), so it is not repeated in this set. TRAINING_SLOT_TURN_STORE, TRAINING_SUPERVISOR_DIRECTORY, ...TRAINING_EVALUATOR_ROOTS.map((entry) => entry.path) diff --git a/src/compiler/training/broker/projection.ts b/src/compiler/training/broker/projection.ts index b1616e78..14642183 100644 --- a/src/compiler/training/broker/projection.ts +++ b/src/compiler/training/broker/projection.ts @@ -6,7 +6,7 @@ import { } from "../../../runtime/daimon/contractManifest.js"; import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; import type { TrainingBrokerDeclaration } from "./declaration.js"; -import { TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME } from "./paths.js"; +import { TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME } from "./paths.js"; /** * The slice of Daimon's public `@noopolis/daimon/runtime` export the slot @@ -88,6 +88,9 @@ export const resolveTrainingGrokProjection = async ( usageLedgerPath: registration.usageLedgerPath, limits: declaration.limits, acceptanceStorePath: TRAINING_SLOT_ACCEPTANCE_STORE, + // Daimon masks the store through the slot state root it lives in; the store's own parent is + // `2000:2000 0700`, so bubblewrap could not materialize a deny target inside it as the worker uid. + acceptanceStoreDenyPath: TRAINING_SLOT_STATE_ROOT, denyPaths: registration.denyPaths, seccompProfileSha256: declaration.seccompProfileSha256, profileSha256: registration.profileSha256 diff --git a/src/compiler/training/broker/registration.test.ts b/src/compiler/training/broker/registration.test.ts index 56e9dcf2..6f2a923a 100644 --- a/src/compiler/training/broker/registration.test.ts +++ b/src/compiler/training/broker/registration.test.ts @@ -11,6 +11,8 @@ import { TRAINING_INFERENCE_DIRECTORY, TRAINING_INFERENCE_LEDGER, TRAINING_REALM_MOUNT, + TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_SLOT_USAGE_LEDGER @@ -28,13 +30,25 @@ describe("training Grok slot registration", () => { } for (const entry of [...TRAINING_EVALUATOR_ROOTS.map((role) => role.path), "/run/daimon-engine-broker", "/etc/daimon-engine-broker", TRAINING_GRANT_HOME_ROOT, TRAINING_INFERENCE_DIRECTORY, - TRAINING_SLOT_TURN_STORE, TRAINING_REALM_MOUNT]) { + TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, TRAINING_REALM_MOUNT]) { expect(denied, entry).toContain(entry); } expect(denied).toEqual([...denied].sort()); expect(new Set(denied).size).toBe(denied.length); }); + it("masks the wake-acceptance store through the slot state root, never the store itself", () => { + // The store's parent is `2000:2000 0700`, and Grok 1.0.34 materializes every deny target inside + // bubblewrap as the worker uid, so the store itself is unplaceable and would make Grok refuse the + // whole profile — the defect P5 hit live (`.runtime/grok-deny-placement/EVIDENCE.md`). + const denied = slot().denyPaths; + expect(denied).toContain(TRAINING_SLOT_STATE_ROOT); + expect(denied).not.toContain(TRAINING_SLOT_ACCEPTANCE_STORE); + expect(TRAINING_SLOT_ACCEPTANCE_STORE.startsWith(`${TRAINING_SLOT_STATE_ROOT}/`)).toBe(true); + // And the store can never come back as an added entry: the state root already covers it. + expect(() => resolveTrainingGrokDenyPaths([...TRAINING_ADDED_DENY_PATHS, TRAINING_SLOT_ACCEPTANCE_STORE])).toThrow(/masks cannot nest/u); + }); + it("keeps the subject's own workspace, worker home and runtime home reachable", () => { const registration = slot(); for (const own of [registration.workspace, registration.home, registration.grokHome, registration.runtimeHome, registration.privateTmp, registration.spillDirectory]) { diff --git a/src/compiler/training/broker/registration.ts b/src/compiler/training/broker/registration.ts index 881f0f8e..05ab5c4e 100644 --- a/src/compiler/training/broker/registration.ts +++ b/src/compiler/training/broker/registration.ts @@ -24,9 +24,9 @@ import { TRAINING_BOOTSTRAP_MOUNT, TRAINING_DEFERRED_DENY_PATHS, TRAINING_REALM_MOUNT, - TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_INDEX, TRAINING_SLOT_RUNTIME_HOME, + TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_USAGE_LEDGER, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME, @@ -41,8 +41,8 @@ const within = (candidate: string, root: string): boolean => candidate === root /** * The one training slot's deny list: Daimon's own protected set for a - * single-agent organization (Grok bootstrap, realm, wake-acceptance store) - * plus everything this container provisions that the subject must not read — + * single-agent organization (Grok bootstrap, realm, and the slot state root that + * holds the wake-acceptance store) plus everything this container provisions that the subject must not read — * the evaluator roots, the caller's protected `/run/paideia` paths, the broker * control and registration roots, the grant home root, the inference ledger, * the per-slot ledger, turn store and supervisor socket directory. @@ -53,7 +53,11 @@ const within = (candidate: string, root: string): boolean => candidate === root * entry is a provisioning bug rather than something to silently drop. */ export const resolveTrainingGrokDenyPaths = (added: readonly string[] = TRAINING_ADDED_DENY_PATHS): string[] => { - const daimonOwn = [TRAINING_BOOTSTRAP_MOUNT, TRAINING_REALM_MOUNT, TRAINING_SLOT_ACCEPTANCE_STORE]; + // The wake-acceptance store is masked through `TRAINING_SLOT_STATE_ROOT`, never directly: Grok 1.0.34 + // materializes every deny target inside bubblewrap AS THE WORKER UID, and the store's parent is + // `2000:2000 0700`, so the store itself is unplaceable and would make Grok refuse the whole profile + // (`.runtime/grok-deny-placement/EVIDENCE.md`). The state root covers it and nothing else lives there. + const daimonOwn = [TRAINING_BOOTSTRAP_MOUNT, TRAINING_REALM_MOUNT, TRAINING_SLOT_STATE_ROOT]; const grokHome = path.posix.join(TRAINING_WORKER_HOME, DAIMON_GROK_WORKER_HOME_DIRECTORY); const grants = [ ...DAIMON_GROK_BASE_PROFILE_GRANTS, TRAINING_SLOT_WORKSPACE, grokHome, From 2adc92c6395a1259a7d20919c96aa3fb20007323 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:59:13 +0200 Subject: [PATCH 55/84] docs: record the Grok deny-placement rule in the runtimes spec --- specs/RUNTIMES.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specs/RUNTIMES.md b/specs/RUNTIMES.md index 3340b159..d8980467 100644 --- a/specs/RUNTIMES.md +++ b/specs/RUNTIMES.md @@ -248,7 +248,8 @@ list was empty and unix modes were the only boundary. On 1.0.34 the deny list is enforced inside bubblewrap for both shell and `read_file`, and it is mandatory: Grok's strict base reads all of `/run`, `/var`, `/tmp`, and `/etc`, and macOS bind mounts ignore unix modes. Each worker denies the Grok bootstrap -and realm, AGY realm and unlock secret when present, the wake-acceptance store, +and realm, AGY realm and unlock secret when present, the organization `state` +directory that holds the wake-acceptance store, every peer agent's runtime home and workspace, the organization config directory, every persistent mount (its own tool state, credential home and memory included), other runtime instance roots, the shared Moltnet, agent-token @@ -258,7 +259,14 @@ path not linked into its own workspace, every other worker's home, the broker's `/run/secrets`, `/run/spawnfile`, `/run/spawnfile-secrets`, `/run/world`. It reaches Moltnet and memory only through Daimon's MCP tools. Every deny entry and registration path must be canonical (present, not a symlink, its own -realpath) at provisioning or the container refuses to start. +realpath) at provisioning or the container refuses to start, and must be +*placeable*: Grok materializes each deny target inside bubblewrap as the worker +uid, so the worker must be able to search every ancestor directory and the +target must already exist. A single unplaceable entry makes Grok refuse the +whole profile, failing every turn of that worker. Provisioning asserts this once +every mode is final; where a protected path sits under a private parent — the +wake-acceptance store under the `0700 2000:2000` organization state directory — +the mask is lifted to that parent rather than opening it with `o+x`. Workspace skills are not emitted for Grok agents: the worker workspace stays untrusted and Daimon overrides the system prompt, so Grok loads neither From ccbe537090f047552be451ece2fde4d243ec0a80 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:02:47 +0200 Subject: [PATCH 56/84] fix: pass the slot state root as the projection's acceptance-store mask --- src/compiler/training/broker/projection.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/compiler/training/broker/projection.ts b/src/compiler/training/broker/projection.ts index 14642183..d0248e7f 100644 --- a/src/compiler/training/broker/projection.ts +++ b/src/compiler/training/broker/projection.ts @@ -6,7 +6,7 @@ import { } from "../../../runtime/daimon/contractManifest.js"; import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; import type { TrainingBrokerDeclaration } from "./declaration.js"; -import { TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME } from "./paths.js"; +import { TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_HOME } from "./paths.js"; /** * The slice of Daimon's public `@noopolis/daimon/runtime` export the slot @@ -87,10 +87,10 @@ export const resolveTrainingGrokProjection = async ( architecture: declaration.architecture, usageLedgerPath: registration.usageLedgerPath, limits: declaration.limits, - acceptanceStorePath: TRAINING_SLOT_ACCEPTANCE_STORE, - // Daimon masks the store through the slot state root it lives in; the store's own parent is - // `2000:2000 0700`, so bubblewrap could not materialize a deny target inside it as the worker uid. - acceptanceStoreDenyPath: TRAINING_SLOT_STATE_ROOT, + // The mask that protects the wake-acceptance store is the slot state root it lives in: the store's + // own parent is `2000:2000 0700`, so bubblewrap could not materialize a deny target inside it as + // the worker uid. Every party that recomputes this projection must pass the same value. + acceptanceStorePath: TRAINING_SLOT_STATE_ROOT, denyPaths: registration.denyPaths, seccompProfileSha256: declaration.seccompProfileSha256, profileSha256: registration.profileSha256 From dc9607d321aeb4ed848770e6da159cc6aa76b20e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:30:35 +0200 Subject: [PATCH 57/84] fix: let the training image mode check run on a symlinked tmpdir and non-nested listing roots --- scripts/training-image-modes.test.ts | 9 ++++++++- scripts/training-image-modes.ts | 7 +++++++ scripts/verify-training-image-modes.ts | 10 ++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/training-image-modes.test.ts b/scripts/training-image-modes.test.ts index 897bf003..6fcf0277 100644 --- a/scripts/training-image-modes.test.ts +++ b/scripts/training-image-modes.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes } from "./training-image-modes.ts"; +import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_ROOTS } from "./training-image-modes.ts"; const row = (entry: string, mode = "755", type = "d", target = "") => `${entry}\t${mode}\troot\troot\t${type}\t${target}`; const base = [row("/opt/training"), row("/opt/training/paideia/bridges/dspy/.venv"), row("/opt/training/bin/grok", "555", "f"), @@ -41,3 +41,10 @@ test("recipes must actually differ in the recursive closure", () => { assert.throws(() => assertControlRecipe("# chmod -R a+rX /opt/training\nRUN true"), /Control recipe/u); assert.doesNotThrow(() => assertNewRecipe("# Exactly `chmod -R a+rX /opt/training`, but change-only\nRUN find /opt/training")); }); + +test("the listing roots never nest, so find cannot emit a duplicate entry", () => { + for (const root of LISTING_ROOTS) { + assert.equal(LISTING_ROOTS.some(other => other !== root && root.startsWith(other + "/")), false, root); + } + for (const required of MODE_ROOTS) assert.equal(LISTING_ROOTS.some(root => required === root || required.startsWith(root + "/")), true, required); +}); diff --git a/scripts/training-image-modes.ts b/scripts/training-image-modes.ts index c8c26889..7792c394 100644 --- a/scripts/training-image-modes.ts +++ b/scripts/training-image-modes.ts @@ -3,6 +3,13 @@ /** One `find -printf` record per entry: path, octal mode, owner, group, type, link target. */ export const MODE_LISTING_FORMAT = "%p\\t%m\\t%u\\t%g\\t%y\\t%l\\n"; export const MODE_ROOTS = ["/opt/training", "/opt/training/paideia/bridges/dspy/.venv"] as const; +/** + * `find` arguments. Only non-nested roots: `find a a/b` walks `a/b` twice and every + * entry beneath it would arrive duplicated, which `parse` rejects. The venv stays a + * required entry above, so its absence is still a failure. + */ +export const LISTING_ROOTS = MODE_ROOTS.filter((root, _index, all) => + !all.some(other => other !== root && root.startsWith(other + "/"))); /** The former recipe's whole-tree closure; a control recipe without it is not a control. */ export const CONTROL_CLOSURE = "chmod -R a+rX /opt/training"; diff --git a/scripts/verify-training-image-modes.ts b/scripts/verify-training-image-modes.ts index 9b2a1283..611e00f6 100644 --- a/scripts/verify-training-image-modes.ts +++ b/scripts/verify-training-image-modes.ts @@ -7,11 +7,11 @@ * [--root ] [--control-ref 47050be] [--docker-context default] [--keep-images] */ import { execFileSync, spawnSync } from "node:child_process"; -import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, MODE_LISTING_FORMAT, MODE_ROOTS } from "./training-image-modes.ts"; +import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_LISTING_FORMAT } from "./training-image-modes.ts"; type ImageModule = typeof import("../src/compiler/training/preparation/image.js"); type FilesModule = typeof import("../src/compiler/training/preparation/files.js"); @@ -50,7 +50,9 @@ async function main(): Promise { assertNewRecipe(plan.dockerfile); assertControlRecipe(controlRecipe); - const parent = await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-modes-")); + // `copySealed` re-seals every staged file through `exactPath`, which refuses a symlinked + // ancestor; macOS `os.tmpdir()` is `/var/folders/...` and `/var` is a symlink. + const parent = await realpath(await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-modes-"))); const tags = { next: "spawnfile-training-modes:new", control: "spawnfile-training-modes:control" }; try { // Mirror buildTrainingImage staging; only the new side normalizes modes, as in each recipe's era. @@ -68,7 +70,7 @@ async function main(): Promise { docker(dockerContext, ["build", ...buildArgs, "--tag", tags.next, await stage("new", plan.dockerfile, true)]); docker(dockerContext, ["build", ...buildArgs, "--tag", tags.control, await stage("control", controlRecipe, false)]); const listing = (tag: string) => docker(dockerContext, ["run", "--rm", "--platform", build.platform, "--user", "0:0", "--network", "none", - "--entrypoint", "find", tag, ...MODE_ROOTS, "-printf", MODE_LISTING_FORMAT], true); + "--entrypoint", "find", tag, ...LISTING_ROOTS, "-printf", MODE_LISTING_FORMAT], true); const diff = diffModeListings(listing(tags.next), listing(tags.control)); if (!identicalModes(diff)) { for (const entry of diff.onlyNew.slice(0, 50)) console.error(`only in new: ${entry}`); From b521b2dd1e25f105d16684f06c0199cab212a176 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:46:02 +0200 Subject: [PATCH 58/84] fix: install the native parent's pinned Grok and engine broker at the paths the broker contract names --- runtime-images/training/Dockerfile | 7 +++++++ specs/TRAINING_CONTAINERS.md | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index b0e70395..8d8bee47 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -47,8 +47,15 @@ RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \ # Exactly `chmod -R a+rX /opt/training`, but only entries that need a change are # touched, so unchanged lower-layer files are never copied up into a new layer. +# This layer also puts the native parent's own pinned Grok and engine broker at the two fixed paths +# `GROK_ENGINE_BROKER` names, exactly as a generated organization image does (`src/runtime/container.ts`). +# No second build is introduced: both are installed from the parent's runtime install, and the slot +# provisioning and the slot supervisor both refuse an executable that is not a manifest-pinned one. RUN find /opt/training \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} + \ && mkdir -m 0755 /opt/training/bin /opt/training/integration \ + && mkdir -p /opt/daimon/bin \ + && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/grok /usr/local/bin/grok \ + && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker \ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \ && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \ diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 7af75700..7cd28d0c 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -167,7 +167,8 @@ lock, executable and recipe bytes, parent images, architecture and entrypoint; reuse also verifies the image ID and recipe label in the selected Docker context. Staged build contexts normalize modes and times before `docker build`: directories -0755, files `a+rX`-closed, `grok` and the entrypoint 0555, mtimes fixed. Together +0755, files `a+rX`-closed, both entrypoints (`train`, `train-broker`) 0555, mtimes +fixed. Together with the recipe's change-only `a+rX` closure this reproduces the former recursive chmod's in-image modes without a whole-tree RUN after every distribution copy (`runtime-images/training/AGENTS.md`). The recipe text is part of the image digest, From 028612fab5ff6610d496f30c6030edc52aec2746 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:50:43 +0200 Subject: [PATCH 59/84] test: derive the training image mode control from the recipe under test --- runtime-images/training/AGENTS.md | 15 ++++++++++++++- runtime-images/training/Dockerfile | 4 +++- scripts/training-image-modes.test.ts | 17 ++++++++++++++++- scripts/training-image-modes.ts | 26 ++++++++++++++++++++++++++ scripts/verify-training-image-modes.ts | 16 +++++++++------- 5 files changed, 68 insertions(+), 10 deletions(-) diff --git a/runtime-images/training/AGENTS.md b/runtime-images/training/AGENTS.md index a730e762..0e6727a0 100644 --- a/runtime-images/training/AGENTS.md +++ b/runtime-images/training/AGENTS.md @@ -40,4 +40,17 @@ chmod -R a+rX /opt/training`, and owners stay COPY/RUN defaults (root): integration `node_modules` layout is created after the closure, as before. - `contextModes.test.ts` runs the rendered closure against real `chmod -R a+rX` and asserts staged modes, recipe order and symlinks. Build-cache and timing - effects require a Docker-enabled measurement. + effects require a Docker-enabled measurement + (`npm run verify:training-image-modes`, evidence in `.runtime/grok-p5/build/`). + +## Broker contract paths + +The layout RUN also installs the native parent's own pinned Grok and engine +broker at `/usr/local/bin/grok` and `/opt/daimon/bin/daimon-engine-broker`, the +two paths `GROK_ENGINE_BROKER` fixes, exactly as a generated organization image +does (`src/runtime/container.ts`). It introduces no second build — both come +from `/opt/spawnfile/runtime-installs/daimon/bin` — and a parent whose Grok is +not a manifest-pinned 1.0.34 build is refused at slot provisioning and again by +the slot supervisor, by digest. `spawnfile/node_modules/@noopolis/daimon` links +to the same install so the root entrypoint can import Daimon's public +`/runtime` export for the broker projection. diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 8d8bee47..49a85f18 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -62,7 +62,9 @@ RUN find /opt/training \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l && mkdir -p /opt/training/integration/node_modules/@noopolis \ && ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia \ && ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile \ - && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon + && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon \ + && mkdir -p /opt/training/spawnfile/node_modules/@noopolis \ + && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/spawnfile/node_modules/@noopolis/daimon COPY bridge /opt/training/paideia/bridges/dspy RUN /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy \ diff --git a/scripts/training-image-modes.test.ts b/scripts/training-image-modes.test.ts index 6fcf0277..deea91a9 100644 --- a/scripts/training-image-modes.test.ts +++ b/scripts/training-image-modes.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; import { test } from "node:test"; -import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_ROOTS } from "./training-image-modes.ts"; +import { assertControlRecipe, assertNewRecipe, CONTROL_CLOSURE, deriveControlRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_ROOTS } from "./training-image-modes.ts"; const row = (entry: string, mode = "755", type = "d", target = "") => `${entry}\t${mode}\troot\troot\t${type}\t${target}`; const base = [row("/opt/training"), row("/opt/training/paideia/bridges/dspy/.venv"), row("/opt/training/bin/grok", "555", "f"), @@ -48,3 +49,17 @@ test("the listing roots never nest, so find cannot emit a duplicate entry", () = } for (const required of MODE_ROOTS) assert.equal(LISTING_ROOTS.some(root => required === root || required.startsWith(root + "/")), true, required); }); + +test("derives a control recipe that reapplies the whole-tree closure this recipe replaced", async () => { + const recipe = await readFile(new URL("../runtime-images/training/Dockerfile", import.meta.url), "utf8"); + assertNewRecipe(recipe); + const control = deriveControlRecipe(recipe); + assertControlRecipe(control); + assert.equal(/-exec chmod a\+rX \{\} \+/u.test(control), false, "no change-only closure may survive in the control"); + assert.ok(control.includes("RUN chmod 0555 /opt/training/bin/train /opt/training/bin/train-broker")); + // The closure must RUN after the last COPY, where the former recipe ran it. `indexOf` would find the + // recipe's own comment about the closure, which sits near the top. + assert.ok(control.indexOf("\nRUN chmod 0555 ") > control.lastIndexOf("\nCOPY ")); + assert.ok(control.lastIndexOf(CONTROL_CLOSURE) > control.lastIndexOf("\nCOPY ")); + assert.throws(() => deriveControlRecipe(control), /no change-only a\+rX closure/u); +}); diff --git a/scripts/training-image-modes.ts b/scripts/training-image-modes.ts index 7792c394..d3a0b191 100644 --- a/scripts/training-image-modes.ts +++ b/scripts/training-image-modes.ts @@ -22,6 +22,32 @@ export function assertControlRecipe(dockerfile: string): void { if (!instructions(dockerfile).includes(CONTROL_CLOSURE)) throw Error(`Control recipe must run "${CONTROL_CLOSURE}"`); } +/** + * The control recipe: this exact recipe with the old whole-tree closure back. + * + * A git-ref control only works while some commit carries the same layout with + * the recursive chmod, which stops being true the moment the recipe gains a + * layer. Deriving it keeps the comparison about the one thing under test — the + * mode mechanism — instead of about everything else that changed since. + * Each change-only `find` closure becomes a no-op, and the former final + * `chmod 0555 && chmod -R a+rX /opt/training` runs after the last + * COPY, exactly where the old recipe ran it. + */ +export function deriveControlRecipe(dockerfile: string): string { + const closures = dockerfile.match(/find \S+ \\\( .*? -exec chmod a\+rX \{\} \+/gu) ?? []; + if (closures.length === 0) throw Error("Recipe has no change-only a+rX closure to replace"); + let control = closures.reduce((text, closure) => text.replace(closure, "true"), dockerfile); + const anchor = control.indexOf("\nENV PATH="); + if (anchor === -1) throw Error("Recipe has no trailing ENV PATH to anchor the control closure"); + const restore = `\nRUN chmod 0555 ${ENTRYPOINT_PATHS.join(" ")} \\\n && ${CONTROL_CLOSURE}\n`; + control = control.slice(0, anchor) + restore + control.slice(anchor + 1); + assertControlRecipe(control); + return control; +} + +/** The entrypoints the former recipe forced to 0555 before its whole-tree closure. */ +export const ENTRYPOINT_PATHS = ["/opt/training/bin/train", "/opt/training/bin/train-broker"] as const; + export function assertNewRecipe(dockerfile: string): void { if (instructions(dockerfile).includes(CONTROL_CLOSURE)) throw Error("New recipe still runs the recursive chmod; nothing would be compared"); } diff --git a/scripts/verify-training-image-modes.ts b/scripts/verify-training-image-modes.ts index 8de5c783..4d823090 100644 --- a/scripts/verify-training-image-modes.ts +++ b/scripts/verify-training-image-modes.ts @@ -4,14 +4,14 @@ * compares mode, owner, group, type and link target of every entry. * * npx tsx scripts/verify-training-image-modes.ts --build-config \ - * [--root ] [--control-ref 47050be] [--docker-context default] [--keep-images] + * [--root ] [--control-ref ] [--docker-context default] [--keep-images] */ import { execFileSync, spawnSync } from "node:child_process"; import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { assertControlRecipe, assertNewRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_LISTING_FORMAT } from "./training-image-modes.ts"; +import { assertControlRecipe, assertNewRecipe, deriveControlRecipe, diffModeListings, identicalModes, LISTING_ROOTS, MODE_LISTING_FORMAT } from "./training-image-modes.ts"; type ImageModule = typeof import("../src/compiler/training/preparation/image.js"); type FilesModule = typeof import("../src/compiler/training/preparation/files.js"); @@ -35,10 +35,10 @@ function docker(context: string, args: string[], capture = false): string { async function main(): Promise { const configPath = option("--build-config"); if (!configPath) throw Error("--build-config is required"); - // The control must stage the same file set as the recipe under test, or its build fails before a - // single mode is compared. `ccbe537` is the last `spawnfile.training-container.v3` recipe that still - // ran `chmod -R a+rX /opt/training`: same `train`/`train-broker` context, no Grok binary. - const controlRef = option("--control-ref") ?? "ccbe537"; + // Default: derive the control from the recipe under test, so the only difference between the two + // images is the mode mechanism. `--control-ref ` still compares against a historical recipe, + // which only works while that commit stages the same file set. + const controlRef = option("--control-ref"); const dockerContext = option("--docker-context") ?? "default"; const { planTrainingImage } = await import("../src/compiler/training/preparation/image.js") as ImageModule; const { copySealed } = await import("../src/compiler/training/preparation/files.js") as FilesModule; @@ -49,8 +49,10 @@ async function main(): Promise { const build = trainingBuildSchema.parse(raw.image?.build ?? raw); const root = path.resolve(option("--root") ?? path.dirname(configPath)); const plan = await planTrainingImage(build, root, [], repository); - const controlRecipe = execFileSync("git", ["-C", repository, "show", `${controlRef}:runtime-images/training/Dockerfile`], { encoding: "utf8" }); assertNewRecipe(plan.dockerfile); + const controlRecipe = controlRef + ? execFileSync("git", ["-C", repository, "show", `${controlRef}:runtime-images/training/Dockerfile`], { encoding: "utf8" }) + : deriveControlRecipe(plan.dockerfile); assertControlRecipe(controlRecipe); // `copySealed` re-seals every staged file through `exactPath`, which refuses a symlinked From 54d20951e0a47c41851a416bd2514290cbed9c54 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:52:51 +0200 Subject: [PATCH 60/84] test: pin the broker contract paths and the Daimon runtime link in the training recipe --- src/compiler/training/preparation/contextModes.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/compiler/training/preparation/contextModes.test.ts b/src/compiler/training/preparation/contextModes.test.ts index b1f67642..35a24378 100644 --- a/src/compiler/training/preparation/contextModes.test.ts +++ b/src/compiler/training/preparation/contextModes.test.ts @@ -99,5 +99,10 @@ it("hands Docker a normalized context and never runs a layer after the distribut "ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile", "ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia", "ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile", - "ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon"]); + "ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon", + // The v3 root entrypoint imports Daimon's public /runtime export for the broker projection. + "ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/spawnfile/node_modules/@noopolis/daimon"]); + // Both fixed broker executable paths come from the native parent's own runtime install, never a second build. + expect(dockerfile).toContain("install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/grok /usr/local/bin/grok"); + expect(dockerfile).toContain("install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker"); }); From 5a3486de000979fcc7e45ada4155208fb888e8ca Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 18:38:20 +0200 Subject: [PATCH 61/84] fix: move the training broker temp out of the control root and clear around mount points --- src/compiler/containerDaimonBrokerRender.ts | 52 +++++++++- .../training/broker/mountAwareClear.test.ts | 94 +++++++++++++++++++ src/compiler/training/broker/paths.ts | 16 ++++ src/compiler/training/broker/processes.ts | 7 +- src/compiler/training/broker/provisioning.ts | 6 +- .../training/broker/registration.test.ts | 38 ++++++++ src/compiler/training/broker/runtime.ts | 8 +- src/compiler/training/container/security.ts | 4 +- src/compiler/training/preparation/prepare.ts | 3 +- .../training/preparation/scratch.test.ts | 58 ++++++++++++ src/compiler/training/preparation/scratch.ts | 47 ++++++++++ 11 files changed, 319 insertions(+), 14 deletions(-) create mode 100644 src/compiler/training/broker/mountAwareClear.test.ts create mode 100644 src/compiler/training/preparation/scratch.test.ts create mode 100644 src/compiler/training/preparation/scratch.ts diff --git a/src/compiler/containerDaimonBrokerRender.ts b/src/compiler/containerDaimonBrokerRender.ts index 691c735f..ca297518 100644 --- a/src/compiler/containerDaimonBrokerRender.ts +++ b/src/compiler/containerDaimonBrokerRender.ts @@ -85,6 +85,44 @@ export const renderDaimonUsageLedgerProvisioning = (): string[] => { ]; }; +/** + * Empties a directory without touching a mount point inside it. + * + * `rm -rf` and `find -delete` both fail `EBUSY` on a mount point — it is + * another filesystem's root, and its entry cannot be unlinked while it is + * mounted. A deployment that mounts a tmpfs below a directory it later clears + * would otherwise abort provisioning on a bare `find` failure, which is exactly + * how a live training launch died. Mount points are left in place and descended + * into; anything else that cannot be removed is reported by name. + * + * Mount points come from `/proc/self/mountinfo` field 5. The kernel + * octal-escapes space, tab, newline and backslash there; every path this + * compiler provisions is free of all four, and a path that is not would simply + * fail to match and be treated as an ordinary directory. + */ +export const MOUNT_AWARE_CLEAR_HELPER = [ + "spawnfile_mount_points=$(awk '{print $5}' /proc/self/mountinfo)", + "spawnfile_is_mount() { printf '%s\\n' \"$spawnfile_mount_points\" | grep -qxF \"$1\"; }", + "spawnfile_holds_mount() { printf '%s\\n' \"$spawnfile_mount_points\" | grep -qE \"^$(printf '%s' \"$1\" | sed 's/[][\\.*^$/]/\\\\&/g')(/|$)\"; }", + "spawnfile_clear_tree() {", + " spawnfile_root=$1", + " [ -d \"$spawnfile_root\" ] || return 0", + " for spawnfile_entry in \"$spawnfile_root\"/* \"$spawnfile_root\"/.[!.]* \"$spawnfile_root\"/..?*; do", + " if [ ! -e \"$spawnfile_entry\" ] && [ ! -L \"$spawnfile_entry\" ]; then continue; fi", + " if spawnfile_is_mount \"$spawnfile_entry\"; then continue; fi", + " if spawnfile_holds_mount \"$spawnfile_entry\"; then spawnfile_clear_tree \"$spawnfile_entry\"; continue; fi", + " rm -rf \"$spawnfile_entry\" || { echo \"cannot clear $spawnfile_entry: it is in use; a mount below a provisioned root must be declared outside it\" >&2; return 1; }", + " done", + "}", + // Removes the target outright, unless it is or contains a mount point: then its contents go and the + // mount points and the directories holding them stay, because neither can be unlinked while mounted. + "spawnfile_remove_tree() {", + " if [ ! -e \"$1\" ] && [ ! -L \"$1\" ]; then return 0; fi", + " if spawnfile_holds_mount \"$1\"; then spawnfile_clear_tree \"$1\"; return $?; fi", + " rm -rf \"$1\" || { echo \"cannot remove $1: it is in use\" >&2; return 1; }", + "}" +]; + export const renderDaimonBrokerProvisioning = (plans: RuntimeTargetPlan[]): string[] => { const registrations = resolveDaimonGrokRegistrations(plans); if (registrations.length === 0) return []; @@ -132,7 +170,15 @@ export const renderDaimonBrokerProvisioningProgram = ( */ rootReset: "remove" | "clear" = "remove", /** Deny targets the worker provisioning creates when absent; see `renderDaimonGrokWorkerProvisioning`. */ - optionalDenyPaths?: readonly string[] + optionalDenyPaths?: readonly string[], + /** + * Where the broker and relay get their private `TMPDIR`. Production keeps it + * inside the control root, which it removes and recreates exactly once. A + * deployment that re-provisions that root — the training slot recycle — must + * pass a path outside it: a `tmp/` inside a cleared root is a mount point in + * the training launch, and clearing a mount point fails `EBUSY`. + */ + brokerTmpdir: string = DAIMON_BROKER_TMPDIR ): string[] => { const program = [ "const crypto = require('node:crypto'); const fs = require('node:fs');", @@ -168,9 +214,9 @@ export const renderDaimonBrokerProvisioningProgram = ( "if [ -d /run/daimon-engine-broker ]; then chmod u+rwx /run/daimon-engine-broker; fi", ...(rootReset === "remove" ? ["rm -rf /etc/daimon-engine-broker /run/daimon-engine-broker"] - : ["for broker_root in /etc/daimon-engine-broker /run/daimon-engine-broker; do if [ -d \"$broker_root\" ]; then find \"$broker_root\" -mindepth 1 -delete; fi; done"]), + : [...MOUNT_AWARE_CLEAR_HELPER, "for broker_root in /etc/daimon-engine-broker /run/daimon-engine-broker; do spawnfile_clear_tree \"$broker_root\"; done"]), `install -d -o root -g ${DAIMON_BROKER_UID} -m 0731 /run/daimon-engine-broker`, - `install -d -o ${DAIMON_BROKER_UID} -g ${DAIMON_BROKER_UID} -m 0700 ${DAIMON_BROKER_TMPDIR}`, + `install -d -o ${DAIMON_BROKER_UID} -g ${DAIMON_BROKER_UID} -m 0700 ${brokerTmpdir}`, "node <<'SPAWNFILE_DAIMON_BROKER_PROVISION'", program, "SPAWNFILE_DAIMON_BROKER_PROVISION" diff --git a/src/compiler/training/broker/mountAwareClear.test.ts b/src/compiler/training/broker/mountAwareClear.test.ts new file mode 100644 index 00000000..d98daa84 --- /dev/null +++ b/src/compiler/training/broker/mountAwareClear.test.ts @@ -0,0 +1,94 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +import { MOUNT_AWARE_CLEAR_HELPER } from "../../containerDaimonBrokerRender.js"; + +const run = promisify(execFile); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); + +/** + * Exercises the rendered shell itself against a real tree, with the mount table + * injected instead of read from `/proc` — the helper's only input, and the only + * part of it a developer machine cannot produce. `bash` on macOS is `sh`-level + * POSIX here, which is exactly what the container's `bash --noprofile --norc` + * runs the script as. + */ +const exercise = async (root: string, mounts: readonly string[], command: string): Promise<{ stdout: string; stderr: string }> => { + const script = [ + // Single quotes keep the real newlines; `JSON.stringify` would hand bash a literal backslash-n. + ...MOUNT_AWARE_CLEAR_HELPER.map(line => line.replace("spawnfile_mount_points=$(awk '{print $5}' /proc/self/mountinfo)", + `spawnfile_mount_points='${mounts.join("\n")}'`)), + command + ].join("\n"); + return run("/bin/bash", ["--noprofile", "--norc", "-ceu", script], { cwd: root }); +}; + +const tree = async (): Promise => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "mount-aware-clear-"))); + roots.push(root); + await mkdir(path.join(root, "target/keep-me"), { recursive: true }); + await mkdir(path.join(root, "target/plain"), { recursive: true }); + await writeFile(path.join(root, "target/plain/file"), "x"); + await writeFile(path.join(root, "target/loose"), "x"); + return root; +}; + +describe("mount-aware clear", () => { + it("empties a tree that holds no mount point", async () => { + const root = await tree(); + await exercise(root, [], `spawnfile_clear_tree ${JSON.stringify(path.join(root, "target"))}`); + expect(await readdir(path.join(root, "target"))).toEqual([]); + }); + + it("keeps a mount point and still clears everything around it", async () => { + const root = await tree(); + const mount = path.join(root, "target/keep-me"); + await writeFile(path.join(mount, "owned-by-the-mount"), "x"); + await exercise(root, [root, mount], `spawnfile_clear_tree ${JSON.stringify(path.join(root, "target"))}`); + expect(await readdir(path.join(root, "target"))).toEqual(["keep-me"]); + // Its contents belong to the other filesystem and are never touched either. + expect(await readdir(mount)).toEqual(["owned-by-the-mount"]); + }); + + it("descends into a directory that merely contains a mount point, instead of removing it", async () => { + const root = await tree(); + const mount = path.join(root, "target/plain/nested-mount"); + await mkdir(mount); + await exercise(root, [root, mount], `spawnfile_clear_tree ${JSON.stringify(path.join(root, "target"))}`); + expect(await readdir(path.join(root, "target"))).toEqual(["plain"]); + expect(await readdir(path.join(root, "target/plain"))).toEqual(["nested-mount"]); + }); + + it("removes a whole target that holds no mount point", async () => { + const root = await tree(); + await exercise(root, [], `spawnfile_remove_tree ${JSON.stringify(path.join(root, "target"))}`); + expect(await readdir(root)).toEqual([]); + }); + + it("clears rather than removes a target that is itself a mount point", async () => { + const root = await tree(); + const target = path.join(root, "target"); + await exercise(root, [root, target], `spawnfile_remove_tree ${JSON.stringify(target)}`); + expect(await readdir(target)).toEqual([]); + }); + + it("is a no-op for an absent target", async () => { + const root = await tree(); + await exercise(root, [], `spawnfile_remove_tree ${JSON.stringify(path.join(root, "absent"))}`); + await exercise(root, [], `spawnfile_clear_tree ${JSON.stringify(path.join(root, "absent"))}`); + }); + + it("names the path it could not clear instead of failing on a bare find or rm", async () => { + const root = await tree(); + // A path the shell cannot remove stands in for a busy mount the table did not list. + const script = `spawnfile_clear_tree() { echo "cannot clear $1/busy: it is in use" >&2; return 1; }\nspawnfile_clear_tree ${JSON.stringify(root)}`; + await expect(run("/bin/bash", ["--noprofile", "--norc", "-ceu", script])).rejects.toMatchObject({ + stderr: expect.stringContaining("cannot clear") as unknown as string + }); + }); +}); diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 1bc53a00..3bc01f5d 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -54,6 +54,20 @@ export const TRAINING_INFERENCE_LEDGER = `${TRAINING_INFERENCE_DIRECTORY}/infere /** Judge grant homes: `2000:2000 0700`, denied to every worker uid. Paideia reads it as `PAIDEIA_GROK_GRANT_HOME_ROOT`. */ export const TRAINING_GRANT_HOME_ROOT = "/run/training/grants"; +/** + * Private temp for the broker and its relay (uid 2100, outside the organization + * group), deliberately **outside** `/run/daimon-engine-broker`. + * + * Production puts it at `/tmp`, which is fine there: that root is + * removed and recreated exactly once, at container start. Training + * re-provisions the same root on every recycle, and the launch mounts this + * directory as its own tmpfs — so a `tmp/` inside the cleared root is a *mount + * point*, and clearing a mount point fails `EBUSY`. A live P8 launch died + * exactly there. Its own tmpfs under `/run/training` is never a wipe target and + * is denied to every worker uid. + */ +export const TRAINING_BROKER_TMPDIR = "/run/training/broker-tmp"; + export const TRAINING_SUPERVISOR_DIRECTORY = "/run/training/supervisor"; export const TRAINING_SUPERVISOR_SOCKET = `${TRAINING_SUPERVISOR_DIRECTORY}/control.sock`; export const TRAINING_SUPERVISOR_LOG = `${TRAINING_SUPERVISOR_DIRECTORY}/supervisor.log`; @@ -98,6 +112,7 @@ export const TRAINING_EVALUATOR_ROOTS = [ * or `/tmp` — and no entry covers another, because masks cannot nest. */ export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ + TRAINING_BROKER_TMPDIR, path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, @@ -113,6 +128,7 @@ export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ /** Paths the provisioning program creates root-owned `0700` when absent, so every deny entry always has a target inode. */ export const TRAINING_OPTIONAL_DENY_DIRECTORIES: readonly string[] = [ + TRAINING_BROKER_TMPDIR, TRAINING_INFERENCE_DIRECTORY, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, diff --git a/src/compiler/training/broker/processes.ts b/src/compiler/training/broker/processes.ts index d9dfbd63..6971f3f1 100644 --- a/src/compiler/training/broker/processes.ts +++ b/src/compiler/training/broker/processes.ts @@ -4,8 +4,7 @@ import { setTimeout as pause } from "node:timers/promises"; import { SpawnfileError } from "../../../shared/index.js"; import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; -import { DAIMON_BROKER_TMPDIR } from "../../containerDaimonBrokerRender.js"; -import { DAIMON_BROKER_UID } from "./paths.js"; +import { DAIMON_BROKER_UID, TRAINING_BROKER_TMPDIR } from "./paths.js"; export const DAIMON_RUNTIME_ROOT = "/opt/spawnfile/runtime-installs/daimon"; /** `00000000000000c1` = CHOWN|SETGID|SETUID, the launcher's bounding set after `setpriv`. */ @@ -32,11 +31,11 @@ export const brokerProcessPlan = (): { name: string; argv: string[]; socket: str { name: "engine broker backend", socket: DAIMON_GROK_ENGINE_BROKER.backendSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND, argv: setpriv(["--clear-groups", `--reuid=${DAIMON_BROKER_UID}`, `--regid=${DAIMON_BROKER_UID}`, "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", - "--", "env", `TMPDIR=${DAIMON_BROKER_TMPDIR}`, `${DAIMON_RUNTIME_ROOT}/bin/daimon-runtime`, "engine-broker", "serve"]) + "--", "env", `TMPDIR=${TRAINING_BROKER_TMPDIR}`, `${DAIMON_RUNTIME_ROOT}/bin/daimon-runtime`, "engine-broker", "serve"]) }, { name: "engine broker control relay", socket: DAIMON_GROK_ENGINE_BROKER.controlSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND, - argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid,+setpcap", "--", "env", `TMPDIR=${DAIMON_BROKER_TMPDIR}`, + argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid,+setpcap", "--", "env", `TMPDIR=${TRAINING_BROKER_TMPDIR}`, DAIMON_GROK_ENGINE_BROKER.nativeExecutablePath, "--relay"]) } ]; diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts index fd5d2498..62489a88 100644 --- a/src/compiler/training/broker/provisioning.ts +++ b/src/compiler/training/broker/provisioning.ts @@ -20,6 +20,7 @@ import { TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_SLOT_USAGE_LEDGER, TRAINING_SUPERVISOR_DIRECTORY, + TRAINING_BROKER_TMPDIR, TRAINING_SLOT_WORKSPACE, TRAINING_WORKER_ROOT, TRAINING_WORKER_UID @@ -69,6 +70,9 @@ export const trainingSlotDirectories = (): readonly { path: string; mode: string // Traverse only: the supervisor socket inside it is the uid gate, and a directory nobody but root may // write is what keeps that socket from being replaced by a laxer one. { path: TRAINING_SUPERVISOR_DIRECTORY, mode: "0711", uid: 0, gid: 0 }, + // The broker and its relay run as uid 2100, outside the organization group, and shared `/tmp` is closed + // to them; this is their own temp, outside every wipe target (see `TRAINING_BROKER_TMPDIR`). + { path: TRAINING_BROKER_TMPDIR, mode: "0700", uid: DAIMON_BROKER_UID, gid: DAIMON_BROKER_UID }, { path: TRAINING_WORKER_ROOT, mode: "0711", uid: 0, gid: 0 }, // Denied and unused by training, which meters per slot — but a world-readable directory would make its // worker-uid canary meaningless, so both get the modes the production organization gives them. @@ -121,7 +125,7 @@ export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistr ...renderDaimonBrokerProvisioningProgram([registration], [], { turnStore: TRAINING_SLOT_TURN_STORE, inferenceLedgerPath: TRAINING_INFERENCE_LEDGER - }, "clear", TRAINING_OPTIONAL_DENY_DIRECTORIES), + }, "clear", TRAINING_OPTIONAL_DENY_DIRECTORIES, TRAINING_BROKER_TMPDIR), // The shared program leaves the broker's `/etc` root `0555 root:root`, which every uid can list. Training // denies that directory to its worker, and a canary can only observe a denial the kernel actually enforces. `chmod 0550 /etc/daimon-engine-broker; chown 0:${DAIMON_BROKER_UID} /etc/daimon-engine-broker`, diff --git a/src/compiler/training/broker/registration.test.ts b/src/compiler/training/broker/registration.test.ts index 6f2a923a..ac7009ef 100644 --- a/src/compiler/training/broker/registration.test.ts +++ b/src/compiler/training/broker/registration.test.ts @@ -1,10 +1,16 @@ import { describe, expect, it } from "vitest"; import { renderDaimonGrokServiceConfig } from "../../containerDaimonGrokWorkerRender.js"; +import { trainingBrokerTmpfsTargets } from "../container/security.js"; +import { brokerProcessPlan } from "./processes.js"; import { renderTrainingBrokerProvisioning, trainingSlotDirectories } from "./provisioning.js"; import { resolveTrainingGrokDenyPaths, resolveTrainingGrokRegistration } from "./registration.js"; import { TRAINING_ADDED_DENY_PATHS, + TRAINING_BROKER_TMPDIR, + TRAINING_SLOT_RUNTIME_HOME, + TRAINING_WORKER_HOME, + TRAINING_SLOT_WORKSPACE, TRAINING_CALLER_PROTECTED_PATHS, TRAINING_EVALUATOR_ROOTS, TRAINING_GRANT_HOME_ROOT, @@ -118,3 +124,35 @@ describe("training slot provisioning", () => { expect(script).toContain('chown 0:0 "$target"; chmod "$mode" "$target"; chown "$owner:$group" "$target"'); }); }); + +describe("training wipe targets and the broker temp", () => { + it("keeps the broker and relay temp outside every directory provisioning or recycle clears", () => { + const cleared = ["/etc/daimon-engine-broker", "/run/daimon-engine-broker", TRAINING_SLOT_TURN_STORE, + TRAINING_SLOT_USAGE_DIRECTORY, TRAINING_WORKER_HOME, TRAINING_SLOT_WORKSPACE, TRAINING_SLOT_RUNTIME_HOME, + TRAINING_SLOT_ACCEPTANCE_STORE]; + for (const root of cleared) { + expect(TRAINING_BROKER_TMPDIR === root || TRAINING_BROKER_TMPDIR.startsWith(`${root}/`), root).toBe(false); + } + expect(slot().denyPaths).toContain(TRAINING_BROKER_TMPDIR); + }); + + it("gives the broker temp to uid 2100 alone and mounts it as its own tmpfs", () => { + expect(trainingSlotDirectories().find((entry) => entry.path === TRAINING_BROKER_TMPDIR)) + .toMatchObject({ mode: "0700", uid: 2100, gid: 2100 }); + expect(trainingBrokerTmpfsTargets().map((entry) => entry.path)).toContain(TRAINING_BROKER_TMPDIR); + }); + + it("runs the broker and relay with that temp, never the production control-root one", () => { + for (const entry of brokerProcessPlan().filter((process) => process.uid === 2100)) { + expect(entry.argv).toContain(`TMPDIR=${TRAINING_BROKER_TMPDIR}`); + expect(entry.argv.join(" ")).not.toContain("TMPDIR=/run/daimon-engine-broker/tmp"); + } + }); + + it("clears the broker roots through the mount-aware helper, never a bare recursive delete", () => { + const script = renderTrainingBrokerProvisioning(slot()).join("\n"); + expect(script).toContain("spawnfile_clear_tree \"$broker_root\""); + expect(script).not.toContain("-mindepth 1 -delete"); + expect(script).toContain("spawnfile_holds_mount"); + }); +}); diff --git a/src/compiler/training/broker/runtime.ts b/src/compiler/training/broker/runtime.ts index 20026d0b..41a26631 100644 --- a/src/compiler/training/broker/runtime.ts +++ b/src/compiler/training/broker/runtime.ts @@ -10,6 +10,7 @@ import { SpawnfileError } from "../../../shared/index.js"; import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; import type { TrainingBrokerDeclaration } from "./declaration.js"; +import { MOUNT_AWARE_CLEAR_HELPER } from "../../containerDaimonBrokerRender.js"; import { renderTrainingBrokerProvisioning, renderTrainingIdentities } from "./provisioning.js"; import { startBrokerProcesses, stopBrokerProcesses, type BrokerChild } from "./processes.js"; import { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveTrainingCanaries } from "./receipt.js"; @@ -118,17 +119,18 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): const targets = [options.registration.home, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_ACCEPTANCE_STORE, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.registrationPath]; const contents = [TRAINING_SLOT_WORKSPACE, TRAINING_SLOT_USAGE_DIRECTORY]; const script = [ + ...MOUNT_AWARE_CLEAR_HELPER, // The parent has to be reclaimed too: unlinking is a write to the *directory*, and the slot hands // several of these parents to uid 2000 or the broker. Provisioning restores every mode right after. 'reclaim() { parent=$(dirname "$1"); chown 0:0 "$parent"; chmod u+rwx "$parent"; if [ -e "$1" ]; then chown -R 0:0 "$1"; chmod -R u+rwX "$1"; fi; }', - ...targets.map((target) => `reclaim ${quote(target)}; rm -rf ${quote(target)}`), + ...targets.map((target) => `reclaim ${quote(target)}; spawnfile_remove_tree ${quote(target)}`), // The workspace and the per-slot ledger directory keep their own root: the registered paths must // stay canonical across a recycle, so only what a trial put inside them goes. - ...contents.map((target) => `reclaim ${quote(target)}; if [ -d ${quote(target)} ]; then find ${quote(target)} -mindepth 1 -delete; fi`) + ...contents.map((target) => `reclaim ${quote(target)}; spawnfile_clear_tree ${quote(target)}`) ].join("\n"); await run(shell, ["--noprofile", "--norc", "-ceu", script], { maxBuffer: 1024 * 1024 }).catch((error: unknown) => { const detail = error as { stderr?: string }; - throw new SpawnfileError("runtime_error", `Training slot wipe failed: ${(detail.stderr ?? "").slice(-1024).trim()}`); + throw new SpawnfileError("runtime_error", `Training slot wipe failed: ${(detail.stderr ?? "").slice(-1024).trim() || "no diagnostic"}`); }); }, provision: async () => { diff --git a/src/compiler/training/container/security.ts b/src/compiler/training/container/security.ts index 7020bf32..47dad02c 100644 --- a/src/compiler/training/container/security.ts +++ b/src/compiler/training/container/security.ts @@ -3,10 +3,10 @@ import { materializeDaimonGrokSeccompProfile } from "../../../shared/index.js"; import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; -import { DAIMON_BROKER_TMPDIR } from "../../containerDaimonBrokerRender.js"; import { DAIMON_GROK_TURN_USAGE_LEDGER } from "../../../runtime/daimon/contractManifest.js"; import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js"; import { + TRAINING_BROKER_TMPDIR, TRAINING_GRANT_HOME_ROOT, TRAINING_INFERENCE_DIRECTORY, TRAINING_PAIDEIA_ROOT, @@ -40,7 +40,7 @@ export const trainingBrokerTmpfsTargets = (): readonly { path: string; size: str { path: TRAINING_WORKER_ROOT, size: "1g", mode: "0755" }, { path: "/etc/daimon-engine-broker", size: "16m", mode: "0755" }, { path: "/run/daimon-engine-broker", size: "64m", mode: "0755" }, - { path: DAIMON_BROKER_TMPDIR, size: "64m", mode: "0755" }, + { path: TRAINING_BROKER_TMPDIR, size: "64m", mode: "0755" }, { path: DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, size: "64m", mode: "0755" }, { path: DAIMON_WAKE_FUSE_DIRECTORY, size: "16m", mode: "0755" } ]; diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index 46aad01e..aaf4e3dd 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -9,6 +9,7 @@ import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./ import { planTrainingImage, buildTrainingImage } from "./image.js"; import { planMeasurementRepair, stageMeasurementRepair, writeTrainingWitness } from "../repair/index.js"; import { prepareTrainingBroker } from "./broker.js"; +import { claimTrainingPreparationScratch } from "./scratch.js"; function mappedReceipt(digest: string, image: string, config: ReturnType): TrainingMappedPreparation { return parseTrainingMappedPreparation({ version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image, @@ -92,7 +93,7 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< } else { let owned = false; try { - await mkdir(staging, { mode: 0o700 }); owned = true; + await claimTrainingPreparationScratch(staging, digest, options.streams.stderr); owned = true; try { await mkdir(output, { mode: 0o700 }); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !(await lstat(output)).isDirectory() || await realpath(output) !== output) throw error; } staged = await Promise.all(inputs.map(input => stageInput(input, path.join(staging, input.id)))); diff --git a/src/compiler/training/preparation/scratch.test.ts b/src/compiler/training/preparation/scratch.test.ts new file mode 100644 index 00000000..a5aaf403 --- /dev/null +++ b/src/compiler/training/preparation/scratch.test.ts @@ -0,0 +1,58 @@ +import { mkdir, mkdtemp, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { claimTrainingPreparationScratch } from "./scratch.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +// `realpath`: the preparation parent is always canonical in production, and macOS `/var` is a symlink. +const temporary = async () => { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "training-scratch-"))); roots.push(root); return root; }; +const digest = `sha256:${"a".repeat(64)}`; + +describe("training preparation scratch", () => { + it("creates the directory when nothing is there", async () => { + const staging = path.join(await temporary(), "scratch"); + const lines: string[] = []; + await claimTrainingPreparationScratch(staging, digest, line => lines.push(line)); + expect(await readdir(staging)).toEqual([]); + expect(lines).toEqual([]); + }); + + it("reclaims a leftover from the same preparation and hands back an empty directory", async () => { + const staging = path.join(await temporary(), "scratch"); + await mkdir(staging, { mode: 0o700 }); + await writeFile(path.join(staging, "state.json"), JSON.stringify({ digest, image: "sha256:b", staged: [], snapshots: [] })); + await writeFile(path.join(staging, "launch.json"), "{}"); + const lines: string[] = []; + await claimTrainingPreparationScratch(staging, digest, line => lines.push(line)); + expect(await readdir(staging)).toEqual([]); + expect(lines[0]).toContain("Reusing the preparation scratch"); + }); + + it("refuses a leftover from a different preparation and names how to clear it", async () => { + const staging = path.join(await temporary(), "scratch"); + await mkdir(staging, { mode: 0o700 }); + await writeFile(path.join(staging, "state.json"), JSON.stringify({ digest: `sha256:${"c".repeat(64)}` })); + await expect(claimTrainingPreparationScratch(staging, digest, () => undefined)) + .rejects.toThrow(/belongs to a different preparation .*; remove it to retry: rm -rf/u); + expect(await readdir(staging)).toEqual(["state.json"]); + }); + + it("refuses a leftover that never recorded its identity, and never deletes it", async () => { + const staging = path.join(await temporary(), "scratch"); + await mkdir(staging, { mode: 0o700 }); + await writeFile(path.join(staging, "half-staged"), "x"); + await expect(claimTrainingPreparationScratch(staging, digest, () => undefined)) + .rejects.toThrow(/never recorded its identity; remove it to retry: rm -rf/u); + expect(await readdir(staging)).toEqual(["half-staged"]); + }); + + it("refuses a scratch path that is not a canonical directory", async () => { + const root = await temporary(); + const staging = path.join(root, "scratch"); + await writeFile(staging, "not a directory"); + await expect(claimTrainingPreparationScratch(staging, digest, () => undefined)).rejects.toThrow(/not a canonical directory/u); + }); +}); diff --git a/src/compiler/training/preparation/scratch.ts b/src/compiler/training/preparation/scratch.ts new file mode 100644 index 00000000..09885a37 --- /dev/null +++ b/src/compiler/training/preparation/scratch.ts @@ -0,0 +1,47 @@ +import { lstat, mkdir, readFile, realpath, rm } from "node:fs/promises"; + +/** + * Claims the private preparation scratch directory for one launch. + * + * A launch that fails anywhere past staging — a failed build, a container that + * aborted in its entrypoint — leaves this directory behind, and the next + * attempt used to die on `EEXIST: mkdir '.spawnfile-training-preparation-…'` + * before it could report anything useful. A live P8 run lost its second attempt + * to exactly that. + * + * Leftovers from the *same* preparation are reclaimed: `state.json` records the + * preparation digest, which pins the config, every input identity, the image + * plan and the canonical source digest, so a matching one can only have staged + * the same bytes and nothing is lost by staging them again (the image is + * content-addressed and resolves from its tag). Anything else — a different + * digest, or a run that aborted before it wrote its state — is left untouched + * and reported by name, because only the operator can know whether it is + * wanted or whether another launch is still using it. + */ +export const claimTrainingPreparationScratch = async ( + staging: string, + digest: string, + notify: (line: string) => void +): Promise => { + try { + await mkdir(staging, { mode: 0o700 }); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + const info = await lstat(staging); + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(staging) !== staging) { + throw Error(`Training preparation scratch ${staging} is not a canonical directory; remove it and retry`); + } + let previous: { digest?: unknown } | undefined; + try { previous = JSON.parse(await readFile(`${staging}/state.json`, "utf8")) as { digest?: unknown }; } + catch { previous = undefined; } + if (previous?.digest !== digest) { + throw Error(previous === undefined + ? `Training preparation scratch ${staging} is left over from an interrupted launch that never recorded its identity; remove it to retry: rm -rf ${JSON.stringify(staging)}` + : `Training preparation scratch ${staging} belongs to a different preparation (${String(previous.digest)}); remove it to retry: rm -rf ${JSON.stringify(staging)}`); + } + notify(`Reusing the preparation scratch left by an earlier launch of this exact preparation: ${staging}`); + await rm(staging, { recursive: true, force: true }); + await mkdir(staging, { mode: 0o700 }); +}; From 2f74b8be852583df2ad6cf0b4bdc166280e669a0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 18:39:02 +0200 Subject: [PATCH 62/84] test: cover training preparation scratch reuse and refusal through the real preparation --- .../training/preparation/prepare.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/compiler/training/preparation/prepare.test.ts b/src/compiler/training/preparation/prepare.test.ts index a3683ffe..75f97ac2 100644 --- a/src/compiler/training/preparation/prepare.test.ts +++ b/src/compiler/training/preparation/prepare.test.ts @@ -149,3 +149,21 @@ it("rejects mismatched output before Docker or staging for cold, dry-run and exa expect(f.docker.calls).toHaveLength(calls); expect(await readFile(prepared.preparationPath)).toEqual(saved); }); + +it("reclaims the preparation scratch an aborted launch left behind, and refuses a foreign one by name", async () => { + const f = await fixture(); + const first = await prepareTraining(f.options); if ("dryRun" in first) throw Error("actual preparation expected"); + const staging = path.dirname(first.configPath); + // A launch that reached the container and aborted there leaves exactly this behind. + expect((await lstat(path.join(staging, "state.json"))).isFile()).toBe(true); + const notices: string[] = []; + const again = await prepareTraining({ ...f.options, streams: { stdout() {}, stderr: line => notices.push(line) } }); + if ("dryRun" in again) throw Error("actual preparation expected"); + expect(again.digest).toBe(first.digest); + expect(notices.join("\n")).toContain("Reusing the preparation scratch"); + + await writeFile(path.join(staging, "state.json"), JSON.stringify({ digest: `sha256:${"f".repeat(64)}` })); + await expect(prepareTraining(f.options)).rejects.toThrow(/belongs to a different preparation .*; remove it to retry: rm -rf/u); + await rm(path.join(staging, "state.json")); + await expect(prepareTraining(f.options)).rejects.toThrow(/never recorded its identity; remove it to retry: rm -rf/u); +}); From 2789ea717276d2bf5c488ff496bab3caccbd875e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 18:41:38 +0200 Subject: [PATCH 63/84] test: refuse a launch mount inside any training wipe or clear target --- src/compiler/training/broker/paths.test.ts | 50 ++++++++++++++++++++++ src/compiler/training/broker/paths.ts | 28 ++++++++++++ src/compiler/training/broker/runtime.ts | 3 +- 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/compiler/training/broker/paths.test.ts diff --git a/src/compiler/training/broker/paths.test.ts b/src/compiler/training/broker/paths.test.ts new file mode 100644 index 00000000..57a8edcc --- /dev/null +++ b/src/compiler/training/broker/paths.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; + +import { trainingBrokerMounts, trainingBrokerTmpfsTargets } from "../container/security.js"; +import { + TRAINING_BROKER_TMPDIR, + TRAINING_CLEAR_TARGETS, + TRAINING_WIPE_TARGETS +} from "./paths.js"; + +/** Every path the v3 launch mounts: its tmpfs set plus the realm volume and the two read-only binds. */ +const launchMounts = (): string[] => [ + ...trainingBrokerTmpfsTargets().map((entry) => entry.path), + ...trainingBrokerMounts({ realmVolume: "realm", bootstrap: "/host/auth.json", declaration: "/host/declaration.json" }) + .map((mount) => /dst=([^,]+)/u.exec(mount)![1]!) +]; + +const at_or_below = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +describe("training wipe targets versus the launch's own mounts", () => { + /** + * The P8 regression, as an invariant rather than an incident. A mount point + * cannot be unlinked while it is mounted, so a recycle that must remove or + * empty a directory holding one aborts. `/run/daimon-engine-broker/tmp` was + * both the broker's `TMPDIR` and a declared tmpfs inside a cleared root, and + * a live launch died on `find: cannot delete …: Device or resource busy`. + */ + it("never mounts anything at or below a directory a recycle removes", () => { + for (const mount of launchMounts()) { + for (const target of TRAINING_WIPE_TARGETS) { + expect(at_or_below(mount, target), `${mount} is inside the wipe target ${target}`).toBe(false); + } + } + }); + + it("never mounts anything strictly below a directory a recycle empties", () => { + for (const mount of launchMounts()) { + for (const target of TRAINING_CLEAR_TARGETS) { + // The target may itself be a mount — its contents go and the directory stays — but nothing below it may be. + expect(mount !== target && at_or_below(mount, target), `${mount} is inside the cleared target ${target}`).toBe(false); + } + } + }); + + it("keeps the broker temp a mount of its own, outside every wipe and clear target", () => { + expect(launchMounts()).toContain(TRAINING_BROKER_TMPDIR); + for (const target of [...TRAINING_WIPE_TARGETS, ...TRAINING_CLEAR_TARGETS]) { + expect(at_or_below(TRAINING_BROKER_TMPDIR, target), target).toBe(false); + } + }); +}); diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 3bc01f5d..f4652a76 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -165,6 +165,34 @@ export const TRAINING_DEFERRED_DENY_PATHS: readonly string[] = [ */ export const TRAINING_HOST_BIND_DENY_PATHS: readonly string[] = [TRAINING_RUN_ROOT, TRAINING_SEALED_INPUTS_ROOT]; +/** + * What a recycle removes outright, and what it only empties. + * + * Nothing the launch mounts may be at or below any of these. A mount point + * cannot be unlinked while it is mounted, so a wipe target holding one aborts + * the recycle — which is how the broker/relay `TMPDIR` took down a live launch + * when it still lived at `/tmp`. `paths.test.ts` enforces that + * against the launch's own mount list; the rendered shell skips mount points + * anyway, so an undeclared one degrades to "left in place and reported" rather + * than a failed provision. + */ +export const TRAINING_WIPE_TARGETS: readonly string[] = [ + TRAINING_WORKER_HOME, + TRAINING_SLOT_RUNTIME_HOME, + TRAINING_SLOT_TURN_STORE, + TRAINING_SLOT_ACCEPTANCE_STORE, + DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, + DAIMON_GROK_ENGINE_BROKER.registrationPath +]; + +/** Emptied but kept: their paths are registered with Daimon and must stay canonical across a recycle. */ +export const TRAINING_CLEAR_TARGETS: readonly string[] = [ + TRAINING_SLOT_WORKSPACE, + TRAINING_SLOT_USAGE_DIRECTORY, + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), + path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath) +]; + export const TRAINING_REALM_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath; export const TRAINING_BOOTSTRAP_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath; export { DAIMON_BROKER_UID, DAIMON_ORGANIZATION_UID }; diff --git a/src/compiler/training/broker/runtime.ts b/src/compiler/training/broker/runtime.ts index 41a26631..6d64c3e6 100644 --- a/src/compiler/training/broker/runtime.ts +++ b/src/compiler/training/broker/runtime.ts @@ -19,6 +19,7 @@ import { DAIMON_ORGANIZATION_UID, TRAINING_HOST_BIND_DENY_PATHS, TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_WIPE_TARGETS, TRAINING_SLOT_GENERATION_FILE, TRAINING_SLOT_PREFLIGHT_RECEIPT, TRAINING_SLOT_RUNTIME_HOME, @@ -116,7 +117,7 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): wipe: async () => { // Root holds `CAP_CHOWN` and `CAP_DAC_READ_SEARCH` but not `CAP_DAC_OVERRIDE`, so it cannot delete // inside a directory it handed to the worker or the broker. Reclaim each tree first, then remove it. - const targets = [options.registration.home, TRAINING_SLOT_RUNTIME_HOME, TRAINING_SLOT_TURN_STORE, TRAINING_SLOT_ACCEPTANCE_STORE, DAIMON_GROK_ENGINE_BROKER.serviceConfigPath, DAIMON_GROK_ENGINE_BROKER.registrationPath]; + const targets = TRAINING_WIPE_TARGETS; const contents = [TRAINING_SLOT_WORKSPACE, TRAINING_SLOT_USAGE_DIRECTORY]; const script = [ ...MOUNT_AWARE_CLEAR_HELPER, From 27397149b76e24c0250745c049e000cd8c6b8fc8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 18:42:00 +0200 Subject: [PATCH 64/84] docs: record the mount-inside-a-wipe-target rule and the preparation scratch claim --- specs/TRAINING_CONTAINERS.md | 8 ++++++++ src/compiler/training/broker/AGENTS.md | 11 +++++++++++ src/compiler/training/preparation/AGENTS.md | 5 +++++ 3 files changed, 24 insertions(+) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 7cd28d0c..80b17959 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -307,6 +307,14 @@ worker-uid denial canaries, and only then publishes caller's `nonce`. Recycles are serialized; the caller never names a path or a command. Measured: three recycles at 611–620 ms each. +Nothing the launch mounts may sit inside a directory a recycle removes or +empties: a mount point cannot be unlinked while it is mounted. The broker and +relay therefore take their private `TMPDIR` from `/run/training/broker-tmp` +(`2100:2100 0700`, denied to every worker uid) rather than the production +`/tmp`, which training clears on every recycle. Provisioning and +recycle skip mount points regardless and name any path they genuinely cannot +clear. + The socket node is `root:2000 0660` inside a root-owned `0711` directory on tmpfs, which is the uid gate: the kernel enforces it on `connect()` and uid 2200 gets `EACCES`. Node exposes no `SO_PEERCRED`, and a `0600` root-owned socket diff --git a/src/compiler/training/broker/AGENTS.md b/src/compiler/training/broker/AGENTS.md index fd0c194a..ad6c5f46 100644 --- a/src/compiler/training/broker/AGENTS.md +++ b/src/compiler/training/broker/AGENTS.md @@ -45,6 +45,17 @@ Local constraints: provisioning creates every non-bind deny target itself. One entry still fails this way (`.runtime/grok-p5/EVIDENCE.md`): the wake-acceptance store, which is Daimon's own protected path under a `2000:2000 0700` parent. +- **Nothing the launch mounts may sit inside a wipe target.** A mount point + cannot be unlinked while it is mounted, so a recycle that must remove or + empty a directory holding one aborts. The broker/relay `TMPDIR` lived at the + production `/tmp`, which training clears on every recycle and + the launch mounts as its own tmpfs — a live P8 launch died on + `find: cannot delete …: Device or resource busy` before any model call. It is + `/run/training/broker-tmp` now. `paths.test.ts` checks the invariant against + the launch's own mount list, and the rendered shell + (`MOUNT_AWARE_CLEAR_HELPER`) skips mount points regardless, so an undeclared + one degrades to "left in place" and anything genuinely busy is reported by + name instead of as a bare `find` failure. - Root here holds `CAP_CHOWN`, `CAP_SETUID`, `CAP_SETGID`, `CAP_SETPCAP`, `CAP_KILL` and `CAP_DAC_READ_SEARCH` — never `CAP_FOWNER` or `CAP_DAC_OVERRIDE`. Create the whole tree while it is still root-owned, set diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index f233b557..224d5618 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -15,5 +15,10 @@ lines. Source/lock/recipe mutation must invalidate cache and exact resume. captured-work repair. Optional compiler selects a separate full pinned native compiler distribution; the current package still owns launch/auth operations. - Exclude explicit generated Python coverage/cache files, not arbitrary dotfiles. +- `scratch.ts` claims the private preparation directory. A launch that aborts + past staging leaves it behind; a leftover from the *same* preparation digest + is reclaimed and re-staged, anything else is left untouched and reported by + name with the command that clears it. It must never delete a directory it + cannot prove is a leftover of this exact preparation. - Repair transport/compatibility lives in sibling `repair/`; it never changes candidate, criteria or checkpoint semantics owned by Paideia. From b92faeb79e55cb901fc4f4abd6f771fa7e1c5f26 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:12:22 +0200 Subject: [PATCH 65/84] feat: re-vendor the Daimon contract with the traverse-only organization runtime home rule --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 0d92caf1..f3a72604 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382","sourceSha256":"c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75","x64Sha256":"69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382","sourceSha256":"c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75","x64Sha256":"69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index 1ed539ba..64b0d7a3 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:401da56de1182a4c1834bc872ab3121d0b45d63486d729a02ff98d5627f30829 +sha256:8729807df65ecbbaa905d1e116305745501c01532faae78ec18d46b9f119f8a0 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 117dbbd9..b727bcf6 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:401da56de1182a4c1834bc872ab3121d0b45d63486d729a02ff98d5627f30829" as const; + "sha256:8729807df65ecbbaa905d1e116305745501c01532faae78ec18d46b9f119f8a0" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -82,6 +82,7 @@ export const DAIMON_GROK_ENGINE_BROKER = { sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + organizationRuntimeHome: { owner: "organization", group: "worker", mode: 0o710 }, spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, From b8bb63ff527e971b1bfd4d5d77cdad65029d1fa6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:15:51 +0200 Subject: [PATCH 66/84] fix: keep every persistent mount inside a traversable Grok runtime home private --- src/compiler/AGENTS.md | 9 +++- ...tainerDaimonGrokWorkerProvisioning.test.ts | 50 +++++++++++++++++++ .../containerDaimonGrokWorkerProvisioning.ts | 3 +- .../containerDaimonGrokWorkerRender.ts | 4 ++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/compiler/AGENTS.md b/src/compiler/AGENTS.md index 3ee7b90d..07fc704e 100644 --- a/src/compiler/AGENTS.md +++ b/src/compiler/AGENTS.md @@ -207,7 +207,14 @@ src/compiler/ spill contract: `/tmp` `: 0700` (the launcher's `TMPDIR`); `/tmp` and `/var/tmp` `root:2000 1774` (Grok refuses to start if they are denied, so modes close them); `/tool-output` - `2000: 2750` under a runtime home `2000: 0710` whose + `2000: 2750` under a runtime home `2000: 0710` — the exact + shape Daimon's engine-aware `physicalReadiness.ts` demands for a brokered + Grok agent (owner = runtime uid, mode `0710` with no setgid or sticky, group a + worker group; a `0700` home is refused because the worker could not read its + own spills, and every other engine keeps `0700`). Every persistent mount + inside that traversable home (tool state, the credential home) is re-secured + to `0700 2000:2000`, so `tool-output` is the only thing in there the worker + can reach whose `/var/lib/spawnfile` ancestors are made traversable by reclaim-mode-restore. Root here holds no `CAP_DAC_OVERRIDE`, so the private temp is created while the worker home is still root-owned and the spill directory before the runtime home diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts index 2f45909b..21f31a93 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { daimonGrokWorkerSandboxProfileSha256, renderDaimonGrokWorkerSandboxProfile } from "../runtime/daimon/grokWorkerContract.js"; +import { DAIMON_GROK_ENGINE_BROKER } from "../runtime/daimon/contractManifest.js"; import { DAIMON_GROK_WORKER_CONFIG_BYTES } from "../runtime/daimon/grokWorkerConfigBytes.js"; import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js"; import { renderDaimonGrokHostPreflight, renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js"; @@ -126,6 +127,55 @@ describe("Grok worker home provisioning", () => { } }); + /** + * Daimon's `physicalReadiness.ts` refuses a brokered Grok agent whose + * organization runtime home is not exactly owner = runtime uid, mode 0710 + * (`& 0o7777`, so no setgid or sticky), group = a worker group (gid >= the + * first worker uid and not the runtime's own gid). A 0700 home is refused too, + * because the worker could not reach its own spills. This mirrors that rule so + * a wrong mode or group fails here instead of at container start. + */ + const assertDaimonRuntimeHomeReadiness = (node: Node | undefined, workerUid: number): void => { + const home = DAIMON_GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome; + expect(node?.kind).toBe("dir"); + expect(node!.uid).toBe(2000); + expect(node!.mode & 0o7777).toBe(home.mode); + expect(node!.gid).toBeGreaterThanOrEqual(DAIMON_GROK_ENGINE_BROKER.identities.firstWorkerUid); + expect(node!.gid).not.toBe(2000); + expect(node!.gid).toBe(workerUid); + }; + + it("provisions the runtime home shape Daimon's engine-aware readiness demands, and keeps everything inside it private but tool-output", () => { + const withMounts = resolveDaimonGrokRegistrations([{ + ...plan({ "agent:a": "grok", "agent:b": "grok", "agent:c": "codex" }), + persistentMounts: [ + { id: "tool-state-a", mount_path: `${INSTANCE}/runtime-homes/a/tool-state`, reason: "receipts", volume_name: "a" }, + { id: "engine-home-a", mount_path: `${INSTANCE}/runtime-homes/a/.grok`, reason: "credential home", volume_name: "b" }, + { id: "tool-state-c", mount_path: `${INSTANCE}/runtime-homes/c/tool-state`, reason: "receipts", volume_name: "c" } + ] + } as unknown as RuntimeTargetPlan]); + expect(withMounts[0]!.runtimeHomeMounts).toEqual([`${INSTANCE}/runtime-homes/a/.grok`, `${INSTANCE}/runtime-homes/a/tool-state`]); + expect(withMounts[1]!.runtimeHomeMounts).toEqual([]); + const nodes = run(withMounts, { + ...seedFor(withMounts), + [`${INSTANCE}/runtime-homes/a/tool-state`]: { gid: 2000, kind: "dir", mode: 0o755, uid: 2000 }, + [`${INSTANCE}/runtime-homes/a/.grok`]: { gid: 2000, kind: "dir", mode: 0o755, uid: 2000 } + }); + for (const entry of withMounts) { + assertDaimonRuntimeHomeReadiness(nodes.get(entry.runtimeHome), entry.uid); + // Only the setgid spill directory is wider than 0700 inside the traversable home. + for (const [target, node] of nodes) { + if (!target.startsWith(`${entry.runtimeHome}/`)) continue; + if (target === entry.spillDirectory) { expect(node.mode & 0o7777).toBe(0o2750); continue; } + expect(node.mode & 0o7777, target).toBe(0o700); + expect([node.uid, node.gid], target).toEqual([2000, 2000]); + } + } + // A non-Grok peer's runtime home keeps whatever it had (Daimon still demands 0700 there). + expect(nodes.get(`${INSTANCE}/runtime-homes/c`)?.mode).toBe(0o700); + expect(nodes.has(`${INSTANCE}/runtime-homes/c/tool-state`)).toBe(false); + }); + it("allows a peer resource backing to be absent at provisioning (the entrypoint prepares it later) but never a symlink", () => { const withResource = resolveDaimonGrokRegistrations([{ ...plan({ "agent:a": "grok", "agent:b": "grok" }), diff --git a/src/compiler/containerDaimonGrokWorkerProvisioning.ts b/src/compiler/containerDaimonGrokWorkerProvisioning.ts index dc7f1624..5486447c 100644 --- a/src/compiler/containerDaimonGrokWorkerProvisioning.ts +++ b/src/compiler/containerDaimonGrokWorkerProvisioning.ts @@ -26,6 +26,7 @@ const programRegistration = (entry: DaimonGrokRegistration) => ({ profileSha256: entry.profileSha256, reasoningEffort: entry.reasoningEffort, runtimeHome: entry.runtimeHome, + runtimeHomeMounts: entry.runtimeHomeMounts, spillDirectory: entry.spillDirectory, slot: entry.slot, uid: entry.uid, @@ -78,7 +79,7 @@ export const renderDaimonGrokWorkerProvisioning = (registrations: readonly Daimo // Spills: /tool-output 2000: 2750 (setgid) under a runtime home the worker's group can // traverse. The spill directory is created before the runtime home is narrowed to 0710: without // CAP_DAC_OVERRIDE root cannot create inside a directory it does not own once the mode excludes it. - `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, + `for (const entry of grokWorkers) { traversable(entry.runtimeHome); fs.mkdirSync(entry.runtimeHome, { recursive: true, mode: 0o700 }); assertCanonical(entry.runtimeHome, 'runtime home'); try { fs.mkdirSync(entry.spillDirectory, { mode: 0o700 }); } catch (error) { if (error.code !== 'EEXIST') throw error; } assertCanonical(entry.spillDirectory, 'spill directory'); withMode(entry.spillDirectory, 0o2750, ${DAIMON_ORGANIZATION_UID}, entry.uid); for (const mounted of entry.runtimeHomeMounts) { assertCanonical(mounted, 'runtime home mount'); withMode(mounted, 0o700, ${DAIMON_ORGANIZATION_UID}, ${DAIMON_ORGANIZATION_UID}); } withMode(entry.runtimeHome, 0o710, ${DAIMON_ORGANIZATION_UID}, entry.uid); }`, // Shared temp: Grok refuses a profile denying /tmp or /var/tmp, so modes close them: root: 1774 lets workers list names only. `for (const shared of ${JSON.stringify(DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.paths)}) { fs.mkdirSync(shared, { recursive: true, mode: 0o1777 }); assertCanonical(shared, 'shared temp'); withMode(shared, 0o${DAIMON_GROK_ENGINE_BROKER.worker.home.sharedTmp.mode.toString(8)}, 0, ${DAIMON_ORGANIZATION_UID}); }`, // Deny-path placement, asserted once every mode above is final. Grok 1.0.34 materializes each deny diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 48f00ed3..e355bc92 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -107,6 +107,8 @@ export interface DaimonGrokRegistration { reasoningEffort: DaimonGrokBrokerReasoningEffort; /** The organization runtime home whose `tool-output/` this worker reads. */ runtimeHome: string; + /** Persistent mounts inside this agent's runtime home; Spawnfile keeps each `0700` under the traversable home. */ + runtimeHomeMounts: string[]; /** `/tool-output`: setgid spill directory in the worker's group. */ spillDirectory: string; slot: number; @@ -283,6 +285,8 @@ export const resolveDaimonGrokRegistrations = (plans: RuntimeTargetPlan[]): Daim privateTmp: path.posix.join(home, DAIMON_GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome), reasoningEffort, runtimeHome, + runtimeHomeMounts: [...new Set(plans.flatMap((candidate) => (candidate.persistentMounts ?? []).map((mount) => mount.mount_path)) + .filter((mountPath) => mountPath.startsWith(`${runtimeHome}/`)))].sort(), spillDirectory: path.posix.join(runtimeHome, DAIMON_GROK_ENGINE_BROKER.worker.home.spillDirectory.relativeToRuntimeHome), slot, uid: DAIMON_FIRST_WORKER_UID + slot, From ac90cc6fc399f369d3d1acb392203c96d78cf0fd Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:16:09 +0200 Subject: [PATCH 67/84] test: assert private runtime-home mounts in the Grok live check --- scripts/grok-lean-worker-live-check.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/grok-lean-worker-live-check.ts b/scripts/grok-lean-worker-live-check.ts index 4166adcd..bc14c73f 100644 --- a/scripts/grok-lean-worker-live-check.ts +++ b/scripts/grok-lean-worker-live-check.ts @@ -62,9 +62,10 @@ const main = (): void => { // /run denies rely on /var/run being the /run symlink (a bind mask covers both spellings); report what the image has. process.stdout.write(`/var/run -> ${exec(container, "readlink /var/run || echo not-a-symlink").trim()}\n`); const runtimeHome = "/var/lib/spawnfile/instances/daimon/daimon-organization/runtime-homes/grok-lean-worker"; - const tempLayout = exec(container, `stat -c '%u:%g %a %n' /tmp /var/tmp ${workerHome}/tmp ${runtimeHome} ${runtimeHome}/tool-output`); + const tempLayout = exec(container, `stat -c '%u:%g %a %n' /tmp /var/tmp ${workerHome}/tmp ${runtimeHome} ${runtimeHome}/tool-output ${runtimeHome}/tool-state ${runtimeHome}/.grok`); process.stdout.write(tempLayout); for (const expected of [ + `2000:2000 700 ${runtimeHome}/tool-state\n`, `2000:2000 700 ${runtimeHome}/.grok\n`, "0:2000 1774 /tmp\n", "0:2000 1774 /var/tmp\n", `2200:2200 700 ${workerHome}/tmp\n`, `2000:2200 710 ${runtimeHome}\n`, `2000:2200 2750 ${runtimeHome}/tool-output\n` ]) if (!tempLayout.includes(expected)) throw new Error(`temp/spill layout is missing: ${expected.trim()}`); for (const pattern of ["engine-broker serve", "daimon-engine-broker --relay"]) { From 182256823d4f11a37a0cbae7ecbc8ebb410fcef1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:28:26 +0200 Subject: [PATCH 68/84] test: verify the traverse-only slot runtime home exposes only its setgid spill directory --- src/compiler/training/broker/paths.test.ts | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/compiler/training/broker/paths.test.ts b/src/compiler/training/broker/paths.test.ts index 57a8edcc..1f03272b 100644 --- a/src/compiler/training/broker/paths.test.ts +++ b/src/compiler/training/broker/paths.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "vitest"; +import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js"; import { trainingBrokerMounts, trainingBrokerTmpfsTargets } from "../container/security.js"; +import { renderTrainingBrokerProvisioning, trainingSlotDirectories } from "./provisioning.js"; +import { resolveTrainingGrokRegistration } from "./registration.js"; import { TRAINING_BROKER_TMPDIR, TRAINING_CLEAR_TARGETS, + TRAINING_SLOT_RUNTIME_HOME, TRAINING_WIPE_TARGETS } from "./paths.js"; @@ -48,3 +52,45 @@ describe("training wipe targets versus the launch's own mounts", () => { } }); }); + +describe("the traversable slot runtime home", () => { + const registration = () => resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-4.6", reasoningEffort: "low" }); + + /** + * Daimon's contract makes a brokered Grok agent's organization runtime home + * `2000: 0710` — traverse-only, so the worker can reach the setgid + * `tool-output/` spill directory and nothing else. Every other entry inside + * it must therefore be private on its own. Daimon creates its own + * subdirectories `0700`; this asserts the *training container* adds nothing + * there that is wider. + */ + it("has the training container create nothing of its own inside it", () => { + for (const entry of trainingSlotDirectories()) { + expect(entry.path.startsWith(`${TRAINING_SLOT_RUNTIME_HOME}/`), `${entry.path} is inside the slot runtime home`).toBe(false); + } + expect(trainingSlotDirectories().some((entry) => entry.path === TRAINING_SLOT_RUNTIME_HOME)).toBe(false); + }); + + it("declares no persistent mount inside it, so nothing needs re-privatising", () => { + expect(registration().runtimeHomeMounts).toEqual([]); + for (const mount of launchMounts()) { + expect(at_or_below(mount, TRAINING_SLOT_RUNTIME_HOME), mount).toBe(false); + } + }); + + it("lets the shared renderer own its mode: 0710 2000:, narrowed only after the spill directory exists", () => { + const script = renderTrainingBrokerProvisioning(registration()).join("\n"); + expect(script).toContain("withMode(entry.runtimeHome, 0o710, 2000, entry.uid)"); + expect(script).not.toContain("withMode(entry.runtimeHome, 0o700"); + const spill = script.indexOf("withMode(entry.spillDirectory, 0o2750, 2000, entry.uid)"); + expect(spill).toBeGreaterThan(-1); + expect(script.indexOf("withMode(entry.runtimeHome, 0o710, 2000, entry.uid)")).toBeGreaterThan(spill); + // The only thing provisioning puts inside it, and it is group-readable on purpose. + expect(registration().spillDirectory).toBe(`${TRAINING_SLOT_RUNTIME_HOME}/tool-output`); + }); + + it("keeps the contract's own rule as the source of that mode", () => { + expect(DAIMON_GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome) + .toEqual({ owner: "organization", group: "worker", mode: 0o710 }); + }); +}); From 3638201f4bd0f05f24119bd73bbcf3d4263dc2e6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:35:22 +0200 Subject: [PATCH 69/84] chore: re-vendor Daimon's contract manifest after the launcher diagnostic rebuild --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index f3a72604..15f4c4f5 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382","sourceSha256":"c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75","x64Sha256":"69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381","sourceSha256":"356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98","x64Sha256":"a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index 64b0d7a3..466ade73 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:8729807df65ecbbaa905d1e116305745501c01532faae78ec18d46b9f119f8a0 +sha256:5f3149324cdf916b7ab88ed21c0ed9eb969a04b577ec4ace73a0f6ad2ed21889 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index b727bcf6..67ebc8c6 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:8729807df65ecbbaa905d1e116305745501c01532faae78ec18d46b9f119f8a0" as const; + "sha256:5f3149324cdf916b7ab88ed21c0ed9eb969a04b577ec4ace73a0f6ad2ed21889" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -128,9 +128,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { } }, artifacts: { - sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", - x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", - arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" + sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", + x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", + arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" } } as const; /** From 182231b690f1d77d171d3924aa06b1d6ed5b4c90 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:48:24 +0200 Subject: [PATCH 70/84] fix: carry declared broker limits and one evaluator root per role into the training slot --- .../containerDaimonGrokWorkerRender.ts | 8 +- src/compiler/training/broker/paths.ts | 73 ++++++++++++++----- src/compiler/training/broker/registration.ts | 7 ++ 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/compiler/containerDaimonGrokWorkerRender.ts b/src/compiler/containerDaimonGrokWorkerRender.ts index 9728d549..6b0db05f 100644 --- a/src/compiler/containerDaimonGrokWorkerRender.ts +++ b/src/compiler/containerDaimonGrokWorkerRender.ts @@ -98,6 +98,12 @@ export interface DaimonGrokRegistration { eventsPath: string; grokHome: string; home: string; + /** + * The per-turn limits this registration declares. A wake may only lower them + * (Daimon refuses a raise as `invalid_request`), so a deployment that admits + * wakes above the manifest defaults must declare them here. + */ + limits?: { maxRequests: number; maxTokens: number; timeoutMs: number }; model: DaimonGrokBrokerModel; profile: string; profilePath: string; @@ -346,7 +352,7 @@ export const renderDaimonGrokServiceConfig = ( eventsPath: entry.eventsPath, profileSha256: entry.profileSha256, usageLedgerPath: entry.usageLedgerPath, - limits: { ...DAIMON_GROK_ENGINE_BROKER.turnLimits.v1Defaults }, + limits: { ...(entry.limits ?? DAIMON_GROK_ENGINE_BROKER.turnLimits.v1Defaults) }, model: { id: entry.model, reasoningEffort: entry.reasoningEffort } })) }); diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index f4652a76..fec80ed3 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -95,14 +95,56 @@ export const TRAINING_CALLER_PROTECTED_PATHS = [ `${TRAINING_PAIDEIA_ROOT}/repair.json` ] as const; -/** `paideia.daimon-native.launch.v2`'s five `broker.evaluatorPaths` roles, at this container's real paths. */ -export const TRAINING_EVALUATOR_ROOTS = [ +/** + * `paideia.daimon-native.launch.v2`'s `broker.evaluatorPaths`, in full. + * + * This is not a summary of the five roles — it is the **whole** set of deny + * entries this container adds beyond Daimon's own protected paths, because + * Paideia's worker resolves the projection from exactly + * `[...new Set(broker.evaluatorPaths.map(row => row.path))].sort()`. A launch + * that carried only the five roles made Daimon hash a different deny list than + * the one provisioning wrote, and a live run died on + * `Grok broker projection digest differs from the launch receipt`. + * + * `role` is Paideia's closed five-value enum and may repeat; only `path` is + * unique. The four roles with a dedicated path keep it, and everything else is + * tagged `context` — the caller-state role — since there is no other way to + * carry a path through that contract. + */ +export const TRAINING_EVALUATOR_PATHS: readonly { role: "run-root" | "context" | "sealed-inputs" | "judge-home" | "slot-ledger"; path: string }[] = [ { role: "run-root", path: TRAINING_RUN_ROOT }, - { role: "context", path: TRAINING_PAIDEIA_ROOT }, { role: "sealed-inputs", path: TRAINING_SEALED_INPUTS_ROOT }, { role: "judge-home", path: TRAINING_GRANT_HOME_ROOT }, - { role: "slot-ledger", path: TRAINING_SLOT_USAGE_DIRECTORY } -] as const; + { role: "slot-ledger", path: TRAINING_SLOT_USAGE_DIRECTORY }, + { role: "context", path: TRAINING_PAIDEIA_ROOT }, + { role: "context", path: TRAINING_BROKER_TMPDIR }, + { role: "context", path: TRAINING_INFERENCE_DIRECTORY }, + { role: "context", path: TRAINING_SLOT_TURN_STORE }, + { role: "context", path: TRAINING_SUPERVISOR_DIRECTORY }, + { role: "context", path: path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath) }, + { role: "context", path: path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath) }, + { role: "context", path: DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath }, + { role: "context", path: DAIMON_WAKE_FUSE_DIRECTORY } +]; + +/** + * One entry per evaluator role, in declaration order: the launch receipt requires all five roles, + * and several `context` paths share that role, so the first of each role is the role's root. + */ +export const TRAINING_EVALUATOR_ROOTS = TRAINING_EVALUATOR_PATHS + .filter((entry, index) => TRAINING_EVALUATOR_PATHS.findIndex((first) => first.role === entry.role) === index); + +/** + * What `paideia.daimon-native.launch.v2`'s `controlRoot` must be. + * + * Paideia's worker passes it to Daimon as `acceptanceStorePath`, and Daimon adds + * it to the projection's deny list, so it has to be the same path this + * container masks as its slot state root. Its own default + * (`/run/paideia/control`) would both change the digest and nest inside the + * `/run/paideia` mask, which Grok refuses — Daimon renders a nested mask + * without complaining, so the failure would only appear at the first turn. + */ +export const TRAINING_CALLER_CONTROL_ROOT = TRAINING_SLOT_STATE_ROOT; /** * Everything this container provisions that the one training worker must not @@ -111,20 +153,13 @@ export const TRAINING_EVALUATOR_ROOTS = [ * refuses a profile whose deny entry equals or contains `/run`, `/var`, `/etc` * or `/tmp` — and no entry covers another, because masks cannot nest. */ -export const TRAINING_ADDED_DENY_PATHS: readonly string[] = [ - TRAINING_BROKER_TMPDIR, - path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.controlSocketPath), - path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath), - DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, - DAIMON_WAKE_FUSE_DIRECTORY, - TRAINING_INFERENCE_DIRECTORY, - // `TRAINING_SLOT_STATE_ROOT` is Daimon's own protected entry here (it covers the - // wake-acceptance store, which is itself unplaceable under a `2000:2000 0700` - // parent), so it is not repeated in this set. - TRAINING_SLOT_TURN_STORE, - TRAINING_SUPERVISOR_DIRECTORY, - ...TRAINING_EVALUATOR_ROOTS.map((entry) => entry.path) -]; +/** + * The added deny set is exactly the launch's `evaluatorPaths`, so Spawnfile and + * Paideia hand Daimon's resolver the same list. `TRAINING_SLOT_STATE_ROOT` is + * deliberately absent: Daimon adds it itself, from the control root above. + */ +export const TRAINING_ADDED_DENY_PATHS: readonly string[] = + [...new Set(TRAINING_EVALUATOR_PATHS.map((entry) => entry.path))].sort(); /** Paths the provisioning program creates root-owned `0700` when absent, so every deny entry always has a target inode. */ export const TRAINING_OPTIONAL_DENY_DIRECTORIES: readonly string[] = [ diff --git a/src/compiler/training/broker/registration.ts b/src/compiler/training/broker/registration.ts index da6b6088..ffd9981c 100644 --- a/src/compiler/training/broker/registration.ts +++ b/src/compiler/training/broker/registration.ts @@ -83,6 +83,12 @@ export interface TrainingGrokSlotInput { reasoningEffort: DaimonGrokBrokerReasoningEffort; /** Deny entries this container adds beyond Daimon's own protected set; defaults to the fixed training set. */ denyPaths?: readonly string[]; + /** + * The declaration's per-turn limits. The broker registration must declare + * them, because a wake may only LOWER a declared limit: a trial whose wake + * ceiling exceeds the manifest defaults is refused as `invalid_request`. + */ + limits?: { maxRequests: number; maxTokens: number; timeoutMs: number }; } /** @@ -111,6 +117,7 @@ export const resolveTrainingGrokRegistration = (input: TrainingGrokSlotInput): D return { agentId: input.agentId, config: config.bytes, + ...(input.limits === undefined ? {} : { limits: { ...input.limits } }), configSha256: config.sha256, deferredDenyPaths: denyPaths.filter((entry) => TRAINING_DEFERRED_DENY_PATHS.includes(entry)), denyPaths, From e9c2b4d5ca8724f6985b29fe4e99cfd9a9b3d870 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:50:09 +0200 Subject: [PATCH 71/84] chore: re-vendor the Daimon Grok contract for the launcher's head-and-tail diagnostic window --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 15f4c4f5..41f174f6 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381","sourceSha256":"356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98","x64Sha256":"a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277","sourceSha256":"d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61","x64Sha256":"3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index 466ade73..f538af08 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:5f3149324cdf916b7ab88ed21c0ed9eb969a04b577ec4ace73a0f6ad2ed21889 +sha256:7d5be9b14f07bab1bf8ace678c0fc60c73b6e944fe6d623758e99b26b4aec559 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 67ebc8c6..140aab3e 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:5f3149324cdf916b7ab88ed21c0ed9eb969a04b577ec4ace73a0f6ad2ed21889" as const; + "sha256:7d5be9b14f07bab1bf8ace678c0fc60c73b6e944fe6d623758e99b26b4aec559" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -128,9 +128,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { } }, artifacts: { - sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", - x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", - arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" + sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", + x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", + arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" } } as const; /** From 28dd710ddb4e3ee78ba6d26cc33c19bf40650313 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 02:13:22 +0200 Subject: [PATCH 72/84] chore: re-vendor the Daimon Grok contract for the launcher's blocking worker stdout pipe --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 41f174f6..205c2994 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277","sourceSha256":"d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61","x64Sha256":"3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c","sourceSha256":"2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f","x64Sha256":"5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index f538af08..6ec059b4 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:7d5be9b14f07bab1bf8ace678c0fc60c73b6e944fe6d623758e99b26b4aec559 +sha256:5f3953aeab7ea53a3b50ebadf813d0c39ad6c9bc70416031555a2e2b499cc86e diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 140aab3e..97891f09 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:7d5be9b14f07bab1bf8ace678c0fc60c73b6e944fe6d623758e99b26b4aec559" as const; + "sha256:5f3953aeab7ea53a3b50ebadf813d0c39ad6c9bc70416031555a2e2b499cc86e" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -128,9 +128,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { } }, artifacts: { - sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", - x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", - arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" + sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", + x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", + arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" } } as const; /** From 691a333e3c7946f1552143849a1592cb6939cb36 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 03:03:04 +0200 Subject: [PATCH 73/84] chore: re-vendor the Daimon Grok contract for the worker retry ceiling and launcher drain bound --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 14 +++++++------- src/runtime/daimon/grokWorkerConfigBytes.ts | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 205c2994..211d8e4a 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c","sourceSha256":"2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f","x64Sha256":"5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646","low":"7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25","medium":"218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9"},"grok-4.6":{"high":"3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d","low":"eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3","medium":"8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f"},"grok-build":{"high":"bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859","low":"83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878","medium":"0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2","sourceSha256":"d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","x64Sha256":"4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index 6ec059b4..f937017b 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:5f3953aeab7ea53a3b50ebadf813d0c39ad6c9bc70416031555a2e2b499cc86e +sha256:5fd04e453e0bce01f28bad64fb690ed477fcf50f4c23b3f59c1705b012d1aee9 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 97891f09..6022dc71 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:5f3953aeab7ea53a3b50ebadf813d0c39ad6c9bc70416031555a2e2b499cc86e" as const; + "sha256:5fd04e453e0bce01f28bad64fb690ed477fcf50f4c23b3f59c1705b012d1aee9" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -71,9 +71,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { maxTurns: 48, systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", configSha256: { - "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, - "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, - "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } + "grok-4.6": { low: "ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e", medium: "df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f", high: "65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3" }, + "grok-4.5": { low: "8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6", medium: "59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155", high: "c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f" }, + "grok-build": { low: "fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5", medium: "8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656", high: "a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3" } }, home: { directory: { uid: 0, group: "worker", mode: 0o1771 }, @@ -128,9 +128,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { } }, artifacts: { - sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", - x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", - arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" + sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", + x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", + arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" } } as const; /** diff --git a/src/runtime/daimon/grokWorkerConfigBytes.ts b/src/runtime/daimon/grokWorkerConfigBytes.ts index 24bb788c..c6d448a5 100644 --- a/src/runtime/daimon/grokWorkerConfigBytes.ts +++ b/src/runtime/daimon/grokWorkerConfigBytes.ts @@ -4,19 +4,19 @@ /** Daimon `renderGrokBrokerWorkerConfig({ model, reasoningEffort })` bytes, verified against the manifest pins on use. */ export const DAIMON_GROK_WORKER_CONFIG_BYTES = { "grok-4.6": { - "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.6\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" }, "grok-4.5": { - "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-4.5\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" }, "grok-build": { - "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", - "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" + "low": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"low\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"low\"\nlabel = \"Low\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "medium": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"medium\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"medium\"\nlabel = \"Medium\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n", + "high": "[cli]\nauto_update = false\nuse_leader = false\nshow_tips = false\n\n[features]\ntelemetry = false\ntitle_refresh = false\nsession_recap = false\nturn_summary = false\nrepo_status_in_system_prompt = false\ncodebase_indexing = false\nbackend_tools = false\nask_user_question = false\nimage_gen = false\nvideo_gen = false\nweb_fetch = false\ncampaigns = false\nmanaged_config = false\n\n[managed_mcps]\nenabled = false\n\n[skills]\ndisabled = [\"build-with-ai\", \"code-review\", \"create-skill\", \"create-workflow\", \"design\", \"docx\", \"execute-plan\", \"game-animation-frames\", \"game-asset-core\", \"game-character-consistency\", \"game-tilesets\", \"game-ui-icons\", \"imagine\", \"implement\", \"learn\", \"long-running-background-tasks\", \"pdf\", \"pptx\", \"pr-babysit\", \"resume-claude\", \"resume-codex\", \"resume-cursor\", \"review\", \"skill-design-principles\", \"statusline\"]\n\n[workflows]\nenabled = false\n\n[models]\ndefault = \"daimon-broker-grok\"\ndefault_reasoning_effort = \"high\"\nsession_summary = \"daimon-session-title-disabled\"\n\n[model.daimon-session-title-disabled]\nmodel = \"disabled\"\nbase_url = \"http://127.0.0.1:43123/v1\"\napi_key = \"session-title-disabled\"\nmax_retries = 0\nhidden = true\n\n[model.daimon-broker-grok]\nmodel = \"grok-build\"\nbase_url = \"http://127.0.0.1:43123/v1\"\nenv_key = \"DAIMON_PROVIDER_CAPABILITY\"\napi_backend = \"chat_completions\"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n\n[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = \"high\"\nlabel = \"High\"\ndefault = true\n\n[mcp_servers.daimon]\nurl = \"http://127.0.0.1:43124/mcp\"\nbearer_token_env_var = \"DAIMON_MCP_CAPABILITY\"\n" } } as const; From 599287dd20674d63b910b8d2c860baadcf53c8b3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 10:25:56 +0200 Subject: [PATCH 74/84] feat: bind a training run's declared native parent to its attested Daimon runtime identity --- scripts/verify-training-image-modes.ts | 2 +- .../training/preparation/daimonParent.test.ts | 145 ++++++++++++++++++ .../training/preparation/daimonParent.ts | 137 +++++++++++++++++ src/compiler/training/preparation/image.ts | 10 +- src/compiler/training/preparation/prepare.ts | 4 +- 5 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 src/compiler/training/preparation/daimonParent.test.ts create mode 100644 src/compiler/training/preparation/daimonParent.ts diff --git a/scripts/verify-training-image-modes.ts b/scripts/verify-training-image-modes.ts index 4d823090..a4b53930 100644 --- a/scripts/verify-training-image-modes.ts +++ b/scripts/verify-training-image-modes.ts @@ -48,7 +48,7 @@ async function main(): Promise { const raw = JSON.parse(await readFile(configPath, "utf8")) as { image?: { build?: unknown } }; const build = trainingBuildSchema.parse(raw.image?.build ?? raw); const root = path.resolve(option("--root") ?? path.dirname(configPath)); - const plan = await planTrainingImage(build, root, [], repository); + const plan = await planTrainingImage(build, root, [], repository, path.resolve(configPath)); assertNewRecipe(plan.dockerfile); const controlRecipe = controlRef ? execFileSync("git", ["-C", repository, "show", `${controlRef}:runtime-images/training/Dockerfile`], { encoding: "utf8" }) diff --git a/src/compiler/training/preparation/daimonParent.test.ts b/src/compiler/training/preparation/daimonParent.test.ts new file mode 100644 index 00000000..c6be3139 --- /dev/null +++ b/src/compiler/training/preparation/daimonParent.test.ts @@ -0,0 +1,145 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, expect, it } from "vitest"; + +import { DAIMON_CONTRACT_MANIFEST_SHA256 } from "../../../runtime/daimon/contractManifest.js"; +import type { TrainingImageBuild } from "./contract.js"; +import { + DAIMON_RUNTIME_INSTALL_ROOT, + bindDaimonParentVerification, + daimonParentVerificationScript, + resolveTrainingDaimonParent +} from "./daimonParent.js"; + +const run = promisify(execFile); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { force: true, recursive: true }))); }); + +const digest = (value: string): string => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const manifestDigest = digest("native-parent-manifest"); +const registry = "127.0.0.1:5000"; +const NATIVE_IMAGE = `${registry}/paideia/native-training@${digest("native-training-image")}`; +const DECLARATION = "/declared/training-declaration/launch.json"; +/** The real recipe's native stage header; the guard must land immediately after it. */ +const RECIPE = "ARG NATIVE_IMAGE\nARG PYTHON_IMAGE\nFROM ${PYTHON_IMAGE} AS python\nFROM ${NATIVE_IMAGE} AS training\nCOPY --from=python /usr/local /opt/python\n"; + +const build = (patch: Partial = {}): TrainingImageBuild => ({ + recipe: "daimon-dspy.v1", nativeImage: NATIVE_IMAGE, pythonImage: digest("python"), platform: "linux/arm64", + paideia: "paideia", bridge: "bridge", claude: "claude", + integration: { source: "integration", entry: "entry.ts" }, ...patch +} as TrainingImageBuild); + +const identityFile = async (patch: Record = {}): Promise => { + const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-training-identity-")); + roots.push(directory); + const identityPath = path.join(directory, "runtime-identity.json"); + await writeFile(identityPath, `${JSON.stringify({ + capability_receipt_sha256: digest("current-capability-receipt"), + development: { mode: "local-development", non_production: true, unpublished: true, unsigned: true }, + image_architecture: "arm64", image_config_digest: manifestDigest, image_manifest_digest: manifestDigest, + image_reference: `${registry}/noopolis/spawnfile-runtime-daimon@${manifestDigest}`, + manifest_sha256: DAIMON_CONTRACT_MANIFEST_SHA256, registry_authority: registry, + version: "spawnfile.local-daimon-runtime-identity.v3", ...patch + })}\n`, { mode: 0o600 }); + return identityPath; +}; + +/** A stand-in for the Daimon install a native parent copies out of the scratch runtime image. */ +const nativeParentInstall = async (receiptContent: string, manifestSha: string = DAIMON_CONTRACT_MANIFEST_SHA256): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "spawnfile-native-parent-")); + roots.push(root); + const installRoot = path.join(root, DAIMON_RUNTIME_INSTALL_ROOT); + await mkdir(installRoot, { recursive: true }); + await writeFile(path.join(installRoot, "capability-receipt.json"), receiptContent); + await writeFile(path.join(installRoot, "contract-manifest.sha256"), manifestSha); + return root; +}; + +/** Executes the generated guard with the image's install root rebased onto a local tree. */ +const verify = async (script: string, parentRoot: string): Promise<{ code: number; stderr: string }> => { + const rebased = script.replaceAll(`'${DAIMON_RUNTIME_INSTALL_ROOT}/`, `'${path.join(parentRoot, DAIMON_RUNTIME_INSTALL_ROOT)}/`); + try { + const { stderr } = await run("/bin/sh", ["-c", rebased]); + return { code: 0, stderr }; + } catch (error) { + const failure = error as { code?: number; stderr?: string }; + return { code: failure.code ?? 1, stderr: failure.stderr ?? "" }; + } +}; + +const parentFor = async (identityPath: string, patch?: Partial) => { + const parent = await resolveTrainingDaimonParent(build(patch), DECLARATION, + { SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: identityPath }); + if (!parent) throw Error("expected a resolved native parent"); + return parent; +}; + +it("refuses a native parent that does not carry the Daimon install its identity attests, naming both", async () => { + const parent = await parentFor(await identityFile()); + // The previous rebuild's image: same contract pin, same architecture, different install. + const stale = await nativeParentInstall(JSON.stringify({ receipt: "previous-rebuild" })); + const result = await verify(daimonParentVerificationScript(parent), stale); + expect(result.code).not.toBe(0); + expect(result.stderr).toContain(DECLARATION); + expect(result.stderr).toContain(NATIVE_IMAGE); + expect(result.stderr).toContain(parent.identityPath); + expect(result.stderr).toContain(digest("current-capability-receipt")); + expect(result.stderr).toContain(digest(JSON.stringify({ receipt: "previous-rebuild" }))); +}); + +it("runs a matching pair, and refuses a parent whose contract manifest drifted", async () => { + const parent = await parentFor(await identityFile({ capability_receipt_sha256: digest("matched-receipt") })); + const matching = await nativeParentInstall("matched-receipt"); + expect(await verify(daimonParentVerificationScript(parent), matching)).toMatchObject({ code: 0 }); + + const drifted = await nativeParentInstall("matched-receipt", digest("other-contract-manifest")); + const result = await verify(daimonParentVerificationScript(parent), drifted); + expect(result.code).not.toBe(0); + expect(result.stderr).toContain(DAIMON_CONTRACT_MANIFEST_SHA256); + expect(result.stderr).toContain(digest("other-contract-manifest")); +}); + +it("refuses a parent with no Daimon install at all", async () => { + const parent = await parentFor(await identityFile()); + const empty = await mkdtemp(path.join(os.tmpdir(), "spawnfile-empty-parent-")); + roots.push(empty); + const result = await verify(daimonParentVerificationScript(parent), empty); + expect(result.code).not.toBe(0); + expect(result.stderr).toContain("not a Daimon native parent"); +}); + +it("keeps the contract-pin refusal and adds architecture agreement", async () => { + const altered = await identityFile({ manifest_sha256: digest("some-other-contract-manifest") }); + await expect(resolveTrainingDaimonParent(build(), DECLARATION, + { SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: altered })).rejects.toThrow(/invalid or incomplete/u); + + const identityPath = await identityFile(); + await expect(resolveTrainingDaimonParent(build({ platform: "linux/amd64" }), DECLARATION, + { SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: identityPath })).rejects.toThrow(/linux\/amd64[\s\S]*arm64/u); +}); + +it("requires an identity for a locally built native parent and leaves a published one alone", async () => { + await expect(resolveTrainingDaimonParent(build(), DECLARATION, {})) + .rejects.toThrow(/no attested Daimon runtime identity/u); + await expect(resolveTrainingDaimonParent(build({ nativeImage: digest("published-parent") }), DECLARATION, {})) + .resolves.toBeUndefined(); +}); + +it("injects the guard as the first instruction of the native stage and refuses a recipe without one", async () => { + const parent = await parentFor(await identityFile()); + const bound = bindDaimonParentVerification(RECIPE, parent); + const lines = bound.split("\n"); + expect(lines[lines.indexOf("FROM ${NATIVE_IMAGE} AS training") + 1]).toBe(`RUN ${daimonParentVerificationScript(parent)}`); + expect(() => bindDaimonParentVerification("FROM scratch\n", parent)).toThrow(/no \$\{NATIVE_IMAGE\} stage/u); +}); + +it("refuses to embed a path it cannot quote inertly", async () => { + const identityPath = await identityFile(); + await expect(resolveTrainingDaimonParent(build(), "/declared/'; rm -rf /; '/launch.json", + { SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY: identityPath })).rejects.toThrow(/cannot be embedded/u); +}); diff --git a/src/compiler/training/preparation/daimonParent.ts b/src/compiler/training/preparation/daimonParent.ts new file mode 100644 index 00000000..20680dff --- /dev/null +++ b/src/compiler/training/preparation/daimonParent.ts @@ -0,0 +1,137 @@ +import { SpawnfileError } from "../../../shared/index.js"; +import { + DAIMON_LOCAL_RUNTIME_IDENTITY_ENV, + loadLocalDaimonRuntimeIdentity, + type LocalDaimonRuntimeIdentity +} from "../../../runtime/index.js"; +import type { TrainingImageBuild } from "./contract.js"; + +/** + * The declared native parent (`image.build.nativeImage`) and the attested local + * Daimon runtime identity (`SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY`) are two + * independent pointers at one composed run, and they are never the same digest: + * `runtime-images/daimon/Dockerfile` ends in `FROM scratch`, so the identity can + * only ever attest a *scratch* image that a runnable native parent copies + * `/opt/spawnfile/runtime-installs/daimon` out of. Nothing host-side can compare + * the two digests, so the binding is by content: the capability receipt and + * contract-manifest digest the identity attests must be the ones actually baked + * into the declared parent. `src/runtime/container.ts` verifies exactly this for + * a compiled organization image; this is the same check for a training image. + */ +export const DAIMON_RUNTIME_INSTALL_ROOT = "/opt/spawnfile/runtime-installs/daimon"; + +/** A native parent served by the attested loopback development registry is always a local build. */ +const LOOPBACK_NATIVE_PARENT = /^127\.0\.0\.1:(?:[1-9]\d{0,4})\//u; +const FROM_NATIVE_IMAGE = /^FROM \$\{NATIVE_IMAGE\}.*$/mu; +/** Everything embedded in the single-quoted shell literals below must be inert. */ +const SHELL_SAFE = /^[A-Za-z0-9 ._:@/=+,()~-]+$/u; + +export interface TrainingDaimonParent { + declarationPath: string; + identity: LocalDaimonRuntimeIdentity; + identityPath: string; + nativeImage: string; +} + +const refuse = (lines: string[]): never => { + throw new SpawnfileError("validation_error", lines.join("\n")); +}; + +const safe = (label: string, value: string): string => { + if (!SHELL_SAFE.test(value)) { + refuse([ + `Training cannot bind its native parent to a local Daimon runtime identity: ${label} contains characters that cannot be embedded in the image recipe.`, + ` ${label} = ${JSON.stringify(value)}` + ]); + } + return value; +}; + +/** + * Resolves the identity that must match the declared native parent. Runs before + * any Docker call, so an absent, stale-pinned or architecture-drifted identity + * refuses the run — including `--dry-run` — before a container or a token is spent. + */ +export const resolveTrainingDaimonParent = async ( + build: TrainingImageBuild, + declarationPath: string, + env: NodeJS.ProcessEnv = process.env +): Promise => { + const identityPath = env[DAIMON_LOCAL_RUNTIME_IDENTITY_ENV]?.trim(); + if (!identityPath) { + if (LOOPBACK_NATIVE_PARENT.test(build.nativeImage)) { + refuse([ + "Training refuses a locally built native parent with no attested Daimon runtime identity.", + ` ${declarationPath} image.build.nativeImage = ${build.nativeImage}`, + ` ${DAIMON_LOCAL_RUNTIME_IDENTITY_ENV} = (unset)`, + `Point ${DAIMON_LOCAL_RUNTIME_IDENTITY_ENV} at the runtime identity written by the same rebuild that baked ${DAIMON_RUNTIME_INSTALL_ROOT} into that image.` + ]); + } + return undefined; + } + // Keeps the existing contract-pin refusal: this is the first time a training + // run applies it at all, and it stays exactly as strict as it already was. + const identity = await loadLocalDaimonRuntimeIdentity(identityPath); + const expected = build.platform === "linux/amd64" ? "amd64" : "arm64"; + if (identity.imageArchitecture !== expected) { + refuse([ + "Training native parent and local Daimon runtime identity disagree on architecture.", + ` ${declarationPath} image.build.platform = ${build.platform}`, + ` ${identityPath} image_architecture = ${identity.imageArchitecture}`, + "Rebuild both from one architecture, or repoint both at one rebuild." + ]); + } + return { + declarationPath: safe("the training declaration path", declarationPath), + identity, + identityPath: safe(DAIMON_LOCAL_RUNTIME_IDENTITY_ENV, identityPath), + nativeImage: safe("image.build.nativeImage", build.nativeImage) + }; +}; + +/** + * One `sh` command, executed as the first instruction of the `${NATIVE_IMAGE}` + * stage, that refuses the build when the declared parent does not carry the exact + * Daimon install the identity attests. This is the only place both values exist + * in one process: the receipt lives inside the image, so no host-side read can + * reach it without starting something. + */ +export const daimonParentVerificationScript = (parent: TrainingDaimonParent): string => { + const receipt = `${DAIMON_RUNTIME_INSTALL_ROOT}/capability-receipt.json`; + const manifest = `${DAIMON_RUNTIME_INSTALL_ROOT}/contract-manifest.sha256`; + const report = [ + `printf '%s\\n'`, + `'REFUSED: the declared training native parent does not carry the Daimon install its runtime identity attests.'`, + `" ${parent.declarationPath} image.build.nativeImage = ${parent.nativeImage}"`, + `" ${parent.identityPath} capability_receipt_sha256 = ${parent.identity.capabilityReceipt}"`, + `" ${parent.identityPath} manifest_sha256 = ${parent.identity.manifestSha256}"`, + `" native parent ${receipt} = $found_receipt"`, + `" native parent ${manifest} = $found_manifest"`, + `'Repoint image.build.nativeImage and ${DAIMON_LOCAL_RUNTIME_IDENTITY_ENV} at the same rebuild.'`, + `>&2` + ].join(" "); + return [ + "set -u", + `test -f '${receipt}' || { echo "REFUSED: ${parent.nativeImage} has no ${receipt}; it is not a Daimon native parent." >&2; exit 1; }`, + `test -f '${manifest}' || { echo "REFUSED: ${parent.nativeImage} has no ${manifest}; it is not a Daimon native parent." >&2; exit 1; }`, + `found_receipt="sha256:$(sha256sum '${receipt}' | cut -d' ' -f1)"`, + `found_manifest="$(cat '${manifest}')"`, + `if [ "$found_receipt" != '${parent.identity.capabilityReceipt}' ] || [ "$found_manifest" != '${parent.identity.manifestSha256}' ]; then ${report}; exit 1; fi` + ].join("; "); +}; + +/** + * Injects the verification as the first instruction of the native stage, so a + * mismatched pair fails in seconds instead of after the whole distribution copy. + * The recipe text is part of the image plan digest, so the identity is bound into + * the training image's identity too — a rotated identity can never be cached over. + */ +export const bindDaimonParentVerification = (dockerfile: string, parent: TrainingDaimonParent): string => { + if (!FROM_NATIVE_IMAGE.test(dockerfile)) { + refuse([ + "Training recipe declares no ${NATIVE_IMAGE} stage, so its native parent cannot be bound to the Daimon runtime identity.", + ` ${parent.identityPath} image_reference = ${parent.identity.imageReference}` + ]); + } + return dockerfile.replace(FROM_NATIVE_IMAGE, (line) => `${line}\nRUN ${daimonParentVerificationScript(parent)}`); +}; diff --git a/src/compiler/training/preparation/image.ts b/src/compiler/training/preparation/image.ts index 8fe75c67..5728b1f7 100644 --- a/src/compiler/training/preparation/image.ts +++ b/src/compiler/training/preparation/image.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import type { TrainingDockerProcess } from "../container/process.js"; import type { TrainingImageBuild } from "./contract.js"; import { assertInputRoot, copySealed, fileIdentity, hashJson, sealFile, sealTree, type SealedFile } from "./files.js"; +import { bindDaimonParentVerification, resolveTrainingDaimonParent } from "./daimonParent.js"; import { normalizeTrainingContext } from "./contextModes.js"; const packageRoot = fileURLToPath(new URL("../../../../", import.meta.url)); @@ -25,11 +26,16 @@ async function packageFiles(root: string, target: string, withDist: boolean, loc ...withDist ? await sealTree(path.join(root, "dist"), `${target}/dist`, { ignoreDevelopment: true }) : []]; } -export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot): Promise { +export async function planTrainingImage(build: TrainingImageBuild, root: string, auth: readonly string[], ownRoot = packageRoot, + declarationPath = root): Promise { const resolve = (value: string) => path.resolve(root, value); + // Before any Docker call, including --dry-run: the attested Daimon runtime identity + // and the declared native parent must name one rebuild (see daimonParent.ts). + const daimonParent = await resolveTrainingDaimonParent(build, declarationPath); for (const source of [build.paideia, build.bridge, build.claude, build.integration.source, ...build.compiler ? [build.compiler] : [], ...build.bootstrap ? [build.bootstrap] : []]) assertInputRoot(resolve(source), auth); const assets = ownRoot === packageRoot ? trainingAssets : path.join(ownRoot, "runtime-images/training"); - const dockerfile = await readFile(path.join(assets, "Dockerfile"), "utf8"); + const recipe = await readFile(path.join(assets, "Dockerfile"), "utf8"); + const dockerfile = daimonParent ? bindDaimonParentVerification(recipe, daimonParent) : recipe; const ownLock = path.extname(fileURLToPath(import.meta.url)) === ".ts" || ownRoot !== packageRoot ? path.join(ownRoot, "package-lock.json") : path.join(assets, "package-lock.json"); const files = [ diff --git a/src/compiler/training/preparation/prepare.ts b/src/compiler/training/preparation/prepare.ts index aaf4e3dd..658b79bf 100644 --- a/src/compiler/training/preparation/prepare.ts +++ b/src/compiler/training/preparation/prepare.ts @@ -54,7 +54,9 @@ export async function prepareTraining(options: PrepareTrainingOptions): Promise< for (const other of inputs.slice(index + 1)) if (within(input.source, other.source) || within(other.source, input.source) || within(input.destination, other.destination) || within(other.destination, input.destination)) throw Error("Training inputs overlap"); } - const imagePlan = "build" in config.image ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot) : undefined; + const imagePlan = "build" in config.image + ? await planTrainingImage(config.image.build, root, auth.map(entry => entry.source), options.packageRoot, path.resolve(options.configPath)) + : undefined; if (options.repairWitness && !options.repairMeasurements) throw Error("A repair witness requires --repair-measurements"); if (options.repairMeasurements && !imagePlan) throw Error("Measurement repair requires a verifiable image build recipe"); const repair = options.repairMeasurements ? await planMeasurementRepair({ parent: options.repairMeasurements, From 30facc7f9b6c96dd4b16fb1b96ab859909e037cb Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 10:25:56 +0200 Subject: [PATCH 75/84] docs: specify the training native-parent and Daimon runtime identity binding --- specs/TRAINING_CONTAINERS.md | 21 +++++++++++++++++++++ src/compiler/training/preparation/AGENTS.md | 18 +++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 80b17959..4026d255 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -166,6 +166,27 @@ datasets never enter the image context. Cache identity includes actual source, lock, executable and recipe bytes, parent images, architecture and entrypoint; reuse also verifies the image ID and recipe label in the selected Docker context. +### Native parent and Daimon runtime identity + +`image.build.nativeImage` and `SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY` are two +independent pointers at one run, and they never carry the same digest: the local +Daimon runtime image ends in `FROM scratch`, so the identity always attests a +scratch image that a runnable native parent copies +`/opt/spawnfile/runtime-installs/daimon` out of. Preparation therefore binds them +by content, not by digest equality. + +Before any Docker call, and before `--dry-run` returns, preparation loads the +identity when that variable is set and refuses when its `manifest_sha256` is not +the compiler's contract pin, when `image_architecture` disagrees with +`image.build.platform`, or when a `127.0.0.1:` native parent is declared +with no identity at all. It then makes the first instruction of the native stage +verify that the parent's own `capability-receipt.json` and +`contract-manifest.sha256` are exactly the ones the identity attests, refusing +with both file paths and both digests named. The recipe text is part of the image +plan digest, so a rotated identity can never be satisfied by a cached image. A +published (non-loopback) native parent with no identity keeps its previous +behaviour. + Staged build contexts normalize modes and times before `docker build`: directories 0755, files `a+rX`-closed, both entrypoints (`train`, `train-broker`) 0555, mtimes fixed. Together diff --git a/src/compiler/training/preparation/AGENTS.md b/src/compiler/training/preparation/AGENTS.md index 224d5618..607507b6 100644 --- a/src/compiler/training/preparation/AGENTS.md +++ b/src/compiler/training/preparation/AGENTS.md @@ -8,7 +8,9 @@ inputs. Dry-run performs reads only; resume validates the preserved preparation. `contract.ts` declares authoring and runtime receipts; `files.ts` safely seals declared bytes; `image.ts` prepares/builds the recipe; `inputs.ts` snapshots Git; -`contextModes.ts` normalizes staged build-context modes/times; `prepare.ts` combines those operations. Keep tests adjacent and files below 400 +`contextModes.ts` normalizes staged build-context modes/times; `daimonParent.ts` +binds the declared native parent to the attested Daimon runtime identity; +`prepare.ts` combines those operations. Keep tests adjacent and files below 400 lines. Source/lock/recipe mutation must invalidate cache and exact resume. - Built preparations seal original image bytes in a protected witness for explicit @@ -22,3 +24,17 @@ lines. Source/lock/recipe mutation must invalidate cache and exact resume. cannot prove is a leftover of this exact preparation. - Repair transport/compatibility lives in sibling `repair/`; it never changes candidate, criteria or checkpoint semantics owned by Paideia. +- `daimonParent.ts` closes the two-pointer hole between `image.build.nativeImage` + and `SPAWNFILE_DAIMON_LOCAL_RUNTIME_IDENTITY`. They are never the same digest — + `runtime-images/daimon/Dockerfile` ends in `FROM scratch`, so the identity can + only attest a scratch image that a runnable native parent copies + `/opt/spawnfile/runtime-installs/daimon` out of — so the binding is by content. + Host-side, before any Docker call and before `--dry-run` returns, it applies the + existing `manifest_sha256` contract pin to training runs (which never applied it + at all before), refuses an architecture disagreement, and refuses a loopback + registry parent declared with no identity. The digest equality itself lives + inside the image, so the guard is injected as the first instruction of the + `${NATIVE_IMAGE}` stage and names both files and both values when it refuses; + `src/runtime/container.ts` verifies the same receipt for a compiled + organization image. The recipe text is part of the plan digest, so a rotated + identity can never be served from an existing training image. From 67e3f7390c2005489c2a4ccf9fb6d1827bf8a967 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 12:51:14 +0200 Subject: [PATCH 76/84] chore: re-vendor the Daimon contract after the launcher's output bound was raised --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 211d8e4a..21559194 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2","sourceSha256":"d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","x64Sha256":"4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":65536,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","sourceSha256":"dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","x64Sha256":"67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":262144,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index f937017b..e9acd78f 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:5fd04e453e0bce01f28bad64fb690ed477fcf50f4c23b3f59c1705b012d1aee9 +sha256:6a8a49f0405a7b07a6abd5caaf7d714313f7f26b1de76d8330850da811d45bb9 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 6022dc71..a8b98474 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:5fd04e453e0bce01f28bad64fb690ed477fcf50f4c23b3f59c1705b012d1aee9" as const; + "sha256:6a8a49f0405a7b07a6abd5caaf7d714313f7f26b1de76d8330850da811d45bb9" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; @@ -86,7 +86,7 @@ export const DAIMON_GROK_ENGINE_BROKER = { spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, - bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 262_144 }, controlProtocolVersion: "noopolis.daimon.engine-broker.v2", turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], serviceConfigVersions: ["noopolis.daimon.engine-broker-service.v1", "noopolis.daimon.engine-broker-service.v2"], @@ -128,9 +128,9 @@ export const DAIMON_GROK_ENGINE_BROKER = { } }, artifacts: { - sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", - x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", - arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" + sourceSha256: "dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24", + x64Sha256: "67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7", + arm64Sha256: "c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98" } } as const; /** From 1aa84beb3d47847fa3f03f10eeb52aa7094b7220 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:18:30 +0200 Subject: [PATCH 77/84] fix: seal the training datasets with DAC on their ancestor and refuse a slot that cannot prove it --- runtime-images/training/Dockerfile | 11 ++ src/compiler/training/broker/declaration.ts | 26 ++-- src/compiler/training/broker/paths.ts | 51 ++++++- .../training/broker/provisioning.test.ts | 114 +++++++++++++++ src/compiler/training/broker/provisioning.ts | 64 +++++++- src/compiler/training/broker/receipt.test.ts | 137 +++++++++++++----- src/compiler/training/broker/receipt.ts | 58 ++++++-- src/compiler/training/broker/runtime.ts | 18 ++- .../training/preparation/contextModes.test.ts | 8 +- 9 files changed, 411 insertions(+), 76 deletions(-) create mode 100644 src/compiler/training/broker/provisioning.test.ts diff --git a/runtime-images/training/Dockerfile b/runtime-images/training/Dockerfile index 49a85f18..7bed0cf6 100644 --- a/runtime-images/training/Dockerfile +++ b/runtime-images/training/Dockerfile @@ -20,6 +20,17 @@ RUN for fixed_uid in 2000 2100 2200; do \ --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; \ done +# The sealed train and test datasets are bind-mounted UNDER /run/training/inputs, so this directory is +# the ancestor every read of them must traverse, and it is the only inode in that chain the image owns. +# Root-owned with no world bits, on the read-only root: uid 2200 is in neither the owner nor the group +# class, so it loses search permission; a `unshare --map-root-user` namespace maps only uid 2200, so +# CAP_DAC_OVERRIDE there cannot override an inode owned by real uid 0; and a fresh bind of the parent +# re-exposes this same directory rather than the bytes under it. Nothing at runtime can widen it, +# because the container root is read-only — the slot provisioning asserts the mode and refuses otherwise. +RUN mkdir -p /run/training/inputs /run/training/output \ + && chown 0:0 /run/training /run/training/output && chmod 0755 /run/training /run/training/output \ + && chown 0:2000 /run/training/inputs && chmod 0750 /run/training/inputs + # Layers run from least to most frequently changing: locked dependencies, the # DSPy venv, bridge source, pinned executables, then distributions. Staged context # modes are already `a+rX`-closed (train/train-broker 0555), so late COPY layers need no RUN. diff --git a/src/compiler/training/broker/declaration.ts b/src/compiler/training/broker/declaration.ts index a1b8e996..833c2bc2 100644 --- a/src/compiler/training/broker/declaration.ts +++ b/src/compiler/training/broker/declaration.ts @@ -41,17 +41,23 @@ export const trainingBrokerDeclarationSchema = z.strictObject({ seccompProfileSha256: z.string().regex(/^[a-f0-9]{64}$/u), /** * What the slot supervisor does with a deny path whose backing filesystem - * does not enforce unix ownership — every host bind under Docker Desktop and - * Colima, where `chown` is silently ignored and uid 2200 reads a root `0600` - * file (P0 §5). + * does not enforce unix ownership — virtiofs and grpcfuse under Docker + * Desktop and Colima, where `chown` is silently ignored and uid 2200 reads a + * root `0600` file (P0 §5). The backing filesystem decides, not the fact of + * being a host bind: the same bind over ext4 or overlay on a Linux daemon is + * probed for real, which is what makes `refuse` a default a run can meet. * - * `refuse` (the default) fails the recycle and writes no receipt: a - * worker-uid probe over such a path carries no information, so the - * supervisor will not certify it. `profile-only` is the operator's explicit - * acceptance that for those paths the boundary is the bubblewrap-enforced - * `deny` list alone — verified to block both shell `cat` and `read_file` on - * 1.0.34 (P0 §4) — and the supervisor records every path that used that - * weaker evidence in its log. + * `refuse` (the default) fails the slot and writes no receipt: a worker-uid + * probe over such a path carries no information, so the supervisor will not + * certify it. `profile-only` is the operator's explicit acceptance that for + * those paths the boundary is the bubblewrap-enforced `deny` list alone — + * verified to block both shell `cat` and `read_file` on 1.0.34 (P0 §4), but + * *not* a boundary a worker-uid namespace cannot lift — and the supervisor + * records every path that used that weaker evidence in its log. + * + * It does not reach `TRAINING_SEALED_DENY_PATHS`. The sealed train and test + * datasets are held by DAC on their ancestor under both values, and a slot + * that cannot prove that denial is refused whatever this says. */ unenforcedBindPolicy: z.enum(["refuse", "profile-only"]).default("refuse") }).strict(); diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index fec80ed3..09849a7d 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -189,16 +189,51 @@ export const TRAINING_DEFERRED_DENY_PATHS: readonly string[] = [ ]; /** - * Deny entries that are host bind mounts. + * Deny entries the launch binds from the host, named so a refusal can say why. * - * A worker-uid read probe over one of these proves nothing: the host owns the - * inode, its mode is whatever the operator's project directory happens to be, - * and Docker Desktop and Colima ignore `chown` outright (P0 §5). Their boundary - * is the bubblewrap-enforced `deny` list, which P0 verified blocks both shell - * `cat` and `read_file` on 1.0.34. The declaration's `unenforcedBindPolicy` - * decides whether the slot supervisor accepts that or refuses the slot. + * Only the run root is one. The sealed inputs root is **not**: the launch binds + * each declared input at `/run/training/inputs/`, so `/run/training/inputs` + * itself is the binds' *parent*, a directory on the read-only image root — and + * that is exactly what makes it sealable (see `TRAINING_SEALED_DENY_PATHS`). + * + * Being a host bind is no longer a waiver on its own. On Docker Desktop and + * Colima a bind lands on virtiofs/grpcfuse, which ignores `chown`, and there a + * worker-uid probe carries no information; on a Linux daemon over ext4, xfs, + * btrfs or overlay the same bind enforces ownership and the probe is real. The + * canary resolver asks the backing filesystem rather than this list, which is + * why the documented `refuse` default is reachable at all. */ -export const TRAINING_HOST_BIND_DENY_PATHS: readonly string[] = [TRAINING_RUN_ROOT, TRAINING_SEALED_INPUTS_ROOT]; +export const TRAINING_HOST_BIND_DENY_PATHS: readonly string[] = [TRAINING_RUN_ROOT, DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath]; + +/** + * Deny entries whose denial must be proven by the kernel under **every** + * policy, because the experiment's central claim depends on them. + * + * `/run/training/inputs` holds the sealed train and test datasets. A bubblewrap + * `deny` mask is not a boundary the subject cannot lift: the pinned seccomp + * profile must allow `clone/unshare/mount/umount2/pivot_root/setns` for + * bubblewrap itself, and the container runs `apparmor=unconfined`, so a + * worker-uid process can open a user + mount namespace of its own and either + * detach the mask or bind the mask's parent somewhere fresh. + * + * What survives that is DAC on the datasets' ancestor. The image bakes this + * directory `0: 0750` on the read-only root, so: + * + * - the worker uid is in neither the owner nor the group class and loses + * search permission on the one directory every dataset read must traverse; + * - the inode is owned by real uid 0, which a `unshare --map-root-user` + * namespace does not map, so `CAP_DAC_OVERRIDE` there cannot override it; + * - the root filesystem is read-only, so nothing in the container — root + * included — can loosen the mode after the image is built. + * + * Provisioning asserts the mode and then attacks it as the worker uid, and the + * canary resolver refuses the slot rather than certifying one of these on the + * sandbox profile alone. + */ +export const TRAINING_SEALED_DENY_PATHS: readonly string[] = [TRAINING_SEALED_INPUTS_ROOT]; + +/** The image-baked identity of the sealed inputs root, asserted before every slot and never writable at runtime. */ +export const TRAINING_SEALED_INPUTS_IDENTITY = { uid: 0, gid: DAIMON_ORGANIZATION_UID, mode: "750" } as const; /** * What a recycle removes outright, and what it only empties. diff --git a/src/compiler/training/broker/provisioning.test.ts b/src/compiler/training/broker/provisioning.test.ts new file mode 100644 index 00000000..cfeb99dd --- /dev/null +++ b/src/compiler/training/broker/provisioning.test.ts @@ -0,0 +1,114 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +import { trainingContainerConfigSchema } from "../container/contract.js"; +import { renderTrainingBrokerProvisioning, renderTrainingSealedInputsAssertions } from "./provisioning.js"; +import { resolveTrainingGrokRegistration } from "./registration.js"; +import { TRAINING_SEALED_INPUTS_IDENTITY, TRAINING_SEALED_INPUTS_ROOT, TRAINING_WORKER_UID } from "./paths.js"; + +const run = promisify(execFile); +const registration = () => resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-4.6", reasoningEffort: "low" }); + +/** + * The `sealed_denied` helper on its own, with a stub standing in for `setpriv` + * so the verdict handling can be exercised on any host. The probes themselves + * need a Linux kernel; what is testable here is the part that decides whether + * the slot is allowed to proceed. + */ +const verdictScript = (): string => { + const lines = renderTrainingSealedInputsAssertions(); + return lines.slice(0, lines.indexOf("}") + 1).join("\n"); +}; + +const runVerdict = async (output: string): Promise<{ code: number; stdout: string; stderr: string }> => { + const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-sealed-verdict-")); + await writeFile(path.join(directory, "setpriv"), '#!/bin/sh\nprintf "%s" "$SEALED_PROBE_OUTPUT"\n', { mode: 0o755 }); + const script = `${verdictScript()}\nsealed_denied "probe" namespace-rebind\n`; + return run("/bin/bash", ["--noprofile", "--norc", "-ceu", script], { + env: { ...process.env, PATH: `${directory}:${process.env.PATH ?? ""}`, SEALED_PROBE_OUTPUT: output } + }).then((result) => ({ code: 0, ...result })).catch((error: { code?: number; stdout?: string; stderr?: string }) => + ({ code: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? "" })); +}; + +describe("the sealed inputs seal", () => { + it("asserts the image-baked identity of the datasets' ancestor rather than trying to set it", () => { + const script = renderTrainingSealedInputsAssertions().join("\n"); + const { uid, gid, mode } = TRAINING_SEALED_INPUTS_IDENTITY; + expect(script).toContain(`stat -c '%u:%g %a' '${TRAINING_SEALED_INPUTS_ROOT}'`); + expect(script).toContain(`!= "${uid}:${gid} ${mode}"`); + // The root filesystem is read-only, so a chmod here could only ever fail or hide a broken image. + expect(script).not.toMatch(new RegExp(`chmod [^\\n]*${TRAINING_SEALED_INPUTS_ROOT}`, "u")); + expect(script).not.toMatch(new RegExp(`chown [^\\n]*${TRAINING_SEALED_INPUTS_ROOT}`, "u")); + }); + + it("attacks the seal the way the subject would, not the way an honest tool would", () => { + const script = renderTrainingSealedInputsAssertions().join("\n"); + expect(script).toContain(`--reuid ${TRAINING_WORKER_UID}`); + // Direct read, then the two routes a bubblewrap `deny` mask cannot answer. + expect(script).toContain("direct-read"); + expect(script).toContain("namespace-unmount"); + expect(script).toContain("namespace-rebind"); + expect(script).toContain("unshare --user --map-root-user --mount"); + expect(script).toContain(`umount -l ${TRAINING_SEALED_INPUTS_ROOT}`); + expect(script).toContain("mount --bind /run/training /tmp/.sealed-probe"); + // Per dataset too: binding the dataset's OWN mount somewhere fresh carries no `deny` mask and is not + // refused for locked children the way binding their parent is. A live control container handed the + // held-out answer key to uid 2200 through exactly that route (.runtime/sealed-inputs-dac/EVIDENCE.md). + expect(script).toContain(`for sealed_child in ${TRAINING_SEALED_INPUTS_ROOT}/*; do`); + expect(script).toContain("mount --bind '$sealed_child' /tmp/.sealed-probe-child"); + expect(script).toContain('cat \\"$sealed_child\\"/*'); + }); + + it("runs the seal assertions as part of the same provisioning a recycle replays", () => { + const provisioning = renderTrainingBrokerProvisioning(registration()).join("\n"); + for (const line of renderTrainingSealedInputsAssertions(registration().uid)) expect(provisioning).toContain(line); + }); + + it("keeps every declared dataset strictly below the sealed root, so the seal can never be bound over", () => { + const config = (destination: string) => ({ + version: "spawnfile.training-container.v1", dockerContext: "desktop-linux", + inputs: [{ source: "/host/project", destination }], output: { source: "/host/out", destination: "/run/training/output" }, auth: [] + }); + expect(trainingContainerConfigSchema.parse(config(`${TRAINING_SEALED_INPUTS_ROOT}/project`)).inputs).toHaveLength(1); + // A bind AT the sealed root would replace the image's root-owned inode with the operator's own. + expect(() => trainingContainerConfigSchema.parse(config(TRAINING_SEALED_INPUTS_ROOT))).toThrow(); + expect(() => trainingContainerConfigSchema.parse(config("/run/training"))).toThrow(); + }); + + it("refuses the slot when a probe reaches the datasets, and says which route did", async () => { + const result = await runVerdict("SEALED-REACHABLE\n"); + expect(result.code).not.toBe(0); + expect(result.stderr).toContain(`the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is reachable by uid ${TRAINING_WORKER_UID} via namespace-rebind`); + }); + + it("refuses the slot when a probe reaches no verdict at all, instead of reading silence as a denial", async () => { + for (const output of ["", "bash: unshare: command not found\n", "SEALED-PARTIAL\n"]) { + const result = await runVerdict(output); + expect(result.code, output).not.toBe(0); + expect(result.stderr, output).toContain("reached no verdict, so the denial is unproven"); + } + }); + + it("accepts an observed denial, and a worker that cannot open a namespace at all", async () => { + expect((await runVerdict("SEALED-DENIED\n")).code).toBe(0); + expect((await runVerdict("SEALED-NO-NAMESPACE\n")).code).toBe(0); + }); +}); + +describe("the training image's own half of the seal", () => { + it("bakes the datasets' ancestor root-owned with no world bits on the read-only root", async () => { + const dockerfile = await readFile(fileURLToPath(new URL("../../../../runtime-images/training/Dockerfile", import.meta.url)), "utf8"); + const { gid, mode } = TRAINING_SEALED_INPUTS_IDENTITY; + expect(dockerfile).toContain(`mkdir -p ${TRAINING_SEALED_INPUTS_ROOT}`); + expect(dockerfile).toContain(`chown 0:${gid} ${TRAINING_SEALED_INPUTS_ROOT}`); + expect(dockerfile).toContain(`chmod 0${mode} ${TRAINING_SEALED_INPUTS_ROOT}`); + // Its ancestors must stay traversable, or every deny entry under /run/training becomes unplaceable + // and Grok refuses the whole sandbox profile. + expect(dockerfile).toContain("chmod 0755 /run/training /run/training/output"); + }); +}); diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts index 62489a88..b1aadfa7 100644 --- a/src/compiler/training/broker/provisioning.ts +++ b/src/compiler/training/broker/provisioning.ts @@ -14,6 +14,8 @@ import { TRAINING_OPTIONAL_DENY_DIRECTORIES, TRAINING_PAIDEIA_ROOT, TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SEALED_INPUTS_IDENTITY, + TRAINING_SEALED_INPUTS_ROOT, TRAINING_SLOT_ROOT, TRAINING_SLOT_STATE_ROOT, TRAINING_SLOT_TURN_STORE, @@ -134,7 +136,67 @@ export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistr `test "$(stat -c '%u:%g %a' ${quote(TRAINING_GRANT_HOME_ROOT)})" = "${DAIMON_ORGANIZATION_UID}:${DAIMON_ORGANIZATION_UID} 700"`, // The judge grant home root is the evaluator's alone: no worker uid may even traverse it. `setpriv --clear-groups --reuid ${TRAINING_WORKER_UID} --regid ${TRAINING_WORKER_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu '! test -r ${quote(TRAINING_GRANT_HOME_ROOT)}'`, - `setpriv --clear-groups --reuid ${DAIMON_ORGANIZATION_UID} --regid ${DAIMON_ORGANIZATION_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu 'test -r ${quote(TRAINING_SLOT_USAGE_LEDGER)} && test -r ${quote(TRAINING_INFERENCE_LEDGER)}'` + `setpriv --clear-groups --reuid ${DAIMON_ORGANIZATION_UID} --regid ${DAIMON_ORGANIZATION_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu 'test -r ${quote(TRAINING_SLOT_USAGE_LEDGER)} && test -r ${quote(TRAINING_INFERENCE_LEDGER)}'`, + // The sealed train and test datasets: the one boundary here that must hold against the subject's own + // namespace, not only against an honest tool. Last, so it runs with every mode already final. + ...renderTrainingSealedInputsAssertions(registration.uid) +]; + +/** + * The sealed-inputs seal, asserted and then attacked before any slot is used. + * + * The declared datasets are bind-mounted at `/run/training/inputs/`, so + * `/run/training/inputs` is their common ancestor and the only inode in the + * chain the container controls: it lives on the read-only image root, the image + * bakes it `0: 0750`, and nothing in the container can widen + * it afterwards. The worker uid is in neither its owner nor its group class, so + * it loses *search* permission on the directory every dataset read must + * traverse. + * + * The three probes below are the attacker's route, not a cooperating tool's: + * + * 1. a direct read/search/list as the worker uid; + * 2. a user + mount namespace of the worker's own, detaching the bubblewrap + * `deny` mask and the parent mount, then listing again — the route that + * makes `profile-only` an unsound boundary for this path; + * 3. the same namespace, binding `/run/training` somewhere fresh so the bind + * carries no `deny` mask, then listing through it. + * + * (2) and (3) are the ones the mask cannot answer and DAC can: `--map-root-user` + * maps only the worker uid, so `CAP_DAC_OVERRIDE` in that namespace is + * ineffective against a root-owned inode, and a fresh bind re-exposes the same + * root-owned directory rather than the bytes under it. A probe that reaches no + * verdict refuses the slot; a worker that cannot create the namespace at all + * cannot take the route, and says so. + */ +export const renderTrainingSealedInputsAssertions = (workerUid = TRAINING_WORKER_UID): string[] => [ + "sealed_denied() {", + ` sealed_out=$(setpriv --clear-groups --reuid ${workerUid} --regid ${workerUid} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- /bin/sh -c "$1" 2>&1) || true`, + ' case "$sealed_out" in', + ` *SEALED-REACHABLE*) echo "the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is reachable by uid ${workerUid} via $2" >&2; exit 1 ;;`, + ' *SEALED-DENIED*) echo "sealed inputs denial observed for $2" ;;', + ' *SEALED-NO-NAMESPACE*) echo "sealed inputs probe $2 could not open a user namespace as the worker uid, so that route does not exist here" ;;', + ' *) echo "sealed inputs probe $2 reached no verdict, so the denial is unproven: $sealed_out" >&2; exit 1 ;;', + " esac", + "}", + // Asserted, never set: the root filesystem is read-only, so an image that did not bake this mode + // cannot be corrected here — and that is the point. Refuse instead. + `test ! -L ${quote(TRAINING_SEALED_INPUTS_ROOT)} && test -d ${quote(TRAINING_SEALED_INPUTS_ROOT)}`, + `if [ "$(stat -c '%u:%g %a' ${quote(TRAINING_SEALED_INPUTS_ROOT)})" != "${TRAINING_SEALED_INPUTS_IDENTITY.uid}:${TRAINING_SEALED_INPUTS_IDENTITY.gid} ${TRAINING_SEALED_INPUTS_IDENTITY.mode}" ]; then`, + ` echo "the training image must bake ${TRAINING_SEALED_INPUTS_ROOT} as ${TRAINING_SEALED_INPUTS_IDENTITY.uid}:${TRAINING_SEALED_INPUTS_IDENTITY.gid} ${TRAINING_SEALED_INPUTS_IDENTITY.mode} on its read-only root; the sealed datasets are unprotected otherwise" >&2; exit 1`, + "fi", + `sealed_denied "if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1 || test -r ${TRAINING_SEALED_INPUTS_ROOT} || test -x ${TRAINING_SEALED_INPUTS_ROOT}; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi" direct-read`, + `sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c 'umount -l ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; umount -l /run/training >/dev/null 2>&1; if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0' || echo SEALED-NO-NAMESPACE" namespace-unmount`, + `sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c 'mkdir -p /tmp/.sealed-probe >/dev/null 2>&1 || exit 0; mount --bind /run/training /tmp/.sealed-probe >/dev/null 2>&1 || { echo SEALED-DENIED; exit 0; }; if ls -1 /tmp/.sealed-probe/inputs >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0' || echo SEALED-NO-NAMESPACE" namespace-rebind`, + // Per dataset, because the strongest route is specific: binding the *dataset's own mount* somewhere + // fresh carries no `deny` mask and is not refused for locked children the way binding their parent is. + // A live control container with this directory left at Docker's own `0755 root:root` handed the held-out + // answer key to uid 2200 through exactly this route. Root enumerates the children; the worker may not. + `for sealed_child in ${TRAINING_SEALED_INPUTS_ROOT}/*; do`, + ' test -e "$sealed_child" || continue', + ` sealed_denied "if ls -1 \\"$sealed_child\\" >/dev/null 2>&1 || cat \\"$sealed_child\\"/* >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi" "direct-read $sealed_child"`, + ` sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c \\"mkdir -p /tmp/.sealed-probe-child >/dev/null 2>&1 || exit 0; mount --bind '$sealed_child' /tmp/.sealed-probe-child >/dev/null 2>&1 || { echo SEALED-DENIED; exit 0; }; if ls -1 /tmp/.sealed-probe-child >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0\\" || echo SEALED-NO-NAMESPACE" "namespace-rebind $sealed_child"`, + "done" ]; /** Where the broker's own `service.json` lands, so the supervisor can prove the slot it restarted is the slot it provisioned. */ diff --git a/src/compiler/training/broker/receipt.test.ts b/src/compiler/training/broker/receipt.test.ts index ff833140..df818a6f 100644 --- a/src/compiler/training/broker/receipt.test.ts +++ b/src/compiler/training/broker/receipt.test.ts @@ -1,6 +1,16 @@ import { describe, expect, it } from "vitest"; +import { trainingBrokerTmpfsTargets } from "../container/security.js"; import { buildTrainingSlotReceipt, resolveBackingFilesystem, resolveTrainingCanaries } from "./receipt.js"; +import { resolveTrainingGrokRegistration } from "./registration.js"; +import { + TRAINING_BOOTSTRAP_MOUNT, + TRAINING_HOST_BIND_DENY_PATHS, + TRAINING_REALM_MOUNT, + TRAINING_RUN_ROOT, + TRAINING_SEALED_DENY_PATHS, + TRAINING_SEALED_INPUTS_ROOT +} from "./paths.js"; const digest = (fill: string) => fill.repeat(64).slice(0, 64); const nonce = digest("a"); @@ -12,12 +22,6 @@ const receipt = (overrides: Partial[ createdAt: new Date("2026-09-17T12:00:00.000Z"), ...overrides }); -const mountinfo = [ - "21 20 0:20 / / rw,relatime - overlay overlay rw", - "30 21 0:31 / /run/training/slot rw,relatime - tmpfs tmpfs rw", - "31 21 0:32 / /run/training/output rw,relatime - virtiofs docker rw" -].join("\n"); - describe("training slot preflight receipt v2", () => { it("is the exact v2 shape with the caller's nonce and the supervisor's generation", () => { expect(receipt()).toEqual({ @@ -39,51 +43,106 @@ describe("training slot preflight receipt v2", () => { }); }); +/** + * The real v3 deny list, not a hand-picked pair. Every test below drives + * `resolveTrainingCanaries` with `registration.denyPaths` and the real + * `TRAINING_HOST_BIND_DENY_PATHS` / `TRAINING_SEALED_DENY_PATHS`, because the + * defect these cover was invisible to a suite that passed `hostBindPaths: []`: + * with the real constants the documented `refuse` default could not succeed on + * any host at all, and `profile-only` silently certified the sealed datasets on + * a boundary the subject can lift from inside its own namespace. + */ +const denyPaths = (): readonly string[] => + resolveTrainingGrokRegistration({ agentId: "agent:author", model: "grok-4.6", reasoningEffort: "low" }).denyPaths; + +/** + * `/proc/self/mountinfo` as the v3 launch actually produces it: an overlay + * image root, one tmpfs per declared target, the realm volume, and the host + * binds on whatever the daemon's filesystem is. `/run/training/inputs` is + * deliberately *not* a mount — the launch binds each dataset at + * `/run/training/inputs/`, so the sealed root is their parent on the + * read-only image root. + */ +const mountinfoFor = (bindFstype: string, sealedRootFstype?: string): string => [ + "21 20 0:20 / / rw,relatime - overlay overlay rw", + ...trainingBrokerTmpfsTargets().map((entry, index) => `${30 + index} 21 0:${60 + index} / ${entry.path} rw,relatime - tmpfs tmpfs rw`), + `90 21 0:90 / ${TRAINING_RUN_ROOT} rw,relatime - ${bindFstype} docker rw`, + `91 21 0:91 / ${TRAINING_SEALED_INPUTS_ROOT}/project ro,relatime - ${bindFstype} docker ro`, + `92 21 0:92 / ${TRAINING_BOOTSTRAP_MOUNT} ro,relatime - ${bindFstype} docker ro`, + `93 21 0:93 / ${TRAINING_REALM_MOUNT} rw,relatime - ext4 /dev/vda1 rw`, + ...sealedRootFstype === undefined ? [] : [`94 21 0:94 / ${TRAINING_SEALED_INPUTS_ROOT} ro,relatime - ${sealedRootFstype} docker ro`] +].join("\n"); + +const canaryOptions = (overrides: Partial[0]> = {}) => ({ + denyPaths: denyPaths(), hostBindPaths: TRAINING_HOST_BIND_DENY_PATHS, sealedPaths: TRAINING_SEALED_DENY_PATHS, + mountinfo: mountinfoFor("ext4"), unenforcedBindPolicy: "refuse" as const, + probe: async () => true, log: () => undefined, ...overrides +}); + describe("training slot canaries", () => { - it("requires a worker-uid denial for every deny path on a filesystem that enforces ownership", async () => { + it("lets the documented refuse default succeed over the real deny list when the daemon's filesystem enforces ownership", async () => { const probed: string[] = []; - const canaries = await resolveTrainingCanaries({ - denyPaths: ["/run/training/slot/turns", "/run/training/slot/usage"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", - probe: async (target) => { probed.push(target); return true; }, log: () => undefined - }); - expect(probed).toEqual(["/run/training/slot/turns", "/run/training/slot/usage"]); - expect(canaries).toHaveLength(2); + const canaries = await resolveTrainingCanaries(canaryOptions({ + probe: async (target: string) => { probed.push(target); return true; } + })); + expect(canaries.map((entry) => entry.path)).toEqual([...denyPaths()]); + // The whole point: nothing was waived, so every deny entry — the two host binds included — was probed. + expect(probed).toEqual([...denyPaths()]); + expect(probed).toContain(TRAINING_RUN_ROOT); + expect(probed).toContain(TRAINING_SEALED_INPUTS_ROOT); }); - it("refuses to certify a path the worker uid can still read", async () => { - await expect(resolveTrainingCanaries({ - denyPaths: ["/run/training/slot/turns"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", - probe: async () => false, log: () => undefined - })).rejects.toThrow(/still readable by the worker uid/u); + it("probes a host bind on an ownership-enforcing filesystem instead of waiving it", async () => { + await expect(resolveTrainingCanaries(canaryOptions({ + probe: async (target: string) => target !== TRAINING_RUN_ROOT + }))).rejects.toThrow(new RegExp(`${TRAINING_RUN_ROOT} is still readable by the worker uid`, "u")); }); - it("refuses a declared host bind by default, because its mode is the operator's and not the container's", async () => { - await expect(resolveTrainingCanaries({ - denyPaths: ["/run/training/inputs"], mountinfo, hostBindPaths: ["/run/training/inputs"], unenforcedBindPolicy: "refuse", - probe: async () => { throw Error("the probe must never run on a host bind"); }, log: () => undefined - })).rejects.toThrow(/is a host bind mount/u); + it("refuses under the default when a deny path lands on a filesystem that ignores ownership", async () => { + await expect(resolveTrainingCanaries(canaryOptions({ mountinfo: mountinfoFor("virtiofs") }))) + .rejects.toThrow(/ignores unix ownership/u); }); - it("refuses a filesystem that ignores unix ownership by default, even when the launch did not declare it", async () => { - await expect(resolveTrainingCanaries({ - denyPaths: ["/run/training/output"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "refuse", - probe: async () => { throw Error("the probe must never run on an unenforced filesystem"); }, log: () => undefined - })).rejects.toThrow(/virtiofs, which ignores unix ownership/u); + it("accepts the run root on the profile's deny entry alone only when the operator declared that, and says so", async () => { + const lines: string[] = []; + const canaries = await resolveTrainingCanaries(canaryOptions({ + mountinfo: mountinfoFor("virtiofs"), unenforcedBindPolicy: "profile-only", log: (line: string) => { lines.push(line); } + })); + expect(canaries.map((entry) => entry.path)).toEqual([...denyPaths()]); + expect(lines.join("\n")).toContain(`canary ${TRAINING_RUN_ROOT} certified by the enforced sandbox profile only (a host bind mount on virtiofs`); }); - it("accepts a host bind on the profile's deny entry alone only when the operator declared that, and says so", async () => { - const lines: string[] = []; - const canaries = await resolveTrainingCanaries({ - denyPaths: ["/run/training/output"], mountinfo, hostBindPaths: [], unenforcedBindPolicy: "profile-only", - probe: async () => false, log: (line) => lines.push(line) - }); - expect(canaries).toEqual([canary("/run/training/output")]); - expect(lines[0]).toContain("certified by the enforced sandbox profile only"); + it("never lets any policy waive the sealed inputs root, even when it lands on an unenforced filesystem", async () => { + for (const unenforcedBindPolicy of ["refuse", "profile-only"] as const) { + await expect(resolveTrainingCanaries(canaryOptions({ + mountinfo: mountinfoFor("virtiofs", "virtiofs"), unenforcedBindPolicy + }))).rejects.toThrow(new RegExp(`sealed canary ${TRAINING_SEALED_INPUTS_ROOT} .*no unenforcedBindPolicy waives this`, "su")); + } + }); + + it("requires the sealed inputs root to be unenterable, not merely unreadable", async () => { + // A directory the worker cannot `open()` but can still `search` hands over every dataset it can + // name — `test.paideia.yaml` included — so `read` denial alone must not certify a sealed root. + await expect(resolveTrainingCanaries(canaryOptions({ + probe: async (_target: string, depth: "read" | "enter") => depth === "read" + }))).rejects.toThrow(new RegExp(`sealed canary ${TRAINING_SEALED_INPUTS_ROOT} is still reachable`, "u")); + }); + + it("keeps the sealed set inside the real deny list and out of the host-bind waiver list", () => { + for (const sealed of TRAINING_SEALED_DENY_PATHS) { + expect(denyPaths()).toContain(sealed); + expect(TRAINING_HOST_BIND_DENY_PATHS).not.toContain(sealed); + } }); it("resolves the longest matching mount point", () => { - expect(resolveBackingFilesystem("/run/training/slot/usage/usage.jsonl", mountinfo)).toBe("tmpfs"); - expect(resolveBackingFilesystem("/run/training/output/runs", mountinfo)).toBe("virtiofs"); - expect(resolveBackingFilesystem("/etc/daimon-engine-broker", mountinfo)).toBe("overlay"); + expect(resolveBackingFilesystem("/run/training/slot/usage/usage.jsonl", mountinfoFor("virtiofs"))).toBe("tmpfs"); + expect(resolveBackingFilesystem(`${TRAINING_RUN_ROOT}/runs`, mountinfoFor("virtiofs"))).toBe("virtiofs"); + // The sealed root is the datasets' parent on the image root, never a mount of its own. + expect(resolveBackingFilesystem(TRAINING_SEALED_INPUTS_ROOT, mountinfoFor("virtiofs"))).toBe("overlay"); + // `/etc/daimon-engine-broker` is one of the declared tmpfs targets, so the image root only shows + // through for a path no mount covers at all. + expect(resolveBackingFilesystem("/etc/daimon-engine-broker", mountinfoFor("ext4"))).toBe("tmpfs"); + expect(resolveBackingFilesystem("/opt/training/bin/train", mountinfoFor("ext4"))).toBe("overlay"); }); }); diff --git a/src/compiler/training/broker/receipt.ts b/src/compiler/training/broker/receipt.ts index aaeed365..d33f2c1d 100644 --- a/src/compiler/training/broker/receipt.ts +++ b/src/compiler/training/broker/receipt.ts @@ -31,42 +31,70 @@ export const resolveBackingFilesystem = (target: string, mountinfo: string): str }; export interface TrainingCanaryProbe { - /** Runs one worker-uid `open()` attempt; resolves true when the read was denied. */ - (target: string): Promise; + /** + * Runs one worker-uid attempt and resolves true when it was denied. + * + * `read` is an `open()` for reading. `enter` additionally requires that the + * worker uid cannot *search* the directory, which is the property a sealed + * root actually needs: its protection is that every dataset read has to + * traverse it, not that the directory listing itself is unreadable. + */ + (target: string, depth: "read" | "enter"): Promise; } export interface TrainingCanaryOptions { denyPaths: readonly string[]; probe: TrainingCanaryProbe; mountinfo: string; - /** Deny entries the launch bound from the host; a worker-uid probe over one of them proves nothing. */ + /** Deny entries the launch bound from the host. Diagnostics only: the backing filesystem decides, not this list. */ hostBindPaths: readonly string[]; + /** Deny entries no policy may waive; each one must be observed unenterable by the worker uid. */ + sealedPaths: readonly string[]; unenforcedBindPolicy: "refuse" | "profile-only"; log(line: string): void; } /** - * One denied canary per deny path, or a refusal naming the first path that is - * still reachable. + * One denied canary per deny path, or a refusal naming the first path whose + * denial this container cannot prove. * - * A path on a filesystem that enforces unix ownership must be observed denied - * to the worker uid. A path on a host bind that ignores ownership cannot be: - * the declaration's `unenforcedBindPolicy` decides whether that refuses the - * recycle or is accepted on the attested profile's deny entry alone, and every - * path taking the weaker route is named in the supervisor log. + * Three cases, in decreasing strength: + * + * - A **sealed** path (the datasets' root) must sit on a filesystem that + * enforces unix ownership and must be observed *unenterable* by the worker + * uid. `unenforcedBindPolicy` does not reach it: the sandbox profile alone + * is a boundary the worker can lift from inside a namespace of its own, and + * the held-out test set is the one thing that cannot rest on it. + * - Any other path on an ownership-enforcing filesystem — a host bind over + * ext4/xfs/btrfs/overlay included — gets a real worker-uid read probe. This + * is what makes the documented `refuse` default reachable: a host bind is + * only unprovable where the filesystem says so. + * - A path on a filesystem that ignores ownership (virtiofs and grpcfuse under + * Docker Desktop and Colima, 9p, nfs, cifs, fuse) cannot be probed at all. + * `refuse` — the default — fails the recycle and writes no receipt; + * `profile-only` accepts the attested deny entry as that path's only + * boundary and names every such path in the supervisor log. */ export const resolveTrainingCanaries = async (options: TrainingCanaryOptions): Promise<{ path: string; method: "sandboxed-read"; result: "denied" }[]> => { const canaries: { path: string; method: "sandboxed-read"; result: "denied" }[] = []; for (const target of options.denyPaths) { const fstype = resolveBackingFilesystem(target, options.mountinfo); - const hostBind = options.hostBindPaths.includes(target); - if (hostBind || isUnenforced(fstype)) { + const origin = `${options.hostBindPaths.includes(target) ? "a host bind mount on " : ""}${fstype || "an unknown filesystem"}`; + if (options.sealedPaths.includes(target)) { + if (isUnenforced(fstype)) { + throw new SpawnfileError("runtime_error", + `Grok slot sealed canary ${target} is backed by ${origin}, which ignores unix ownership, so the worker uid's denial cannot be proven; the sealed datasets must sit under a directory on an ownership-enforcing filesystem and no unenforcedBindPolicy waives this`); + } + if (!await options.probe(target, "enter")) { + throw new SpawnfileError("runtime_error", `Grok slot sealed canary ${target} is still reachable by the worker uid`); + } + } else if (isUnenforced(fstype)) { if (options.unenforcedBindPolicy === "refuse") { throw new SpawnfileError("runtime_error", - `Grok slot canary ${target} is ${hostBind ? "a host bind mount" : `backed by ${fstype || "an unknown filesystem"}, which ignores unix ownership`}; declare unenforcedBindPolicy "profile-only" to accept the bubblewrap deny list as its only boundary`); + `Grok slot canary ${target} is backed by ${origin}, which ignores unix ownership; declare unenforcedBindPolicy "profile-only" to accept the bubblewrap deny list as its only boundary`); } - options.log(`canary ${target} certified by the enforced sandbox profile only (${hostBind ? "host bind mount" : `${fstype} ignores unix ownership`})`); - } else if (!await options.probe(target)) { + options.log(`canary ${target} certified by the enforced sandbox profile only (${origin} ignores unix ownership)`); + } else if (!await options.probe(target, "read")) { throw new SpawnfileError("runtime_error", `Grok slot canary ${target} is still readable by the worker uid`); } canaries.push({ path: target, method: "sandboxed-read", result: "denied" }); diff --git a/src/compiler/training/broker/runtime.ts b/src/compiler/training/broker/runtime.ts index 6d64c3e6..685b1fb0 100644 --- a/src/compiler/training/broker/runtime.ts +++ b/src/compiler/training/broker/runtime.ts @@ -18,6 +18,7 @@ import type { TrainingSlotRuntime } from "./supervisor.js"; import { DAIMON_ORGANIZATION_UID, TRAINING_HOST_BIND_DENY_PATHS, + TRAINING_SEALED_DENY_PATHS, TRAINING_SLOT_ACCEPTANCE_STORE, TRAINING_WIPE_TARGETS, TRAINING_SLOT_GENERATION_FILE, @@ -90,10 +91,22 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): process.stderr.write(`[slot-supervisor] ${entry}`); try { appendFileSync(logPath, entry, { mode: 0o640 }); } catch { /* the log is diagnostics, never the gate */ } }; - const probe = async (target: string): Promise => { + /** + * One worker-uid reachability attempt. The command below *succeeds* when the + * worker can still reach the target, so a non-zero exit is the denial. + * + * `enter` is the stronger question a sealed root needs answered: a directory + * can be unreadable (`test -r` fails) and still searchable, and a subject + * that knows `test.paideia.yaml` by name needs only search permission. + */ + const probe = async (target: string, depth: "read" | "enter"): Promise => { + const quoted = JSON.stringify(target); + const reachable = depth === "enter" + ? `test -r ${quoted} || test -x ${quoted} || ls -1 ${quoted} >/dev/null 2>&1` + : `test -r ${quoted}`; try { await run("setpriv", ["--clear-groups", `--reuid=${options.registration.uid}`, `--regid=${options.registration.uid}`, - "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", "--", "/bin/sh", "-c", `exec test -r ${JSON.stringify(target)}`], { timeout: 10_000 }); + "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all", "--", "/bin/sh", "-c", reachable], { timeout: 10_000 }); return false; } catch { return true; } }; @@ -144,6 +157,7 @@ export const createTrainingSlotRuntime = (options: TrainingSlotRuntimeOptions): start: async () => { children = await startBrokerProcesses({ log, procRoot: options.procRoot }); }, canaries: async () => resolveTrainingCanaries({ denyPaths: options.registration.denyPaths, probe, log, hostBindPaths: TRAINING_HOST_BIND_DENY_PATHS, + sealedPaths: TRAINING_SEALED_DENY_PATHS, mountinfo: await readFile(`${options.procRoot ?? "/proc"}/self/mountinfo`, "utf8"), unenforcedBindPolicy: options.declaration.unenforcedBindPolicy }), diff --git a/src/compiler/training/preparation/contextModes.test.ts b/src/compiler/training/preparation/contextModes.test.ts index 35a24378..090f4f41 100644 --- a/src/compiler/training/preparation/contextModes.test.ts +++ b/src/compiler/training/preparation/contextModes.test.ts @@ -83,7 +83,13 @@ it("hands Docker a normalized context and never runs a layer after the distribut const dockerfile = await recipe(); const instructions = dockerfile.split(/\n(?! )/u).map(line => line.trim()).filter(line => line && !line.startsWith("#")); - expect(instructions.join("\n")).not.toMatch(/chmod (-R|0)/u); + // The staged context already carries its modes, so no layer may re-chmod what it copies. A numeric + // chmod is allowed only on a runtime mount point under /run/training — the sealed-inputs seal, which + // is baked before any COPY and is the one mode the read-only root can never repair at runtime. + expect(instructions.join("\n")).not.toMatch(/chmod -R/u); + for (const numeric of instructions.join("\n").match(/chmod 0[0-7]{3}[^\\\n]*/gu) ?? []) { + expect(numeric, "a numeric chmod may only touch a runtime mount point").toMatch(/^chmod 0[0-7]{3}(\s+\/run\/training\S*)+\s*$/u); + } const firstLate = instructions.indexOf("COPY train /opt/training/bin/train"); expect(instructions.slice(firstLate).map(line => line.split(" ")[0])).toEqual(["COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "COPY", "ENV", "ENV", "WORKDIR", "ENTRYPOINT"]); const copies = instructions.filter(line => line.startsWith("COPY ") && !line.startsWith("COPY --from")); From bbcc4f1143ee0be1ab988c1d322afd686d0bd37f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:18:30 +0200 Subject: [PATCH 78/84] fix: canary the training slot and publish its preflight receipt before the first trial --- src/compiler/training/broker/entrypoint.ts | 12 ++++++--- .../training/broker/supervisor.test.ts | 24 +++++++++++++++++- src/compiler/training/broker/supervisor.ts | 25 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/compiler/training/broker/entrypoint.ts b/src/compiler/training/broker/entrypoint.ts index dcdb0cd0..c7f961e6 100644 --- a/src/compiler/training/broker/entrypoint.ts +++ b/src/compiler/training/broker/entrypoint.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; import { lstat, readFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -9,7 +10,7 @@ import { parseTrainingBrokerDeclaration, type TrainingBrokerDeclaration } from " import { loadDaimonProjectionModule, resolveTrainingGrokProjection, type DaimonProjectionModule } from "./projection.js"; import { resolveTrainingGrokRegistration } from "./registration.js"; import { createTrainingSlotRuntime } from "./runtime.js"; -import { createTrainingSlotSupervisor, serveTrainingSlotSupervisor } from "./supervisor.js"; +import { createTrainingSlotSupervisor, serveTrainingSlotSupervisor, startTrainingSlot } from "./supervisor.js"; import { DAIMON_ORGANIZATION_UID, TRAINING_BOOTSTRAP_MOUNT, @@ -81,11 +82,14 @@ export const runTrainingBrokerEntrypoint = async (options: TrainingEntrypointOpt // Daimon's projection is I/O-free, so the digest the receipt will bind is known before anything is provisioned. const { projectionSha256 } = await resolveTrainingGrokProjection(declaration, registration, daimon); const runtime = createTrainingSlotRuntime({ declaration, registration, projectionSha256 }); - await runtime.provision(); - await runtime.start(); + // Provision, start, canary and publish the slot preflight receipt before `train` exists at all: the + // first trial must not be the one trial that runs on no worker-uid denial evidence, and a refusal + // here costs nothing while the same refusal after the first wake costs that trial's spend. + const preflight = await startTrainingSlot(runtime, randomBytes(32).toString("hex")); const supervisor = createTrainingSlotSupervisor({ runtime, organizationUid: declaration.organizationUid }); const server = serveTrainingSlotSupervisor(supervisor, TRAINING_SUPERVISOR_SOCKET, log, declaration.organizationUid); - log(`slot ${registration.slot} provisioned for ${registration.agentId} (${registration.model}/${registration.reasoningEffort}), projection ${projectionSha256}`); + log(`slot ${registration.slot} provisioned for ${registration.agentId} (${registration.model}/${registration.reasoningEffort}), projection ${projectionSha256}, ` + + `generation ${preflight.generation} with ${preflight.canaries} denied canaries at ${preflight.receipt}`); const status = await runTrainingChild(options.argv, declaration, log); server.close(); await runtime.stop(); diff --git a/src/compiler/training/broker/supervisor.test.ts b/src/compiler/training/broker/supervisor.test.ts index c536af35..8b3ffeaf 100644 --- a/src/compiler/training/broker/supervisor.test.ts +++ b/src/compiler/training/broker/supervisor.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; -import { createTrainingSlotSupervisor, type TrainingSlotRuntime } from "./supervisor.js"; +import { createTrainingSlotSupervisor, startTrainingSlot, type TrainingSlotRuntime } from "./supervisor.js"; const nonce = "a".repeat(64); @@ -75,3 +75,25 @@ describe("training slot recycle", () => { await expect(supervisor.recycle("c".repeat(64), 2000)).resolves.toMatchObject({ ok: true }); }); }); + +describe("training slot start-up", () => { + /** + * The first trial is the one whose sealed datasets have never been probed, so + * it is the one that most needs the evidence. Start-up used to be + * `provision → start` and nothing else: `canaries` and `publishReceipt` were + * reachable only through `recycle`, so trial 1 ran with no worker-uid denial + * evidence at all and a refusal surfaced only after that trial's spend. + */ + it("canaries the slot and publishes the preflight receipt before it hands the slot over", async () => { + const { runtime, order } = stubRuntime(); + const result = await startTrainingSlot(runtime, nonce); + expect(order).toEqual(["provision", "start", "canaries", "generation", "receipt"]); + expect(result).toEqual({ generation: 1, receipt: "/run/training/slot/preflight.json", canaries: 1 }); + }); + + it("never reaches a receipt when a start-up canary is still reachable", async () => { + const { runtime, order } = stubRuntime({ canaries: async () => { throw Error("sealed canary is still reachable"); } }); + await expect(startTrainingSlot(runtime, nonce)).rejects.toThrow(/sealed canary is still reachable/u); + expect(order).not.toContain("receipt"); + }); +}); diff --git a/src/compiler/training/broker/supervisor.ts b/src/compiler/training/broker/supervisor.ts index 343d2327..11d4df57 100644 --- a/src/compiler/training/broker/supervisor.ts +++ b/src/compiler/training/broker/supervisor.ts @@ -34,6 +34,31 @@ export interface TrainingSlotRuntime { log(line: string): void; } +/** + * Bringing a slot up, receipt included — the only way this container starts one. + * + * `provision → start` alone was the whole start-up path, and `canaries` and + * `publishReceipt` were reachable only from `recycle`. The first trial of every + * run therefore executed with no worker-uid denial evidence at all, and a + * refusal that should have cost nothing surfaced only after that trial's spend. + * Trial 1 is exactly the trial whose sealed datasets have never been probed, so + * it is the one that most needs the evidence. + * + * The receipt this publishes carries generation 1..N and a nonce of the + * container's own, so an evaluator that reads `preflight.json` before its first + * wake sees the same `noopolis.daimon.grok-slot-preflight.v2` shape a recycle + * publishes, from the same canaries. + */ +export const startTrainingSlot = async (runtime: TrainingSlotRuntime, nonce: string): Promise<{ generation: number; receipt: string; canaries: number }> => { + await runtime.provision(); + await runtime.start(); + const canaries = await runtime.canaries(); + const generation = await runtime.nextGeneration(); + const receipt = await runtime.publishReceipt({ generation, nonce, canaries }); + runtime.log(`slot start generation=${generation} canaries=${canaries.length} receipt=${receipt}`); + return { generation, receipt, canaries: canaries.length }; +}; + export interface TrainingSlotSupervisorOptions { runtime: TrainingSlotRuntime; /** Only this uid may recycle. Paideia, DSPy and the judges all run as it; every worker uid is refused. */ From 4cadee3c691397681f1a9b81a3a3863490299bac Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:18:30 +0200 Subject: [PATCH 79/84] docs: specify the sealed-inputs DAC boundary and the start-up slot receipt --- specs/TRAINING_CONTAINERS.md | 71 +++++++++++++++++++++++--- src/compiler/training/broker/AGENTS.md | 30 ++++++++--- 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 4026d255..00dced56 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -407,9 +407,68 @@ the ancestor with `o+x`, gives the worker no additional reach. Root provisioning asserts placement for every entry once all modes are final — at container start and on every recycle — and refuses the slot otherwise. -A canary is a worker-uid `open()` that must fail. For a deny entry that is a -host bind mount, or sits on a filesystem that ignores unix ownership (Docker -Desktop and Colima both do), that probe proves nothing, so -`unenforcedBindPolicy` decides: `refuse` (the default) fails the recycle and -writes no receipt; `profile-only` accepts the bubblewrap-enforced `deny` list as -that path's only boundary and names every such path in the supervisor log. +A canary is a worker-uid attempt that must fail. What decides whether it means +anything is the **backing filesystem**, not the declaration: a bind on +ext4/xfs/btrfs/overlay enforces unix ownership and is probed for real, while +virtiofs, grpcfuse, 9p, nfs, cifs and fuse ignore `chown` outright — Docker +Desktop and Colima both land every host bind there. Only on those does +`unenforcedBindPolicy` apply: `refuse` (the default) fails the slot and writes +no receipt; `profile-only` accepts the bubblewrap-enforced `deny` list as that +path's only boundary and names every such path in the supervisor log. The +default is therefore reachable — on a Linux daemon over a native filesystem +every deny entry is kernel-probed — and on Docker Desktop the one entry that +forces `profile-only` is `/run/training/output`. + +### The sealed datasets + +`/run/training/inputs` is not covered by that policy at all, and no declaration +can waive it. It holds the sealed train and test datasets, and the bubblewrap +`deny` list is not a boundary the subject cannot lift: the pinned seccomp +profile has to allow `clone/clone3/unshare/mount/umount2/pivot_root/setns` for +bubblewrap itself, and the container runs `apparmor=unconfined`, so a worker-uid +process can open a user and mount namespace of its own and either detach the +mask or bind the mask's parent somewhere fresh. Evidence gathered with `cat` and +`read_file` says nothing about that route. + +What holds against it is DAC on the datasets' ancestor. Every declared input is +bound at `/run/training/inputs/` — the launch schema admits no other +destination — so `/run/training/inputs` itself is their parent, a directory on +the read-only image root. The image bakes it `0:2000 0750`: + +- uid 2200 is in neither the owner nor the group class, so it loses *search* + permission on the one directory every dataset read must traverse; +- the inode is owned by real uid 0, which `unshare --map-root-user` does not + map, so `CAP_DAC_OVERRIDE` inside the worker's own namespace cannot override + it, and a fresh `mount --bind` of the parent re-exposes this same directory + rather than the bytes beneath it; +- the root filesystem is read-only, so nothing in the container — root included + — can widen the mode after the image is built. + +Root provisioning therefore asserts that identity instead of setting it, and +refuses the slot when an image did not bake it. It then attacks the seal as the +worker uid, at container start and on every recycle: a direct read/search/list, +the same list after `unshare --user --map-root-user --mount` plus a lazy unmount +of the mask and its parent, and the same list through a fresh `mount --bind` of +`/run/training` made inside that namespace — and, per dataset, a read and a +bind of the dataset's own mount, the route that carries no mask and is not +refused for locked children the way binding their parent is. A control +container left at Docker's own `0755 root:root` hands the held-out answer key +to uid 2200 through exactly that route, and the sealed one refuses all four +(`.runtime/sealed-inputs-dac/EVIDENCE.md`). A probe that reaches no verdict +refuses the slot; a worker that cannot open a namespace at all cannot take the +route, and the supervisor log says so. The slot canary for this path +additionally requires the worker uid to be unable to *enter* the directory, not +merely to read it: a searchable directory hands over every dataset the subject +can name. + +### Evidence before the first trial + +`provision → start → canaries → generation → receipt` is the whole start-up +path, in that order, and it is the only way this container brings a slot up. +Start-up used to stop after `start()`, with the canaries and the +`noopolis.daimon.grok-slot-preflight.v2` receipt reachable only through +`recycle`, so the first trial of every run — the one trial whose sealed +datasets had never been probed — executed on no worker-uid denial evidence at +all, and a refusal surfaced only after that trial's spend. The start-up receipt +carries the container's own nonce and generation 1..N, in the same shape a +recycle publishes and from the same canaries. diff --git a/src/compiler/training/broker/AGENTS.md b/src/compiler/training/broker/AGENTS.md index ad6c5f46..994341af 100644 --- a/src/compiler/training/broker/AGENTS.md +++ b/src/compiler/training/broker/AGENTS.md @@ -18,8 +18,9 @@ container that runs a brokered Grok subject beside the evaluator. - `processes.ts` starts the launcher (root), broker and relay (2100) and proves each one's socket and post-drop uid/`CapBnd` before the next starts. - `runtime.ts` implements drain, wipe, provision, start, canaries and receipt. -- `supervisor.ts` is the one-verb root socket server. One verb, one argument, - no caller-supplied path or command. +- `supervisor.ts` is the one-verb root socket server (one verb, one argument, + no caller-supplied path or command) and owns `startTrainingSlot`, the only + way a slot comes up: `provision → start → canaries → generation → receipt`. - `receipt.ts` writes `noopolis.daimon.grok-slot-preflight.v2`. - `entrypoint.ts`/`main.ts` are the image's root entrypoint. @@ -35,10 +36,27 @@ Local constraints: - The supervisor socket's uid gate is the socket node (`root:2000 0660` in a root-owned `0711` directory on tmpfs), because Node exposes no `SO_PEERCRED`. A `0600` root-owned socket would deny the one caller it exists for. -- A worker-uid canary over a host bind proves nothing — the host owns the inode, - and Docker Desktop and Colima ignore `chown` outright — so - `unenforcedBindPolicy` decides between refusing the slot and accepting the - bubblewrap deny list as that path's only boundary. +- What decides whether a worker-uid canary means anything is the **backing + filesystem**, not the fact of being a host bind: virtiofs, grpcfuse, 9p, nfs, + cifs and fuse ignore `chown` outright (Docker Desktop and Colima), while the + same bind over ext4 or overlay is probed for real. Only on the former does + `unenforcedBindPolicy` choose between refusing the slot and accepting the + bubblewrap deny list as that path's only boundary — which is why the + documented `refuse` default is reachable at all. +- **`/run/training/inputs` is never that.** It holds the sealed train and test + datasets, every input is bound strictly below it, and the image bakes the + directory itself `0:2000 0750` on the read-only root. The worker uid loses + *search* permission on the datasets' one common ancestor; the inode is owned + by real uid 0, which `unshare --map-root-user` does not map, so + `CAP_DAC_OVERRIDE` in the worker's own namespace cannot override it and a + fresh `mount --bind` of the parent re-exposes this same directory. A + bubblewrap `deny` mask alone would not survive that route — the seccomp + profile must allow `unshare`/`mount`/`umount2` for bubblewrap itself — so + provisioning asserts the mode, attacks it as the worker uid over every route — + direct, namespace unmount, namespace rebind of the parent, and a namespace + rebind of each dataset's own mount, which is the one a mask cannot answer — + and no `unenforcedBindPolicy` waives it + (`.runtime/sealed-inputs-dac/EVIDENCE.md`). - Grok 1.0.34 materializes every `deny` target inside bubblewrap as the worker uid, so a deny entry it cannot create makes the whole profile fail. That is why `/run/paideia` is masked as a directory rather than file by file, and why From 5debf84ec57e8e931ceb537032112f9122df3283 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:19:21 +0200 Subject: [PATCH 80/84] docs: state the reachability limits of the refuse policy and the unsealed run root --- specs/TRAINING_CONTAINERS.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 00dced56..676820e1 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -415,9 +415,22 @@ Desktop and Colima both land every host bind there. Only on those does `unenforcedBindPolicy` apply: `refuse` (the default) fails the slot and writes no receipt; `profile-only` accepts the bubblewrap-enforced `deny` list as that path's only boundary and names every such path in the supervisor log. The -default is therefore reachable — on a Linux daemon over a native filesystem -every deny entry is kernel-probed — and on Docker Desktop the one entry that -forces `profile-only` is `/run/training/output`. +default is therefore reachable: on a Linux daemon over a native filesystem every +deny entry is kernel-probed, and preparation creates the output root `0700` +owned by the invoking user, so uid 2200 is genuinely denied there. On Docker +Desktop and Colima the one entry that forces `profile-only` is +`/run/training/output`, whose inode is the operator's and whose filesystem +ignores `chown`; the sealed datasets never depend on that choice. + +Two limits of that are worth stating rather than discovering. A `0700` +host-owned output root also denies uid 2000, so a v3 run on a native +filesystem needs an output directory the in-container organization uid can +write — group `2000` mode `0750`, or an equivalent id mapping — before it can +complete; and under `profile-only` the run root's only boundary is the +bubblewrap `deny` list, which the same namespace route that motivated the +sealed-inputs seal can lift. Sealing the run root the same way means moving the +bind below a baked `0750` ancestor, which changes a container path +`paideia.daimon-native.launch.v2` names, so it is deliberately not done here. ### The sealed datasets From f81ea527035d8d447806e1c27c8618a9d37e6ce2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 16:17:20 +0200 Subject: [PATCH 81/84] fix: probe the sealed inputs from the worker's private tmp and record each route's denial mechanism --- src/compiler/training/broker/paths.ts | 3 + .../training/broker/provisioning.test.ts | 133 ++++++++++++++---- src/compiler/training/broker/provisioning.ts | 108 +++++++++----- src/compiler/training/broker/seccompRoutes.ts | 41 ++++++ 4 files changed, 225 insertions(+), 60 deletions(-) create mode 100644 src/compiler/training/broker/seccompRoutes.ts diff --git a/src/compiler/training/broker/paths.ts b/src/compiler/training/broker/paths.ts index 09849a7d..298a308d 100644 --- a/src/compiler/training/broker/paths.ts +++ b/src/compiler/training/broker/paths.ts @@ -232,6 +232,9 @@ export const TRAINING_HOST_BIND_DENY_PATHS: readonly string[] = [TRAINING_RUN_RO */ export const TRAINING_SEALED_DENY_PATHS: readonly string[] = [TRAINING_SEALED_INPUTS_ROOT]; +/** Spawnfile-owned per-route seal attestation, written by provisioning beside the slot preflight receipt. */ +export const TRAINING_SEALED_INPUTS_ATTESTATION = `${TRAINING_SLOT_ROOT}/sealed-inputs.json`; + /** The image-baked identity of the sealed inputs root, asserted before every slot and never writable at runtime. */ export const TRAINING_SEALED_INPUTS_IDENTITY = { uid: 0, gid: DAIMON_ORGANIZATION_UID, mode: "750" } as const; diff --git a/src/compiler/training/broker/provisioning.test.ts b/src/compiler/training/broker/provisioning.test.ts index cfeb99dd..238492ee 100644 --- a/src/compiler/training/broker/provisioning.test.ts +++ b/src/compiler/training/broker/provisioning.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { trainingContainerConfigSchema } from "../container/contract.js"; import { renderTrainingBrokerProvisioning, renderTrainingSealedInputsAssertions } from "./provisioning.js"; import { resolveTrainingGrokRegistration } from "./registration.js"; +import { pinnedProfileAllowsNamespaceRoutes, trainingNamespaceDenialMechanism } from "./seccompRoutes.js"; import { TRAINING_SEALED_INPUTS_IDENTITY, TRAINING_SEALED_INPUTS_ROOT, TRAINING_WORKER_UID } from "./paths.js"; const run = promisify(execFile); @@ -20,17 +21,28 @@ const registration = () => resolveTrainingGrokRegistration({ agentId: "agent:aut * need a Linux kernel; what is testable here is the part that decides whether * the slot is allowed to proceed. */ -const verdictScript = (): string => { +const sealHarness = (): string => { const lines = renderTrainingSealedInputsAssertions(); - return lines.slice(0, lines.indexOf("}") + 1).join("\n"); + // Everything up to and including the namespace gate: the helpers and the classifier, without the + // routes themselves, which need a Linux kernel and a worker uid. + return lines.slice(0, lines.findIndex((line) => line.startsWith("seal_ns_route()")) + 1).join("\n"); }; -const runVerdict = async (output: string): Promise<{ code: number; stdout: string; stderr: string }> => { +/** + * Drives the verdict classifier with one token, under a stub `setpriv` so it + * runs on any host. What is testable here is the decision the slot hangs on: + * which verdicts are allowed to proceed and which refuse. + */ +const runVerdict = async (verdict: string, namespaceAvailable = true): Promise<{ code: number; stdout: string; stderr: string }> => { const directory = await mkdtemp(path.join(os.tmpdir(), "spawnfile-sealed-verdict-")); - await writeFile(path.join(directory, "setpriv"), '#!/bin/sh\nprintf "%s" "$SEALED_PROBE_OUTPUT"\n', { mode: 0o755 }); - const script = `${verdictScript()}\nsealed_denied "probe" namespace-rebind\n`; + await writeFile(path.join(directory, "setpriv"), '#!/bin/sh\nprintf "%s" "$SEAL_VERDICT"\nexit "${SEAL_STATUS:-0}"\n', { mode: 0o755 }); + const script = [ + `seal_diagnostic=${JSON.stringify(path.join(directory, "seal.err"))}`, + sealHarness().split("\n").filter((line) => !line.startsWith("seal_diagnostic=")).join("\n"), + 'seal_ns_route namespace-rebind-parent "probe"' + ].join("\n"); return run("/bin/bash", ["--noprofile", "--norc", "-ceu", script], { - env: { ...process.env, PATH: `${directory}:${process.env.PATH ?? ""}`, SEALED_PROBE_OUTPUT: output } + env: { ...process.env, PATH: `${directory}:${process.env.PATH ?? ""}`, SEAL_VERDICT: verdict, SEAL_STATUS: namespaceAvailable ? "0" : "1" } }).then((result) => ({ code: 0, ...result })).catch((error: { code?: number; stdout?: string; stderr?: string }) => ({ code: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? "" })); }; @@ -47,21 +59,21 @@ describe("the sealed inputs seal", () => { }); it("attacks the seal the way the subject would, not the way an honest tool would", () => { - const script = renderTrainingSealedInputsAssertions().join("\n"); + const entry = registration(); + const script = renderTrainingSealedInputsAssertions(entry.uid, entry.privateTmp).join("\n"); expect(script).toContain(`--reuid ${TRAINING_WORKER_UID}`); - // Direct read, then the two routes a bubblewrap `deny` mask cannot answer. - expect(script).toContain("direct-read"); - expect(script).toContain("namespace-unmount"); - expect(script).toContain("namespace-rebind"); + // Direct read, then the three routes a bubblewrap `deny` mask cannot answer. + for (const route of ["direct-read", "namespace-unmount", "namespace-rebind-parent", "namespace-rebind $sealed_child"]) { + expect(script, route).toContain(route); + } expect(script).toContain("unshare --user --map-root-user --mount"); expect(script).toContain(`umount -l ${TRAINING_SEALED_INPUTS_ROOT}`); - expect(script).toContain("mount --bind /run/training /tmp/.sealed-probe"); - // Per dataset too: binding the dataset's OWN mount somewhere fresh carries no `deny` mask and is not - // refused for locked children the way binding their parent is. A live control container handed the - // held-out answer key to uid 2200 through exactly that route (.runtime/sealed-inputs-dac/EVIDENCE.md). + expect(script).toContain(`mount --bind /run/training ${entry.privateTmp}/seal-parent`); + // Per dataset, because binding the dataset's OWN mount carries no `deny` mask and is not refused for + // locked children the way binding its parent is. A live control container handed the held-out answer + // key to uid 2200 through exactly that route (.runtime/sealed-inputs-dac/EVIDENCE.md). expect(script).toContain(`for sealed_child in ${TRAINING_SEALED_INPUTS_ROOT}/*; do`); - expect(script).toContain("mount --bind '$sealed_child' /tmp/.sealed-probe-child"); - expect(script).toContain('cat \\"$sealed_child\\"/*'); + expect(script).toContain(`mount --bind \\"$sealed_child\\" ${entry.privateTmp}/seal-child`); }); it("runs the seal assertions as part of the same provisioning a recycle replays", () => { @@ -80,23 +92,42 @@ describe("the sealed inputs seal", () => { expect(() => trainingContainerConfigSchema.parse(config("/run/training"))).toThrow(); }); - it("refuses the slot when a probe reaches the datasets, and says which route did", async () => { - const result = await runVerdict("SEALED-REACHABLE\n"); + it("refuses the slot when a route reaches the datasets, and says which one did", async () => { + const result = await runVerdict("reachable\n"); expect(result.code).not.toBe(0); - expect(result.stderr).toContain(`the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is reachable by uid ${TRAINING_WORKER_UID} via namespace-rebind`); + expect(result.stderr).toContain(`the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is REACHABLE by uid ${TRAINING_WORKER_UID} via namespace-rebind-parent`); }); - it("refuses the slot when a probe reaches no verdict at all, instead of reading silence as a denial", async () => { - for (const output of ["", "bash: unshare: command not found\n", "SEALED-PARTIAL\n"]) { - const result = await runVerdict(output); - expect(result.code, output).not.toBe(0); - expect(result.stderr, output).toContain("reached no verdict, so the denial is unproven"); + it("accepts the two denials that mean the kernel refused something", async () => { + for (const verdict of ["denied at-read", "denied at-mount"]) { + const result = await runVerdict(`${verdict}\n`); + expect(result.code, verdict).toBe(0); + expect(result.stdout, verdict).toContain(`sealed inputs route namespace-rebind-parent: ${verdict}`); } }); - it("accepts an observed denial, and a worker that cannot open a namespace at all", async () => { - expect((await runVerdict("SEALED-DENIED\n")).code).toBe(0); - expect((await runVerdict("SEALED-NO-NAMESPACE\n")).code).toBe(0); + /** + * A route the kernel will not even let the worker attempt is a *stronger* + * denial than DAC, so it must be a verdict of its own rather than a + * no-verdict refusal — and it must name the layer that refused, because a + * filtered syscall recorded as a bare "denied" reads as though DAC held. + */ + it("treats an unopenable namespace as a real verdict and names the mechanism", async () => { + const result = await runVerdict("", false); + expect(result.code).toBe(0); + expect(result.stdout).toContain(`sealed inputs route namespace-rebind-parent: unavailable ${trainingNamespaceDenialMechanism()}`); + expect(result.stdout).toContain("provably cannot happen"); + // Never merged with a DAC denial: the two tokens are distinct in the log and in the attestation. + expect(result.stdout).not.toContain("denied at-read"); + }); + + it("still refuses a genuine no-verdict, and prints the diagnostic that made it unreadable before", async () => { + for (const verdict of ["", "bash: unshare: command not found\n", "denied somehow-else\n", "SEALED-PARTIAL\n"]) { + const result = await runVerdict(verdict); + expect(result.code, verdict).not.toBe(0); + expect(result.stderr, verdict).toContain("reached no verdict, so the denial is unproven"); + expect(result.stderr, verdict).toContain("stderr:"); + } }); }); @@ -112,3 +143,49 @@ describe("the training image's own half of the seal", () => { expect(dockerfile).toContain("chmod 0755 /run/training /run/training/output"); }); }); + +describe("where the seal probes are allowed to work", () => { + /** + * The first live run refused every trial on + * `sealed inputs probe namespace-rebind reached no verdict`. The cause was + * not the pinned seccomp profile — which allows `unshare`, `mount`, + * `umount2` and `setns` outright — but the probe's own workspace: Daimon's + * broker provisioning closes the shared temps to `root:2000 1774` so a + * worker "lists names only", and the probe was doing `mkdir /tmp/...` as uid + * 2200. The `mkdir` failed, the route exited with no sentinel, and + * fail-closed did the rest. Observed in the real runner image: + * `mkdir: cannot create directory '/tmp/x': Permission denied`. + * + * The worker's private tmp (`/tmp`, `2200:2200 0700`) is the one + * directory it can write, and it is also the faithful attacker workspace. + */ + it("mounts only inside the worker's own private tmp, never the closed shared temps", () => { + const entry = registration(); + const script = renderTrainingSealedInputsAssertions(entry.uid, entry.privateTmp).join("\n"); + const targets = [...script.matchAll(/mount --bind \S+ (\S+)/gu)].map((match) => match[1]!); + expect(targets.length).toBeGreaterThan(0); + for (const target of targets) { + expect(target.startsWith(`${entry.privateTmp}/`), `${target} must sit inside ${entry.privateTmp}`).toBe(true); + } + for (const closed of ["/tmp/", "/var/tmp/"]) expect(script).not.toContain(`mkdir -p ${closed}`); + }); + + it("names the mechanism behind an unavailable namespace instead of merging it into a denial", () => { + // The pinned profile allows the route's syscalls, so a refusal cannot be blamed on seccomp. + expect(pinnedProfileAllowsNamespaceRoutes()).toBe(true); + expect(trainingNamespaceDenialMechanism()).toBe("kernel"); + const filtered = JSON.stringify({ syscalls: [{ names: ["mount", "umount2", "setns"], action: "SCMP_ACT_ALLOW" }] }); + expect(pinnedProfileAllowsNamespaceRoutes(filtered)).toBe(false); + expect(trainingNamespaceDenialMechanism(filtered)).toBe("seccomp"); + // An allow that only holds with CAP_SYS_ADMIN is not an allow here: the container drops it. + const capped = JSON.stringify({ syscalls: [{ names: [...["unshare", "mount", "umount2", "setns"]], action: "SCMP_ACT_ALLOW", includes: { caps: ["CAP_SYS_ADMIN"] } }] }); + expect(pinnedProfileAllowsNamespaceRoutes(capped)).toBe(false); + }); + + it("renders the mechanism into the script, so an unavailable route is a verdict and not a refusal", () => { + const entry = registration(); + const script = renderTrainingSealedInputsAssertions(entry.uid, entry.privateTmp).join("\n"); + expect(script).toContain(`unavailable ${trainingNamespaceDenialMechanism()}`); + expect(script).toContain("unshare --user --map-root-user --mount true"); + }); +}); diff --git a/src/compiler/training/broker/provisioning.ts b/src/compiler/training/broker/provisioning.ts index b1aadfa7..68030085 100644 --- a/src/compiler/training/broker/provisioning.ts +++ b/src/compiler/training/broker/provisioning.ts @@ -1,5 +1,6 @@ import path from "node:path"; +import { DAIMON_GROK_SECCOMP_PROFILE_SHA256 } from "../../../shared/daimonGrokSeccompProfile.js"; import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_TURN_USAGE_LEDGER } from "../../../runtime/daimon/contractManifest.js"; import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js"; import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js"; @@ -14,6 +15,7 @@ import { TRAINING_OPTIONAL_DENY_DIRECTORIES, TRAINING_PAIDEIA_ROOT, TRAINING_SLOT_ACCEPTANCE_STORE, + TRAINING_SEALED_INPUTS_ATTESTATION, TRAINING_SEALED_INPUTS_IDENTITY, TRAINING_SEALED_INPUTS_ROOT, TRAINING_SLOT_ROOT, @@ -27,6 +29,7 @@ import { TRAINING_WORKER_ROOT, TRAINING_WORKER_UID } from "./paths.js"; +import { trainingNamespaceDenialMechanism } from "./seccompRoutes.js"; const quote = (value: string): string => `'${value.replace(/'/g, `'"'"'`)}'`; @@ -139,7 +142,7 @@ export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistr `setpriv --clear-groups --reuid ${DAIMON_ORGANIZATION_UID} --regid ${DAIMON_ORGANIZATION_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu 'test -r ${quote(TRAINING_SLOT_USAGE_LEDGER)} && test -r ${quote(TRAINING_INFERENCE_LEDGER)}'`, // The sealed train and test datasets: the one boundary here that must hold against the subject's own // namespace, not only against an honest tool. Last, so it runs with every mode already final. - ...renderTrainingSealedInputsAssertions(registration.uid) + ...renderTrainingSealedInputsAssertions(registration.uid, registration.privateTmp) ]; /** @@ -153,50 +156,91 @@ export const renderTrainingBrokerProvisioning = (registration: DaimonGrokRegistr * it loses *search* permission on the directory every dataset read must * traverse. * - * The three probes below are the attacker's route, not a cooperating tool's: + * Four attacker routes, not a cooperating tool's read: a direct + * read/search/list; a user + mount namespace of the worker's own with the + * bubblewrap mask and its parent lazily unmounted; a fresh `mount --bind` of + * `/run/training` made inside that namespace; and, per dataset, a `mount --bind` + * of the dataset's *own* mount, which carries no mask and — unlike its parent — + * is not refused for locked children. A live control container left at Docker's + * own `0755 root:root` hands the held-out answer key to uid 2200 through routes + * one, two and four (`.runtime/sealed-inputs-dac/EVIDENCE.md`). * - * 1. a direct read/search/list as the worker uid; - * 2. a user + mount namespace of the worker's own, detaching the bubblewrap - * `deny` mask and the parent mount, then listing again — the route that - * makes `profile-only` an unsound boundary for this path; - * 3. the same namespace, binding `/run/training` somewhere fresh so the bind - * carries no `deny` mask, then listing through it. + * Every route reports one verdict, and the verdicts are deliberately not + * interchangeable: * - * (2) and (3) are the ones the mask cannot answer and DAC can: `--map-root-user` - * maps only the worker uid, so `CAP_DAC_OVERRIDE` in that namespace is - * ineffective against a root-owned inode, and a fresh bind re-exposes the same - * root-owned directory rather than the bytes under it. A probe that reaches no - * verdict refuses the slot; a worker that cannot create the namespace at all - * cannot take the route, and says so. + * - `reachable` — the bytes were read. Refuse. + * - `denied at-read` — the route ran and the kernel's permission check refused + * the open or the list. This is the DAC seal doing the work. + * - `denied at-mount` — the mount the route needs was refused although the + * syscall is available. + * - `unavailable seccomp` / `unavailable kernel` — the worker uid cannot open + * the namespace the route needs at all, so the route provably cannot happen. + * That is a *stronger* denial than DAC, and naming which layer refused is + * the point: `EPERM` from a seccomp filter and `EPERM` from a kernel or LSM + * policy are indistinguishable by errno, so the mechanism is derived from + * the pinned profile Spawnfile ships and the declaration's digest binds + * (`seccompRoutes.ts`), never guessed. + * - anything else — no verdict. Refuse. "Provably cannot happen" and "could + * not tell" must never collapse into one pass. + * + * The probe's workspace is the worker's **private** tmp, never `/tmp`: the + * shared temps are provisioned `root: 1774` so a worker lists + * names only, and probing from there is what made the first live run refuse + * every trial on `namespace-rebind reached no verdict` — the `mkdir` failed, not + * the seal. The private tmp is also the workspace the subject itself has. */ -export const renderTrainingSealedInputsAssertions = (workerUid = TRAINING_WORKER_UID): string[] => [ - "sealed_denied() {", - ` sealed_out=$(setpriv --clear-groups --reuid ${workerUid} --regid ${workerUid} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- /bin/sh -c "$1" 2>&1) || true`, - ' case "$sealed_out" in', - ` *SEALED-REACHABLE*) echo "the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is reachable by uid ${workerUid} via $2" >&2; exit 1 ;;`, - ' *SEALED-DENIED*) echo "sealed inputs denial observed for $2" ;;', - ' *SEALED-NO-NAMESPACE*) echo "sealed inputs probe $2 could not open a user namespace as the worker uid, so that route does not exist here" ;;', - ' *) echo "sealed inputs probe $2 reached no verdict, so the denial is unproven: $sealed_out" >&2; exit 1 ;;', +export const renderTrainingSealedInputsAssertions = ( + workerUid = TRAINING_WORKER_UID, + privateTmp = path.posix.join(TRAINING_WORKER_ROOT, String(TRAINING_WORKER_UID), "tmp"), + mechanism = trainingNamespaceDenialMechanism() +): string[] => [ + // Diagnostics go to a file rather than into the verdict: a route's stderr is util-linux prose that + // would turn every real verdict into "unknown", but a no-verdict refusal with no diagnostic is what + // made the first live failure unreadable, so the unknown branch prints it back. + `seal_diagnostic=${quote(`${TRAINING_SLOT_ROOT}/seal-probe.err`)}`, + `seal_worker() { setpriv --clear-groups --reuid ${workerUid} --regid ${workerUid} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- /bin/sh -c "$1" 2>"$seal_diagnostic"; }`, + "seal_rows=''", + 'seal_record() { if [ -n "$seal_rows" ]; then seal_rows="$seal_rows,"; fi; seal_rows="$seal_rows{\\"route\\":\\"$1\\",\\"verdict\\":\\"$2\\"}"; }', + "seal_route() {", + ' case "$2" in', + ` reachable) echo "the sealed inputs root ${TRAINING_SEALED_INPUTS_ROOT} is REACHABLE by uid ${workerUid} via $1" >&2; exit 1 ;;`, + ' "denied at-read"|"denied at-mount") echo "sealed inputs route $1: $2"; seal_record "$1" "$2" ;;', + ' "unavailable seccomp"|"unavailable kernel") echo "sealed inputs route $1: $2 — the worker uid cannot open the namespace this route needs, so it provably cannot happen"; seal_record "$1" "$2" ;;', + ' *) echo "sealed inputs route $1 reached no verdict, so the denial is unproven: ${2:-no stdout} [stderr: $(cat "$seal_diagnostic" 2>/dev/null | tr "\\n" " " | cut -c1-400)]" >&2; exit 1 ;;', " esac", "}", + // One availability determination for every namespace route, so a filtered syscall is reported as the + // verdict it is instead of surfacing as a route that mysteriously produced nothing. + "seal_namespace=available", + `if ! seal_worker 'unshare --user --map-root-user --mount true' >/dev/null 2>&1; then seal_namespace='unavailable ${mechanism}'; fi`, + 'seal_ns_route() { if [ "$seal_namespace" != available ]; then seal_route "$1" "$seal_namespace"; else seal_route "$1" "$(seal_worker "$2")"; fi; }', // Asserted, never set: the root filesystem is read-only, so an image that did not bake this mode // cannot be corrected here — and that is the point. Refuse instead. `test ! -L ${quote(TRAINING_SEALED_INPUTS_ROOT)} && test -d ${quote(TRAINING_SEALED_INPUTS_ROOT)}`, `if [ "$(stat -c '%u:%g %a' ${quote(TRAINING_SEALED_INPUTS_ROOT)})" != "${TRAINING_SEALED_INPUTS_IDENTITY.uid}:${TRAINING_SEALED_INPUTS_IDENTITY.gid} ${TRAINING_SEALED_INPUTS_IDENTITY.mode}" ]; then`, ` echo "the training image must bake ${TRAINING_SEALED_INPUTS_ROOT} as ${TRAINING_SEALED_INPUTS_IDENTITY.uid}:${TRAINING_SEALED_INPUTS_IDENTITY.gid} ${TRAINING_SEALED_INPUTS_IDENTITY.mode} on its read-only root; the sealed datasets are unprotected otherwise" >&2; exit 1`, "fi", - `sealed_denied "if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1 || test -r ${TRAINING_SEALED_INPUTS_ROOT} || test -x ${TRAINING_SEALED_INPUTS_ROOT}; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi" direct-read`, - `sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c 'umount -l ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; umount -l /run/training >/dev/null 2>&1; if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0' || echo SEALED-NO-NAMESPACE" namespace-unmount`, - `sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c 'mkdir -p /tmp/.sealed-probe >/dev/null 2>&1 || exit 0; mount --bind /run/training /tmp/.sealed-probe >/dev/null 2>&1 || { echo SEALED-DENIED; exit 0; }; if ls -1 /tmp/.sealed-probe/inputs >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0' || echo SEALED-NO-NAMESPACE" namespace-rebind`, - // Per dataset, because the strongest route is specific: binding the *dataset's own mount* somewhere - // fresh carries no `deny` mask and is not refused for locked children the way binding their parent is. - // A live control container with this directory left at Docker's own `0755 root:root` handed the held-out - // answer key to uid 2200 through exactly this route. Root enumerates the children; the worker may not. + `seal_route direct-read "$(seal_worker 'if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1 || test -r ${TRAINING_SEALED_INPUTS_ROOT} || test -x ${TRAINING_SEALED_INPUTS_ROOT}; then echo reachable; else echo denied at-read; fi')"`, + `seal_ns_route namespace-unmount "unshare --user --map-root-user --mount -- /bin/sh -c 'umount -l ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; umount -l /run/training >/dev/null 2>&1; if ls -1 ${TRAINING_SEALED_INPUTS_ROOT} >/dev/null 2>&1; then echo reachable; else echo denied at-read; fi; exit 0'"`, + `seal_ns_route namespace-rebind-parent "unshare --user --map-root-user --mount -- /bin/sh -c 'mkdir -p ${privateTmp}/seal-parent 2>/dev/null || { echo unknown private-tmp-unwritable; exit 0; }; if mount --bind /run/training ${privateTmp}/seal-parent 2>/dev/null; then ls -1 ${privateTmp}/seal-parent/inputs >/dev/null 2>&1 && echo reachable || echo denied at-read; else echo denied at-mount; fi; exit 0'"`, + // Per dataset, because the strongest route is specific. Root enumerates the children; the worker may not. `for sealed_child in ${TRAINING_SEALED_INPUTS_ROOT}/*; do`, ' test -e "$sealed_child" || continue', - ` sealed_denied "if ls -1 \\"$sealed_child\\" >/dev/null 2>&1 || cat \\"$sealed_child\\"/* >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi" "direct-read $sealed_child"`, - ` sealed_denied "unshare --user --map-root-user --mount -- /bin/sh -c \\"mkdir -p /tmp/.sealed-probe-child >/dev/null 2>&1 || exit 0; mount --bind '$sealed_child' /tmp/.sealed-probe-child >/dev/null 2>&1 || { echo SEALED-DENIED; exit 0; }; if ls -1 /tmp/.sealed-probe-child >/dev/null 2>&1; then echo SEALED-REACHABLE; else echo SEALED-DENIED; fi; exit 0\\" || echo SEALED-NO-NAMESPACE" "namespace-rebind $sealed_child"`, - "done" + ` seal_route "direct-read $sealed_child" "$(seal_worker "if ls -1 '$sealed_child' >/dev/null 2>&1 || cat '$sealed_child'/* >/dev/null 2>&1; then echo reachable; else echo denied at-read; fi")"`, + ` seal_ns_route "namespace-rebind $sealed_child" "unshare --user --map-root-user --mount -- /bin/sh -c 'mkdir -p ${privateTmp}/seal-child 2>/dev/null || { echo unknown private-tmp-unwritable; exit 0; }; if mount --bind \\"$sealed_child\\" ${privateTmp}/seal-child 2>/dev/null; then ls -1 ${privateTmp}/seal-child >/dev/null 2>&1 && echo reachable || echo denied at-read; else echo denied at-mount; fi; exit 0'"`, + "done", + // A Spawnfile-owned attestation beside the slot receipt, readable by the organization uid. The + // cross-repo `noopolis.daimon.grok-slot-preflight.v2` canary shape is deliberately untouched — its + // schema lives in Daimon and no consumer here can validate an added member — so the per-route + // mechanism is recorded here rather than implied by a bare `result: "denied"` there. + `seal_attestation=${quote(TRAINING_SEALED_INPUTS_ATTESTATION)}`, + `printf '{"version":"spawnfile.training-sealed-inputs.v1","root":"%s","worker_uid":%s,"identity":"%s","pinned_seccomp_profile_sha256":"%s","namespace_routes":"%s","routes":[%s]}\\n' ` + + `${quote(TRAINING_SEALED_INPUTS_ROOT)} ${workerUid} ` + + `"$(stat -c '%u:%g %a' ${quote(TRAINING_SEALED_INPUTS_ROOT)})" ${quote(DAIMON_GROK_SECCOMP_PROFILE_SHA256)} ` + + '"$seal_namespace" "$seal_rows" > "$seal_attestation.tmp"', + 'chown 0:0 "$seal_attestation.tmp"; chmod 0640 "$seal_attestation.tmp"; chown 0:' + String(DAIMON_ORGANIZATION_UID) + ' "$seal_attestation.tmp"', + 'mv "$seal_attestation.tmp" "$seal_attestation"', + 'echo "sealed inputs attestation written to $seal_attestation"' ]; /** Where the broker's own `service.json` lands, so the supervisor can prove the slot it restarted is the slot it provisioned. */ diff --git a/src/compiler/training/broker/seccompRoutes.ts b/src/compiler/training/broker/seccompRoutes.ts new file mode 100644 index 00000000..0bf79e13 --- /dev/null +++ b/src/compiler/training/broker/seccompRoutes.ts @@ -0,0 +1,41 @@ +import { DAIMON_GROK_SECCOMP_PROFILE_BYTES } from "../../../shared/daimonGrokSeccompProfile.js"; + +/** + * The syscalls a worker-uid namespace escape needs before it can even be + * attempted: a new user namespace, a new mount namespace, and the mount and + * unmount calls inside them. + * + * `clone`/`clone3` are deliberately absent. The pinned profile allows them only + * under an argument filter, so a static "is it allowed" question has no honest + * yes/no answer for them, and `unshare` already gates the route. + */ +export const TRAINING_NAMESPACE_ROUTE_SYSCALLS = ["unshare", "mount", "umount2", "setns"] as const; + +/** + * Whether the pinned seccomp profile lets the worker uid attempt a namespace + * escape at all — an **unconditional** `SCMP_ACT_ALLOW`, with no + * `includes.caps` (the container drops `CAP_SYS_ADMIN`) and no argument filter. + * + * This exists to keep one specific lie out of the slot receipt. When a probe + * route cannot run, the honest record is *which layer refused it*: a syscall + * the profile filters is a stronger denial than DAC and must be named as + * `seccomp`, while the same syscall refused by a profile that allows it came + * from the kernel or an LSM and must be named `kernel`. Merging the two into + * "denied" would let a filtered-syscall run read as though DAC had held. + */ +export const pinnedProfileAllowsNamespaceRoutes = (bytes: string = DAIMON_GROK_SECCOMP_PROFILE_BYTES): boolean => { + const profile = JSON.parse(bytes) as { syscalls?: { names?: string[]; action?: string; includes?: { caps?: string[] }; args?: unknown[] }[] }; + return TRAINING_NAMESPACE_ROUTE_SYSCALLS.every((syscall) => (profile.syscalls ?? []).some((rule) => + (rule.names ?? []).includes(syscall) && rule.action === "SCMP_ACT_ALLOW" + && (rule.includes?.caps ?? []).length === 0 && (rule.args ?? []).length === 0)); +}; + +/** + * What to blame when the worker uid cannot open the namespace a route needs. + * + * Derived, never guessed: seccomp and a DAC/LSM refusal both surface as + * `EPERM`, so errno cannot tell them apart. The pinned profile can — Spawnfile + * ships its bytes and the declaration binds their digest. + */ +export const trainingNamespaceDenialMechanism = (bytes?: string): "seccomp" | "kernel" => + pinnedProfileAllowsNamespaceRoutes(bytes) ? "kernel" : "seccomp"; From f959cc8e17d74e3e13e9138ea07c659588fcf5d8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 16:17:20 +0200 Subject: [PATCH 82/84] docs: specify the seal probe verdicts, their mechanisms and the sealed-inputs attestation --- specs/TRAINING_CONTAINERS.md | 59 ++++++++++++++++++++------ src/compiler/training/broker/AGENTS.md | 17 ++++++++ 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/specs/TRAINING_CONTAINERS.md b/specs/TRAINING_CONTAINERS.md index 676820e1..ce2c81a5 100644 --- a/specs/TRAINING_CONTAINERS.md +++ b/specs/TRAINING_CONTAINERS.md @@ -459,20 +459,51 @@ the read-only image root. The image bakes it `0:2000 0750`: Root provisioning therefore asserts that identity instead of setting it, and refuses the slot when an image did not bake it. It then attacks the seal as the -worker uid, at container start and on every recycle: a direct read/search/list, -the same list after `unshare --user --map-root-user --mount` plus a lazy unmount -of the mask and its parent, and the same list through a fresh `mount --bind` of -`/run/training` made inside that namespace — and, per dataset, a read and a -bind of the dataset's own mount, the route that carries no mask and is not -refused for locked children the way binding their parent is. A control -container left at Docker's own `0755 root:root` hands the held-out answer key -to uid 2200 through exactly that route, and the sealed one refuses all four -(`.runtime/sealed-inputs-dac/EVIDENCE.md`). A probe that reaches no verdict -refuses the slot; a worker that cannot open a namespace at all cannot take the -route, and the supervisor log says so. The slot canary for this path -additionally requires the worker uid to be unable to *enter* the directory, not -merely to read it: a searchable directory hands over every dataset the subject -can name. +worker uid, at container start and on every recycle, from the worker's **own +private tmp** — never `/tmp`, which the broker provisioning closes to +`root:2000 1774` so a worker lists names only, and probing from there refused +the first live run on a `mkdir` rather than on the seal. The routes are a direct +read/search/list; the same list after `unshare --user --map-root-user --mount` +plus a lazy unmount of the mask and its parent; the same list through a fresh +`mount --bind` of `/run/training` inside that namespace; and, per dataset, a +read and a `mount --bind` of the dataset's *own* mount, the route that carries +no mask and is not refused for locked children the way binding its parent is. A +control container left at Docker's own `0755 root:root` hands the held-out +answer key to uid 2200 through three of those; the sealed one refuses all of +them (`.runtime/sealed-inputs-dac/EVIDENCE.md`). + +Each route yields exactly one verdict, and they are deliberately not +interchangeable: + +| verdict | meaning | +| --- | --- | +| `reachable` | the bytes were read — refuse the slot | +| `denied at-read` | the route ran and the kernel's permission check refused the open or list; the DAC seal did the work | +| `denied at-mount` | the mount the route needs was refused although the syscall is available | +| `unavailable seccomp` / `unavailable kernel` | the worker uid cannot open the namespace the route needs, so the route provably cannot happen — a *stronger* denial than DAC, with the layer that refused it named | +| anything else | no verdict — refuse the slot, and print the probe's stderr | + +"Provably cannot happen" and "could not tell" never collapse into one pass. The +mechanism behind an unavailable namespace is derived, not guessed: a seccomp +`EPERM` and a kernel or LSM `EPERM` are indistinguishable by errno, so +Spawnfile answers it from the pinned profile it ships and the declaration's +digest binds — `seccomp` when that profile does not allow `unshare`, `mount`, +`umount2` and `setns` unconditionally, `kernel` when it does and the syscall was +still refused. The pinned `seccomp-default-plus-userns` profile allows all four +outright, because bubblewrap needs them, so in production an unavailable +namespace is reported as `kernel`. + +Provisioning records every route and its verdict in +`/run/training/slot/sealed-inputs.json`, a Spawnfile-owned +`spawnfile.training-sealed-inputs.v1` attestation, `root:2000 0640` beside the +slot preflight receipt. It is separate on purpose: the canary members of +`noopolis.daimon.grok-slot-preflight.v2` are a cross-repo contract whose schema +lives in Daimon, and a bare `result: "denied"` there must not be read as "DAC +held" when a filtered syscall is what refused the route. + +The slot canary for this path additionally requires the worker uid to be unable +to *enter* the directory, not merely to read it: a searchable directory hands +over every dataset the subject can name. ### Evidence before the first trial diff --git a/src/compiler/training/broker/AGENTS.md b/src/compiler/training/broker/AGENTS.md index 994341af..fc617dae 100644 --- a/src/compiler/training/broker/AGENTS.md +++ b/src/compiler/training/broker/AGENTS.md @@ -22,6 +22,9 @@ container that runs a brokered Grok subject beside the evaluator. no caller-supplied path or command) and owns `startTrainingSlot`, the only way a slot comes up: `provision → start → canaries → generation → receipt`. - `receipt.ts` writes `noopolis.daimon.grok-slot-preflight.v2`. +- `seccompRoutes.ts` answers, from the pinned profile bytes alone, whether the + worker uid may even attempt a namespace escape — the only honest way to say + `seccomp` rather than `kernel` when a route is unavailable. - `entrypoint.ts`/`main.ts` are the image's root entrypoint. Local constraints: @@ -57,6 +60,20 @@ Local constraints: rebind of each dataset's own mount, which is the one a mask cannot answer — and no `unenforcedBindPolicy` waives it (`.runtime/sealed-inputs-dac/EVIDENCE.md`). +- **Probe from the worker's private tmp, never `/tmp`.** The broker + provisioning closes the shared temps to `root:2000 1774` ("workers list names + only"), so a `mkdir /tmp/...` as the worker uid fails — which is how the first + live run with the seal refused every trial on `namespace-rebind reached no + verdict`. The cause was the probe's workspace, not the seal and not seccomp: + the pinned profile allows `unshare`/`mount`/`umount2`/`setns` outright. +- **Four verdicts, never merged** (`reachable` / `denied at-read` / + `denied at-mount` / `unavailable seccomp|kernel`), plus no-verdict, which + refuses and prints the probe's stderr. A route the kernel will not let the + worker attempt is a stronger denial than DAC and is recorded as such, with the + layer named — derived from the pinned profile in `seccompRoutes.ts`, because + errno cannot tell a seccomp `EPERM` from a kernel one. Per-route verdicts land + in `/run/training/slot/sealed-inputs.json`; the cross-repo + `grok-slot-preflight.v2` canary shape is deliberately untouched. - Grok 1.0.34 materializes every `deny` target inside bubblewrap as the worker uid, so a deny entry it cannot create makes the whole profile fail. That is why `/run/paideia` is masked as a directory rather than file by file, and why From 4d4c31a62d220368e444fb40ada892708ba19c02 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 16:47:45 +0200 Subject: [PATCH 83/84] chore: re-vendor Daimon's runtime contract manifest at 36ce9f0 Co-Authored-By: Claude Opus 5 (1M context) --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 21559194..2caad472 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","sourceSha256":"dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","x64Sha256":"67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":262144,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"state":{"enum":["running","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","sourceSha256":"dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","x64Sha256":"67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":262144,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index e9acd78f..f9776614 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:6a8a49f0405a7b07a6abd5caaf7d714313f7f26b1de76d8330850da811d45bb9 +sha256:6cf2c185e7e187ae80288bd281e596391f01802edc8a8422411d1a5b70e7e5ea diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index a8b98474..60eeaeca 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:6a8a49f0405a7b07a6abd5caaf7d714313f7f26b1de76d8330850da811d45bb9" as const; + "sha256:6cf2c185e7e187ae80288bd281e596391f01802edc8a8422411d1a5b70e7e5ea" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon"; From 98de64124f74c7f73802f24ffb8b364d787ed1f6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 19:15:39 +0200 Subject: [PATCH 84/84] chore: re-vendor Daimon's runtime contract manifest with the widened receipt code enum --- src/runtime/daimon/contract-manifest.json | 2 +- src/runtime/daimon/contract-manifest.sha256 | 2 +- src/runtime/daimon/contractManifest.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/daimon/contract-manifest.json b/src/runtime/daimon/contract-manifest.json index 2caad472..74fe09d8 100644 --- a/src/runtime/daimon/contract-manifest.json +++ b/src/runtime/daimon/contract-manifest.json @@ -1 +1 @@ -{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"state":{"enum":["running","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","sourceSha256":"dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","x64Sha256":"67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":262144,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} +{"activityResponseSchema":{"additionalProperties":false,"properties":{"items":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"id":{"format":"uuid","pattern":"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","type":"string"},"kind":{"enum":["wake_started","wake_completed","wake_rejected","wake_aborted","agent_stopped"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["id","agentId","kind","occurredAt"],"type":"object"},"maxItems":100,"type":"array"},"nextCursor":{"maxLength":16,"minLength":1,"pattern":"^(0|[1-9][0-9]{0,15})$","type":"string"},"version":{"const":"noopolis.daimon.organization-runtime-activity.v1"}},"required":["version","items"],"type":"object"},"activityV2ResponseSchema":{"additionalProperties":false,"properties":{"executions":{"items":{"additionalProperties":false,"properties":{"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"delivery_ids":{"items":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"maxItems":32,"type":"array"},"execution_id":{"type":"string"},"state":{"const":"running"}},"required":["agent_id","execution_id","state","delivery_ids"],"type":"object"},"maxItems":32,"type":"array"},"items":{"items":{"additionalProperties":false,"properties":{"acceptance_id":{"type":"string"},"accepted_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"active":{"type":"boolean"},"agent_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["engine_failed","host_stopped","host_stopping","queued_wake_stopped","active_wake_aborted","queue_full","unknown_agent"]},"deferred":{"type":"boolean"},"delivery_id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"execution_id":{"type":"string"},"queue_position":{"minimum":1,"type":"integer"},"request_digest":{"type":"string"},"state":{"enum":["accepted","running","completed","failed","stopped"]},"text":{"maxLength":16384,"type":"string"},"updated_at":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"version":{"const":"noopolis.daimon.wake-receipt-status.v2"}},"required":["version","acceptance_id","agent_id","delivery_id","request_digest","state","accepted_at","updated_at","active"],"type":"object"},"maxItems":2112,"type":"array"},"state":{"enum":["running","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-activity.v2"}},"required":["version","items"],"type":"object"},"agySubscriptionRealm":{"directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/agy-subscription-realm","fileMode":384,"maxUnlockBytes":4096,"unlockMountPath":"/var/lib/spawnfile/daimon/agy-unlock-secret","unlockSourceSlot":"agy-unlock-secret"},"attention":{"accounting":"execution-start-reservations","busyDispatch":"bounded-pending-message-batch","completion":"explicit-per-delivery","defaultMaxBatchBytes":12000,"defaultMaxBatchMessages":8,"enabledBy":"agents[].attention","idleDispatch":"immediate","unhandled":"deferred-until-new-input"},"consumedConfigFields":["version","host.bindHost","host.port","host.controlTokenEnv","agents[].id","agents[].name","agents[].instructions","agents[].workspacePath","agents[].runtimeHomePath","agents[].engine.kind","agents[].engine.model","agents[].engine.reasoningEffort","agents[].engine.codexSandbox","agents[].schedule.kind","agents[].schedule.interval_ms","agents[].schedule.cron","agents[].schedule.timezone","agents[].schedule.prompt","agents[].schedule.jitter_seconds","agents[].mcp","agents[].moltnet","agents[].memory","agents[].attention"],"deliverySemantics":{"activeDeliveryIdempotency":"unbounded-until-terminal","concurrentSameAgentTurns":false,"externalEffectsExactlyOnce":false,"recovery":"at-least-once-with-stable-wake-id","terminalReceiptHorizon":2048},"engineCredentialMaterial":{"codex":{"destinationRelativePath":".codex/auth.json","directoryMode":448,"fileMode":384,"sourceRelativePath":".daimon-inbound/codex-auth","sourceSlot":"codex-auth"}},"grokEngineBroker":{"artifacts":{"arm64Sha256":"c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","sourceSha256":"dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","x64Sha256":"67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7"},"backendSocketPath":"/run/daimon-engine-broker/backend.sock","bounds":{"capabilityBundleBytes":8196,"capabilityBytes":4096,"outputBytes":262144,"promptBytes":65536},"controlProtocolVersion":"noopolis.daimon.engine-broker.v2","controlSocketPath":"/run/daimon-engine-broker/control.sock","credentialHomePath":"/var/lib/spawnfile/daimon/grok-subscription-realm","grokCliArtifacts":{"arm64":{"bytes":136090504,"sha256":"39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64"},"x64":{"bytes":163035648,"sha256":"be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d","url":"https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64"}},"grokCliBuild":"3736acbc8658","grokCliVersion":"1.0.34","grokExecutablePath":"/usr/local/bin/grok","identities":{"brokerUid":2100,"firstWorkerUid":2200,"organizationUid":2000},"inferenceGrants":{"bodyMembers":["messages","model","reasoning_effort","response_format","stream","stream_options"],"client":{"configSha256":{"grok-4.5":{"high":"ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2","low":"a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74","medium":"1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9"},"grok-4.6":{"high":"5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac","low":"79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f","medium":"6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8"},"grok-build":{"high":"98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c","low":"ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1","medium":"01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d"}},"envKey":"DAIMON_INFERENCE_GRANT","modelId":"daimon-inference-grok"},"failureCodes":["auth_stale","grant_limit","invalid_request","unavailable"],"ledgerDedupeKey":["grant","request"],"ledgerVersion":"noopolis.daimon.inference-usage.v1","limits":{"maxRequests":64,"maxTokens":2000000},"maxInFlightRequestsPerGrant":1,"maxLiveGrants":8,"messageRoles":["system","user","assistant"],"purposes":["judge","optimizer"],"requestKinds":["request_inference_grant","release_inference_grant"],"tokenPrefix":"inference_","ttlMs":600000},"launcherSocketPath":"/run/daimon-engine-broker/launcher.sock","mcpFacade":{"host":"127.0.0.1","path":"/mcp","port":43124},"nativeAbiVersion":2,"nativeExecutablePath":"/opt/daimon/bin/daimon-engine-broker","projectionVersion":"noopolis.daimon.grok-broker-projection.v1","providerProxy":{"host":"127.0.0.1","port":43123},"registrationPath":"/etc/daimon-engine-broker/registrations.bin","serviceConfigPath":"/etc/daimon-engine-broker/service.json","serviceConfigVersions":["noopolis.daimon.engine-broker-service.v1","noopolis.daimon.engine-broker-service.v2"],"slotPreflightVersion":"noopolis.daimon.grok-slot-preflight.v2","turnLimits":{"bounds":{"maxRequests":[1,48],"maxTokens":[1,10000000],"timeoutMs":[1000,3600000]},"keys":["maxRequests","maxTokens","timeoutMs"],"limitReasons":["tokens","requests","timeout","none"],"maxInFlightRequests":1,"missingUsageEstimate":{"inputBytesPerToken":2,"outputTokens":4096},"requestUsageMaxTokens":500000,"tokenCeilingOvershoot":"at-most-one-request","v1Defaults":{"maxRequests":32,"maxTokens":300000,"timeoutMs":240000},"wakeMayOnlyLower":true},"turnRecordVersions":["noopolis.daimon.engine-broker-turn.v1","noopolis.daimon.engine-broker-turn.v2"],"turnStorePath":"/var/lib/spawnfile/daimon/grok-subscription-realm/turns","wakeLimitEnvironment":{"maxTokens":"DAIMON_ENGINE_WAKE_TOKEN_CEILING","timeoutMs":"DAIMON_ENGINE_WAKE_TIMEOUT_MS"},"worker":{"configSha256":{"grok-4.5":{"high":"c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f","low":"8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6","medium":"59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155"},"grok-4.6":{"high":"65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3","low":"ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e","medium":"df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f"},"grok-build":{"high":"a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3","low":"fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5","medium":"8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656"}},"defaultModel":"grok-4.6","defaultReasoningEffort":"low","home":{"directory":{"group":"worker","mode":1017,"uid":0},"organizationRuntimeHome":{"group":"worker","mode":456,"owner":"organization"},"privateTmp":{"mode":448,"owner":"worker","relativeToWorkerHome":"tmp"},"readOnlyFiles":{"gid":0,"mode":292,"names":["config.toml","managed_config.toml","requirements.toml","sandbox.toml","trusted_folders.toml"],"uid":0},"sandboxEvents":{"group":"broker","mode":416,"owner":"worker","relativePath":"sessions/sandbox-events.jsonl"},"sessionsDirectory":{"group":"worker","mode":1017,"relativePath":"sessions","uid":0},"sharedTmp":{"maxGroupExclusive":2200,"mode":1020,"otherMode":4,"paths":["/tmp","/var/tmp"],"uid":0},"spillDirectory":{"fileMode":416,"group":"worker","mode":1512,"owner":"organization","relativeToRuntimeHome":"tool-output"}},"maxTurns":48,"modelId":"daimon-broker-grok","models":["grok-4.6","grok-4.5","grok-build"],"reasoningEfforts":["low","medium","high"],"systemPromptSha256":"2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8","toolIds":["run_terminal_cmd","read_file","grep","list_dir","search_tool","use_tool"],"visibleTools":["grep","list_dir","read_file","run_terminal_command","search_tool","use_tool"]}},"grokSubscriptionRealm":{"agentCredentialRelativePath":".grok/auth.json","bootstrapMountPath":"/var/lib/spawnfile/daimon/grok-bootstrap-auth","bootstrapSourceSlot":"grok-auth","directoryMode":448,"durableMountPath":"/var/lib/spawnfile/daimon/grok-subscription-realm","fileMode":384,"maxCredentialBytes":65536},"healthResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"state":{"enum":["starting","running","stopping","stopped","idle","failed"]}},"required":["agentId","state"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["starting","running","stopping","stopped"]},"version":{"const":"noopolis.daimon.organization-runtime-health.v1"}},"required":["version","state","agents"],"type":"object"},"organizationRuntimeConfigSchema":{"$id":"noopolis.daimon.organization-runtime.v1","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v1"}},"required":["version","host","agents"],"type":"object"},"organizationRuntimeConfigV2Schema":{"$id":"noopolis.daimon.organization-runtime.v2","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"attention":{"additionalProperties":false,"properties":{"maxBatchBytes":{"maximum":12000,"minimum":1024,"type":"integer"},"maxBatchMessages":{"maximum":32,"minimum":1,"type":"integer"},"maxExecutions":{"maximum":9007199254740991,"minimum":1,"type":"integer"},"maxTokens":{"maximum":9007199254740991,"minimum":1,"type":"integer"}},"type":"object"},"engine":{"additionalProperties":false,"allOf":[{"if":{"properties":{"kind":{"const":"grok"}}},"then":{"dependentRequired":{"model":["reasoningEffort"],"reasoningEffort":["model"]},"properties":{"codexSandbox":false,"model":{"enum":["grok-4.6","grok-4.5","grok-build"]},"reasoningEffort":{"enum":["low","medium","high"]}}}},{"if":{"properties":{"kind":{"const":"agy"}}},"then":{"properties":{"codexSandbox":false,"model":false,"reasoningEffort":false}}}],"properties":{"codexSandbox":{"additionalProperties":false,"properties":{"mode":{"const":"workspace-write"},"networkAccess":{"const":false},"webSearch":{"const":"disabled"}},"required":["mode","networkAccess","webSearch"],"type":"object"},"kind":{"enum":["codex","grok","agy"]},"model":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"reasoningEffort":{"enum":["none","minimal","low","medium","high","xhigh","max","ultra","persistent"]}},"required":["kind"],"type":"object"},"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"instructions":{"maxLength":16384,"minLength":1,"pattern":"\\S","type":"string"},"mcp":{"items":{"additionalProperties":false,"properties":{"args":{"items":{"maxLength":4096,"type":"string"},"maxItems":32,"type":"array"},"authSecretEnv":{"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"command":{"pattern":"^/","type":"string"},"env":{"additionalProperties":{"type":"string"},"type":"object"},"name":{"maxLength":4096,"minLength":1,"type":"string"},"tools":{"items":{"maxLength":4096,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"transport":{"enum":["stdio","sse","streamable_http"]},"url":{"type":"string"}},"required":["name","transport","args","env","tools"],"type":"object"},"maxItems":8,"type":"array"},"memory":{"additionalProperties":false,"properties":{"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"source":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"tokenBudget":{"maximum":1000000,"minimum":1,"type":"integer"}},"required":["runtimeHomePath"],"type":"object"},"moltnet":{"additionalProperties":false,"properties":{"cliPath":{"pattern":"^/","type":"string"},"configPath":{"pattern":"^/","type":"string"},"networks":{"items":{"additionalProperties":false,"properties":{"dms":{"type":"boolean"},"id":{"minLength":1,"type":"string"},"rooms":{"items":{"minLength":1,"type":"string"},"type":"array","uniqueItems":true}},"required":["id","rooms","dms"],"type":"object"},"maxItems":16,"type":"array"}},"required":["cliPath","configPath","networks"],"type":"object"},"name":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"runtimeHomePath":{"maxLength":4096,"pattern":"^/","type":"string"},"schedule":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"disabled"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"interval_ms":{"maximum":31536000000,"minimum":1,"type":"integer"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"every"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","interval_ms","prompt"],"type":"object"},{"additionalProperties":false,"properties":{"cron":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"jitter_seconds":{"maximum":3600,"minimum":0,"type":"integer"},"kind":{"const":"cron"},"prompt":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"timezone":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["kind","cron","timezone","prompt"],"type":"object"}]},"workspacePath":{"maxLength":4096,"pattern":"^/","type":"string"}},"required":["id","name","instructions","workspacePath","runtimeHomePath","engine","schedule"],"type":"object"},"maxItems":32,"minItems":1,"type":"array"},"host":{"additionalProperties":false,"properties":{"bindHost":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"controlTokenEnv":{"maxLength":4096,"pattern":"^[A-Za-z_][A-Za-z0-9_]*$","type":"string"},"port":{"maximum":65535,"minimum":1,"type":"integer"}},"required":["bindHost","port","controlTokenEnv"],"type":"object"},"version":{"const":"noopolis.daimon.organization-runtime.v2"}},"required":["version","host","agents"],"type":"object"},"supportedEngineKinds":["agy","codex","grok"],"version":"noopolis.daimon.runtime-contract-manifest.v3","wakeAcceptanceTypes":["manual","message","schedule","external"],"wakeRequestSchema":{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"event":{"additionalProperties":false,"properties":{"id":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"kind":{"enum":["manual","message","schedule","external"]},"occurredAt":{"format":"date-time","pattern":"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$","type":"string"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake.v1"}},"required":["version","id","kind","text","occurredAt"],"type":"object"}},"required":["agentId","event"],"type":"object"},"wakeResultSchema":{"oneOf":[{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"durationMs":{"minimum":0,"type":"integer"},"status":{"const":"completed"},"text":{"maxLength":4096,"type":"string"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","text","durationMs"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["unauthorized","unknown_agent","queue_full","durable_inbox_required"]},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"type":"string"},"code":{"const":"invalid_request"},"status":{"const":"rejected"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"enum":["host_stopping","host_stopped","queued_wake_stopped","active_wake_aborted"]},"status":{"const":"stopped"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"},{"additionalProperties":false,"properties":{"agentId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"},"code":{"const":"engine_failed"},"status":{"const":"failed"},"version":{"const":"noopolis.daimon.wake-result.v1"},"wakeId":{"maxLength":4096,"minLength":1,"pattern":"\\S","type":"string"}},"required":["version","status","agentId","wakeId","code"],"type":"object"}]},"workAvailabilityResponseSchema":{"additionalProperties":false,"properties":{"agents":{"items":{"additionalProperties":false,"properties":{"agent_id":{"type":"string"},"budget":{"additionalProperties":false,"properties":{"agent_executions_remaining":{"minimum":0,"type":"integer"},"agent_executions_used":{"minimum":0,"type":"integer"},"agent_tokens_remaining":{"minimum":0,"type":"integer"},"agent_tokens_used":{"minimum":0,"type":"integer"},"armed":{"type":"boolean"},"epoch":{"type":"string"},"executions_remaining":{"minimum":0,"type":"integer"},"executions_used":{"minimum":0,"type":"integer"},"reason":{"type":"string"},"state":{"enum":["available","paused","stopped"]},"tokens_remaining":{"minimum":0,"type":"integer"},"tokens_used":{"minimum":0,"type":"integer"}},"required":["armed","epoch","state","executions_used","executions_remaining","tokens_used","tokens_remaining","agent_executions_used","agent_tokens_used"],"type":"object"},"deferred":{"minimum":0,"type":"integer"},"error":{"type":"string"},"pending":{"minimum":0,"type":"integer"},"running":{"type":"boolean"}},"required":["agent_id","pending","running","deferred","budget"],"type":"object"},"maxItems":32,"type":"array"},"state":{"enum":["running","paused","stopped"]},"version":{"const":"noopolis.daimon.work-availability.v1"}},"required":["version","state","agents"],"type":"object"},"workBlockedSchema":{"additionalProperties":false,"properties":{"reason":{"enum":["operator_stop","ledger_unavailable","host_stopping","host_stopped","queue_full"]},"retry_after_ms":{"maximum":300000,"minimum":1000,"type":"integer"},"version":{"const":"noopolis.daimon.work-blocked.v1"}},"required":["version","reason","retry_after_ms"],"type":"object"}} diff --git a/src/runtime/daimon/contract-manifest.sha256 b/src/runtime/daimon/contract-manifest.sha256 index f9776614..a3155f30 100644 --- a/src/runtime/daimon/contract-manifest.sha256 +++ b/src/runtime/daimon/contract-manifest.sha256 @@ -1 +1 @@ -sha256:6cf2c185e7e187ae80288bd281e596391f01802edc8a8422411d1a5b70e7e5ea +sha256:c672ed47e4a01a75f1482ea6e4b6ecb28205aab344741a10cad8783f5501b6c4 diff --git a/src/runtime/daimon/contractManifest.ts b/src/runtime/daimon/contractManifest.ts index 60eeaeca..bed96d41 100644 --- a/src/runtime/daimon/contractManifest.ts +++ b/src/runtime/daimon/contractManifest.ts @@ -7,7 +7,7 @@ import { SpawnfileError } from "../../shared/index.js"; export const DAIMON_CONTRACT_MANIFEST_VERSION = "noopolis.daimon.runtime-contract-manifest.v3" as const; export const DAIMON_CONTRACT_MANIFEST_SHA256 = - "sha256:6cf2c185e7e187ae80288bd281e596391f01802edc8a8422411d1a5b70e7e5ea" as const; + "sha256:c672ed47e4a01a75f1482ea6e4b6ecb28205aab344741a10cad8783f5501b6c4" as const; export const DAIMON_CONTRACT_MANIFEST_FILE = "contract-manifest.json"; export const DAIMON_CONTRACT_MANIFEST_DIGEST_FILE = "contract-manifest.sha256"; export const DAIMON_RUNTIME_HOME_ROOT = "/var/lib/spawnfile/instances/daimon";