From 4eb663a03f715e539ab02efbc0c9a45bf11c33ff Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 12:55:00 -0400 Subject: [PATCH 1/7] feat(pi): map Agent/Task frontmatter tools to the pi Agent tool An agent definition that lists Agent or Task under tools: now activates both pi tool names (the extension registers Agent with Task as its legacy alias) instead of reporting them as unsupported. Adds the enablement predicate (no tools: entry, or Agent/Task listed) and the read-only Explore tool set shared with the extension via the manifest. Refs: #6527, #6464 Assisted-by: Claude Signed-off-by: Wayne Sun --- internal/runtime/pi_agent.go | 35 +++++++++++++++++++++++++++++++ internal/runtime/pi_agent_test.go | 29 +++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/internal/runtime/pi_agent.go b/internal/runtime/pi_agent.go index fa17f555e5..55f60da7de 100644 --- a/internal/runtime/pi_agent.go +++ b/internal/runtime/pi_agent.go @@ -210,6 +210,29 @@ var claudeToolForPi = map[string]string{ "ls": "LS", } +// Sub-agent tool. Claude Code's Agent tool (legacy name Task) has no pi +// counterpart in core; the embedded fullsend-agent.js extension registers +// both names with the same contract (prompt, description, model, +// subagent_type) so the fleet's skills dispatch unchanged (#6527). +const ( + piAgentToolName = "Agent" + piAgentToolAlias = "Task" +) + +// piExploreTools is the read-only tool set a child gets when dispatched +// with subagent_type "Explore" (Claude Code's built-in read-only agent). +// Bootstrap writes it into the manifest so the extension and the runner +// agree on it. +var piExploreTools = []string{"read", "grep", "find", "ls"} + +// piAgentToolEnabled reports whether the runtime registers the Agent tool +// for this agent definition: an absent tools: entry means the default set, +// which includes it (as under Claude Code); a declared list must name +// Agent or Task. +func piAgentToolEnabled(def *piAgentDef) bool { + return def.Tools == nil || hasTool(def.Tools, piAgentToolName) || hasTool(def.Tools, piAgentToolAlias) +} + func hasTool(tools []string, name string) bool { for _, t := range tools { if t == name { @@ -232,6 +255,18 @@ func piToolsFor(claudeTools []string) (tools, unsupported []string) { if ct == "Skill" { continue } + if ct == piAgentToolName || ct == piAgentToolAlias { + // Both names are registered by the extension; --tools is a + // strict allowlist across built-in and extension tools, so + // activating one requires naming both. + for _, pt := range []string{piAgentToolName, piAgentToolAlias} { + if !seen[pt] { + seen[pt] = true + tools = append(tools, pt) + } + } + continue + } pt, ok := piToolForClaude[ct] if !ok { unsupported = append(unsupported, ct) diff --git a/internal/runtime/pi_agent_test.go b/internal/runtime/pi_agent_test.go index 30a9c05944..e5e21140b9 100644 --- a/internal/runtime/pi_agent_test.go +++ b/internal/runtime/pi_agent_test.go @@ -147,14 +147,39 @@ func TestPiToolsFor(t *testing.T) { assert.Nil(t, unsupported) tools, unsupported = piToolsFor([]string{"Read", "Edit", "MultiEdit", "Glob", "WebFetch", "Task", "LS"}) - assert.Equal(t, []string{"read", "edit", "find", "ls"}, tools) - assert.Equal(t, []string{"WebFetch", "Task"}, unsupported) + assert.Equal(t, []string{"read", "edit", "find", "Agent", "Task", "ls"}, tools, "Task is the legacy alias of Agent; both pi tool names are activated, in place") + assert.Equal(t, []string{"WebFetch"}, unsupported) + + tools, unsupported = piToolsFor([]string{"Agent", "Task", "Bash"}) + assert.Equal(t, []string{"Agent", "Task", "bash"}, tools, "Agent and Task activate the same pair once") + assert.Nil(t, unsupported) tools, unsupported = piToolsFor([]string{"Skill"}) assert.Equal(t, []string{}, tools, "restriction with no pi tools stays non-nil") assert.Nil(t, unsupported) } +func TestPiAgentToolEnabled(t *testing.T) { + t.Parallel() + assert.True(t, piAgentToolEnabled(&piAgentDef{}), "no tools: frontmatter means the default set, which includes the Agent tool") + assert.True(t, piAgentToolEnabled(&piAgentDef{Tools: []string{"Bash", "Agent"}})) + assert.True(t, piAgentToolEnabled(&piAgentDef{Tools: []string{"Task"}})) + assert.False(t, piAgentToolEnabled(&piAgentDef{Tools: []string{"Bash", "Skill"}})) + assert.False(t, piAgentToolEnabled(&piAgentDef{Tools: []string{}})) +} + +// The Explore tool set the extension hands read-only children must stay +// inside pi's built-in set the sandbox activates. +func TestPiExploreTools_SubsetOfDefaults(t *testing.T) { + t.Parallel() + for _, name := range piExploreTools { + assert.Contains(t, piDefaultTools, name) + } + assert.NotContains(t, piExploreTools, "bash") + assert.NotContains(t, piExploreTools, "write") + assert.NotContains(t, piExploreTools, "edit") +} + func TestPiToolNameMapsAreInverse(t *testing.T) { t.Parallel() for claude, pi := range piToolForClaude { From 2c621afd23777fac96999bf5b1d1fdd3c40c4e8c Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 12:58:47 -0400 Subject: [PATCH 2/7] feat(pi): add the fullsend-agent.js extension providing the Agent tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pi extension that registers Claude Code's Agent tool (and its legacy alias Task) with the same contract — prompt, description, model, subagent_type, run_in_background accepted and ignored — so the fleet's pr-review and retro-analysis skills dispatch unchanged on the pi runtime. Each call runs one child pi --print --mode json to completion with the parent's flag set (trust off, --no-extensions plus the manifest's provider extensions and the hook adapter, strict --tools, its own session dir) and FULLSEND_SUBAGENT_DEPTH=1; the extension refuses to register when that variable is already set, so recursion is impossible. Models are translated through the manifest alias table (opus|sonnet| haiku, Claude ids with an @suffix, anthropic/ and xai/ direct-API forms) and anything the run cannot serve — an invented Claude id, a provider without credentials — is rejected with the accepted forms rather than passed through. Explore gives the read-only tool set; a concurrency cap, a timeout that kills the child's process group, and a usage.jsonl line per child (for metrics.json) complete it. Failed children surface as isError with the child's message. Refs: #6527, #6464 Assisted-by: Claude Signed-off-by: Wayne Sun --- .../runtime/pi_extension/fullsend-agent.js | 741 ++++++++++++++++ .../pi_extension/fullsend-agent.test.mjs | 829 ++++++++++++++++++ 2 files changed, 1570 insertions(+) create mode 100644 internal/runtime/pi_extension/fullsend-agent.js create mode 100644 internal/runtime/pi_extension/fullsend-agent.test.mjs diff --git a/internal/runtime/pi_extension/fullsend-agent.js b/internal/runtime/pi_extension/fullsend-agent.js new file mode 100644 index 0000000000..2ab9968cbd --- /dev/null +++ b/internal/runtime/pi_extension/fullsend-agent.js @@ -0,0 +1,741 @@ +// fullsend-agent.js — pi extension that provides Claude Code's `Agent` tool +// (legacy alias `Task`) on the pi runtime, so skills written for Claude +// Code's sub-agent roster (pr-review, retro-analysis) dispatch unchanged +// (fullsend#6527, #6464). +// +// Each call runs one child `pi --print --mode json` to completion inside the +// sandbox, with the same flag set PiRuntime.Run gives the parent: trust off, +// no auto-discovered extensions, the manifest's provider extensions and the +// hook adapter with -e, a strict --tools allowlist, its own session dir. +// pi runs sibling tool calls from one assistant message concurrently, which +// is the parallel dispatch the skills ask for; `run_in_background` is +// accepted and ignored. Children never receive this extension (and refuse +// to register it if they did — FULLSEND_SUBAGENT_DEPTH), so recursion is +// impossible. +// +// The prompt goes over the child's stdin, never as a positional argument. +// Without a terminator pi's argv parser (0.84.4 dist/cli/args.js) reads a +// leading "-" as an unknown option (a startup error), a leading "--" as an +// unknown flag that swallows the next word, and a leading "@" as a file +// argument. pi does honour a "--" end-of-options terminator (args.js:23), +// but that is not enough to make argv usable here: after it a positional +// starting with "@" is *still* taken as a file argument (args.js:25), and +// argv is capped by the kernel either way (spawn E2BIG above ~128 KiB on +// Linux), which a context package easily exceeds. In --print mode pi reads +// a non-TTY stdin to EOF (dist/main.js readPipedStdin) and +// buildInitialMessage (dist/cli/initial-message.js) uses it as the initial +// message, so stdin carries an arbitrary prompt verbatim, whatever it +// starts with and however long it is. +// +// A child is stopped with SIGTERM, escalated to SIGKILL after a grace +// period — never SIGKILL first. pi's bash tool spawns commands `detached` +// in their own process group and kills them from its SIGTERM handler +// (dist/modes/print-mode.js -> killTrackedDetachedChildren, then exit 143); +// an unhandleable SIGKILL would leave those grandchildren running. +// +// Loaded explicitly by PiRuntime.Run with `-e` after the hook adapter. +// Everything it needs is the `agent` block of the manifest +// PiRuntime.Bootstrap wrote (FULLSEND_PI_MANIFEST). +import { spawn as nodeSpawn } from "node:child_process"; +import { appendFileSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname } from "node:path"; + +export const DEFAULT_MANIFEST_PATH = "/sandbox/pi-config/fullsend-manifest.json"; +export const DEPTH_ENV = "FULLSEND_SUBAGENT_DEPTH"; +// RESULT_MAX_BYTES caps the text handed back to the parent; pi's own +// built-in tools truncate at 50 KB, and a reviewer's findings fit well +// within this. +export const RESULT_MAX_BYTES = 64 * 1024; +const TRUNCATED_MARKER = "\n[truncated]"; +const STDERR_TAIL_BYTES = 4 * 1024; +const LOG_PREFIX = "[fullsend-agent]"; +const DEFAULT_MAX_CONCURRENT = 4; +const DEFAULT_TIMEOUT_SECONDS = 900; +const DEFAULT_THINKING = "medium"; +const DEFAULT_EXPLORE_TOOLS = ["read", "grep", "find", "ls"]; +const TOOL_NAME = "Agent"; +const TOOL_ALIAS = "Task"; +// Providers pi serves without an extension and with the same Vertex ADC +// the sandbox already carries. +const BUILTIN_PROVIDERS = ["google-vertex"]; +// DEFAULT_KILL_GRACE_MS is how long a child gets to handle SIGTERM (kill +// its own detached bash grandchildren and flush the session) before SIGKILL. +export const DEFAULT_KILL_GRACE_MS = 3000; +// MAX_STDOUT_LINE_CHARS bounds one --mode json line, mirroring the Go +// transcript reader's 1 MB cap. A longer line is dropped rather than +// buffered: a child that emits an unbounded line must not grow the +// orchestrator's heap without limit. +export const MAX_STDOUT_LINE_CHARS = 1024 * 1024; +// MAX_DESCRIPTION_BYTES caps the label copied into the usage file. Children +// append to one file concurrently, and only a write below PIPE_BUF (4096 on +// Linux) is atomic; the rest of a record is bounded by construction. +export const MAX_DESCRIPTION_BYTES = 512; +// PI_THINKING_LEVELS are pi's --thinking values, used to recognise (and +// drop) the "provider/id:high" shorthand pi's model resolver accepts +// (0.84.4 dist/core/model-resolver.js parseModelPattern): left in place it +// would silently override the manifest's thinking level for the child. +const PI_THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]); +// CHILD_SYSTEM_NOTE replaces the parent's APPEND_SYSTEM.md for a child. +// pi discovers PI_CODING_AGENT_DIR/APPEND_SYSTEM.md only when no +// --append-system-prompt was given (0.84.4 dist/core/resource-loader.js: +// the discovery is guarded by `if (!appendSources)`), and children share the +// parent's config dir — so without this flag every child would inherit the +// orchestrator persona, including its "make several Agent calls in one +// message" dispatch note, for a tool it does not have. +export const CHILD_SYSTEM_NOTE = + "You are a sub-agent dispatched by another agent. Carry out the single task in the message you " + + "were given and then stop. You have no sub-agent tool of your own and cannot dispatch further " + + "work. Your final assistant message is the entire result handed back to the agent that " + + "dispatched you, so make it self-contained."; + +// AGENT_TOOL_PARAMETERS is Claude Code's Agent tool input shape as plain +// JSON Schema (pi validates with typebox 1.x, which is JSON-Schema-native, +// so no typebox import is needed and the file stays runnable under node +// alone). +export const AGENT_TOOL_PARAMETERS = { + type: "object", + properties: { + prompt: { type: "string", description: "The full task for the sub-agent, including all context it needs; it starts with no memory of this conversation." }, + description: { type: "string", description: "A short (3-5 word) label for the task, shown in progress output." }, + model: { type: "string", description: "Model for the sub-agent: opus, sonnet, haiku, or a provider/id spec available in this run. Omit to inherit the current model." }, + subagent_type: { type: "string", description: "Explore gives a read-only sub-agent (read, grep, find, ls); any other value or omission gives the current tool set." }, + run_in_background: { type: "boolean", description: "Accepted for compatibility; sub-agents always run to completion inside the call." }, + }, + required: ["prompt"], +}; + +export function loadManifest(path = process.env.FULLSEND_PI_MANIFEST || DEFAULT_MANIFEST_PATH) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function cut(s, sep) { + const i = s.indexOf(sep); + return i < 0 ? [s, "", false] : [s.slice(0, i), s.slice(i + sep.length), true]; +} + +function providerOf(spec) { + return cut(spec, "/")[0].toLowerCase(); +} + +// allowedProviders are the provider prefixes a child may name directly: +// those of the manifest's model table, those whose extension the child is +// given (directory basename, the way the sandbox image names them), pi's +// credential-free built-ins on Vertex, and the provider the parent is +// actually running on. +// +// The parent's live provider is in the set so that naming the parent's own +// model explicitly is not stricter than omitting `model` (which inherits +// that same spec). It is also the provider whose credentials this +// iteration demonstrably has: the runner picked it, the child shares the +// sandbox config dir (pi's auth.json) and gets the provider extensions the +// image probe found. Without it a run on a provider that needs no -e +// extension and is not in the model table — openai, whose key the runner +// seeds into auth.json — could not dispatch a sub-agent on its own model +// at all. +function allowedProviders(agent, parentSpec) { + const out = new Set(BUILTIN_PROVIDERS); + if (typeof parentSpec === "string" && parentSpec.includes("/")) out.add(providerOf(parentSpec)); + for (const spec of Object.values(agent?.models ?? {})) { + if (typeof spec === "string" && spec.includes("/")) out.add(providerOf(spec)); + } + for (const ext of agent?.extensions ?? []) { + if (typeof ext !== "string" || ext.endsWith(".js") || ext.endsWith(".ts")) continue; + const base = ext.replace(/\/+$/, "").split("/").pop(); + if (base) out.add(base.toLowerCase()); + } + return out; +} + +// stripThinkingSuffix removes pi's ":" shorthand. +// Only a suffix that names a real level is removed; anything else stays so +// the model table rejects it by name instead of silently losing a segment. +function stripThinkingSuffix(spec) { + const i = spec.lastIndexOf(":"); + if (i < 0) return spec; + if (!PI_THINKING_LEVELS.has(spec.slice(i + 1).toLowerCase())) return spec; + return spec.slice(0, i); +} + +// resolveModel turns the orchestrator's `model` argument into the pi spec +// a child is started with. Accepted: empty (the parent's model, else the +// manifest default), the Claude aliases, a Claude id the fleet personas +// use ("claude-sonnet-4-6@default" — the @suffix is dropped), a +// "provider/id" this run can actually serve, and the direct-API forms +// "anthropic/" and "xai/" translated the way the runner +// translates them for the parent. Anything else — notably a Claude-shaped id +// the model invented — is rejected with the accepted forms, so the +// orchestrator can correct itself instead of losing the dispatch to a +// provider with no credentials. +// +// "This run can serve it" is a closed set, not a provider-prefix check: the +// manifest's model table, the parent's own spec, and the ids the manifest +// lists per provider in providerModels (pi's built-in google-vertex and the +// vendored xai-vertex extension, neither of which has a model-table entry). +// A provider prefix alone is not enough — an invented id under an allowed +// provider ("anthropic-vertex/claude-sonnet-4-20250514", +// "google-vertex/gemini-9", "xai/grok-9") would otherwise be handed to the +// API, which is exactly the failure the rejection exists to prevent for the +// direct-API forms. A manifest without providerModels therefore serves no +// bare provider ids at all: the extension and the manifest are written by +// the same Bootstrap, so the field is absent only when something rewrote it. +// +// A trailing ":" (pi's own "provider/id:high" shorthand) is +// dropped rather than passed through: the child's reasoning effort is the +// manifest's --thinking, and a spec-borne level would silently override it. +export function resolveModel(agent, spec, parentSpec) { + const models = agent?.models ?? {}; + const fallback = (typeof parentSpec === "string" && parentSpec.trim()) || models.default || ""; + let s = typeof spec === "string" ? spec.trim() : ""; + if (s === "") return fallback; + s = cut(s, "@")[0]; + s = stripThinkingSuffix(s); + if (s === "") return fallback; + + const aliases = {}; + const byBareID = {}; + const known = new Set(); + for (const [name, value] of Object.entries(models)) { + if (typeof value !== "string" || value === "") continue; + known.add(value); + if (name !== "default") aliases[name.toLowerCase()] = value; + byBareID[cut(value, "/")[2] ? cut(value, "/")[1].toLowerCase() : value.toLowerCase()] = value; + } + const reject = (what) => { + const names = Object.keys(aliases).join(", "); + throw new Error(`model "${spec}": ${what}; use ${names || "the parent's model"}, or one of ${[...known].join(", ")}`); + }; + const bare = (id) => { + const key = id.toLowerCase(); + if (aliases[key]) return aliases[key]; + if (byBareID[key]) return byBareID[key]; + return reject(`"${id}" is not available in this sandbox`); + }; + + const [head, rest, hasSlash] = cut(s, "/"); + if (!hasSlash) return bare(s); + let provider = head.toLowerCase(); + if (provider === "anthropic") return bare(rest); + // Mirror the runner (normalizeXaiVertexModel): the xai-vertex extension + // registers publisher-qualified ids ("xai/grok-4.6"), so both the short + // vendor form and a two-segment spec land on the three-segment one. Only + // the spelling is normalized here — the result then goes through the same + // closed set as every other provider, because an allowed provider prefix + // is not a licence to name an id under it: "xai-vertex/xai/grok-9" would + // otherwise be handed to Vertex as an unknown model, exactly the failure + // the closed set exists to prevent. + let normalized = s; + if (provider === "xai" || provider === "xai-vertex") { + provider = "xai-vertex"; + normalized = `xai-vertex/xai/${rest.toLowerCase().startsWith("xai/") ? rest.slice(4) : rest}`; + } + const allowed = allowedProviders(agent, parentSpec); + // The normalized name, not the one written: "xai/grok-4.6" is a spec for + // the xai-vertex provider and must be reported as one. + if (!allowed.has(provider)) return reject(`provider "${provider}" is not available in this run`); + const canonical = servableSpecs(agent, parentSpec).get(normalized.toLowerCase()); + if (canonical) return canonical; + return reject(`"${rest}" is not a model this run serves on "${provider}"`); +} + +// servableSpecs maps the lowercased form of every full "provider/id" spec +// this run can serve to the spec as written: the manifest's model table, +// the parent's own model (whose credentials this iteration demonstrably +// has), and each provider id listed in the manifest's providerModels. +function servableSpecs(agent, parentSpec) { + const out = new Map(); + const add = (spec) => { + if (typeof spec === "string" && spec.includes("/")) out.set(spec.trim().toLowerCase(), spec.trim()); + }; + for (const spec of Object.values(agent?.models ?? {})) add(spec); + add(typeof parentSpec === "string" ? parentSpec.trim() : ""); + for (const [prov, ids] of Object.entries(agent?.providerModels ?? {})) { + if (!Array.isArray(ids)) continue; + for (const id of ids) { + if (typeof id === "string" && id !== "") add(`${prov}/${id}`); + } + } + return out; +} + +// childTools is the --tools allowlist for a child: Explore gets the +// read-only set intersected with the parent's, anything else the parent's +// built-in set. The Agent tool itself is never in it (children cannot +// dispatch). +// +// The intersection matters because a sub-agent must never be able to reach +// past its parent: an agent whose tools: frontmatter withheld `grep` would +// otherwise get it back by dispatching an Explore child. It is a no-op for +// an agent that declared no tools:, whose set is pi's defaults and already +// a superset of the read-only tools. +export function childTools(agent, subagentType) { + const explore = typeof subagentType === "string" && subagentType.trim().toLowerCase() === "explore"; + const parent = (agent?.tools ?? []).filter((t) => t !== TOOL_NAME && t !== TOOL_ALIAS); + if (!explore) return parent; + return (agent?.exploreTools ?? DEFAULT_EXPLORE_TOOLS).filter((t) => parent.includes(t)); +} + +// childArgs renders the child's argv (without the binary): the parent's +// flag set minus the shell hygiene the parent already did. The prompt is +// not here — it goes over stdin (see the file header). +export function childArgs(agent, { seq, modelSpec, tools }) { + const args = [ + "--print", "--mode", "json", "--no-approve", "--no-extensions", "--no-prompt-templates", "--no-themes", + "--session-dir", `${agent.sessionsDir}/agent-${seq}`, + ]; + for (const ext of agent.extensions ?? []) args.push("-e", ext); + if (tools.length === 0) { + // Mirror the runner: an empty allowlist means "no built-in tools", not + // "the default set". `--tools ''` would be read as one empty name. + args.push("--no-builtin-tools"); + } else { + args.push("--tools", tools.join(",")); + } + args.push("--model", modelSpec); + args.push("--thinking", agent.thinking || DEFAULT_THINKING); + // Replaces the discovered APPEND_SYSTEM.md, which is the parent's + // orchestrator persona (see CHILD_SYSTEM_NOTE). + args.push("--append-system-prompt", CHILD_SYSTEM_NOTE); + return args; +} + +// childEnv is the environment one child runs with. The runner's provider +// hygiene (buildPiRunCommand) is applied to the shell that launched the +// *parent*, so it only ever matched the parent's provider; a child on a +// different provider would otherwise inherit, say, a stray +// ANTHROPIC_API_KEY (which the bundled SDK sends to Vertex as X-Api-Key) or +// an ambient GOOGLE_CLOUD_PROJECT. The rules below are the same ones +// pi_run.go applies, per resolved child provider. +// +// google-vertex and openai have no rules here on purpose, matching +// buildPiRunCommand: google-vertex reads the ADC the sandbox already +// carries plus GOOGLE_CLOUD_PROJECT/GOOGLE_CLOUD_LOCATION, which are the +// runner's own exports rather than another provider's credential; and +// pi's built-in openai provider resolves its key from the runner-seeded +// auth.json, which the run deliberately leaves as the only source (Run +// unsets OPENAI_API_KEY itself). There is nothing a child on either of +// them could inherit that a scrub would take away. +export function childEnv(base, modelSpec) { + const env = { ...base, [DEPTH_ENV]: "1" }; + const provider = providerOf(modelSpec); + if (provider === "anthropic-vertex") { + delete env.ANTHROPIC_API_KEY; + delete env.ANTHROPIC_AUTH_TOKEN; + delete env.ANTHROPIC_BASE_URL; + delete env.ANTHROPIC_VERTEX_BASE_URL; + const project = base.ANTHROPIC_VERTEX_PROJECT_ID || base.GOOGLE_CLOUD_PROJECT; + if (project) env.GOOGLE_CLOUD_PROJECT = project; + } else if (provider === "xai-vertex") { + delete env.XAI_API_KEY; + const project = base.XAI_VERTEX_PROJECT_ID || base.ANTHROPIC_VERTEX_PROJECT_ID || base.GOOGLE_CLOUD_PROJECT; + if (project) env.XAI_VERTEX_PROJECT_ID = project; + } + return env; +} + +function joinText(content) { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter((b) => b && b.type === "text" && typeof b.text === "string") + .map((b) => b.text) + .join("\n"); +} + +function newStreamState() { + return { + text: "", + stopReason: "", + errorMessage: "", + model: "", + provider: "", + ended: false, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 }, + }; +} + +function noteAssistant(state, msg) { + if (!msg || msg.role !== "assistant") return; + const text = joinText(msg.content); + if (text.trim() !== "") state.text = text; + if (typeof msg.stopReason === "string") state.stopReason = msg.stopReason; + if (typeof msg.errorMessage === "string") state.errorMessage = msg.errorMessage; + if (typeof msg.model === "string" && msg.model) state.model = msg.model; + if (typeof msg.provider === "string" && msg.provider) state.provider = msg.provider; +} + +// consumeLine folds one --mode json event into the stream state: +// message_end accumulates usage and keeps the last assistant text; +// agent_end is the completion marker and its last assistant message is +// authoritative for the stop reason. +export function consumeLine(state, line) { + const trimmed = line.trim(); + if (trimmed === "") return; + let evt; + try { + evt = JSON.parse(trimmed); + } catch { + return; + } + if (!evt || typeof evt !== "object") return; + if (evt.type === "message_end") { + const msg = evt.message; + if (msg && msg.role === "assistant") { + const u = msg.usage ?? {}; + state.usage.input += Number(u.input) || 0; + state.usage.output += Number(u.output) || 0; + state.usage.cacheRead += Number(u.cacheRead) || 0; + state.usage.cacheWrite += Number(u.cacheWrite) || 0; + state.usage.cost += Number(u.cost?.total) || 0; + noteAssistant(state, msg); + } + } else if (evt.type === "agent_end") { + state.ended = true; + const msgs = Array.isArray(evt.messages) ? evt.messages : []; + for (let i = msgs.length - 1; i >= 0; i--) { + if (msgs[i]?.role === "assistant") { + noteAssistant(state, msgs[i]); + break; + } + } + } +} + +function capText(text) { + const trimmed = text.trim(); + if (Buffer.byteLength(trimmed) <= RESULT_MAX_BYTES) return trimmed; + return Buffer.from(trimmed).subarray(0, RESULT_MAX_BYTES).toString("utf8").replace(/�+$/, "") + TRUNCATED_MARKER; +} + +// capBytes truncates on a byte budget without leaving a split code point. +function capBytes(text, max) { + if (Buffer.byteLength(text) <= max) return text; + return Buffer.from(text).subarray(0, max).toString("utf8").replace(/�+$/, ""); +} + +function signalChild(child, signal) { + try { + child.kill(signal); + } catch { + // already gone + } +} + +// createAgentTool builds the dispatcher from a manifest. `spawn`, `log` and +// `now` are injectable for tests. run() never throws for a failed child — +// it returns { isError, error } — so the registered execute() decides how +// to surface it (pi marks a result isError only when execute throws). +export function createAgentTool(manifest, { spawn = nodeSpawn, log = (m) => console.error(m), now = () => Date.now(), env = process.env, killGraceMs = DEFAULT_KILL_GRACE_MS } = {}) { + const agent = manifest?.agent ?? {}; + const maxConcurrent = Math.max(1, Number(agent.maxConcurrent) || DEFAULT_MAX_CONCURRENT); + const timeoutMs = Math.max(1, (Number(agent.timeoutSeconds) || DEFAULT_TIMEOUT_SECONDS) * 1000); + let seq = 0; + let active = 0; + // waiters are dispatches queued behind maxConcurrent. Each is an object + // so shutdown (and an abort while queued) can take a specific one out of + // the queue instead of only ever releasing the head. + const waiters = []; + const running = new Set(); + let shuttingDown = false; + + // A slot is held from the moment a ticket is granted one until exactly one + // release(); `ticket.acquired` records whether this ticket holds one, so + // only a ticket that does ever gives one back. A waiter therefore settles + // one of two ways: grant() (the releasing dispatch hands its slot straight + // over, so `active` is unchanged and the ticket now holds it) or evict() + // (the dispatch is cancelled and will spawn nothing, so it takes no slot + // and must not release one). Claiming a slot for an evicted waiter would + // over-admit: its release() would pass that slot to the next waiter + // without ever decrementing `active`, and maxConcurrent+1 children would + // run at once. + const acquire = (ticket) => { + if (active < maxConcurrent) { + active++; + ticket.acquired = true; + return Promise.resolve(); + } + return new Promise((resolve) => { + ticket.waiter = { + grant: () => { + ticket.acquired = true; + resolve(); + }, + evict: resolve, + }; + waiters.push(ticket.waiter); + }); + }; + const release = () => { + const next = waiters.shift(); + if (next) { + next.grant(); + } else { + active--; + } + }; + const unqueue = (waiter) => { + if (!waiter) return; + const i = waiters.indexOf(waiter); + if (i < 0) return; + waiters.splice(i, 1); + waiter.evict(); + }; + + const recordUsage = (record) => { + if (!agent.usageFile) return; + try { + mkdirSync(dirname(agent.usageFile), { recursive: true }); + appendFileSync(agent.usageFile, JSON.stringify(record) + "\n"); + } catch (err) { + log(`${LOG_PREFIX} cannot write ${agent.usageFile}: ${err.message}`); + } + }; + + // runChild spawns one child and resolves when it is gone. It resolves a + // handle synchronously through `out` so the caller can terminate the + // child on abort or shutdown while it is still running. + const runChild = (id, params, modelSpec, tools, out) => + new Promise((resolve) => { + const state = newStreamState(); + const startedAt = now(); + const args = childArgs(agent, { seq: id, modelSpec, tools }); + let child; + try { + child = spawn(agent.piBin || "pi", args, { + env: childEnv(env, modelSpec), + // stdin carries the prompt; a child in the parent's process group + // so `detached` is not set — a killed parent must not leave + // children spending tokens, and the SIGTERM below is what makes + // pi clean up its own detached bash grandchildren. + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (err) { + resolve({ state, startedAt, exitCode: null, signal: null, spawnError: err.message, stderr: "", timedOut: false, pid: undefined, droppedLines: 0 }); + return; + } + let timedOut = false; + let stderr = ""; + let buffered = ""; + let dropping = false; + let droppedLines = 0; + let killTimer; + // terminate is the whole stop sequence: SIGTERM first so pi runs its + // handler (kills the detached bash processes it tracks, exits 143), + // SIGKILL only if it is still there after the grace period. + const terminate = () => { + if (child.exitCode !== null || child.signalCode !== null) return; + signalChild(child, "SIGTERM"); + if (killTimer !== undefined) return; + killTimer = setTimeout(() => signalChild(child, "SIGKILL"), killGraceMs); + if (typeof killTimer.unref === "function") killTimer.unref(); + }; + const handle = { child, terminate }; + if (out) out.handle = handle; + running.add(handle); + const timer = setTimeout(() => { + timedOut = true; + terminate(); + }, timeoutMs); + // The prompt is written and the pipe closed at once: pi reads stdin + // to EOF before it starts. A child that dies first (or never drains a + // prompt larger than the pipe buffer) makes this EPIPE, which is an + // ordinary outcome here — `close` already carries the verdict — but + // an unhandled "error" on the stream would take the parent down. + child.stdin.on("error", () => {}); + child.stdin.end(params.prompt); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + buffered += chunk; + let nl; + while ((nl = buffered.indexOf("\n")) >= 0) { + const line = buffered.slice(0, nl); + buffered = buffered.slice(nl + 1); + // While dropping, the first complete line is the tail of the + // oversized one; skip it and resume parsing. + if (dropping) { + dropping = false; + droppedLines++; + continue; + } + // A complete oversized line (its newline arrived in the same + // chunk that carried the bulk of it) is dropped here; one that + // has not terminated yet is dropped by the buffer cap below. + if (line.length > MAX_STDOUT_LINE_CHARS) { + droppedLines++; + continue; + } + consumeLine(state, line); + } + if (buffered.length > MAX_STDOUT_LINE_CHARS) { + dropping = true; + buffered = ""; + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { + stderr = (stderr + chunk).slice(-STDERR_TAIL_BYTES); + }); + const finish = (exitCode, signal, spawnError) => { + clearTimeout(timer); + if (killTimer !== undefined) clearTimeout(killTimer); + running.delete(handle); + if (!dropping && buffered.trim() !== "") consumeLine(state, buffered); + if (dropping) droppedLines++; + resolve({ state, startedAt, exitCode, signal, spawnError, stderr, timedOut, pid: child.pid, droppedLines }); + }; + child.once("error", (err) => finish(null, null, err.message)); + child.once("close", (code, signal) => finish(code, signal, undefined)); + }); + + const run = async (params, { parentModel, signal } = {}) => { + const id = ++seq; + const description = typeof params?.description === "string" ? params.description : ""; + let modelSpec; + try { + modelSpec = resolveModel(agent, params?.model, parentModel); + } catch (err) { + return { seq: id, isError: true, error: err.message, text: "", stopReason: "rejected", model: "" }; + } + if (typeof params?.prompt !== "string" || params.prompt.trim() === "") { + return { seq: id, isError: true, error: "prompt is required", text: "", stopReason: "rejected", model: modelSpec }; + } + const cancelled = () => { + if (shuttingDown) return { seq: id, isError: true, error: "the session is shutting down", text: "", stopReason: "aborted", model: modelSpec }; + if (signal?.aborted) return { seq: id, isError: true, error: "the tool call was aborted", text: "", stopReason: "aborted", model: modelSpec }; + return null; + }; + let early = cancelled(); + if (early) return early; + const tools = childTools(agent, params?.subagent_type); + // The ticket carries the queue entry (before a slot is free) and then + // the running child, so an abort reaches whichever stage the dispatch + // is in. The listener is removed on every exit path. + const ticket = {}; + const onAbort = () => { + if (ticket.handle) ticket.handle.terminate(); + else unqueue(ticket.waiter); + }; + signal?.addEventListener?.("abort", onAbort, { once: true }); + let outcome; + try { + await acquire(ticket); + // Re-check: shutdown or an abort can land while this dispatch is + // queued, and shutdown drains the queue rather than leaving waiters + // pending forever. + early = cancelled(); + if (early) { + // Only a ticket that was granted a slot gives one back; one that was + // evicted from the queue never held one. + if (ticket.acquired) release(); + return early; + } + try { + log(`${LOG_PREFIX} #${id} ${modelSpec} start "${description}"`); + outcome = await runChild(id, params, modelSpec, tools, ticket); + } finally { + release(); + } + } finally { + signal?.removeEventListener?.("abort", onAbort); + } + const { state, startedAt, exitCode, signal: exitSignal, spawnError, stderr, timedOut, pid, droppedLines } = outcome; + const durationMs = Math.max(0, now() - startedAt); + const tail = stderr.trim(); + let error = ""; + let stopReason = state.stopReason; + if (spawnError) { + error = `could not start pi: ${spawnError}`; + stopReason = "error"; + } else if (timedOut) { + error = `sub-agent timed out after ${timeoutMs / 1000}s`; + stopReason = "timeout"; + } else if ((shuttingDown || signal?.aborted) && exitCode !== 0) { + error = shuttingDown + ? "sub-agent was killed because the session shut down" + : "sub-agent was killed because the tool call was aborted"; + stopReason = "aborted"; + } else if (exitCode !== 0) { + error = `pi exited ${exitCode ?? `on ${exitSignal}`}${tail ? `: ${tail}` : ""}`; + stopReason = stopReason || "error"; + } else if (stopReason === "error" || stopReason === "aborted") { + error = state.errorMessage || `sub-agent stopped with stopReason ${stopReason}`; + } else if (!state.ended) { + error = `sub-agent produced no agent_end${tail ? `: ${tail}` : ""}`; + stopReason = stopReason || "incomplete"; + } + const isError = error !== ""; + if (droppedLines > 0) log(`${LOG_PREFIX} #${id} dropped ${droppedLines} stdout line(s) over ${MAX_STDOUT_LINE_CHARS} chars`); + log(`${LOG_PREFIX} #${id} done ${durationMs}ms ${stopReason || "unknown"}`); + recordUsage({ + seq: id, + model: modelSpec, + provider: state.provider || providerOf(modelSpec), + description: capBytes(description, MAX_DESCRIPTION_BYTES), + startedAt: new Date(startedAt).toISOString(), + durationMs, + usage: state.usage, + stopReason, + isError, + }); + return { seq: id, isError, error, text: capText(state.text), stopReason, model: modelSpec, pid, durationMs, usage: state.usage }; + }; + + const shutdown = () => { + shuttingDown = true; + for (const handle of running) handle.terminate(); + // Queued dispatches would otherwise never settle: nothing will call + // release() for them once the in-flight children are gone. They are + // evicted, not granted: none of them will spawn a child, so none takes + // a slot (see acquire). + for (const waiter of waiters.splice(0)) waiter.evict(); + }; + + return { run, shutdown, inFlight: () => running.size }; +} + +export default function (pi) { + if (process.env[DEPTH_ENV] !== undefined) { + console.error(`${LOG_PREFIX} ${DEPTH_ENV} is set: this is a sub-agent, the Agent tool is not registered (no recursion)`); + return; + } + let manifest; + try { + manifest = loadManifest(); + } catch (err) { + console.error(`${LOG_PREFIX} cannot read manifest: ${err.message}; the Agent tool is not registered`); + return; + } + if (!manifest?.agent?.enabled) { + console.error(`${LOG_PREFIX} the Agent tool is not enabled for this agent; nothing registered`); + return; + } + const tool = createAgentTool(manifest); + + const execute = async (_toolCallId, params, signal, _onUpdate, ctx) => { + const m = ctx?.model; + const parentModel = m && typeof m.provider === "string" && typeof m.id === "string" ? `${m.provider}/${m.id}` : ""; + // pi aborts a tool call when the turn is cancelled; without this the + // child would keep running (and spending) until its own timeout. + const res = await tool.run(params ?? {}, { parentModel, signal }); + if (res.isError) { + throw new Error(res.text ? `${res.error}\n\n${res.text}` : res.error); + } + return { + content: [{ type: "text", text: res.text }], + details: { seq: res.seq, model: res.model, stopReason: res.stopReason, durationMs: res.durationMs }, + }; + }; + + const description = "Launch a sub-agent that runs a task to completion in its own context and returns its final message. " + + "Several Agent calls in one message run in parallel. The sub-agent starts with no memory of this conversation, so the prompt must carry everything it needs."; + for (const name of [TOOL_NAME, TOOL_ALIAS]) { + pi.registerTool({ + name, + label: name === TOOL_NAME ? "Agent" : "Task (alias of Agent)", + description, + promptSnippet: name === TOOL_NAME ? "Run a sub-agent on a self-contained task; parallel dispatch is several Agent calls in one message" : undefined, + parameters: AGENT_TOOL_PARAMETERS, + execute, + }); + } + pi.on("session_shutdown", () => tool.shutdown()); +} diff --git a/internal/runtime/pi_extension/fullsend-agent.test.mjs b/internal/runtime/pi_extension/fullsend-agent.test.mjs new file mode 100644 index 0000000000..1bb4002950 --- /dev/null +++ b/internal/runtime/pi_extension/fullsend-agent.test.mjs @@ -0,0 +1,829 @@ +// Unit tests for the fullsend pi Agent tool extension. Run with: +// node --test internal/runtime/pi_extension/ +// Children are a fake `pi` (a node script that prints a canned --mode json +// stream chosen by its prompt argument) or, where the test needs +// deterministic control over process lifetime, an injected spawn. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { test } from "node:test"; + +import defaultExport, { + AGENT_TOOL_PARAMETERS, + CHILD_SYSTEM_NOTE, + MAX_DESCRIPTION_BYTES, + MAX_STDOUT_LINE_CHARS, + RESULT_MAX_BYTES, + childArgs, + childEnv, + childTools, + createAgentTool, + resolveModel, +} from "./fullsend-agent.js"; + +// FAKE_PI reads its prompt from stdin, the way real pi does in --print +// mode, and answers by prompt: "ok" prints a successful stream whose final +// text carries its argv, the prompt it received and +// FULLSEND_SUBAGENT_DEPTH; "fail" ends with stopReason error; "crash" exits +// 3; "noend" never emits agent_end; "hang" sleeps forever (the timeout test +// kills it); "hang-spawn" also leaves a detached grandchild behind, the way +// pi's bash tool does; "hugeline" prints one line far over the cap. +const FAKE_PI = `#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; +let stdin = ""; +process.stdin.setEncoding("utf8"); +process.stdin.on("data", (c) => { stdin += c; }); +process.stdin.on("end", () => run(stdin.trim())); +process.stdin.resume(); +function run(prompt) { +const usage = { input: 100, output: 20, cacheRead: 30, cacheWrite: 5, cost: { total: 0.25 } }; +const line = (o) => process.stdout.write(JSON.stringify(o) + "\\n"); +line({ type: "session", version: 3, id: "child" }); +line({ type: "agent_start" }); +if (prompt === "hang") { setTimeout(() => {}, 60_000); } +else if (prompt === "hang-spawn") { + const kid = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { detached: true, stdio: "ignore" }); + kid.unref(); + writeFileSync(process.env.FAKE_PI_KID_FILE, String(kid.pid)); + process.on("SIGTERM", () => { try { process.kill(-kid.pid, "SIGKILL"); } catch {} process.exit(143); }); + setTimeout(() => {}, 60_000); +} +else if (prompt === "crash") { process.stderr.write("boom: provider exploded\\n"); process.exit(3); } +else if (prompt === "fail") { + line({ type: "message_end", message: { role: "assistant", content: [], model: "m", provider: "p", usage, stopReason: "error", errorMessage: "quota exhausted" } }); + line({ type: "agent_end", messages: [{ role: "assistant", content: [], model: "m", provider: "p", usage, stopReason: "error", errorMessage: "quota exhausted" }] }); + line({ type: "agent_settled" }); +} else if (prompt === "noend") { + line({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text: "partial" }], model: "m", provider: "p", usage, stopReason: "stop" } }); +} else if (prompt.startsWith("BIG:")) { + line({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text: JSON.stringify({ big: prompt.length }) }], model: "m", provider: "p", usage, stopReason: "stop" } }); + line({ type: "agent_end", messages: [] }); +} else if (prompt === "hugeline") { + process.stdout.write(JSON.stringify({ type: "junk", pad: "z".repeat(${MAX_STDOUT_LINE_CHARS + 10}) }) + "\\n"); + line({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text: "after the huge line" }], model: "m", provider: "p", usage, stopReason: "stop" } }); + line({ type: "agent_end", messages: [] }); +} else { + const text = JSON.stringify({ argv: process.argv.slice(2), prompt, depth: process.env.FULLSEND_SUBAGENT_DEPTH ?? null }); + line({ type: "message_end", message: { role: "assistant", content: [{ type: "toolCall", id: "c1", name: "read" }], model: "m", provider: "p", usage, stopReason: "toolUse" } }); + line({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text: " " + text + " " }], model: "claude-opus-4-6", provider: "anthropic-vertex", usage, stopReason: "stop" } }); + line({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text }], model: "claude-opus-4-6", provider: "anthropic-vertex", usage, stopReason: "stop" }] }); + line({ type: "agent_settled" }); +} +} +`; + +function fixture() { + const dir = mkdtempSync(join(tmpdir(), "fullsend-agent-")); + const piBin = join(dir, "fake-pi.mjs"); + writeFileSync(piBin, FAKE_PI, { mode: 0o755 }); + const manifest = { + agentName: "review", + tools: null, + hooks: { groups: [], toolNames: {} }, + agent: { + enabled: true, + piBin, + sessionsDir: join(dir, "sessions"), + extensions: ["/usr/local/share/pi-extensions/anthropic-vertex", "/usr/local/share/pi-extensions/xai-vertex", "/sandbox/pi-config/fullsend-hooks.js"], + models: { + default: "anthropic-vertex/claude-opus-4-6", + opus: "anthropic-vertex/claude-opus-4-6", + sonnet: "anthropic-vertex/claude-sonnet-4-6", + haiku: "anthropic-vertex/claude-haiku-4-5", + }, + providerModels: { "google-vertex": ["gemini-3.7-flash", "gemini-3.5-flash", "gemini-2.5-pro"] }, + thinking: "medium", + tools: ["read", "bash", "edit", "write", "grep", "find", "ls"], + exploreTools: ["read", "grep", "find", "ls"], + maxConcurrent: 4, + timeoutSeconds: 900, + usageFile: join(dir, "subagents", "usage.jsonl"), + }, + }; + return { dir, manifest }; +} + +const quiet = { log: () => {} }; + +test("tool contract mirrors Claude Code's Agent tool", () => { + assert.deepEqual(AGENT_TOOL_PARAMETERS.required, ["prompt"]); + assert.deepEqual(Object.keys(AGENT_TOOL_PARAMETERS.properties).sort(), ["description", "model", "prompt", "run_in_background", "subagent_type"]); + assert.equal(AGENT_TOOL_PARAMETERS.type, "object"); +}); + +test("resolveModel: aliases, Claude ids and provider specs", () => { + const { manifest } = fixture(); + const a = manifest.agent; + const parent = "xai-vertex/xai/grok-4.6"; + assert.equal(resolveModel(a, "", parent), parent, "omitted → parent's model"); + assert.equal(resolveModel(a, undefined, ""), a.models.default, "no parent known → manifest default"); + assert.equal(resolveModel(a, "sonnet", parent), "anthropic-vertex/claude-sonnet-4-6"); + assert.equal(resolveModel(a, "Haiku", parent), "anthropic-vertex/claude-haiku-4-5", "aliases are case-insensitive"); + assert.equal(resolveModel(a, "claude-sonnet-4-6@default", parent), "anthropic-vertex/claude-sonnet-4-6", "fleet persona form: @default stripped, bare id matched"); + assert.equal(resolveModel(a, "claude-opus-4-6", parent), "anthropic-vertex/claude-opus-4-6"); + assert.equal(resolveModel(a, "anthropic/claude-sonnet-4-6", parent), "anthropic-vertex/claude-sonnet-4-6", "a direct-API provider prefix is translated to the sandbox provider"); + assert.equal(resolveModel(a, "anthropic-vertex/claude-sonnet-4-6", parent), "anthropic-vertex/claude-sonnet-4-6", "known provider passes through"); + assert.equal(resolveModel(a, "xai/grok-4.6", parent), "xai-vertex/xai/grok-4.6", "short Grok spec is normalized like the runner does"); + assert.equal(resolveModel(a, "xai-vertex/grok-4.6", parent), "xai-vertex/xai/grok-4.6"); + assert.equal(resolveModel(a, "google-vertex/gemini-3.7-flash", parent), "google-vertex/gemini-3.7-flash", "pi's built-in Vertex Gemini provider needs no extension"); + assert.throws(() => resolveModel(a, "anthropic/claude-sonnet-4-20250514", parent), /claude-sonnet-4-20250514.*opus, sonnet, haiku/s, "an invented Claude id is rejected, not passed through"); + assert.throws(() => resolveModel(a, "claude-sonnet-4-20250514", parent), /not available/); + assert.throws(() => resolveModel(a, "openai/gpt-5", parent), /provider "openai"/, "a provider the run has no credentials for is rejected"); + const noXai = { ...a, extensions: ["/sandbox/pi-config/fullsend-hooks.js"], models: { default: "anthropic-vertex/claude-opus-4-6" } }; + assert.throws(() => resolveModel(noXai, "xai-vertex/xai/grok-4.6", "anthropic-vertex/claude-opus-4-6"), /provider "xai-vertex"/, "a provider whose extension is not in the manifest is rejected"); + assert.equal( + resolveModel(noXai, "xai-vertex/xai/grok-4.6", "xai-vertex/xai/grok-4.6"), + "xai-vertex/xai/grok-4.6", + "naming the parent's own provider is never stricter than omitting model (which inherits the same spec)", + ); + assert.equal(resolveModel(noXai, "openai/gpt-5-codex", "openai/gpt-5-codex"), "openai/gpt-5-codex", "a parent on a keyless-in-env provider can still dispatch on its own model"); + + // pi's ":" shorthand would override the manifest --thinking. + assert.equal(resolveModel(a, "anthropic-vertex/claude-sonnet-4-6:high", parent), "anthropic-vertex/claude-sonnet-4-6"); + assert.equal(resolveModel(a, "sonnet:XHIGH", parent), "anthropic-vertex/claude-sonnet-4-6"); + assert.equal(resolveModel(a, ":max", parent), parent, "a bare level is not a model; fall back like an omitted spec"); + assert.throws(() => resolveModel(a, "sonnet:turbo", parent), /not available/, "an unknown suffix is not silently dropped"); +}); + +test("resolveModel: an id is checked against a closed set, not just its provider prefix", () => { + const { manifest } = fixture(); + const a = manifest.agent; + const parent = "xai-vertex/xai/grok-4.6"; + + // An allowed provider is not a licence to name any id under it: the spec + // would reach the API as an unknown model instead of being corrected. + assert.throws( + () => resolveModel(a, "anthropic-vertex/claude-sonnet-4-20250514", parent), + /is not a model this run serves on "anthropic-vertex"/, + "an invented Claude id under the sandbox provider is rejected, not passed through", + ); + assert.throws(() => resolveModel(a, "google-vertex/gemini-9-ultra", parent), /is not a model this run serves on "google-vertex"/); + assert.throws( + () => resolveModel({ ...a, providerModels: undefined }, "google-vertex/gemini-3.7-flash", parent), + /is not a model this run serves/, + "a manifest without providerModels serves no bare provider ids: it is written by the same Bootstrap as this file", + ); + + // What the closed set does accept. + assert.equal(resolveModel(a, "google-vertex/gemini-2.5-pro", parent), "google-vertex/gemini-2.5-pro", "a catalog id the manifest lists"); + assert.equal(resolveModel(a, "GOOGLE-VERTEX/GEMINI-3.7-FLASH", parent), "google-vertex/gemini-3.7-flash", "matched case-insensitively, returned canonical"); + assert.equal(resolveModel(a, parent, parent), parent, "the parent's own spec, whatever provider it is on"); + assert.equal(resolveModel(a, "anthropic-vertex/claude-haiku-4-5", parent), "anthropic-vertex/claude-haiku-4-5", "a model-table entry"); +}); + +test("resolveModel: a Grok spec is normalized and then checked against the closed set", () => { + const { manifest } = fixture(); + // A parent that is not on Grok, so nothing here is served merely because + // it is the parent's own spec: this exercises the providerModels path. + const parent = "anthropic-vertex/claude-opus-4-6"; + const a = { ...manifest.agent, providerModels: { ...manifest.agent.providerModels, "xai-vertex": ["xai/grok-4.6"] } }; + + for (const spec of ["xai/grok-4.6", "xai-vertex/grok-4.6", "xai-vertex/xai/grok-4.6", "XAI/GROK-4.6"]) { + assert.equal(resolveModel(a, spec, parent), "xai-vertex/xai/grok-4.6", `every spelling lands on the three-segment spec: ${spec}`); + } + + // The provider prefix is not a licence to name an id: an invented Grok id + // would reach Vertex as an unknown model, which is what the closed set + // exists to prevent for every other provider. + assert.throws( + () => resolveModel(a, "xai-vertex/xai/grok-invented", parent), + /is not a model this run serves on "xai-vertex"/, + "an invented Grok id under an allowed provider is rejected, not passed through", + ); + assert.throws(() => resolveModel(a, "xai/grok-invented", parent), /is not a model this run serves on "xai-vertex"/); + assert.throws( + () => resolveModel({ ...a, providerModels: undefined }, "xai/grok-4.6", parent), + /is not a model this run serves on "xai-vertex"/, + "a manifest without providerModels serves no bare Grok id either", + ); + + // Without the extension the provider is refused before the id is looked + // up, and it is named as the provider the spec really targets. + const noXai = { ...a, extensions: ["/sandbox/pi-config/fullsend-hooks.js"] }; + for (const spec of ["xai/grok-4.6", "xai-vertex/xai/grok-4.6"]) { + assert.throws(() => resolveModel(noXai, spec, parent), /provider "xai-vertex" is not available in this run/, spec); + } +}); + +test("childTools: Explore is read-only, everything else is the parent's built-ins minus Agent/Task", () => { + const { manifest } = fixture(); + assert.deepEqual(childTools(manifest.agent, "Explore"), ["read", "grep", "find", "ls"]); + assert.deepEqual(childTools(manifest.agent, "explore"), ["read", "grep", "find", "ls"]); + assert.deepEqual(childTools(manifest.agent, "general-purpose"), ["read", "bash", "edit", "write", "grep", "find", "ls"]); + assert.deepEqual(childTools(manifest.agent, undefined), ["read", "bash", "edit", "write", "grep", "find", "ls"]); + assert.deepEqual(childTools({ ...manifest.agent, tools: ["bash", "Agent", "Task", "read"] }, ""), ["bash", "read"]); + assert.deepEqual( + childTools({ ...manifest.agent, tools: ["read", "bash"] }, "Explore"), + ["read"], + "Explore is intersected with the parent's set: a child never reaches past its parent", + ); + assert.deepEqual(childTools({ ...manifest.agent, tools: ["bash"] }, "Explore"), [], "no overlap leaves an empty allowlist"); +}); + +test("childArgs mirrors the runner's pi command line, extensions in manifest order", () => { + const { manifest } = fixture(); + const args = childArgs(manifest.agent, { seq: 3, modelSpec: "anthropic-vertex/claude-sonnet-4-6", tools: ["read", "grep"] }); + assert.deepEqual(args, [ + "--print", "--mode", "json", "--no-approve", "--no-extensions", "--no-prompt-templates", "--no-themes", + "--session-dir", join(manifest.agent.sessionsDir, "agent-3"), + "-e", "/usr/local/share/pi-extensions/anthropic-vertex", + "-e", "/usr/local/share/pi-extensions/xai-vertex", + "-e", "/sandbox/pi-config/fullsend-hooks.js", + "--tools", "read,grep", + "--model", "anthropic-vertex/claude-sonnet-4-6", + "--thinking", "medium", + "--append-system-prompt", CHILD_SYSTEM_NOTE, + ]); + assert.ok(!args.some((a) => a.startsWith("do it")), "the prompt is never in argv; it goes over stdin"); + const loaded = args.filter((_, i) => i > 0 && args[i - 1] === "-e"); + assert.ok(!loaded.some((a) => a.includes("fullsend-agent")), "children never get the Agent extension"); + + // The parent's APPEND_SYSTEM.md must not reach a child: pi only discovers + // it when no --append-system-prompt was given. + assert.ok(!CHILD_SYSTEM_NOTE.includes("Agent calls in one message")); + assert.match(CHILD_SYSTEM_NOTE, /sub-agent/); + + const empty = childArgs(manifest.agent, { seq: 4, modelSpec: "anthropic-vertex/claude-opus-4-6", tools: [] }); + assert.ok(empty.includes("--no-builtin-tools"), "an empty child tool list means no built-ins, like the runner does for the parent"); + assert.ok(!empty.includes("--tools")); +}); + +test("childEnv scrubs the provider credentials the child does not use", () => { + const base = { + PATH: "/usr/bin", + ANTHROPIC_API_KEY: "sk-parent", + ANTHROPIC_AUTH_TOKEN: "tok", + ANTHROPIC_BASE_URL: "https://evil", + ANTHROPIC_VERTEX_BASE_URL: "https://evil", + ANTHROPIC_VERTEX_PROJECT_ID: "claude-proj", + GOOGLE_CLOUD_PROJECT: "ambient-proj", + XAI_API_KEY: "xai-key", + }; + // A Claude child under a Grok parent: the runner's shell only scrubbed + // for the parent's provider, so this is the extension's job. + const claude = childEnv(base, "anthropic-vertex/claude-opus-4-6"); + assert.equal(claude.FULLSEND_SUBAGENT_DEPTH, "1"); + for (const k of ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_VERTEX_BASE_URL"]) { + assert.ok(!(k in claude), `${k} is removed`); + } + assert.equal(claude.GOOGLE_CLOUD_PROJECT, "claude-proj", "pinned to the variable Claude on Vertex is driven by"); + assert.equal(claude.XAI_API_KEY, "xai-key", "another provider's key is left alone"); + + const grok = childEnv(base, "xai-vertex/xai/grok-4.6"); + assert.ok(!("XAI_API_KEY" in grok), "the built-in xai provider must not shadow the extension"); + assert.equal(grok.XAI_VERTEX_PROJECT_ID, "claude-proj", "falls back the way pi_run.go does"); + assert.equal(grok.ANTHROPIC_API_KEY, "sk-parent", "not this child's provider, left alone"); + + assert.equal(childEnv({ ...base, XAI_VERTEX_PROJECT_ID: "grok-proj" }, "xai-vertex/xai/grok-4.6").XAI_VERTEX_PROJECT_ID, "grok-proj", "an explicit value wins"); + const gemini = childEnv(base, "google-vertex/gemini-3.7-flash"); + assert.equal(gemini.GOOGLE_CLOUD_PROJECT, "ambient-proj", "a provider with no rules is untouched"); + assert.equal(base.FULLSEND_SUBAGENT_DEPTH, undefined, "the caller's env object is not mutated"); +}); + +test("run: success returns the child's final text, trimmed, and records usage", async () => { + const { manifest } = fixture(); + const logs = []; + const tool = createAgentTool(manifest, { log: (l) => logs.push(l) }); + const res = await tool.run({ prompt: "ok", description: "unit child", model: "sonnet" }, { parentModel: "anthropic-vertex/claude-opus-4-6" }); + assert.equal(res.isError, false); + assert.equal(res.stopReason, "stop"); + const payload = JSON.parse(res.text); + assert.equal(payload.depth, "1", "children carry FULLSEND_SUBAGENT_DEPTH=1"); + assert.equal(payload.argv[0], "--print"); + assert.ok(payload.argv.includes("--model") && payload.argv[payload.argv.indexOf("--model") + 1] === "anthropic-vertex/claude-sonnet-4-6"); + assert.equal(payload.prompt, "ok", "the prompt arrives on stdin, not in argv"); + assert.ok(!payload.argv.includes("ok"), "and never in argv"); + assert.equal(payload.argv[payload.argv.indexOf("--append-system-prompt") + 1], CHILD_SYSTEM_NOTE, "the child gets its own system note, not the parent's APPEND_SYSTEM.md"); + assert.equal(payload.argv[payload.argv.indexOf("--session-dir") + 1], join(manifest.agent.sessionsDir, "agent-1")); + assert.equal(payload.argv[payload.argv.indexOf("--thinking") + 1], "medium"); + + const usage = readFileSync(manifest.agent.usageFile, "utf8").trim().split("\n").map((l) => JSON.parse(l)); + assert.equal(usage.length, 1); + assert.equal(usage[0].seq, 1); + assert.equal(usage[0].model, "anthropic-vertex/claude-sonnet-4-6"); + assert.equal(usage[0].provider, "anthropic-vertex"); + assert.equal(usage[0].description, "unit child"); + assert.equal(usage[0].stopReason, "stop"); + assert.equal(usage[0].isError, false); + assert.deepEqual(usage[0].usage, { input: 200, output: 40, cacheRead: 60, cacheWrite: 10, cost: 0.5 }, "usage sums every assistant message"); + assert.ok(typeof usage[0].durationMs === "number" && usage[0].durationMs >= 0); + assert.match(usage[0].startedAt, /^\d{4}-\d{2}-\d{2}T/); + + assert.equal(logs.length, 2); + assert.match(logs[0], /^\[fullsend-agent\] #1 anthropic-vertex\/claude-sonnet-4-6 start "unit child"$/); + assert.match(logs[1], /^\[fullsend-agent\] #1 done \d+ms stop$/); +}); + +test("run: error stopReason → isError with the child's message", async () => { + const { manifest } = fixture(); + const tool = createAgentTool(manifest, quiet); + const res = await tool.run({ prompt: "fail" }, {}); + assert.equal(res.isError, true); + assert.equal(res.stopReason, "error"); + assert.match(res.error, /quota exhausted/); + const usage = JSON.parse(readFileSync(manifest.agent.usageFile, "utf8").trim()); + assert.equal(usage.isError, true); + assert.equal(usage.stopReason, "error"); + assert.equal(usage.model, manifest.agent.models.default, "omitted model with no parent model known → manifest default"); +}); + +test("run: non-zero exit and a stream without agent_end are errors", async () => { + const { manifest } = fixture(); + const tool = createAgentTool(manifest, quiet); + const crashed = await tool.run({ prompt: "crash" }, {}); + assert.equal(crashed.isError, true); + assert.match(crashed.error, /exited 3/); + assert.match(crashed.error, /boom: provider exploded/, "stderr tail is surfaced"); + + const noend = await tool.run({ prompt: "noend" }, {}); + assert.equal(noend.isError, true); + assert.match(noend.error, /no agent_end/); + assert.equal(noend.text, "partial", "whatever text arrived is still returned"); +}); + +test("run: timeout stops the child and reports isError", async () => { + const { manifest } = fixture(); + manifest.agent.timeoutSeconds = 0.3; + const tool = createAgentTool(manifest, quiet); + const started = Date.now(); + const res = await tool.run({ prompt: "hang" }, {}); + assert.equal(res.isError, true); + assert.match(res.error, /timed out after 0.3s/); + assert.equal(res.stopReason, "timeout"); + assert.ok(Date.now() - started < 5000, "did not wait for the child's own timer"); + assert.throws(() => process.kill(res.pid, 0), { code: "ESRCH" }, "the child is gone"); +}); + +test("run: a timed-out child gets SIGTERM, so its detached grandchildren die with it", async (t) => { + if (process.platform === "win32") return t.skip("POSIX signals"); + const { dir, manifest } = fixture(); + manifest.agent.timeoutSeconds = 0.3; + const kidFile = join(dir, "kid.pid"); + // The fake pi mimics pi's bash tool: a `detached` grandchild in its own + // process group, reaped from its own SIGTERM handler + // (dist/modes/print-mode.js -> killTrackedDetachedChildren). SIGKILL to + // the child's group would never reach it. + const tool = createAgentTool(manifest, { ...quiet, env: { ...process.env, FAKE_PI_KID_FILE: kidFile } }); + const res = await tool.run({ prompt: "hang-spawn" }, {}); + assert.equal(res.stopReason, "timeout"); + const kid = Number(readFileSync(kidFile, "utf8")); + assert.ok(kid > 0); + for (let i = 0; i < 100; i++) { + try { + process.kill(kid, 0); + } catch (err) { + assert.equal(err.code, "ESRCH"); + return; + } + await new Promise((r) => setTimeout(r, 50)); + } + assert.fail(`grandchild ${kid} survived the child's death`); +}); + +test("run: a prompt that looks like a flag reaches the child verbatim", async () => { + const { manifest } = fixture(); + const tool = createAgentTool(manifest, quiet); + // pi's argv parser reads these as an unknown flag, an unknown option + // (a startup error) and a file argument respectively, so none of them + // can go on the command line unterminated — and a "--" terminator would + // not rescue the "@" one, which stays a file argument after it. + for (const prompt of ["--no-approve then review this", "- a leading dash", "@/etc/passwd"]) { + const res = await tool.run({ prompt }, {}); + assert.equal(res.isError, false, `prompt ${JSON.stringify(prompt)} failed: ${res.error}`); + assert.equal(JSON.parse(res.text).prompt, prompt); + } +}); + +test("run: a prompt larger than the kernel's argv limit still reaches the child", async () => { + const { manifest } = fixture(); + const tool = createAgentTool(manifest, quiet); + // 256 KiB is past the per-argument cap that makes spawn fail E2BIG on + // Linux; a context package for a reviewer sub-agent easily reaches it. + const prompt = "BIG:" + "x".repeat(256 * 1024); + const res = await tool.run({ prompt }, {}); + assert.equal(res.isError, false, res.error); + assert.equal(JSON.parse(res.text).big, prompt.length, "the child received every byte"); +}); + +test("run: a stdout line over the cap is dropped, not buffered, and parsing resumes", async () => { + const { manifest } = fixture(); + const logs = []; + const tool = createAgentTool(manifest, { log: (l) => logs.push(l) }); + const res = await tool.run({ prompt: "hugeline" }, {}); + assert.equal(res.text, "after the huge line", "the events after the oversized line are still parsed"); + assert.ok(logs.some((l) => /dropped 1 stdout line/.test(l))); +}); + +test("run: the usage record's description is capped", async () => { + const { manifest } = fixture(); + const tool = createAgentTool(manifest, quiet); + await tool.run({ prompt: "ok", description: "d".repeat(MAX_DESCRIPTION_BYTES * 3) }, {}); + const rec = JSON.parse(readFileSync(manifest.agent.usageFile, "utf8").trim()); + assert.equal(Buffer.byteLength(rec.description), MAX_DESCRIPTION_BYTES); + // The whole record has to stay under PIPE_BUF for concurrent appends to + // be atomic. + assert.ok(Buffer.byteLength(JSON.stringify(rec) + "\n") < 4096); +}); + +test("run: model rejection is an error before anything is spawned", async () => { + const { manifest } = fixture(); + const spawned = []; + const tool = createAgentTool(manifest, { ...quiet, spawn: (...a) => { spawned.push(a); throw new Error("must not spawn"); } }); + const res = await tool.run({ prompt: "ok", model: "anthropic/claude-sonnet-4-20250514" }, {}); + assert.equal(res.isError, true); + assert.match(res.error, /opus, sonnet, haiku/); + assert.equal(spawned.length, 0); + assert.ok(!existsSync(manifest.agent.usageFile), "no usage line for a dispatch that never ran"); +}); + +// fakeSpawn returns children the test completes by hand, so concurrency can +// be observed deterministically. +function fakeSpawn() { + const children = []; + const spawn = (bin, args, opts) => { + const child = new EventEmitter(); + child.pid = 40000 + children.length; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.exitCode = null; + child.signalCode = null; + child.signals = []; + child.kill = (sig) => { child.signals.push(sig); return true; }; + child.stdinText = ""; + child.stdin.on("data", (c) => { child.stdinText += c; }); + child.finish = (lines, code = 0) => { + child.exitCode = code; + for (const l of lines) child.stdout.write(JSON.stringify(l) + "\n"); + child.stdout.end(); + child.stderr.end(); + child.emit("close", code, null); + }; + children.push({ bin, args, opts, child }); + return child; + }; + return { spawn, children }; +} + +const okStream = (text) => [ + { type: "agent_start" }, + { type: "message_end", message: { role: "assistant", content: [{ type: "text", text }], model: "m", provider: "p", usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, cost: { total: 0.01 } }, stopReason: "stop" } }, + { type: "agent_end", messages: [] }, + { type: "agent_settled" }, +]; + +test("run: at most maxConcurrent children run at once; the rest queue", async () => { + const { manifest } = fixture(); + manifest.agent.maxConcurrent = 2; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn }); + const runs = [1, 2, 3, 4].map((i) => tool.run({ prompt: `p${i}`, subagent_type: i === 4 ? "Explore" : undefined }, {})); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2, "two spawned, two waiting"); + assert.equal(tool.inFlight(), 2); + children[0].child.finish(okStream("one")); + await runs[0]; + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 3, "a slot freed → the third starts"); + children[1].child.finish(okStream("two")); + children[2].child.finish(okStream("three")); + await Promise.all([runs[1], runs[2]]); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 4); + children[3].child.finish(okStream("four")); + const results = await Promise.all(runs); + assert.deepEqual(results.map((r) => r.text), ["one", "two", "three", "four"]); + assert.equal(tool.inFlight(), 0); + + // The Explore child got the read-only set; the others the parent's built-ins. + const toolsOf = (c) => c.args[c.args.indexOf("--tools") + 1]; + assert.equal(toolsOf(children[3]), "read,grep,find,ls"); + assert.equal(toolsOf(children[0]), "read,bash,edit,write,grep,find,ls"); + assert.equal(children[0].opts.env.FULLSEND_SUBAGENT_DEPTH, "1"); + assert.deepEqual(children[0].opts.stdio, ["pipe", "pipe", "pipe"], "stdin is a pipe: it carries the prompt"); + assert.equal(children[0].child.stdinText, "p1", "the prompt is written and the pipe closed"); + assert.equal(children[0].opts.detached, undefined, "children share the parent's process group, so a killed parent takes them with it"); + assert.equal(children[0].bin, manifest.agent.piBin); +}); + +test("run: a queued dispatch aborted before it starts never spawns and still settles", async () => { + const { manifest } = fixture(); + manifest.agent.maxConcurrent = 1; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn }); + const first = tool.run({ prompt: "p1" }, {}); + const ac = new AbortController(); + const queued = tool.run({ prompt: "p2" }, { signal: ac.signal }); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 1, "the second is queued"); + ac.abort(); + const res = await queued; + assert.equal(res.isError, true); + assert.equal(res.stopReason, "aborted"); + assert.equal(children.length, 1, "an aborted dispatch never spawns"); + children[0].child.finish(okStream("one")); + assert.equal((await first).text, "one"); + assert.equal(tool.inFlight(), 0); + // The slot accounting survived the early exit. + const after = tool.run({ prompt: "p3" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2, "the semaphore did not leak the aborted dispatch's slot"); + children[1].child.finish(okStream("three")); + assert.equal((await after).text, "three"); +}); + +test("run: aborting a queued dispatch behind another waiter does not over-admit", async () => { + const { manifest } = fixture(); + manifest.agent.maxConcurrent = 2; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn }); + const first = tool.run({ prompt: "p1" }, {}); + const second = tool.run({ prompt: "p2" }, {}); + const ac = new AbortController(); + const abortedQueued = tool.run({ prompt: "p3" }, { signal: ac.signal }); + const stillQueued = tool.run({ prompt: "p4" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2, "two running, two queued"); + + // Taking the head waiter out of the queue must not claim a slot for it: + // its own release() would then hand the slot to the next waiter without + // giving it back, and that waiter would run as a third concurrent child. + ac.abort(); + const abortedRes = await abortedQueued; + assert.equal(abortedRes.stopReason, "aborted"); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2, "the dispatch behind it must not start while both slots are busy"); + assert.equal(tool.inFlight(), 2); + + children[0].child.finish(okStream("one")); + assert.equal((await first).text, "one"); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 3, "the freed slot goes to the waiter that is still queued"); + children[1].child.finish(okStream("two")); + children[2].child.finish(okStream("four")); + assert.equal((await second).text, "two"); + assert.equal((await stillQueued).text, "four"); + assert.equal(tool.inFlight(), 0); + + // Both slots are free again: two fresh dispatches start at once. + const more = [tool.run({ prompt: "p5" }, {}), tool.run({ prompt: "p6" }, {})]; + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 5, "the aborted dispatch leaked no slot"); + children[3].child.finish(okStream("five")); + children[4].child.finish(okStream("six")); + assert.deepEqual((await Promise.all(more)).map((r) => r.text), ["five", "six"]); +}); + +test("shutdown does not leak the slots of the waiters it drains", async () => { + const { manifest } = fixture(); + manifest.agent.maxConcurrent = 2; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn, killGraceMs: 5 }); + const running = [tool.run({ prompt: "p1" }, {}), tool.run({ prompt: "p2" }, {})]; + const queued = [tool.run({ prompt: "p3" }, {}), tool.run({ prompt: "p4" }, {})]; + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2); + tool.shutdown(); + const drained = await Promise.all(queued); + assert.deepEqual(drained.map((r) => r.stopReason), ["aborted", "aborted"]); + assert.equal(children.length, 2, "a dispatch queued at shutdown never spawns"); + children[0].child.finish([], null); + children[1].child.finish([], null); + await Promise.all(running); + assert.equal(tool.inFlight(), 0); +}); + +test("run: an abort while the child is running terminates it", async () => { + const { manifest } = fixture(); + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn, killGraceMs: 5 }); + const ac = new AbortController(); + const p = tool.run({ prompt: "slow" }, { signal: ac.signal }); + await new Promise((r) => setImmediate(r)); + ac.abort(); + assert.deepEqual(children[0].child.signals, ["SIGTERM"], "SIGTERM first: pi reaps its own detached bash children and exits 143"); + await new Promise((r) => setTimeout(r, 30)); + assert.deepEqual(children[0].child.signals, ["SIGTERM", "SIGKILL"], "escalated after the grace period"); + children[0].child.finish([], 143); + const res = await p; + assert.equal(res.isError, true); + assert.equal(res.stopReason, "aborted"); + assert.match(res.error, /aborted/); +}); + +test("run: an already-aborted signal is refused before anything is spawned", async () => { + const { manifest } = fixture(); + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn }); + const res = await tool.run({ prompt: "p" }, { signal: AbortSignal.abort() }); + assert.equal(res.isError, true); + assert.equal(res.stopReason, "aborted"); + assert.equal(children.length, 0); +}); + +test("run: result text is capped at 64 KB with a marker", async () => { + const { manifest } = fixture(); + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn }); + const p = tool.run({ prompt: "big" }, {}); + await new Promise((r) => setImmediate(r)); + children[0].child.finish(okStream("x".repeat(RESULT_MAX_BYTES + 100))); + const res = await p; + assert.equal(res.isError, false); + assert.ok(res.text.endsWith("\n[truncated]")); + assert.ok(Buffer.byteLength(res.text) <= RESULT_MAX_BYTES + "\n[truncated]".length); +}); + +test("shutdown stops in-flight children, fails their calls and settles the queue", async () => { + const { manifest } = fixture(); + manifest.agent.maxConcurrent = 1; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn, killGraceMs: 5 }); + const running = tool.run({ prompt: "slow" }, {}); + const queued = tool.run({ prompt: "queued" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 1); + tool.shutdown(); + assert.deepEqual(children[0].child.signals, ["SIGTERM"]); + + // A queued dispatch must not hang forever: nothing would call release() + // for it once the in-flight children are gone. + const queuedRes = await queued; + assert.equal(queuedRes.isError, true); + assert.equal(queuedRes.stopReason, "aborted"); + assert.equal(children.length, 1, "a dispatch queued at shutdown never spawns"); + + children[0].child.finish([], null); + const res = await running; + assert.equal(res.isError, true); + assert.match(res.error, /shut down|killed/); +}); + +function registerWith(manifestPath, env = {}) { + const tools = []; + const events = {}; + const lines = []; + const origError = console.error; + console.error = (l) => lines.push(l); + const saved = { ...process.env }; + process.env.FULLSEND_PI_MANIFEST = manifestPath; + delete process.env.FULLSEND_SUBAGENT_DEPTH; + Object.assign(process.env, env); + try { + defaultExport({ registerTool: (d) => tools.push(d), on: (ev, fn) => { events[ev] = fn; } }); + } finally { + console.error = origError; + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); + } + return { tools, events, lines }; +} + +test("default export registers Agent and Task with one handler and a shutdown hook", () => { + const { dir, manifest } = fixture(); + const path = join(dir, "manifest.json"); + writeFileSync(path, JSON.stringify(manifest)); + const { tools, events, lines } = registerWith(path); + assert.deepEqual(tools.map((t) => t.name), ["Agent", "Task"]); + assert.equal(tools[0].execute, tools[1].execute); + assert.deepEqual(tools[0].parameters, AGENT_TOOL_PARAMETERS); + assert.ok(typeof events.session_shutdown === "function"); + assert.deepEqual(lines, []); +}); + +test("default export: missing manifest, disabled block, or a set depth registers nothing and logs once", () => { + const { dir, manifest } = fixture(); + let r = registerWith(join(dir, "nonexistent.json")); + assert.equal(r.tools.length, 0); + assert.equal(r.lines.length, 1); + assert.match(r.lines[0], /cannot read manifest/); + + const disabled = join(dir, "disabled.json"); + writeFileSync(disabled, JSON.stringify({ ...manifest, agent: { ...manifest.agent, enabled: false } })); + r = registerWith(disabled); + assert.equal(r.tools.length, 0); + assert.equal(r.lines.length, 1); + assert.match(r.lines[0], /not enabled/); + + const noBlock = join(dir, "noblock.json"); + writeFileSync(noBlock, JSON.stringify({ ...manifest, agent: undefined })); + r = registerWith(noBlock); + assert.equal(r.tools.length, 0); + assert.equal(r.lines.length, 1); + + const ok = join(dir, "manifest.json"); + writeFileSync(ok, JSON.stringify(manifest)); + r = registerWith(ok, { FULLSEND_SUBAGENT_DEPTH: "1" }); + assert.equal(r.tools.length, 0, "a child never registers the tool (recursion is refused)"); + assert.equal(r.lines.length, 1); + assert.match(r.lines[0], /depth/i); +}); + +test("execute throws on a failed child so pi marks the result isError", async () => { + const { dir, manifest } = fixture(); + const path = join(dir, "manifest.json"); + writeFileSync(path, JSON.stringify(manifest)); + const { tools } = registerWith(path); + const ctx = { model: { provider: "anthropic-vertex", id: "claude-opus-4-6" } }; + await assert.rejects(tools[0].execute("call-1", { prompt: "fail" }, undefined, undefined, ctx), /quota exhausted/); + const ok = await tools[0].execute("call-2", { prompt: "ok", run_in_background: true }, undefined, undefined, ctx); + assert.equal(ok.content[0].type, "text"); + const payload = JSON.parse(ok.content[0].text); + assert.equal(payload.argv[payload.argv.indexOf("--model") + 1], "anthropic-vertex/claude-opus-4-6", "omitted model → the parent's active model from ctx"); + assert.equal(ok.details.seq, 2); + assert.equal(ok.details.model, "anthropic-vertex/claude-opus-4-6"); +}); + +// --- Real pi --------------------------------------------------------------- +// The prompt-delivery contract is a claim about pi's own argv parser, so one +// test checks it against the real binary. CI has no pi, so it is gated: +// npm install --prefix /tmp/pi @earendil-works/pi-coding-agent@0.84.4 \ +// --ignore-scripts +// FULLSEND_TEST_PI_BIN=/tmp/pi/node_modules/.bin/pi \ +// node --test internal/runtime/pi_extension/ +// It runs offline: the model is pointed at a base URL nothing listens on, so +// the turn fails at the first request — after pi has parsed argv, read stdin +// and recorded the initial user message, which is all this asserts. +const REAL_PI = process.env.FULLSEND_TEST_PI_BIN; +const OFFLINE_MODEL = "openai/gpt-5"; + +function realPiEnv(home) { + return { + PATH: process.env.PATH, + HOME: home, + PI_CODING_AGENT_DIR: home, + OPENAI_API_KEY: "sk-not-a-real-key", + OPENAI_BASE_URL: "http://127.0.0.1:1", + }; +} + +function runRealPi(args, { cwd, home, stdin }) { + return new Promise((resolve) => { + const child = spawn(REAL_PI, args, { cwd, env: realPiEnv(home), stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (c) => { stdout += c; }); + child.stderr.on("data", (c) => { stderr += c; }); + child.stdin.on("error", () => {}); + child.stdin.end(stdin ?? ""); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +// userText returns the text of the first user message in a --mode json stream. +function userText(stdout) { + for (const line of stdout.split("\n")) { + if (line.trim() === "") continue; + let evt; + try { + evt = JSON.parse(line); + } catch { + continue; + } + if (evt.type === "message_start" && evt.message?.role === "user") { + return (evt.message.content ?? []).filter((b) => b.type === "text").map((b) => b.text).join(""); + } + } + return undefined; +} + +test("real pi: the prompt goes over stdin because argv cannot carry it", { timeout: 180_000 }, async (t) => { + if (!REAL_PI) return t.skip("set FULLSEND_TEST_PI_BIN to a pi 0.84.4 binary"); + const { manifest } = fixture(); + const home = join(manifest.agent.sessionsDir, "..", "pi-home"); + mkdirSync(home, { recursive: true }); + // The vendored provider extensions only exist in the sandbox image. + const agent = { ...manifest.agent, extensions: [] }; + + // Negative control: the same prompt as a positional argument is what the + // finding is about. pi does accept a "--" terminator, but it is not a way + // out: an "@"-prefixed positional after it is still read as a file + // argument, and the kernel's argv cap applies either way (the 200 KiB + // prompt below). This control uses the childArgs form the runner + // actually builds, which has no terminator in it. + const positional = await runRealPi( + [...childArgs(agent, { seq: 90, modelSpec: OFFLINE_MODEL, tools: ["read"] }), "- a leading dash"], + { cwd: home, home }, + ); + assert.equal(positional.code, 1); + assert.match(positional.stderr, /Unknown option/); + + // Prompts pi's parser would eat as an option, an unknown flag or a file + // argument, plus one past the kernel's argv limit. + const prompts = ["- a leading dash", "--no-approve then review this", "@/etc/passwd", "x".repeat(200 * 1024)]; + for (const [i, prompt] of prompts.entries()) { + const args = childArgs(agent, { seq: 100 + i, modelSpec: OFFLINE_MODEL, tools: ["read"] }); + const res = await runRealPi(args, { cwd: home, home, stdin: prompt }); + const label = JSON.stringify(prompt.slice(0, 32)); + assert.doesNotMatch(res.stderr, /Unknown option/, `pi rejected ${label} as an option`); + assert.equal(res.code, 0, `pi exited ${res.code} on ${label}: ${res.stderr}`); + assert.equal(userText(res.stdout), prompt, `pi did not receive ${label} as the initial message`); + } +}); From 07aba0bc88dc8a1afd253d571ace8f008906dd7e Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 13:02:47 -0400 Subject: [PATCH 3/7] feat(pi): write the agent manifest block and install the Agent extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootstrap now enables the Agent tool when the agent definition has no tools: frontmatter or lists Agent/Task: it uploads fullsend-agent.js, probes the sandbox for the pi binary and the vendored provider extension directories that exist, and writes the `agent` manifest block the extension reads — child extensions (providers present, then the hook adapter when security is on), the model alias table (default = the agent's model translated as for the parent; opus|sonnet|haiku on the Anthropic Vertex provider), thinking (FULLSEND_PI_SUBAGENT_THINKING when it names a pi level, else medium), the child and Explore tool sets, the concurrency cap, timeout and usage file. The hook adapter's tool-name table gains Agent/Task so the scripts see Claude vocabulary, and APPEND_SYSTEM.md gets a note describing the tool and that parallel dispatch is several Agent calls in one message; an agent whose tools: leaves Agent out keeps the single-context note. Refs: #6527, #6464 Assisted-by: Claude Signed-off-by: Wayne Sun --- internal/harness/extension_spec.go | 2 +- internal/runtime/pi_bootstrap.go | 345 ++++++++++++++++++++++++- internal/runtime/pi_bootstrap_test.go | 133 ++++++++++ internal/runtime/pi_extensions_test.go | 4 +- 4 files changed, 474 insertions(+), 10 deletions(-) diff --git a/internal/harness/extension_spec.go b/internal/harness/extension_spec.go index 7cdbb61744..17e718174d 100644 --- a/internal/harness/extension_spec.go +++ b/internal/harness/extension_spec.go @@ -133,7 +133,7 @@ func ExtensionPaths(entries []ExtensionSpec) []string { // bootstrap; the check here is so a harness author learns at load which // entry is the problem. The list lives in this package because // internal/runtime imports it and not the other way round. -var PiReservedExtensionNames = []string{"fullsend-hooks", "anthropic-vertex", "xai-vertex"} +var PiReservedExtensionNames = []string{"fullsend-hooks", "fullsend-agent", "anthropic-vertex", "xai-vertex"} var validExtensionEnvKey = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) diff --git a/internal/runtime/pi_bootstrap.go b/internal/runtime/pi_bootstrap.go index e29de322ad..a268c4f04f 100644 --- a/internal/runtime/pi_bootstrap.go +++ b/internal/runtime/pi_bootstrap.go @@ -1,12 +1,15 @@ package runtime import ( + "crypto/sha256" _ "embed" + "encoding/hex" "encoding/json" "fmt" "maps" "os" "strings" + "sync" "time" "github.com/fullsend-ai/fullsend/internal/sandbox" @@ -23,6 +26,14 @@ const ( // sandbox hook scripts; loaded explicitly with -e, never auto-discovered // (Run passes --no-extensions). piHooksExtensionFile = "fullsend-hooks.js" + // piAgentExtensionFile is the embedded pi extension that provides Claude + // Code's Agent tool (sub-agents as child pi processes); loaded with -e + // after the hook adapter when the agent's tools allow it (#6527). + piAgentExtensionFile = "fullsend-agent.js" + // piAgentUsageFile is where the Agent extension appends one JSON line + // per child (model, usage, stop reason); Run folds it into RunMetrics + // and ExtractTranscripts saves it next to the child session files. + piAgentUsageFile = "subagents/usage.jsonl" // piAppendSystemFile is pi's hook for appending to its default system // prompt (packages/coding-agent README "System Prompt"); the agent body // goes here rather than SYSTEM.md so pi's tool guidance is kept. @@ -34,6 +45,61 @@ const ( //go:embed pi_extension/fullsend-hooks.js var piHooksExtensionJS []byte +//go:embed pi_extension/fullsend-agent.js +var piAgentExtensionJS []byte + +// Agent tool defaults written into the manifest for the extension. +const ( + // piAgentThinkingEnv overrides the --thinking level children run at. + // The default is pi's own "medium", not the parent's "high": the + // verified pr-review roster at high overran the 20-minute review + // budget (research, 2026-08-29). + piAgentThinkingEnv = "FULLSEND_PI_SUBAGENT_THINKING" + piAgentDefaultThinking = "medium" + piAgentMaxConcurrent = 4 + piAgentTimeoutSeconds = 900 + piAgentProbeMaxBytes = 4096 + piAgentProbeFallbackBin = "pi" +) + +// piAgentManifest is the `agent` block of fullsend-manifest.json, read by +// fullsend-agent.js. Absent when the agent's tools: frontmatter leaves out +// Agent/Task, which is also when Run does not load the extension. +type piAgentManifest struct { + Enabled bool `json:"enabled"` + // PiBin is the pi binary the children run, resolved in the sandbox at + // Bootstrap ("pi" when the probe found nothing, i.e. PATH lookup). + PiBin string `json:"piBin"` + SessionsDir string `json:"sessionsDir"` + // Extensions are the -e paths every child gets, in order: the provider + // extensions present in the image, then the hook adapter when security + // is enabled. Never this extension itself. + Extensions []string `json:"extensions"` + // ExtensionDigests is the sha256 (hex) of each Extensions entry that + // Bootstrap itself wrote under the runner-owned config dir. The + // extension re-hashes them before every dispatch; see + // piAgentExtensionDigests for why the vendored ones are absent. + ExtensionDigests map[string]string `json:"extensionDigests,omitempty"` + // Models maps "default" (the agent's model) and the Claude aliases to + // pi model specs; the extension translates a child's `model` through + // it and rejects anything else it cannot serve. + Models map[string]string `json:"models"` + // ProviderModels lists the model ids of the providers pi serves without + // an extension and without a Models entry (google-vertex). The + // extension accepts a `provider/id` spec only when the id is in this + // list, so an id the model invented is rejected at dispatch instead of + // reaching the API as an unknown model. + ProviderModels map[string][]string `json:"providerModels,omitempty"` + Thinking string `json:"thinking"` + // Tools is the built-in set a child gets (the parent's, minus + // Agent/Task); ExploreTools the read-only set for subagent_type Explore. + Tools []string `json:"tools"` + ExploreTools []string `json:"exploreTools"` + MaxConcurrent int `json:"maxConcurrent"` + TimeoutSeconds int `json:"timeoutSeconds"` + UsageFile string `json:"usageFile"` +} + // piManifest is the JSON document at ConfigDir/fullsend-manifest.json. type piManifest struct { AgentName string `json:"agentName"` @@ -58,6 +124,9 @@ type piManifest struct { // (ADR 0094). Informational for the hook adapter; Run's preflight uses // hashes recomputed from the host, not these. Extensions []piManifestExtension `json:"extensions,omitempty"` + // Agent configures the fullsend-agent.js extension; nil when the Agent + // tool is not enabled for this agent. + Agent *piAgentManifest `json:"agent,omitempty"` } type piHooksManifest struct { @@ -79,6 +148,8 @@ func (r PiRuntime) piManifestPath() string { return r.ConfigDir() + "/" + piMani func (r PiRuntime) piSessionsDir() string { return r.ConfigDir() + "/sessions" } +func (r PiRuntime) piAgentUsagePath() string { return r.ConfigDir() + "/" + piAgentUsageFile } + // Bootstrap prepares the runner-owned pi config directory for one agent run: // agent body as APPEND_SYSTEM.md, locked-down settings.json, skills, the // hook scripts plus the fullsend hook extension when the harness enables @@ -121,7 +192,10 @@ func (r PiRuntime) Bootstrap(input BootstrapInput) error { return fmt.Errorf("creating pi config dirs: %w", err) } - if err := uploadBytes(sandboxName, cfg+"/"+piAppendSystemFile, piAppendSystem(agentName, def)); err != nil { + hooksInput, hooksEnabled := input.(SandboxHooksBootstrap) + agentTool := piAgentToolEnabled(def) + + if err := uploadBytes(sandboxName, cfg+"/"+piAppendSystemFile, piAppendSystem(agentName, def, agentTool)); err != nil { return fmt.Errorf("writing %s: %w", piAppendSystemFile, err) } settings, err := piSettingsJSON() @@ -186,7 +260,7 @@ func (r PiRuntime) Bootstrap(input BootstrapInput) error { Extensions: extensions, } - if hooksInput, ok := input.(SandboxHooksBootstrap); ok { + if hooksEnabled { hooks := hooksInput.SandboxHookConfig() if err := installHookScripts(sandboxName, r.piHooksDir(), hooks); err != nil { return err @@ -198,6 +272,23 @@ func (r PiRuntime) Bootstrap(input BootstrapInput) error { return fmt.Errorf("installing hook extension: %w", err) } manifest.Hooks = piHooksManifestFor(r.piHooksDir(), hooks) + if agentTool { + // The adapter hands the scripts Claude-vocabulary names; the + // Agent tool's names already are (Task is the legacy alias). + manifest.Hooks.ToolNames[piAgentToolName] = piAgentToolName + manifest.Hooks.ToolNames[piAgentToolAlias] = piAgentToolAlias + } + } + + if agentTool { + if err := uploadBytes(sandboxName, cfg+"/"+piAgentExtensionFile, piAgentExtensionJS); err != nil { + return fmt.Errorf("installing agent extension: %w", err) + } + block, err := r.piAgentManifestFor(sandboxName, def, tools, hooksEnabled) + if err != nil { + return err + } + manifest.Agent = block } version, err := piPreflightVersion(sandboxName) @@ -213,9 +304,45 @@ func (r PiRuntime) Bootstrap(input BootstrapInput) error { if err := uploadBytes(sandboxName, r.piManifestPath(), manifestJSON); err != nil { return fmt.Errorf("writing %s: %w", piManifestFile, err) } + recordPiManifestHash(sandboxName, manifestJSON) return nil } +// piManifestHashes carries the digest of the manifest Bootstrap wrote from +// Bootstrap to Run. The runner drives both from one process for one +// sandbox (internal/cli/run.go bootstraps, then loops Run), so a package +// map is the seam — no Runtime interface method, and nothing on disk in +// the sandbox for the agent to reach. +// +// Run turns the digest into the shell guard that refuses to start pi on a +// modified manifest. Without an entry (Run reached without this process +// having bootstrapped that sandbox) the guard is simply not emitted: +// failing closed there would break any caller that bootstraps separately, +// and the agent cannot cause the entry to be missing. +// +// So the guard exists only when Bootstrap and Run run in one process. The +// CLI's `run` path does, which is why this is a seam and not a gap — but a +// caller that bootstrapped in one process and ran in another would lose +// both the manifest guard and the FULLSEND_PI_MANIFEST_SHA256 export the +// hook adapter re-checks in every sub-agent, with nothing to say so. +var piManifestHashes sync.Map // sandboxName -> hex sha256 of the manifest bytes + +func recordPiManifestHash(sandboxName string, manifestJSON []byte) { + sum := sha256.Sum256(manifestJSON) + piManifestHashes.Store(sandboxName, hex.EncodeToString(sum[:])) +} + +// piManifestHash returns the digest recorded for a sandbox, or "" when +// Bootstrap did not run in this process. +func piManifestHash(sandboxName string) string { + if v, ok := piManifestHashes.Load(sandboxName); ok { + if s, ok := v.(string); ok { + return s + } + } + return "" +} + // piBashAllowlistEnv selects how the extension treats a Bash(a,b) allowlist // violation: "warn" (default, Claude Code parity) or "enforce". const piBashAllowlistEnv = "FULLSEND_PI_BASH_ALLOWLIST" @@ -230,7 +357,7 @@ func piBashAllowlistMode() string { // piAppendSystem renders the agent definition for APPEND_SYSTEM.md. Claude // Code shows the agent its own name/description; pi gets the same header so // prompts that refer to "this agent" still resolve. -func piAppendSystem(agentName string, def *piAgentDef) []byte { +func piAppendSystem(agentName string, def *piAgentDef, agentTool bool) []byte { var b strings.Builder fmt.Fprintf(&b, "# Agent: %s\n\n", agentName) if def.Description != "" { @@ -239,20 +366,222 @@ func piAppendSystem(agentName string, def *piAgentDef) []byte { } b.WriteString(def.Body) b.WriteString("\n") - b.WriteString(piNoSubagentNote) + if agentTool { + b.WriteString(piSubagentNote) + } else { + b.WriteString(piNoSubagentNote) + } return []byte(b.String()) } -// piNoSubagentNote makes the absence of a sub-agent tool explicit so skills -// written for Claude Code's Agent tool (pr-review, retro) take their -// single-context path deliberately instead of recording a failed dispatch. -// A fullsend-owned Agent tool for pi is tracked on #6527. +// piSubagentNote tells the agent the Agent tool behaves as under Claude +// Code and how parallel dispatch works on pi, so the skills' "dispatch in +// parallel" text maps onto one assistant message with several calls. +const piSubagentNote = "\n## Runtime note\n\n" + + "This agent runs on the pi runtime (FULLSEND_RUNTIME=pi). The Agent tool (alias Task) " + + "dispatches sub-agents as under Claude Code: `prompt` (required), `description`, `model` " + + "(opus, sonnet, haiku, or a provider/id spec available in this run; omit to inherit yours) and " + + "`subagent_type` (`Explore` for a read-only sub-agent). Each call runs to completion and " + + "returns the sub-agent's final message; to run sub-agents in parallel, make several Agent " + + "calls in one message. `run_in_background` is accepted and ignored.\n" + +// piNoSubagentNote makes the absence of a sub-agent tool explicit for an +// agent whose tools: frontmatter leaves out Agent/Task, so skills written +// for Claude Code's Agent tool (pr-review, retro) take their single-context +// path deliberately instead of recording a failed dispatch. const piNoSubagentNote = "\n## Runtime note\n\n" + "This agent runs on the pi runtime (FULLSEND_RUNTIME=pi). No sub-agent tool " + "(Agent/Task) is available. When a skill says to dispatch sub-agents, execute each " + "sub-agent definition yourself, in the listed order, with the same context package, " + "and treat each output as that sub-agent's result.\n" +// piAgentManifestFor builds the manifest block for fullsend-agent.js. tools +// is the parent's --tools list (nil for the default set); children get the +// built-ins from it minus Agent/Task. +func (r PiRuntime) piAgentManifestFor(sandboxName string, def *piAgentDef, tools []string, hooksEnabled bool) (*piAgentManifest, error) { + piBin, providerExts, err := piAgentProbe(sandboxName) + if err != nil { + return nil, err + } + exts := append([]string{}, providerExts...) + hooksExt := r.ConfigDir() + "/" + piHooksExtensionFile + if hooksEnabled { + exts = append(exts, hooksExt) + } + childTools := []string{} + if tools == nil { + childTools = append(childTools, piDefaultTools...) + } else { + for _, t := range tools { + if t != piAgentToolName && t != piAgentToolAlias { + childTools = append(childTools, t) + } + } + } + return &piAgentManifest{ + Enabled: true, + PiBin: piBin, + SessionsDir: r.piSessionsDir(), + Extensions: exts, + ExtensionDigests: piAgentExtensionDigests(hooksExt, hooksEnabled), + Models: piAgentModels(def.Model), + ProviderModels: piAgentProviderModels(), + Thinking: piAgentThinking(), + Tools: childTools, + ExploreTools: append([]string{}, piExploreTools...), + MaxConcurrent: piAgentMaxConcurrent, + TimeoutSeconds: piAgentTimeoutSeconds, + UsageFile: r.piAgentUsagePath(), + }, nil +} + +// piAgentExtensionDigests records the sha256 of every child -e entry that +// Bootstrap itself writes under the runner-owned config dir — today only +// the hook adapter, and the same bytes piHooksGuard checks before pi +// starts. fullsend-agent.js re-hashes them immediately before every +// dispatch: the launch guard fires once, and nothing else re-verifies the +// adapter afterwards, so a parent with `write` could replace it +// mid-iteration and dispatch children whose adapter runs no hooks and +// silently skips its own manifest-digest check. The map travels inside the +// manifest, so the manifest digest already covers it. +// +// The vendored provider extensions under piVertexExtensionPath / +// piXaiVertexExtensionPath are deliberately absent: the image installs them +// root-owned and read-only outside the config dir, so there is nothing +// there for the agent to rewrite and nothing to re-check. +func piAgentExtensionDigests(hooksExt string, hooksEnabled bool) map[string]string { + if !hooksEnabled { + return nil + } + sum := sha256.Sum256(piHooksExtensionJS) + return map[string]string{hooksExt: hex.EncodeToString(sum[:])} +} + +// piAgentModels is the child model table: "default" is the agent +// definition's model translated as Run translates the parent's (honouring +// FULLSEND_PI_PROVIDER; the extension prefers the parent's live model when +// pi reports one), and the Claude aliases always resolve on the Anthropic +// Vertex provider, whatever provider the parent runs on. A persona-style +// "@default" suffix is dropped. +func piAgentModels(defModel string) map[string]string { + base, _, _ := strings.Cut(strings.TrimSpace(defModel), "@") + models := map[string]string{"default": translatePiModel(base)} + for alias, id := range piModelAliases { + models[alias] = piDefaultProvider + "/" + id + } + return models +} + +// piGoogleVertexModels are the model ids pi's built-in google-vertex +// provider registers, verbatim from the catalog the pinned pi bundles +// (@earendil-works/pi-ai 0.84.4, dist/providers/data/google-vertex.json). +// Re-check it on a pi bump, the way the Anthropic ids in pi_run.go are. +// +// Gemini needs no extension and has no entry in the agent's model table, so +// without a closed list the extension would have to pass any +// "google-vertex/" through — including one the model invented, which +// reaches Vertex as an unknown model and loses the dispatch to a confusing +// error instead of the accepted-forms rejection every other bad spec gets. +var piGoogleVertexModels = []string{ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-flash-latest", + "gemini-flash-lite-latest", +} + +// piXaiVertexModels are the model ids the vendored xai-vertex extension +// registers (fullsend-ai/pi-xai-vertex, pinned by PI_XAI_VERTEX_VERSION in +// images/sandbox/Containerfile). They carry the publisher segment Vertex +// wants on the wire, so the full spec is "xai-vertex/xai/grok-4.6" — the +// same three-segment form normalizeXaiVertexModel renders for the parent. +// Re-check the list on an extension bump, the way the Gemini ids are +// re-checked on a pi bump. +// +// Grok, like Gemini, has no entry in the agent's model table unless the +// agent itself runs on it, so without this list the extension would have to +// pass any "xai/" through on the provider prefix alone — including one +// the model invented, which reaches Vertex as an unknown model. +var piXaiVertexModels = []string{"xai/grok-4.6"} + +// piAgentProviderModels is the manifest's per-provider id allowlist for the +// providers a run can serve without an entry in the agent's model table: +// pi's built-in google-vertex and the vendored xai-vertex extension. The +// remaining credential-free provider a run can be on (openai) needs no list +// because the extension always accepts the parent's own model spec. +func piAgentProviderModels() map[string][]string { + return map[string][]string{ + "google-vertex": append([]string(nil), piGoogleVertexModels...), + piXaiVertexProvider: append([]string(nil), piXaiVertexModels...), + } +} + +// piAgentThinking is the children's --thinking level: the env override when +// it names a pi level, else piAgentDefaultThinking. +func piAgentThinking() string { + level := strings.TrimSpace(os.Getenv(piAgentThinkingEnv)) + if level == "" { + return piAgentDefaultThinking + } + if !piThinkingLevels[level] { + fmt.Fprintf(os.Stderr, "%s=%q is not a pi thinking level; sub-agents run at --thinking %s\n", piAgentThinkingEnv, sanitizeOutput(level), piAgentDefaultThinking) + return piAgentDefaultThinking + } + return level +} + +// piAgentProbeCommand resolves, inside the sandbox, the pi binary children +// run and which vendored provider extension directories the image has: +// one line for the binary, then one per existing directory. +func piAgentProbeCommand() string { + return "command -v pi; for d in " + shellQuote(piVertexExtensionPath) + " " + shellQuote(piXaiVertexExtensionPath) + + `; do test -d "$d" && echo "$d"; done; true` +} + +// parsePiAgentProbe reads the probe output. Only the two known extension +// paths are accepted as extensions; a missing binary line falls back to +// PATH lookup when the child is spawned. +func parsePiAgentProbe(stdout string) (piBin string, exts []string) { + piBin = piAgentProbeFallbackBin + binSeen := false + for _, line := range strings.Split(stdout, "\n") { + // Per line: sanitizeOutput folds newlines, so it must not see the + // whole probe output. + line = strings.TrimSpace(sanitizeOutput(line)) + switch { + case line == "": + case line == piVertexExtensionPath || line == piXaiVertexExtensionPath: + // When `command -v pi` printed nothing the first line is an + // extension path; never mistake it for the binary. + exts = append(exts, line) + case !binSeen: + piBin = line + binSeen = true + } + } + return piBin, exts +} + +func piAgentProbe(sandboxName string) (string, []string, error) { + stdout, _, _, err := sandbox.Exec(sandboxName, piAgentProbeCommand(), 10*time.Second) + if err != nil { + return "", nil, fmt.Errorf("probing pi for sub-agents: %w", err) + } + if len(stdout) > piAgentProbeMaxBytes { + stdout = stdout[:piAgentProbeMaxBytes] + } + piBin, exts := parsePiAgentProbe(stdout) + return piBin, exts, nil +} + // piDefaultTools is the built-in tool set activated when the agent lists // no tools: pi 0.84.x itself starts with only read, bash, edit and write // (packages/coding-agent/src/core/sdk.ts defaultActiveToolNames — grep, diff --git a/internal/runtime/pi_bootstrap_test.go b/internal/runtime/pi_bootstrap_test.go index 3837c401b0..b5272f8cd9 100644 --- a/internal/runtime/pi_bootstrap_test.go +++ b/internal/runtime/pi_bootstrap_test.go @@ -36,6 +36,7 @@ if [ "$2" = "exec" ]; then for last; do :; done case "$last" in "pi --version") echo "0.84.2"; exit 0 ;; + "command -v pi"*) printf '%s\n' /usr/bin/pi '/usr/local/share/pi-extensions/anthropic-vertex'; exit 0 ;; cat\ *) f=$(printf '%s' "${last#cat }" | tr -d "'" | tr '/' '_'); cat '` + storeDir + `'/"$f"; exit $? ;; *"--print --mode json"*) cat '` + streamFixture + `'; exit 0 ;; esac @@ -383,6 +384,138 @@ exit 0 assert.Empty(t, PiRuntime{}.ParseTranscriptErrors(out), "clean sessions produce no error annotations") } +// TestPiAgentTool_ManifestBlock covers the `agent` manifest block Bootstrap +// writes for the fullsend-agent.js extension: enabled/disabled cases, the +// model alias table, the extension list from the sandbox probe, and the +// thinking default plus its env override. +func TestPiAgentTool_ManifestBlock(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + t.Setenv(piProviderEnv, "") + t.Setenv(piAgentThinkingEnv, "") + cfg := PiRuntime{}.ConfigDir() + + bootstrap := func(t *testing.T, agentDef string, hooks bool) (*piManifest, string, string) { + t.Helper() + work := t.TempDir() + store := filepath.Join(work, "store") + fakeOpenshellPi(t, filepath.Join(work, "openshell.log"), store, "/dev/null") + base := bootstrapInput{sandboxName: "sb", agentPath: writeAgentFile(t, agentDef), agentName: "review"} + var in BootstrapInput = base + if hooks { + h := &harness.Harness{Security: &harness.SecurityConfig{SandboxHooks: &harness.SandboxHooks{}}} + in = piHooksBootstrapInput{bootstrapInput: base, hooks: security.SandboxHookConfigFromHarness(h)} + } + require.NoError(t, PiRuntime{}.Bootstrap(in)) + var m piManifest + require.NoError(t, json.Unmarshal(storedUpload(t, store, cfg+"/fullsend-manifest.json"), &m)) + return &m, store, string(storedUpload(t, store, cfg+"/APPEND_SYSTEM.md")) + } + + t.Run("enabled without tools frontmatter, hooks on", func(t *testing.T) { + m, store, appendSystem := bootstrap(t, "---\nname: review\nmodel: opus\n---\nReview the PR.", true) + require.NotNil(t, m.Agent) + assert.True(t, m.Agent.Enabled) + assert.Equal(t, "/usr/bin/pi", m.Agent.PiBin, "resolved by the sandbox probe") + assert.Equal(t, cfg+"/sessions", m.Agent.SessionsDir) + assert.Equal(t, []string{piVertexExtensionPath, cfg + "/fullsend-hooks.js"}, m.Agent.Extensions, + "only the provider extensions the image has (the probe found anthropic-vertex, not xai-vertex), then the hook adapter") + assert.Equal(t, map[string]string{ + "default": "anthropic-vertex/claude-opus-4-6", + "opus": "anthropic-vertex/claude-opus-4-6", + "sonnet": "anthropic-vertex/claude-sonnet-4-6", + "haiku": "anthropic-vertex/claude-haiku-4-5", + }, m.Agent.Models) + assert.Equal(t, map[string][]string{ + "google-vertex": piGoogleVertexModels, + piXaiVertexProvider: piXaiVertexModels, + }, m.Agent.ProviderModels, + "the extension needs a closed id list for every provider a run can serve with no model-table entry") + assert.Contains(t, m.Agent.ProviderModels["google-vertex"], "gemini-3.7-flash", "the spec documented in docs/runtimes/pi.md") + assert.Equal(t, []string{"xai/grok-4.6"}, m.Agent.ProviderModels[piXaiVertexProvider], + "the publisher-qualified wire id the vendored extension registers, so xai-vertex/xai/grok-4.6 resolves and an invented Grok id does not") + hooksExt := cfg + "/fullsend-hooks.js" + require.Len(t, m.Agent.ExtensionDigests, 1, + "the one child -e entry Bootstrap wrote into the config dir is digest-covered, so the extension can re-check it before every dispatch") + require.NotEmpty(t, m.Agent.ExtensionDigests[hooksExt]) + assert.Contains(t, piHooksGuard(hooksExt, cfg+"/fullsend-manifest.json"), m.Agent.ExtensionDigests[hooksExt], + "and against the same digest the launch guard checks, so the two cannot drift") + assert.NotContains(t, m.Agent.ExtensionDigests, piVertexExtensionPath, + "the vendored provider extension is root-owned and read-only in the image; nothing to re-check") + assert.Equal(t, "medium", m.Agent.Thinking, "children default to medium: the roster overran the review budget at high") + assert.Equal(t, piDefaultTools, m.Agent.Tools, "no tools: frontmatter → the default built-in set") + assert.Equal(t, piExploreTools, m.Agent.ExploreTools) + assert.Equal(t, piAgentMaxConcurrent, m.Agent.MaxConcurrent) + assert.Equal(t, piAgentTimeoutSeconds, m.Agent.TimeoutSeconds) + assert.Equal(t, cfg+"/subagents/usage.jsonl", m.Agent.UsageFile) + assert.Nil(t, m.Tools, "the parent's --tools stays the default set") + require.NotNil(t, m.Hooks) + assert.Equal(t, "Agent", m.Hooks.ToolNames["Agent"], "the adapter reports Agent calls in Claude vocabulary") + assert.Equal(t, "Task", m.Hooks.ToolNames["Task"]) + + ext := string(storedUpload(t, store, cfg+"/fullsend-agent.js")) + assert.Contains(t, ext, "export default function") + assert.Contains(t, appendSystem, "## Runtime note") + assert.Contains(t, appendSystem, "The Agent tool (alias Task)") + assert.Contains(t, appendSystem, "several Agent calls in one message") + assert.NotContains(t, appendSystem, "No sub-agent tool") + }) + + t.Run("enabled by a tools list naming Task, hooks off", func(t *testing.T) { + m, _, _ := bootstrap(t, "---\nname: review\nmodel: claude-sonnet-4-6@default\ntools: Read, Grep, Task\n---\nbody", false) + require.NotNil(t, m.Agent) + assert.Equal(t, []string{"read", "grep", "Agent", "Task"}, m.Tools, "--tools carries both tool names") + assert.Equal(t, []string{"read", "grep"}, m.Agent.Tools, "children get the built-ins only") + assert.Equal(t, []string{piVertexExtensionPath}, m.Agent.Extensions, "no hook adapter without security") + assert.Empty(t, m.Agent.ExtensionDigests, "and so nothing in the child -e list that Bootstrap wrote, hence no digests") + assert.Equal(t, "anthropic-vertex/claude-sonnet-4-6", m.Agent.Models["default"], "the agent's own model, @suffix stripped, is the default for children") + assert.Nil(t, m.Hooks) + }) + + t.Run("disabled by a tools list without Agent or Task", func(t *testing.T) { + m, store, appendSystem := bootstrap(t, testAgentDef, true) + assert.Nil(t, m.Agent) + assert.NotContains(t, m.Hooks.ToolNames, "Agent") + _, err := os.Stat(filepath.Join(store, strings.ReplaceAll(cfg+"/fullsend-agent.js", "/", "_"))) + assert.True(t, os.IsNotExist(err), "no Agent extension upload when the tool is off") + assert.Contains(t, appendSystem, "No sub-agent tool (Agent/Task) is available", "the single-context note stays for agents that opted out") + }) + + t.Run("thinking env override", func(t *testing.T) { + t.Setenv(piAgentThinkingEnv, "low") + m, _, _ := bootstrap(t, "---\nname: review\n---\nbody", false) + assert.Equal(t, "low", m.Agent.Thinking) + t.Setenv(piAgentThinkingEnv, "turbo") + m, _, _ = bootstrap(t, "---\nname: review\n---\nbody", false) + assert.Equal(t, "medium", m.Agent.Thinking, "an unknown level falls back to the default") + }) + + t.Run("provider env sets the default but not the Claude aliases", func(t *testing.T) { + t.Setenv(piProviderEnv, piXaiVertexProvider) + m, _, _ := bootstrap(t, "---\nname: review\nmodel: grok-4.6\n---\nbody", false) + assert.Equal(t, "xai-vertex/xai/grok-4.6", m.Agent.Models["default"]) + assert.Equal(t, "anthropic-vertex/claude-sonnet-4-6", m.Agent.Models["sonnet"], "aliases are Claude models on the Anthropic Vertex provider regardless of the parent's provider") + }) +} + +func TestPiAgentProbeCommand(t *testing.T) { + t.Parallel() + cmd := piAgentProbeCommand() + assert.True(t, strings.HasPrefix(cmd, "command -v pi"), cmd) + assert.Contains(t, cmd, shellQuote(piVertexExtensionPath)) + assert.Contains(t, cmd, shellQuote(piXaiVertexExtensionPath)) + assert.Contains(t, cmd, `test -d "$d" && echo "$d"`) + + bin, exts := parsePiAgentProbe("/usr/bin/pi\n/usr/local/share/pi-extensions/xai-vertex\n") + assert.Equal(t, "/usr/bin/pi", bin) + assert.Equal(t, []string{piXaiVertexExtensionPath}, exts) + bin, exts = parsePiAgentProbe("") + assert.Equal(t, "pi", bin, "no probe output (the exec failed silently) falls back to PATH lookup at spawn time") + assert.Empty(t, exts) + bin, exts = parsePiAgentProbe("/nonsense\n/etc/passwd\n") + assert.Equal(t, "/nonsense", bin) + assert.Empty(t, exts, "only the known extension paths are accepted") +} + func TestPiRuntimeClearIterationArtifacts(t *testing.T) { work := t.TempDir() logPath := filepath.Join(work, "openshell.log") diff --git a/internal/runtime/pi_extensions_test.go b/internal/runtime/pi_extensions_test.go index d73c4b7a74..be007dcb02 100644 --- a/internal/runtime/pi_extensions_test.go +++ b/internal/runtime/pi_extensions_test.go @@ -436,7 +436,9 @@ func TestPiRuntimeBootstrap_Extensions(t *testing.T) { assert.Contains(t, string(raw), `"extensions"`) require.NoError(t, PiRuntime{}.Bootstrap(bootstrapInput{sandboxName: "sb", agentPath: in.agentPath, agentName: "code"})) raw = storedUpload(t, store, cfg+"/fullsend-manifest.json") - assert.NotContains(t, string(raw), `"extensions"`) + var top map[string]any + require.NoError(t, json.Unmarshal(raw, &top)) + assert.NotContains(t, top, "extensions", "top-level key omitted (the agent block has its own extensions list)") // Name collisions with the runner's own extensions and between entries // fail before anything is uploaded. From 28cb68bc0703320a18a15756520a850b3d5292b3 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 13:04:54 -0400 Subject: [PATCH 4/7] feat(pi): wire the Agent extension into the pi run command Partial: the run-command guard and -e wiring; transcripts, metrics folding and docs follow in the next commits. Assisted-by: Claude Signed-off-by: Wayne Sun --- internal/runtime/pi_bootstrap_test.go | 73 ++++++ .../runtime/pi_extension/fullsend-agent.js | 70 +++++- .../pi_extension/fullsend-agent.test.mjs | 116 ++++++++- .../runtime/pi_extension/fullsend-hooks.js | 45 +++- .../pi_extension/fullsend-hooks.test.mjs | 48 ++++ internal/runtime/pi_extensions_test.go | 10 +- internal/runtime/pi_run.go | 105 +++++++- internal/runtime/pi_run_test.go | 226 ++++++++++++++++-- 8 files changed, 663 insertions(+), 30 deletions(-) diff --git a/internal/runtime/pi_bootstrap_test.go b/internal/runtime/pi_bootstrap_test.go index b5272f8cd9..fc615b6271 100644 --- a/internal/runtime/pi_bootstrap_test.go +++ b/internal/runtime/pi_bootstrap_test.go @@ -2,6 +2,8 @@ package runtime import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "os" "path/filepath" @@ -279,6 +281,47 @@ exit 0 }, ui.New(os.Stderr), time.Now(), &RunMetrics{}) assert.Equal(t, piHooksMissingExit, exit) require.ErrorContains(t, err, "hook adapter or manifest missing") + assert.NotContains(t, err.Error(), "Agent extension", + "the Agent guard has its own exit code; naming it here would send the operator looking at the wrong artifact") +} + +// TestPiRuntimeRun_TamperedAgentExtensionFailsClosed is the counterpart of +// the hook-adapter case: the Agent extension's guard exits with its own +// code, so Run names that extension instead of the hook adapter. Hooks are +// off here, which is exactly the run where the old shared code was +// ambiguous — nothing else in the command line exits 97. +func TestPiRuntimeRun_TamperedAgentExtensionFailsClosed(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + forgetPiManifestHash(t, "sb") + work := t.TempDir() + store := filepath.Join(work, "store") + fakeOpenshellPi(t, filepath.Join(work, "openshell.log"), store, "/dev/null") + // No tools: frontmatter means the default set, which carries the Agent + // tool — so Run emits the extension's -e and its guard. + require.NoError(t, PiRuntime{}.Bootstrap(bootstrapInput{ + sandboxName: "sb", agentPath: writeAgentFile(t, "---\nname: review\nmodel: opus\n---\nReview the PR."), agentName: "review", + })) + binDir := t.TempDir() + script := `#!/bin/sh +if [ "$2" = "exec" ]; then + for last; do :; done + case "$last" in + cat\ *) f=$(printf '%s' "${last#cat }" | tr -d "'" | tr '/' '_'); cat '` + store + `'/"$f"; exit $? ;; + *"exit 94"*) echo 'fullsend: pi Agent extension missing or modified' >&2; exit 94 ;; + esac +fi +exit 0 +` + require.NoError(t, os.WriteFile(filepath.Join(binDir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + exit, err := PiRuntime{}.Run(context.Background(), RunParams{ + SandboxName: "sb", RepoDir: "/r", Timeout: 30 * time.Second, + OnEvent: func(AgentEvent) {}, + }, ui.New(os.Stderr), time.Now(), &RunMetrics{}) + assert.Equal(t, piAgentTamperedExit, exit) + require.ErrorContains(t, err, "Agent extension missing or modified") + assert.NotContains(t, err.Error(), "hook adapter") } func TestPiRuntimeRun_SecurityOnButManifestWithoutHooksFailsFast(t *testing.T) { @@ -396,6 +439,7 @@ func TestPiAgentTool_ManifestBlock(t *testing.T) { bootstrap := func(t *testing.T, agentDef string, hooks bool) (*piManifest, string, string) { t.Helper() + forgetPiManifestHash(t, "sb") work := t.TempDir() store := filepath.Join(work, "store") fakeOpenshellPi(t, filepath.Join(work, "openshell.log"), store, "/dev/null") @@ -538,3 +582,32 @@ func TestPiDefaultTools_CoversToolMap(t *testing.T) { assert.Truef(t, defaults[piName], "pi tool %q (mapped from %s) is missing from piDefaultTools", piName, claudeName) } } + +// forgetPiManifestHash drops the digest Bootstrap recorded for a sandbox, +// before and after the test. piManifestHashes is a package-level map keyed +// by sandbox name and the tests reuse a handful of names, so without this a +// Bootstrap in one test decides whether a later test's buildPiRunCommand +// emits the manifest guard - and against which digest. +func forgetPiManifestHash(t *testing.T, sandboxName string) { + t.Helper() + piManifestHashes.Delete(sandboxName) + t.Cleanup(func() { piManifestHashes.Delete(sandboxName) }) +} + +// TestPiManifestHash covers the Bootstrap-to-Run seam: the digest is +// recorded per sandbox and read back by name, and a sandbox this process +// never bootstrapped yields "" so Run emits no guard rather than failing a +// caller that bootstrapped elsewhere. +func TestPiManifestHash(t *testing.T) { + forgetPiManifestHash(t, "sb-hash") + assert.Empty(t, piManifestHash("sb-hash"), "not bootstrapped in this process") + + body := []byte(`{"agentName":"triage"}`) + recordPiManifestHash("sb-hash", body) + sum := sha256.Sum256(body) + assert.Equal(t, hex.EncodeToString(sum[:]), piManifestHash("sb-hash")) + assert.Empty(t, piManifestHash("another-sandbox"), "the digest is per sandbox") + + forgetPiManifestHash(t, "sb-hash") + assert.Empty(t, piManifestHash("sb-hash"), "and the test helper clears it again") +} diff --git a/internal/runtime/pi_extension/fullsend-agent.js b/internal/runtime/pi_extension/fullsend-agent.js index 2ab9968cbd..9de4e73de9 100644 --- a/internal/runtime/pi_extension/fullsend-agent.js +++ b/internal/runtime/pi_extension/fullsend-agent.js @@ -37,6 +37,7 @@ // Everything it needs is the `agent` block of the manifest // PiRuntime.Bootstrap wrote (FULLSEND_PI_MANIFEST). import { spawn as nodeSpawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { appendFileSync, mkdirSync, readFileSync } from "node:fs"; import { dirname } from "node:path"; @@ -424,7 +425,7 @@ function signalChild(child, signal) { // `now` are injectable for tests. run() never throws for a failed child — // it returns { isError, error } — so the registered execute() decides how // to surface it (pi marks a result isError only when execute throws). -export function createAgentTool(manifest, { spawn = nodeSpawn, log = (m) => console.error(m), now = () => Date.now(), env = process.env, killGraceMs = DEFAULT_KILL_GRACE_MS } = {}) { +export function createAgentTool(manifest, { spawn = nodeSpawn, log = (m) => console.error(m), now = () => Date.now(), env = process.env, killGraceMs = DEFAULT_KILL_GRACE_MS, manifestPath = "", manifestSum = "" } = {}) { const agent = manifest?.agent ?? {}; const maxConcurrent = Math.max(1, Number(agent.maxConcurrent) || DEFAULT_MAX_CONCURRENT); const timeoutMs = Math.max(1, (Number(agent.timeoutSeconds) || DEFAULT_TIMEOUT_SECONDS) * 1000); @@ -480,6 +481,56 @@ export function createAgentTool(manifest, { spawn = nodeSpawn, log = (m) => cons waiter.evict(); }; + // extensionDigests are the digests Bootstrap recorded for the files it + // wrote into agent.extensions that live in the config dir — today only + // the hook adapter, and the same bytes the launch guard checks. They + // travel inside the manifest, so the manifest digest covers them: a + // rewrite that drops or edits them is itself a manifest drift, caught + // first below. The vendored provider extensions under + // /usr/local/share/pi-extensions carry none on purpose — they are + // root-owned and read-only in the image, outside anything the agent can + // write, so there is nothing there to re-check. + const extensionDigests = Object.entries(agent.extensionDigests ?? {}).filter( + ([file, want]) => typeof file === "string" && file !== "" && typeof want === "string" && want !== "", + ); + + // manifestDrift re-reads the manifest and the config-dir extensions it + // names, and reports whether they still hash to what this extension + // loaded. The launch guards in buildPiRunCommand check both once, before + // pi starts; an iteration then runs for minutes with those files sitting + // in a config dir the agent can write to, and between them they name the + // binary a child runs, the -e list it loads, its tool allowlist and where + // its usage is recorded. + // + // The hook adapter is the sharper half: nothing re-verifies it after the + // launch guard, a parent with `write` can replace it mid-iteration, and a + // rewritten adapter simply omits its own manifest-digest check — so every + // child dispatched afterwards would come up unhooked. Both are therefore + // re-checked immediately before every dispatch, inside the slot this + // dispatch holds. The manifest half is skipped when the caller supplied + // no digest — the unit tests build a tool from an object, not a file. + const manifestDrift = () => { + if (manifestPath && manifestSum) { + let sum; + try { + sum = createHash("sha256").update(readFileSync(manifestPath)).digest("hex"); + } catch (err) { + return `cannot re-read ${manifestPath} before dispatching: ${err.message}`; + } + if (sum !== manifestSum) return "manifest changed since load; refusing to dispatch"; + } + for (const [file, want] of extensionDigests) { + let sum; + try { + sum = createHash("sha256").update(readFileSync(file)).digest("hex"); + } catch (err) { + return `cannot re-read ${file} before dispatching: ${err.message}`; + } + if (sum !== want) return "hook adapter changed since load; refusing to dispatch"; + } + return ""; + }; + const recordUsage = (record) => { if (!agent.usageFile) return; try { @@ -629,7 +680,11 @@ export function createAgentTool(manifest, { spawn = nodeSpawn, log = (m) => cons return early; } try { - log(`${LOG_PREFIX} #${id} ${modelSpec} start "${description}"`); + const drift = manifestDrift(); + if (drift) { + return { seq: id, isError: true, error: drift, text: "", stopReason: "rejected", model: modelSpec }; + } + log(`${LOG_PREFIX} #${id} ${modelSpec} start "${capBytes(description, MAX_DESCRIPTION_BYTES)}"`); outcome = await runChild(id, params, modelSpec, tools, ticket); } finally { release(); @@ -697,9 +752,16 @@ export default function (pi) { console.error(`${LOG_PREFIX} ${DEPTH_ENV} is set: this is a sub-agent, the Agent tool is not registered (no recursion)`); return; } + const manifestPath = process.env.FULLSEND_PI_MANIFEST || DEFAULT_MANIFEST_PATH; let manifest; + let manifestSum; try { - manifest = loadManifest(); + // The bytes are read once and both hashed and parsed from that one + // read, so the digest kept for manifestDrift describes exactly the + // configuration in use. + const bytes = readFileSync(manifestPath); + manifestSum = createHash("sha256").update(bytes).digest("hex"); + manifest = JSON.parse(bytes.toString("utf8")); } catch (err) { console.error(`${LOG_PREFIX} cannot read manifest: ${err.message}; the Agent tool is not registered`); return; @@ -708,7 +770,7 @@ export default function (pi) { console.error(`${LOG_PREFIX} the Agent tool is not enabled for this agent; nothing registered`); return; } - const tool = createAgentTool(manifest); + const tool = createAgentTool(manifest, { manifestPath, manifestSum }); const execute = async (_toolCallId, params, signal, _onUpdate, ctx) => { const m = ctx?.model; diff --git a/internal/runtime/pi_extension/fullsend-agent.test.mjs b/internal/runtime/pi_extension/fullsend-agent.test.mjs index 1bb4002950..d5e7362948 100644 --- a/internal/runtime/pi_extension/fullsend-agent.test.mjs +++ b/internal/runtime/pi_extension/fullsend-agent.test.mjs @@ -5,8 +5,9 @@ // deterministic control over process lifetime, an injected spawn. import assert from "node:assert/strict"; import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { EventEmitter } from "node:events"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { PassThrough } from "node:stream"; @@ -430,6 +431,119 @@ test("run: the usage record's description is capped", async () => { assert.ok(Buffer.byteLength(JSON.stringify(rec) + "\n") < 4096); }); +test("run: a manifest rewritten after load stops the next dispatch", async () => { + const { dir, manifest } = fixture(); + const path = join(dir, "manifest.json"); + writeFileSync(path, JSON.stringify(manifest)); + const bytes = readFileSync(path); + const manifestSum = createHash("sha256").update(bytes).digest("hex"); + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(JSON.parse(bytes.toString("utf8")), { ...quiet, spawn, manifestPath: path, manifestSum }); + + const first = tool.run({ prompt: "p1" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 1, "the unmodified manifest dispatches"); + children[0].child.finish(okStream("one")); + assert.equal((await first).text, "one"); + + // The manifest names the binary children run, their -e list (the hook + // adapter among them) and their tool allowlist, and it sits in a dir the + // agent can write to: the launch guard checked it once, minutes ago. + writeFileSync(path, JSON.stringify({ ...manifest, agent: { ...manifest.agent, extensions: [], tools: ["bash"] } })); + const res = await tool.run({ prompt: "p2" }, {}); + assert.equal(res.isError, true); + assert.equal(res.error, "manifest changed since load; refusing to dispatch"); + assert.equal(res.stopReason, "rejected"); + assert.equal(children.length, 1, "nothing was spawned on the rewritten configuration"); + assert.equal(tool.inFlight(), 0); + + // Restoring the bytes restores dispatch: the check is on content, not on + // having seen a write. + writeFileSync(path, bytes); + const third = tool.run({ prompt: "p3" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 2, "and the refused dispatch did not leak its slot"); + children[1].child.finish(okStream("three")); + assert.equal((await third).text, "three"); +}); + +test("run: a manifest that vanishes after load stops the next dispatch", async () => { + const { dir, manifest } = fixture(); + const path = join(dir, "gone.json"); + writeFileSync(path, JSON.stringify(manifest)); + const manifestSum = createHash("sha256").update(readFileSync(path)).digest("hex"); + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool(manifest, { ...quiet, spawn, manifestPath: path, manifestSum }); + rmSync(path); + const res = await tool.run({ prompt: "p" }, {}); + assert.equal(res.isError, true); + assert.match(res.error, /cannot re-read .*gone\.json before dispatching/); + assert.equal(children.length, 0); +}); + +test("run: a hook adapter rewritten after load stops the next dispatch", async () => { + const { dir, manifest } = fixture(); + const adapter = join(dir, "fullsend-hooks.js"); + const bytes = "// the hook adapter Bootstrap wrote\n"; + writeFileSync(adapter, bytes); + const agent = { + ...manifest.agent, + // The vendored provider extension carries no digest: it is root-owned + // and read-only in the image. Only the config-dir file does. + extensions: ["/usr/local/share/pi-extensions/anthropic-vertex", adapter], + extensionDigests: { [adapter]: createHash("sha256").update(bytes).digest("hex") }, + }; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool({ ...manifest, agent }, { ...quiet, spawn }); + + const first = tool.run({ prompt: "p1" }, {}); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 1, "the adapter Bootstrap wrote dispatches"); + children[0].child.finish(okStream("one")); + assert.equal((await first).text, "one"); + + // The launch guard checked this file once, minutes ago. A parent with + // `write` can replace it mid-iteration, and a replacement simply omits + // the adapter's own manifest-digest check — so every child dispatched + // after it would come up with no hooks in it. + writeFileSync(adapter, "// no hooks here\n"); + const res = await tool.run({ prompt: "p2" }, {}); + assert.equal(res.isError, true); + assert.equal(res.error, "hook adapter changed since load; refusing to dispatch"); + assert.equal(res.stopReason, "rejected"); + assert.equal(children.length, 1, "nothing was spawned against the rewritten adapter"); + assert.equal(tool.inFlight(), 0); + + // Restoring the bytes restores dispatch: the check is on content, not on + // having seen a write. maxConcurrent dispatches at once then prove the + // refusal handed its slot back rather than keeping it. + writeFileSync(adapter, bytes); + const rest = []; + for (let i = 0; i < agent.maxConcurrent; i++) rest.push(tool.run({ prompt: `p${i}` }, {})); + await new Promise((r) => setImmediate(r)); + assert.equal(children.length, 1 + agent.maxConcurrent, "the refused dispatch leaked no slot"); + for (let i = 1; i <= agent.maxConcurrent; i++) children[i].child.finish(okStream(`done-${i}`)); + assert.deepEqual((await Promise.all(rest)).map((r) => r.text), ["done-1", "done-2", "done-3", "done-4"]); +}); + +test("run: a hook adapter that vanishes after load stops the next dispatch", async () => { + const { dir, manifest } = fixture(); + const adapter = join(dir, "gone-hooks.js"); + writeFileSync(adapter, "// adapter\n"); + const agent = { + ...manifest.agent, + extensions: [adapter], + extensionDigests: { [adapter]: createHash("sha256").update(readFileSync(adapter)).digest("hex") }, + }; + const { spawn, children } = fakeSpawn(); + const tool = createAgentTool({ ...manifest, agent }, { ...quiet, spawn }); + rmSync(adapter); + const res = await tool.run({ prompt: "p" }, {}); + assert.equal(res.isError, true); + assert.match(res.error, /cannot re-read .*gone-hooks\.js before dispatching/); + assert.equal(children.length, 0); +}); + test("run: model rejection is an error before anything is spawned", async () => { const { manifest } = fixture(); const spawned = []; diff --git a/internal/runtime/pi_extension/fullsend-hooks.js b/internal/runtime/pi_extension/fullsend-hooks.js index 6f7497c637..a2fcfe794d 100644 --- a/internal/runtime/pi_extension/fullsend-hooks.js +++ b/internal/runtime/pi_extension/fullsend-hooks.js @@ -19,9 +19,18 @@ // one's output; a script that cannot be spawned blocks (fail closed) — the // scripts own their individual fail-open cases (tirith). import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; export const DEFAULT_MANIFEST_PATH = "/sandbox/pi-config/fullsend-manifest.json"; +// MANIFEST_SHA256_ENV carries the digest of the manifest the runner wrote, +// exported after .env is sourced (so .env cannot set or clear it) and after +// the shell guard that matched it. That guard runs once, before the parent +// pi starts; this extension also loads inside every sub-agent, minutes +// later, and the manifest sits in the agent-writable config dir — so a +// parent with `write` could otherwise blank hooks.groups mid-iteration and +// dispatch children whose adapter loads a plan with no hooks in it. +export const MANIFEST_SHA256_ENV = "FULLSEND_PI_MANIFEST_SHA256"; const SCRIPT_TIMEOUT_MS = 60_000; const SCRIPT_MAX_BUFFER = 64 * 1024 * 1024; const LOG_PREFIX = "[fullsend-hooks]"; @@ -30,6 +39,19 @@ export function loadManifest(path = process.env.FULLSEND_PI_MANIFEST || DEFAULT_ return JSON.parse(readFileSync(path, "utf8")); } +// manifestDigestError checks the manifest bytes against MANIFEST_SHA256_ENV +// and returns the reason to refuse, or null when the bytes match — or when +// no digest was exported, which is the case for a caller that bootstrapped +// the sandbox in another process (the runner emits neither the guard nor +// the export then; see piManifestHash in pi_bootstrap.go). +export function manifestDigestError(bytes, expected) { + const want = typeof expected === "string" ? expected.trim().toLowerCase() : ""; + if (want === "") return null; + const got = createHash("sha256").update(bytes).digest("hex"); + if (got === want) return null; + return `sha256 ${got} is not the ${want} the runner recorded`; +} + // claudeToolName returns the name the hook scripts expect for a pi tool. // Tools outside the map (extension tools) keep their pi name, so "*" groups // still see them. @@ -290,13 +312,34 @@ export function createHooks(manifest, { spawn = spawnSync, log = (m) => console. } export default function (pi) { + const manifestPath = process.env.FULLSEND_PI_MANIFEST || DEFAULT_MANIFEST_PATH; let manifest = null; let loadError = null; + let bytes = null; try { - manifest = loadManifest(); + bytes = readFileSync(manifestPath); } catch (err) { loadError = err; } + if (bytes !== null) { + // Verified on the bytes just read, then parsed from those same bytes — + // re-reading would leave a window in which the checked file and the + // parsed one differ. A mismatch is fatal rather than fail-closed-at- + // tool-time: the hook plan is not the only thing this file configures, + // and in a sub-agent a non-zero exit is what makes the Agent tool + // report the dispatch as an error instead of returning a result the + // hooks never saw. + const bad = manifestDigestError(bytes, process.env[MANIFEST_SHA256_ENV]); + if (bad) { + console.error(`${LOG_PREFIX} ${manifestPath} ${bad}; refusing to run (did the agent rewrite it during this iteration?)`); + process.exit(1); + } + try { + manifest = JSON.parse(bytes.toString("utf8")); + } catch (err) { + loadError = err; + } + } const hooks = createHooks(manifest); pi.on("session_start", () => { diff --git a/internal/runtime/pi_extension/fullsend-hooks.test.mjs b/internal/runtime/pi_extension/fullsend-hooks.test.mjs index f541009ea3..5972a6ec32 100644 --- a/internal/runtime/pi_extension/fullsend-hooks.test.mjs +++ b/internal/runtime/pi_extension/fullsend-hooks.test.mjs @@ -4,16 +4,20 @@ // no python; one test exercises a real python3 script when available. import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; import { join } from "node:path"; import { test } from "node:test"; import defaultExport, { + MANIFEST_SHA256_ENV, bashAllowlistViolation, claudeToolInput, claudeToolName, createHooks, + manifestDigestError, runScript, } from "./fullsend-hooks.js"; @@ -373,3 +377,47 @@ test("session_start roster names the declared extensions", () => { delete process.env.FULLSEND_PI_MANIFEST; } }); + +test("manifestDigestError: matches, mismatches, and nothing to check", () => { + const bytes = Buffer.from(JSON.stringify(manifest)); + const sum = createHash("sha256").update(bytes).digest("hex"); + assert.equal(manifestDigestError(bytes, sum), null, "the bytes the runner hashed"); + assert.equal(manifestDigestError(bytes, sum.toUpperCase()), null, "hex case does not matter"); + assert.equal(manifestDigestError(bytes, ` ${sum} `), null, "surrounding whitespace does not matter"); + // No digest exported: a caller that bootstrapped the sandbox in another + // process gets neither the shell guard nor this check (pi_bootstrap.go + // piManifestHash), and failing closed here would break it. + assert.equal(manifestDigestError(bytes, undefined), null); + assert.equal(manifestDigestError(bytes, ""), null); + const other = Buffer.from(JSON.stringify({ ...manifest, hooks: { ...manifest.hooks, groups: [] } })); + const bad = manifestDigestError(other, sum); + assert.match(bad, new RegExp(`is not the ${sum} the runner recorded`), "a manifest whose hook plan was emptied is refused"); +}); + +test("the extension exits non-zero when the manifest no longer matches the runner's digest", () => { + const dir = mkdtempSync(join(tmpdir(), "fullsend-hooks-digest-")); + const manifestPath = join(dir, "manifest.json"); + writeFileSync(manifestPath, JSON.stringify(manifest)); + const sum = createHash("sha256").update(JSON.stringify(manifest)).digest("hex"); + const extension = fileURLToPath(new URL("./fullsend-hooks.js", import.meta.url)); + // A child of the Agent tool loads this file minutes into the iteration, + // so the manifest is re-checked there; the exit is what makes the tool + // report the dispatch as an error rather than returning a result no hook + // ever saw. Run out of process: the check calls process.exit. + const load = (env) => + spawnSync(process.execPath, ["-e", `import(${JSON.stringify(extension)}).then((m) => m.default({ on: () => {} }))`], { + encoding: "utf8", + env: { ...process.env, FULLSEND_PI_MANIFEST: manifestPath, ...env }, + }); + + const clean = load({ [MANIFEST_SHA256_ENV]: sum }); + assert.equal(clean.status, 0, clean.stderr); + + writeFileSync(manifestPath, JSON.stringify({ ...manifest, hooks: { ...manifest.hooks, groups: [] } })); + const tampered = load({ [MANIFEST_SHA256_ENV]: sum }); + assert.notEqual(tampered.status, 0, "a rewritten manifest must not produce a hookless process"); + assert.match(tampered.stderr, /refusing to run/); + + const unchecked = load({}); + assert.equal(unchecked.status, 0, "without an exported digest there is nothing to check"); +}); diff --git a/internal/runtime/pi_extensions_test.go b/internal/runtime/pi_extensions_test.go index be007dcb02..79e92e65c9 100644 --- a/internal/runtime/pi_extensions_test.go +++ b/internal/runtime/pi_extensions_test.go @@ -339,7 +339,7 @@ func TestBuildPiRunCommand_Extensions(t *testing.T) { {Name: "pi-fff", Path: "/sandbox/pi-config/extensions/pi-fff", SHA256: strings.Repeat("b", 64), Args: []string{"--fff-mode", "over'ride"}, Env: map[string]string{"FFF_MULTIGREP": "1", "FFF_ROOT": "/sandbox/work space"}}, } - cmd := buildPiRunCommand(params, m, exts) + cmd := buildPiRunCommand(params, m, exts, "") // Preflight: after the pi pin and the hook guard, before .env is sourced. guard := piExtensionsGuard(exts) @@ -371,17 +371,17 @@ func TestBuildPiRunCommand_Extensions(t *testing.T) { // Without extensions nothing is added, and a declared tools: list is // still rendered as before (extension tools are then hidden by pi). m.Tools = []string{"bash", "read"} - plain := buildPiRunCommand(params, m, nil) + plain := buildPiRunCommand(params, m, nil, "") assert.NotContains(t, plain, "pi-config/extensions/") assert.NotContains(t, plain, "exit 96") assert.Contains(t, plain, "--tools 'bash,read'") - withTools := buildPiRunCommand(params, m, exts) + withTools := buildPiRunCommand(params, m, exts, "") assert.Contains(t, withTools, "--tools 'bash,read'") // Hooks disabled: extension guard still runs (it is independent of the // hook adapter) and the adapter is not loaded. params.HooksSettingsPath = "" - noHooks := buildPiRunCommand(params, m, exts) + noHooks := buildPiRunCommand(params, m, exts, "") assert.Contains(t, noHooks, guard) assert.NotContains(t, noHooks, "fullsend-hooks.js") assert.Contains(t, noHooks, `-e '/usr/local/share/pi-extensions/anthropic-vertex' -e '/sandbox/pi-config/extensions/go-diagnostics'`) @@ -569,7 +569,7 @@ func TestPiRuntimeEnvExports_DisablesJitiCache(t *testing.T) { // The export has to survive `. .env`: buildPiRunCommand re-emits // EnvExports() after sourcing it, so the agent cannot turn the cache // back on for the next iteration. - cmd := buildPiRunCommand(RunParams{RepoDir: "/sandbox/workspace/repo"}, &piManifest{}, nil) + cmd := buildPiRunCommand(RunParams{RepoDir: "/sandbox/workspace/repo"}, &piManifest{}, nil, "") env := strings.Index(cmd, ". '/sandbox/workspace/.env'") jiti := strings.Index(cmd, "export JITI_FS_CACHE=false") require.GreaterOrEqual(t, env, 0) diff --git a/internal/runtime/pi_run.go b/internal/runtime/pi_run.go index cfa372356c..e9576683fa 100644 --- a/internal/runtime/pi_run.go +++ b/internal/runtime/pi_run.go @@ -165,6 +165,18 @@ func piThinkingFor(effort string) (string, bool) { // extension would give a hookless iteration that looks healthy. const piHooksMissingExit = 97 +// piAgentTamperedExit is the exit code of the Agent extension's integrity +// guard. Distinct from piHooksMissingExit so Run can name the artifact that +// actually failed: both guards can be in the command line at once, and one +// code for two of them made the message a list of three things to go and +// check ("hook adapter, Agent extension or manifest"). +const piAgentTamperedExit = 94 + +// piManifestTamperedExit is the exit code of the manifest integrity guard: +// the manifest is not byte-identical to the one Bootstrap wrote. Distinct +// from piHooksMissingExit so Run can name the actual cause. +const piManifestTamperedExit = 95 + // piConfigTamperedExit is the exit code of the config-dir integrity guard // for the openai provider (auth.json or models.json present). Distinct from // piHooksMissingExit so Run can name the actual cause instead of reporting a @@ -182,11 +194,23 @@ const piConfigTamperedExit = 98 // the adapter or manifest file is missing. exts are the declared harness // extensions resolved from the host by Run (piResolveRunExtensions): their // preflight hash, -e entries and env exports come from there, not from m. -func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtension) string { +// manifestSum is the digest Bootstrap recorded for the manifest; when it is +// non-empty the command refuses to start pi on a manifest that no longer +// matches it. +func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtension, manifestSum string) string { r := PiRuntime{} envFile := sandbox.SandboxWorkspace + "/.env" hooksEnabled := params.HooksSettingsPath != "" hooksExt := r.ConfigDir() + "/" + piHooksExtensionFile + // The Agent tool is decided from the manifest Bootstrap wrote for this + // agent definition. Both the extension's code and the manifest it reads + // are hash-checked below: the manifest names the binary children run, + // the -e list they load, their tool allowlists and where their usage is + // recorded, and the config dir is agent-writable between iterations — + // so an unchecked manifest would let an agent with Write but no Bash + // (which cannot start pi itself) launch children of its own choosing. + agentEnabled := m.Agent != nil && m.Agent.Enabled + agentExt := r.ConfigDir() + "/" + piAgentExtensionFile model := params.Model if model == "" { @@ -213,6 +237,17 @@ func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtensi // shadow the guard's tools with functions or a PATH entry. parts = append(parts, "&& "+piHooksGuard(hooksExt, r.piManifestPath())) } + if agentEnabled { + // Same block: the Agent extension must be byte-identical to the + // embedded copy before .env can shadow the tools that check it. + parts = append(parts, "&& "+piAgentGuard(agentExt)) + } + if manifestSum != "" { + // Same block, same reason. The hooks guard above only checks that + // the manifest exists; this checks that it is the one Bootstrap + // wrote, and it runs whether or not hooks are enabled. + parts = append(parts, "&& "+piManifestGuard(r.piManifestPath(), manifestSum)) + } if guard := piExtensionsGuard(exts); guard != "" { // Same block, same reason: the extension trees are checked against // the host hashes before .env can shadow find/sort/sha256sum, and @@ -303,6 +338,28 @@ func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtensi // credential leak, not tool misuse. parts = append(parts, "&& unset -f test command grep tr sed printf pi", "&& "+piOpenAIConfigGuard(r.ConfigDir())) } + if manifestSum != "" { + // Second pass, the way piOpenAIConfigGuard has one: .env is + // agent-writable and could have rewritten the manifest just now, + // after the first check. `unset -f` is a special builtin, so a + // function sourced from .env cannot shadow it or survive it. + // + // `[` is in the unset list because the guard uses it. Under bash 4.x + // `unset -f [` is a syntax error, but the sandbox's `sh` is dash + // (which accepts it) and this line is only ever run through `sh -c`. + parts = append(parts, + "&& unset -f test [ command sha256sum cut", + "&& "+piManifestGuard(r.piManifestPath(), manifestSum), + // Exported after .env so .env cannot set it (or clear it) and + // after the guard above so it can only ever carry the digest the + // guard just matched. The hook adapter checks the manifest + // against it when it loads — which for a sub-agent is minutes + // into the iteration, long after this guard ran: without it a + // parent with `write` could empty hooks.groups mid-iteration and + // dispatch children whose adapter loads a hookless plan. + "&& export "+piManifestSumEnv+"="+shellQuote(manifestSum), + ) + } // Declared extensions' env goes last, which protects nothing on its // own: it is exported after the runtime's pins and the provider // hygiene, and pi hands its whole environment to every hook script it @@ -336,6 +393,12 @@ func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtensi if hooksEnabled { parts = append(parts, "-e "+shellQuote(hooksExt)) } + if agentEnabled { + // After the adapter, so its PreToolUse hooks see Agent calls + // (Claude Code runs the same hooks on its Agent tool). Children + // get their own -e list from the manifest, never this file. + parts = append(parts, "-e "+shellQuote(agentExt)) + } // Declared extensions come after the hook adapter: pi runs tool_call // handlers in -e order and the first block wins, so the adapter's // PreToolUse hooks see every call before any declared extension does. @@ -380,6 +443,14 @@ func buildPiRunCommand(params RunParams, m *piManifest, exts []piManifestExtensi // piManifestEnv tells the hook extension where the manifest is. const piManifestEnv = "FULLSEND_PI_MANIFEST" +// piManifestSumEnv carries the digest of the manifest Bootstrap wrote to +// every process that loads fullsend-hooks.js — the parent and, through the +// environment the Agent extension hands its children, each sub-agent. The +// shell guard only fires once, before pi starts; a child reads the manifest +// at its own start, so this is what keeps a mid-iteration rewrite from +// producing a hookless sub-agent. +const piManifestSumEnv = "FULLSEND_PI_MANIFEST_SHA256" + // piBinaryVar holds the absolute path of the pi binary, resolved before // .env is sourced and marked read-only. const piBinaryVar = "FULLSEND_PI_BIN" @@ -520,6 +591,30 @@ func piHooksGuard(hooksExt, manifestPath string) string { shellQuote(hooksExt), shellQuote(manifestPath), shellQuote(hooksExt), shellQuote(hex.EncodeToString(sum[:])), piHooksMissingExit) } +// piAgentGuard is the Agent extension's counterpart of piHooksGuard: the +// file must exist and match the embedded copy, else piAgentTamperedExit — +// its own code, so Run names this extension instead of listing every +// runner-owned artifact the iteration carries. +func piAgentGuard(agentExt string) string { + sum := sha256.Sum256(piAgentExtensionJS) + return fmt.Sprintf(`{ test -f %s && [ "$(command -p sha256sum %s | command -p cut -d' ' -f1)" = %s ] || { echo 'fullsend: pi Agent extension missing or modified; refusing to run' >&2; exit %d; }; }`, + shellQuote(agentExt), shellQuote(agentExt), shellQuote(hex.EncodeToString(sum[:])), piAgentTamperedExit) +} + +// piManifestGuard is the POSIX sh fragment that refuses to start pi when +// fullsend-manifest.json is not byte-identical to the one Bootstrap wrote. +// The manifest is the extensions' whole configuration — the hook plan the +// adapter enforces and, when the Agent tool is on, the binary, -e list and +// tool allowlists of every child — and the config dir is writable by the +// agent between iterations, so it gets the same treatment as the extension +// code itself. +func piManifestGuard(manifestPath, sum string) string { + // `command -p` bypasses shell functions and uses the system default + // PATH; test, [ and echo are builtins. + return fmt.Sprintf(`{ test -f %s && [ "$(command -p sha256sum %s | command -p cut -d' ' -f1)" = %s ] || { echo 'fullsend: pi manifest missing or modified since Bootstrap wrote it; refusing to run' >&2; exit %d; }; }`, + shellQuote(manifestPath), shellQuote(manifestPath), shellQuote(sum), piManifestTamperedExit) +} + // Run executes one agent iteration and normalizes pi's --mode json stream // into AgentEvents. pi exits 0 on model error in json mode, so the stream's // verdict overrides the exit code (#2786/#5361). @@ -549,7 +644,7 @@ func (r PiRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printe if err != nil { return -1, err } - cmd := buildPiRunCommand(params, m, exts) + cmd := buildPiRunCommand(params, m, exts, piManifestHash(params.SandboxName)) stdout, execCmd, cancel, err := sandbox.ExecStreamReader(ctx, params.SandboxName, cmd, params.Timeout, os.Stderr) if err != nil { @@ -626,6 +721,12 @@ func (r PiRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printe if exitCode == piHooksMissingExit && params.HooksSettingsPath != "" { return exitCode, fmt.Errorf("pi hook adapter or manifest missing or modified in %s; refusing to run unhooked (was Bootstrap run, or did the agent change it?)", r.ConfigDir()) } + if exitCode == piAgentTamperedExit && m.Agent != nil && m.Agent.Enabled { + return exitCode, fmt.Errorf("pi Agent extension missing or modified in %s; refusing to run (was Bootstrap run, or did the agent change it?)", r.ConfigDir()) + } + if exitCode == piManifestTamperedExit { + return exitCode, fmt.Errorf("the pi manifest at %s is not the one Bootstrap wrote; refusing to run because it configures the hook plan and the sub-agent children (did the agent or a rewritten .env change it?)", r.piManifestPath()) + } if exitCode == piConfigTamperedExit { return exitCode, fmt.Errorf("pi config dir %s has models.json or an openai entry in auth.json; refusing to run the openai provider because either can redirect or replace the runner's credential (pi's own empty auth.json is fine; did the agent write there between iterations?)", r.ConfigDir()) } diff --git a/internal/runtime/pi_run_test.go b/internal/runtime/pi_run_test.go index ce88d6a443..d3853e87d8 100644 --- a/internal/runtime/pi_run_test.go +++ b/internal/runtime/pi_run_test.go @@ -95,7 +95,7 @@ func TestBuildPiRunCommand_Basic(t *testing.T) { m := &piManifest{AgentName: "triage", Model: "opus", Tools: []string{"bash"}, BashAllowlist: []string{"gh"}, Hooks: &piHooksManifest{}} params := piTestParams() params.HooksSettingsPath = "/sandbox/claude-config/hooks.json" - cmd := buildPiRunCommand(params, m, nil) + cmd := buildPiRunCommand(params, m, nil, "") // The guard runs before the agent-writable .env is sourced; the // runner-owned locations are re-pinned right after it. @@ -130,11 +130,67 @@ func TestBuildPiRunCommand_Basic(t *testing.T) { // pi resolves the provider prefix case-insensitively; so must the gate. params.Model = "Anthropic-Vertex/claude-opus-4-6" - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "&& unset ANTHROPIC_API_KEY") assert.Contains(t, cmd, "--model 'Anthropic-Vertex/claude-opus-4-6'") } +// TestBuildPiRunCommand_AgentTool is the golden for the Agent tool wiring: +// the extension's integrity guard runs before .env whether or not hooks are +// on, its -e comes right after the hook adapter and before declared +// extensions, --tools carries Agent,Task, and the parent's own provider +// -e set is unchanged (the manifest's child extension list is for children). +func TestBuildPiRunCommand_AgentTool(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + t.Setenv(piProviderEnv, "") + agentExt := "/sandbox/pi-config/fullsend-agent.js" + hooksExt := "/sandbox/pi-config/fullsend-hooks.js" + agent := &piAgentManifest{Enabled: true, Extensions: []string{piVertexExtensionPath, piXaiVertexExtensionPath, hooksExt}} + declared := []piManifestExtension{{Name: "go-diagnostics", Path: "/sandbox/pi-config/extensions/go-diagnostics", SHA256: strings.Repeat("a", 64)}} + + // Hooks on, default tool set. + m := &piManifest{AgentName: "review", Hooks: &piHooksManifest{}, Agent: agent} + params := piTestParams() + params.HooksSettingsPath = "/sandbox/claude-config/hooks.json" + cmd := buildPiRunCommand(params, m, declared, "") + assert.Contains(t, cmd, "-e '"+hooksExt+"' -e '"+agentExt+"' -e '/sandbox/pi-config/extensions/go-diagnostics'", + "load order: hook adapter, Agent extension, declared extensions") + assert.Contains(t, cmd, "-e '"+piVertexExtensionPath+"' -e '"+hooksExt+"'", "the provider extension still comes first") + assert.NotContains(t, cmd, "-e '"+piXaiVertexExtensionPath+"'", "the parent loads only its own provider; the child list in the manifest is not the parent's") + assert.NotContains(t, cmd, "--tools", "no tools: frontmatter → pi's default set plus the extension's tools") + guard := piAgentGuard(agentExt) + assert.Contains(t, cmd, "&& "+piHooksGuard(hooksExt, "/sandbox/pi-config/fullsend-manifest.json")+" && "+guard+" && "+piExtensionsGuard(declared), + "the runner-owned guards run in order, ahead of the declared-extension guard") + assert.Contains(t, cmd, piExtensionsGuard(declared)+" && . '/sandbox/workspace/.env'", + "every guard runs before the agent-writable .env is sourced") + assert.Contains(t, guard, fmt.Sprintf("exit %d", piAgentTamperedExit), "its own code, so Run names this extension and not the hook adapter") + assert.NotContains(t, guard, fmt.Sprintf("exit %d", piHooksMissingExit)) + assert.Contains(t, guard, "command -p sha256sum") + sum := sha256.Sum256(piAgentExtensionJS) + assert.Contains(t, guard, hex.EncodeToString(sum[:]), "the guard pins the embedded copy's hash") + + // Hooks off: the Agent guard and -e still apply, on their own. + params.HooksSettingsPath = "" + cmd = buildPiRunCommand(params, m, nil, "") + assert.NotContains(t, cmd, hooksExt) + assert.Contains(t, cmd, "&& "+guard+" && . '/sandbox/workspace/.env'") + assert.Contains(t, cmd, "-e '"+piVertexExtensionPath+"' -e '"+agentExt+"' --model") + + // A declared tools: list naming Agent carries both names into --tools. + m.Tools = []string{"bash", "read", "Agent", "Task"} + cmd = buildPiRunCommand(params, m, nil, "") + assert.Contains(t, cmd, "--tools 'bash,read,Agent,Task'") + + // Tool off: nothing of it in the command line. + off := &piManifest{AgentName: "triage", Tools: []string{"bash"}, Hooks: &piHooksManifest{}} + params.HooksSettingsPath = "/sandbox/claude-config/hooks.json" + cmd = buildPiRunCommand(params, off, nil, "") + assert.NotContains(t, cmd, "fullsend-agent") + assert.Contains(t, cmd, "&& "+piHooksGuard(hooksExt, "/sandbox/pi-config/fullsend-manifest.json")+" && . '/sandbox/workspace/.env'") + disabled := &piManifest{AgentName: "triage", Agent: &piAgentManifest{Enabled: false}} + assert.NotContains(t, buildPiRunCommand(params, disabled, nil, ""), "fullsend-agent") +} + // TestPiHooksGuard runs the rendered guard under a real sh: it must exit 97 // without running what follows when the adapter is missing or modified, and // fall through when both files are intact. @@ -199,6 +255,142 @@ func TestPiHooksGuard(t *testing.T) { assert.NotContains(t, string(out2), "RAN") } +// TestPiAgentGuard runs the rendered Agent-extension guard under a real sh: +// like the hook adapter it must exit before pi starts when the file is +// missing or is not the embedded copy, and the tools it uses must not be +// shadowable from the agent-writable .env. +func TestPiAgentGuard(t *testing.T) { + t.Parallel() + if err := exec.Command("sh", "-c", "command -p sha256sum /dev/null >/dev/null && command -p cut -d' ' -f1 /dev/null").Run(); err != nil { + t.Skip("sha256sum/cut not on the default PATH (stock macOS); the sandbox image has coreutils") + } + dir := t.TempDir() + ext := filepath.Join(dir, "fullsend-agent.js") + + run := func(prefix string) (int, string) { + out, err := exec.Command("sh", "-c", prefix+piAgentGuard(ext)+" && echo RAN").CombinedOutput() + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else { + require.NoError(t, err, string(out)) + } + return code, string(out) + } + + code, out := run("") + assert.Equal(t, piAgentTamperedExit, code, "missing extension") + assert.NotContains(t, out, "RAN") + assert.Contains(t, out, "Agent extension missing or modified") + + require.NoError(t, os.WriteFile(ext, append(append([]byte{}, piAgentExtensionJS...), []byte("\n// tampered\n")...), 0o644)) + code, out = run("") + assert.Equal(t, piAgentTamperedExit, code, "modified extension") + assert.NotContains(t, out, "RAN") + + require.NoError(t, os.WriteFile(ext, piAgentExtensionJS, 0o644)) + code, out = run("") + assert.Equal(t, 0, code) + assert.Contains(t, out, "RAN") + + // A function or PATH entry standing in for sha256sum must not make a + // tampered copy pass: the guard uses `command -p`. + require.NoError(t, os.WriteFile(ext, []byte("// tampered\n"), 0o644)) + sum := sha256.Sum256(piAgentExtensionJS) + hexSum := hex.EncodeToString(sum[:]) + shadowDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(shadowDir, "sha256sum"), []byte("#!/bin/sh\necho '"+hexSum+" x'\n"), 0o755)) + code, out = run("sha256sum() { echo '" + hexSum + " x'; }; cut() { echo '" + hexSum + "'; }; PATH=" + shellQuote(shadowDir) + ":$PATH; ") + assert.Equal(t, piAgentTamperedExit, code, "shadowed sha256sum") + assert.NotContains(t, out, "RAN") + + // Each runner-owned artifact has its own code, so Run names the one that + // failed instead of listing every guard the command line carries. + assert.NotEqual(t, piHooksMissingExit, piAgentTamperedExit) + for _, other := range []int{piManifestTamperedExit, piExtensionTamperedExit, piConfigTamperedExit} { + assert.NotEqual(t, other, piAgentTamperedExit, "the Agent guard's exit code must not collide with another guard's") + } +} + +// TestPiManifestGuard covers the manifest integrity check: the manifest +// names the pi binary children run, the -e list they load and their tool +// allowlists, and the config dir is writable by the agent between +// iterations — so Run refuses to start on a manifest that is not the one +// Bootstrap wrote. +func TestPiManifestGuard(t *testing.T) { + t.Parallel() + if err := exec.Command("sh", "-c", "command -p sha256sum /dev/null >/dev/null && command -p cut -d' ' -f1 /dev/null").Run(); err != nil { + t.Skip("sha256sum/cut not on the default PATH (stock macOS); the sandbox image has coreutils") + } + dir := t.TempDir() + manifest := filepath.Join(dir, "fullsend-manifest.json") + body := []byte(`{"agentName":"triage"}`) + sum := sha256.Sum256(body) + guard := piManifestGuard(manifest, hex.EncodeToString(sum[:])) + + run := func() (int, string) { + out, err := exec.Command("sh", "-c", guard+" && echo RAN").CombinedOutput() + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else { + require.NoError(t, err, string(out)) + } + return code, string(out) + } + + code, out := run() + assert.Equal(t, piManifestTamperedExit, code, "missing manifest") + assert.NotContains(t, out, "RAN") + assert.Contains(t, out, "manifest missing or modified") + + require.NoError(t, os.WriteFile(manifest, body, 0o644)) + code, out = run() + assert.Equal(t, 0, code) + assert.Contains(t, out, "RAN") + + require.NoError(t, os.WriteFile(manifest, []byte(`{"agentName":"triage","agent":{"enabled":true,"piBin":"/tmp/evil"}}`), 0o644)) + code, out = run() + assert.Equal(t, piManifestTamperedExit, code, "a rewritten manifest is refused") + assert.NotContains(t, out, "RAN") +} + +// TestBuildPiRunCommand_ManifestGuard checks where the guard is emitted: +// before .env can shadow the tools it uses, and again after .env, since +// .env could have rewritten the manifest between the two. +func TestBuildPiRunCommand_ManifestGuard(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + m := &piManifest{AgentName: "triage", Model: "opus"} + params := piTestParams() + + assert.NotContains(t, buildPiRunCommand(params, m, nil, ""), "manifest missing or modified", + "no recorded digest (Bootstrap did not run in this process) means no guard, not a failed run") + + cmd := buildPiRunCommand(params, m, nil, "abc123") + assert.Equal(t, 2, strings.Count(cmd, "manifest missing or modified"), "checked before and after .env") + first := strings.Index(cmd, "manifest missing or modified") + envSource := strings.Index(cmd, ". '"+sandbox.SandboxWorkspace+"/.env'") + assert.Less(t, first, envSource, "the first check runs before the agent-writable .env is sourced") + assert.Contains(t, cmd, "unset -f test [ command sha256sum cut", + "the second check restores the real tools first; unset is a special builtin") + assert.Contains(t, cmd, "= 'abc123' ]") + + // The digest is handed to the extensions so a process that loads the + // manifest later in the iteration — a sub-agent's hook adapter — can + // re-check it. Exported after .env, so .env cannot set or clear it, and + // after the second guard, so it can only carry a digest that matched. + export := strings.Index(cmd, "export "+piManifestSumEnv+"='abc123'") + require.Positive(t, export, "the digest is exported for the hook adapter to re-check") + assert.Greater(t, export, envSource, "after .env, which cannot then set it") + assert.Greater(t, export, strings.LastIndex(cmd, "manifest missing or modified"), + "after the post-.env guard, so it only ever carries a digest that just matched") + + assert.NotContains(t, buildPiRunCommand(params, m, nil, ""), piManifestSumEnv, + "no recorded digest means nothing to re-check against either") +} + func TestPiBareModelID(t *testing.T) { t.Parallel() assert.Equal(t, "claude-opus-4-6", piBareModelID("anthropic-vertex/claude-opus-4-6"), "two-segment: strips provider") @@ -215,7 +407,7 @@ func TestBuildPiRunCommand_XaiVertex(t *testing.T) { // Short form: xai/grok-4.6 is normalized to xai-vertex/xai/grok-4.6. params.Model = "xai/grok-4.6" - cmd := buildPiRunCommand(params, m, nil) + cmd := buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "--model 'xai-vertex/xai/grok-4.6'", "normalized model spec") assert.Contains(t, cmd, "-e '"+sandbox.SandboxPiExtensionsDir+"/xai-vertex'", "xai-vertex extension is loaded") @@ -227,7 +419,7 @@ func TestBuildPiRunCommand_XaiVertex(t *testing.T) { // Long form: xai-vertex/xai/grok-4.6 passes through. params.Model = "xai-vertex/xai/grok-4.6" - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "--model 'xai-vertex/xai/grok-4.6'") assert.Contains(t, cmd, "-e '"+sandbox.SandboxPiExtensionsDir+"/xai-vertex'") assert.Contains(t, cmd, "&& unset XAI_API_KEY") @@ -237,7 +429,7 @@ func TestBuildPiRunCommand_XaiVertex(t *testing.T) { // silently sending traffic to xAI's native API instead of Vertex. for _, spec := range []string{"Xai-Vertex/xai/grok-4.6", "XAI/grok-4.6", "Xai/grok-4.6"} { params.Model = spec - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "--model 'xai-vertex/xai/grok-4.6'", "canonical spec for %s", spec) assert.Contains(t, cmd, "&& unset XAI_API_KEY", "XAI_API_KEY unset for %s", spec) assert.Contains(t, cmd, "-e '"+sandbox.SandboxPiExtensionsDir+"/xai-vertex'", "extension loaded for %s", spec) @@ -246,7 +438,7 @@ func TestBuildPiRunCommand_XaiVertex(t *testing.T) { // unset must run after the agent-writable .env is sourced, or the .env // could re-export XAI_API_KEY after we cleared it. params.Model = "xai/grok-4.6" - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Less(t, strings.Index(cmd, ". '"+sandbox.SandboxWorkspace+"/.env'"), strings.Index(cmd, "&& unset XAI_API_KEY"), "XAI_API_KEY is unset after .env is sourced") } @@ -273,7 +465,7 @@ func TestBuildPiRunCommand_OpenAI(t *testing.T) { // openai/gpt-5.6-luna passes through as a two-segment spec. params.Model = "openai/gpt-5.6-luna" - cmd := buildPiRunCommand(params, m, nil) + cmd := buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "--model 'openai/gpt-5.6-luna'", "model spec") assert.NotContains(t, cmd, "--api-key", "no --api-key: it would outrank the auth.json pi re-reads per request and pin the iteration to one placeholder") @@ -294,7 +486,7 @@ func TestBuildPiRunCommand_OpenAI(t *testing.T) { // Case-insensitive gate: pi resolves providers case-insensitively. for _, spec := range []string{"OpenAI/gpt-5.6-luna", "OPENAI/gpt-5.6-luna", "Openai/gpt-5.6-sol"} { params.Model = spec - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "&& "+PiOpenAIAuthSeed(PiRuntime{}.ConfigDir()), "seed for %s", spec) assert.Contains(t, cmd, "&& unset OPENAI_BASE_URL AZURE_OPENAI_API_KEY", "unset for %s", spec) } @@ -303,7 +495,7 @@ func TestBuildPiRunCommand_OpenAI(t *testing.T) { // config-dir guard runs before it (nothing can shadow `test` yet) and // again after it, behind `unset -f test`, in case .env wrote a file. params.Model = "openai/gpt-5.6-luna" - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") envIdx := strings.Index(cmd, ". '"+sandbox.SandboxWorkspace+"/.env'") unsetIdx := strings.Index(cmd, "&& unset OPENAI_BASE_URL") assert.Less(t, envIdx, unsetIdx, "unset after .env sourced") @@ -511,7 +703,7 @@ func TestPiOpenAIConfigGuard(t *testing.T) { func TestBuildPiRunCommand_DirectProviderKeepsAnthropicEnv(t *testing.T) { t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "anthropic") - cmd := buildPiRunCommand(piTestParams(), &piManifest{}, nil) + cmd := buildPiRunCommand(piTestParams(), &piManifest{}, nil, "") assert.Contains(t, cmd, "--model 'anthropic/claude-opus-4-6'") assert.NotContains(t, cmd, "unset ANTHROPIC_API_KEY", "direct Anthropic provider needs its key") assert.NotContains(t, cmd, "GOOGLE_CLOUD_PROJECT") @@ -527,7 +719,7 @@ func TestBuildPiRunCommand_HarnessOverridesAndFlags(t *testing.T) { params.Debug = "*" // A manifest claiming hooks must not matter: the runner's signal decides. m := &piManifest{AgentName: "code", Model: "opus", Tools: nil, Hooks: &piHooksManifest{}} - cmd := buildPiRunCommand(params, m, nil) + cmd := buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, "--model 'anthropic-vertex/claude-sonnet-4-6'", "harness model wins over the agent definition") assert.Contains(t, cmd, "--thinking 'high'") @@ -542,7 +734,7 @@ func TestBuildPiRunCommand_HarnessOverridesAndFlags(t *testing.T) { func TestBuildPiRunCommand_EmptyToolRestriction(t *testing.T) { t.Setenv("FULLSEND_PI_MODEL", "") m := &piManifest{Tools: []string{}} - cmd := buildPiRunCommand(piTestParams(), m, nil) + cmd := buildPiRunCommand(piTestParams(), m, nil, "") assert.Contains(t, cmd, "--no-builtin-tools") assert.NotContains(t, cmd, "--tools ") } @@ -552,7 +744,7 @@ func TestBuildPiRunCommand_QuotesRepoDirAndModel(t *testing.T) { params := piTestParams() params.RepoDir = "/sandbox/workspace/it's" params.Model = "anthropic/claude'x" - cmd := buildPiRunCommand(params, &piManifest{}, nil) + cmd := buildPiRunCommand(params, &piManifest{}, nil, "") assert.Contains(t, cmd, `cd '/sandbox/workspace/it'\''s'`) assert.Contains(t, cmd, `--model 'anthropic/claude'\''x'`) } @@ -575,7 +767,7 @@ func TestPiThinkingFor_DefaultAndUnknown(t *testing.T) { params := piTestParams() params.Effort = "bogus" - cmd := buildPiRunCommand(params, &piManifest{AgentName: "triage", Model: "opus"}, nil) + cmd := buildPiRunCommand(params, &piManifest{AgentName: "triage", Model: "opus"}, nil, "") assert.Contains(t, cmd, "--thinking 'high'", "unknown effort falls back to the default, not to pi's medium") } @@ -584,13 +776,13 @@ func TestBuildPiRunCommand_HonoursPromptOverride(t *testing.T) { m := &piManifest{AgentName: "triage", Model: "opus"} params := piTestParams() - cmd := buildPiRunCommand(params, m, nil) + cmd := buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, shellQuote(DefaultAgentPrompt), "empty prompt falls back to the default") // The validation loop injects the previous iteration's failure here; a // runtime that ignores it turns feedback_mode into a blind retry (#1050). params.Prompt = "Previous iteration failed: tests did not pass.\nFix it; don't repeat it." - cmd = buildPiRunCommand(params, m, nil) + cmd = buildPiRunCommand(params, m, nil, "") assert.Contains(t, cmd, shellQuote(params.Prompt)) assert.NotContains(t, cmd, shellQuote(DefaultAgentPrompt)) assert.True(t, strings.HasSuffix(cmd, " Date: Sat, 29 Aug 2026 13:59:37 -0400 Subject: [PATCH 5/7] feat(pi): extract sub-agent transcripts and fold their cost into metrics A child dispatched through the Agent tool is a separate pi process, so none of its tokens reach the parent's --mode json stream and its session file lands in its own `sessions/agent-/` directory. Without this the run's artifacts and metrics.json show the orchestrator's cost only, and a review whose spend is dominated by its roster looks nearly free. ExtractTranscripts now names child sessions `-sub-` (the sequence number keeps children with equal session basenames apart) and downloads the extension's usage file beside them. Run reads that file after the iteration and folds it into RunMetrics: the totals grow by what the children spent, and per_model_usage attributes them, with the parent's own iteration as one entry so the breakdown sums to the totals. Runs that dispatch nothing keep metrics.json byte-identical. ClearIterationArtifacts removes the usage file, which sits outside the sessions dir the glob already clears, so a retry does not re-count the previous iteration's children. Assisted-by: Claude Signed-off-by: Wayne Sun --- internal/cli/run.go | 17 +++ internal/cli/telemetry_run_test.go | 56 +++++++ internal/runtime/pi_bootstrap_test.go | 201 ++++++++++++++++++++++++- internal/runtime/pi_run.go | 31 +++- internal/runtime/pi_subagents.go | 207 ++++++++++++++++++++++++++ internal/runtime/pi_subagents_test.go | 148 ++++++++++++++++++ internal/runtime/pi_transcript.go | 86 ++++++++++- internal/runtime/runtime.go | 28 ++++ 8 files changed, 764 insertions(+), 10 deletions(-) create mode 100644 internal/runtime/pi_subagents.go create mode 100644 internal/runtime/pi_subagents_test.go diff --git a/internal/cli/run.go b/internal/cli/run.go index 4970a0a96e..fa3f9c5957 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -178,6 +178,13 @@ type aggregateMetrics struct { // "FULLSEND_MODEL", "FULLSEND_PI_MODEL", "harness", "default") so a // silent override is visible after the fact. OverrideSource string `json:"override_source,omitempty"` + // PerModelUsage attributes the totals above to the model specs that + // spent them. Only runtimes that dispatch sub-agents fill it (pi's + // Agent tool), and then on every iteration of such a run — including + // one that dispatched nothing, whose parent entry is what keeps the + // breakdown summing to the totals across a retry. A run on a runtime + // without sub-agents keeps metrics.json as it was. + PerModelUsage map[string]agentruntime.ModelUsage `json:"per_model_usage,omitempty"` } func writeMetricsJSON(dir string, m aggregateMetrics) error { @@ -3104,6 +3111,16 @@ func aggregateRunMetrics(agg *aggregateMetrics, m *agentruntime.RunMetrics, iter if m.Model != "" { agg.Model = m.Model } + // Iterations are retries of the same task, so the per-model entries add + // up the same way the totals do. + for spec, u := range m.PerModelUsage { + if agg.PerModelUsage == nil { + agg.PerModelUsage = make(map[string]agentruntime.ModelUsage, len(m.PerModelUsage)) + } + entry := agg.PerModelUsage[spec] + entry.Add(u) + agg.PerModelUsage[spec] = entry + } } // resolveWorkItemID returns a stable cross-run correlation key for the work diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 7cafb3b29c..4cfdc5127c 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -392,6 +392,10 @@ func TestAggregateRunMetrics(t *testing.T) { m1.CacheCreationInputTokens, m1.CacheReadInputTokens = 1000, 5000 m1.ToolCalls.Store(3) m1.Model = "claude-opus-4-6" + m1.PerModelUsage = map[string]agentruntime.ModelUsage{ + "anthropic-vertex/claude-opus-4-6": {Requests: 1, InputTokens: 10, CostUSD: 0.06}, + "anthropic-vertex/claude-sonnet-4-6": {Requests: 2, InputTokens: 4, CostUSD: 0.04}, + } aggregateRunMetrics(&agg, &m1, 1) var m2 agentruntime.RunMetrics @@ -400,6 +404,9 @@ func TestAggregateRunMetrics(t *testing.T) { m2.ReasoningTokens = 15 m2.CacheCreationInputTokens, m2.CacheReadInputTokens = 200, 900 m2.ToolCalls.Store(2) + m2.PerModelUsage = map[string]agentruntime.ModelUsage{ + "anthropic-vertex/claude-opus-4-6": {Requests: 1, InputTokens: 4, CostUSD: 0.05}, + } aggregateRunMetrics(&agg, &m2, 2) assert.Equal(t, 7, agg.NumTurns) @@ -412,6 +419,55 @@ func TestAggregateRunMetrics(t *testing.T) { assert.Equal(t, 5, agg.ToolCalls) assert.Equal(t, 2, agg.Iterations) assert.Equal(t, "claude-opus-4-6", agg.Model, "last non-empty model is retained") + assert.Equal(t, map[string]agentruntime.ModelUsage{ + "anthropic-vertex/claude-opus-4-6": {Requests: 2, InputTokens: 14, CostUSD: 0.11}, + "anthropic-vertex/claude-sonnet-4-6": {Requests: 2, InputTokens: 4, CostUSD: 0.04}, + }, agg.PerModelUsage, "sub-agent usage accumulates across retry iterations like the totals") + + var noSubagents aggregateMetrics + var m3 agentruntime.RunMetrics + m3.TotalCostUSD = 0.01 + aggregateRunMetrics(&noSubagents, &m3, 1) + assert.Nil(t, noSubagents.PerModelUsage, "runtimes without sub-agents add no breakdown key to metrics.json") +} + +// TestAggregateRunMetrics_MixedIterations is the invariant that makes +// per_model_usage readable: it must sum to the run totals. A retry run +// where only one iteration dispatched sub-agents is the case that breaks +// if an iteration reports totals without also reporting its own entry, so +// the pi runtime folds the parent entry on every iteration. +func TestAggregateRunMetrics_MixedIterations(t *testing.T) { + var agg aggregateMetrics + + // Iteration 1 dispatched two children. + var m1 agentruntime.RunMetrics + m1.TotalCostUSD = 0.30 + m1.InputTokens, m1.OutputTokens = 300, 30 + m1.PerModelUsage = map[string]agentruntime.ModelUsage{ + "anthropic-vertex/claude-opus-4-6": {Requests: 1, InputTokens: 100, OutputTokens: 10, CostUSD: 0.10}, + "anthropic-vertex/claude-sonnet-4-6": {Requests: 2, InputTokens: 200, OutputTokens: 20, CostUSD: 0.20}, + } + aggregateRunMetrics(&agg, &m1, 1) + + // Iteration 2 is a retry that dispatched nothing — but its own tokens + // are still in the totals, so they must be in the breakdown too. + var m2 agentruntime.RunMetrics + m2.TotalCostUSD = 0.07 + m2.InputTokens, m2.OutputTokens = 70, 7 + m2.PerModelUsage = map[string]agentruntime.ModelUsage{ + "anthropic-vertex/claude-opus-4-6": {Requests: 1, InputTokens: 70, OutputTokens: 7, CostUSD: 0.07}, + } + aggregateRunMetrics(&agg, &m2, 2) + + var sum agentruntime.ModelUsage + for _, u := range agg.PerModelUsage { + sum.Add(u) + } + assert.InDelta(t, agg.TotalCostUSD, sum.CostUSD, 1e-9, "the breakdown sums to the run cost") + assert.Equal(t, agg.TokenUsage.Input, sum.InputTokens, "and to the run's input tokens") + assert.Equal(t, agg.TokenUsage.Output, sum.OutputTokens) + assert.Equal(t, 2, agg.PerModelUsage["anthropic-vertex/claude-opus-4-6"].Requests, + "the parent contributes one entry per iteration, dispatching or not") } func testTracer() trace.Tracer { diff --git a/internal/runtime/pi_bootstrap_test.go b/internal/runtime/pi_bootstrap_test.go index fc615b6271..23801e13b7 100644 --- a/internal/runtime/pi_bootstrap_test.go +++ b/internal/runtime/pi_bootstrap_test.go @@ -24,6 +24,15 @@ import ( // under storeDir keyed by the remote path, answers `pi --version` and `cat // ` execs from that store, and streams streamFixture for the pi run // command. Everything else succeeds silently. +// piFakeUsageFile is where the fake serves the Agent extension's usage +// file from: Run reads it with a `mv` + `head -c` fragment +// (piSubagentUsageReadCommand), which the store's remote-path keying cannot +// express. A test that wants sub-agent usage writes this file; absent, the +// read is empty, as for a run that dispatched no sub-agent. The fake +// renames it away once read, the way the real command does, so a test can +// observe that a second read folds nothing. +const piFakeUsageFile = "subagent-usage.jsonl" + func fakeOpenshellPi(t *testing.T, logPath, storeDir, streamFixture string) { t.Helper() require.NoError(t, os.MkdirAll(storeDir, 0o755)) @@ -39,6 +48,7 @@ if [ "$2" = "exec" ]; then case "$last" in "pi --version") echo "0.84.2"; exit 0 ;; "command -v pi"*) printf '%s\n' /usr/bin/pi '/usr/local/share/pi-extensions/anthropic-vertex'; exit 0 ;; + *usage.jsonl*mv*) u='` + storeDir + `'/` + piFakeUsageFile + `; if [ -f "$u" ]; then cat "$u"; mv -f "$u" "$u.read"; fi; exit 0 ;; cat\ *) f=$(printf '%s' "${last#cat }" | tr -d "'" | tr '/' '_'); cat '` + storeDir + `'/"$f"; exit $? ;; *"--print --mode json"*) cat '` + streamFixture + `'; exit 0 ;; esac @@ -228,6 +238,89 @@ func TestPiRuntimeRun_StreamsFixtureAndReportsMetrics(t *testing.T) { assert.Contains(t, string(teed), `"type":"agent_end"`) } +// TestPiRuntimeRun_FoldsSubagentUsage: a child's tokens never reach the +// parent's --mode json stream, so Run reads the Agent extension's usage +// file after the iteration and adds it to the run metrics, with the +// per-model breakdown that makes the totals attributable. +func TestPiRuntimeRun_FoldsSubagentUsage(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + t.Setenv(piProviderEnv, "") + t.Setenv(piAgentThinkingEnv, "") + work := t.TempDir() + store := filepath.Join(work, "store") + fixture, err := filepath.Abs(filepath.Join("testdata", "pi", "basic_run.ndjson")) + require.NoError(t, err) + fakeOpenshellPi(t, filepath.Join(work, "openshell.log"), store, fixture) + + // No tools: frontmatter → the Agent tool is on, so Run reads the file. + require.NoError(t, PiRuntime{}.Bootstrap(bootstrapInput{ + sandboxName: "sb", agentPath: writeAgentFile(t, "---\nname: review\nmodel: opus\n---\nReview."), agentName: "review", + })) + require.NoError(t, os.WriteFile(filepath.Join(store, piFakeUsageFile), []byte( + `{"seq":1,"model":"anthropic-vertex/claude-sonnet-4-6","usage":{"input":300,"output":40,"cacheRead":10,"cacheWrite":5,"cost":0.2},"stopReason":"stop","isError":false}`+"\n"+ + `{"seq":2,"model":"anthropic-vertex/claude-sonnet-4-6","usage":{"input":100,"output":10,"cost":0.1},"stopReason":"error","isError":true}`+"\n"), 0o644)) + + var metrics RunMetrics + exit, err := PiRuntime{}.Run(context.Background(), RunParams{ + SandboxName: "sb", AgentBaseName: "review", RepoDir: "/sandbox/workspace/repo", + Timeout: 30 * time.Second, OnEvent: func(AgentEvent) {}, + }, ui.New(os.Stderr), time.Now(), &metrics) + require.NoError(t, err) + assert.Equal(t, 0, exit) + + require.NotNil(t, metrics.PerModelUsage) + parent := metrics.PerModelUsage["anthropic-vertex/claude-opus-4-6"] + child := metrics.PerModelUsage["anthropic-vertex/claude-sonnet-4-6"] + assert.Equal(t, 1, parent.Requests, "the parent's own iteration is one request, keyed by its full model spec") + assert.Equal(t, 2, child.Requests, "a failed sub-agent still spent tokens") + assert.InDelta(t, 0.3, child.CostUSD, 1e-9) + assert.InDelta(t, parent.CostUSD+child.CostUSD, metrics.TotalCostUSD, 1e-9, "the breakdown sums to the run total") + assert.Equal(t, parent.InputTokens+child.InputTokens, metrics.InputTokens) + assert.Equal(t, parent.CacheReadInputTokens+10, metrics.CacheReadInputTokens) + assert.Equal(t, parent.CacheCreationInputTokens+5, metrics.CacheCreationInputTokens) + assert.Equal(t, "claude-opus-4-6", metrics.Model, "the reported model stays the bare parent id") + + // The read consumes the file, so a retry iteration whose + // ClearIterationArtifacts failed cannot count the same children twice. + var again RunMetrics + _, err = PiRuntime{}.Run(context.Background(), RunParams{ + SandboxName: "sb", AgentBaseName: "review", RepoDir: "/sandbox/workspace/repo", + Timeout: 30 * time.Second, OnEvent: func(AgentEvent) {}, + }, ui.New(os.Stderr), time.Now(), &again) + require.NoError(t, err) + assert.NotContains(t, again.PerModelUsage, "anthropic-vertex/claude-sonnet-4-6", "the children were already folded") + assert.Len(t, again.PerModelUsage, 1, "only the parent's own iteration") +} + +// Without sub-agent usage the totals are exactly what the stream reported, +// and the breakdown is the parent's single entry. +func TestPiRuntimeRun_NoSubagentUsageLeavesMetricsAlone(t *testing.T) { + t.Setenv("FULLSEND_PI_MODEL", "") + t.Setenv(piProviderEnv, "") + work := t.TempDir() + fixture, err := filepath.Abs(filepath.Join("testdata", "pi", "basic_run.ndjson")) + require.NoError(t, err) + fakeOpenshellPi(t, filepath.Join(work, "openshell.log"), filepath.Join(work, "store"), fixture) + require.NoError(t, PiRuntime{}.Bootstrap(bootstrapInput{ + sandboxName: "sb", agentPath: writeAgentFile(t, "---\nname: review\nmodel: opus\n---\nReview."), agentName: "review", + })) + + var metrics RunMetrics + _, err = PiRuntime{}.Run(context.Background(), RunParams{ + SandboxName: "sb", AgentBaseName: "review", RepoDir: "/sandbox/workspace/repo", + Timeout: 30 * time.Second, OnEvent: func(AgentEvent) {}, + }, ui.New(os.Stderr), time.Now(), &metrics) + require.NoError(t, err) + assert.InDelta(t, 0.015, metrics.TotalCostUSD, 0.001) + // The parent's own entry is still recorded: iterations are summed, so + // an iteration missing from the breakdown makes the run's + // per_model_usage stop matching its totals. + require.Len(t, metrics.PerModelUsage, 1) + parent := metrics.PerModelUsage["anthropic-vertex/claude-opus-4-6"] + assert.Equal(t, 1, parent.Requests) + assert.InDelta(t, metrics.TotalCostUSD, parent.CostUSD, 1e-9, "no children means the parent entry is the whole breakdown") +} + func TestPiRuntimeRun_ExitZeroWithStreamErrorReturnsOne(t *testing.T) { t.Setenv("FULLSEND_PI_MODEL", "") work := t.TempDir() @@ -427,6 +520,111 @@ exit 0 assert.Empty(t, PiRuntime{}.ParseTranscriptErrors(out), "clean sessions produce no error annotations") } +// TestPiRuntime_ExtractTranscripts_Children covers the sub-agent artifacts: +// a child session under sessions/agent-/ is saved under a name that +// carries the sequence number (so several children with the same session +// basename do not collide), and the Agent extension's usage file comes down +// beside the transcripts. +func TestPiRuntime_ExtractTranscripts_Children(t *testing.T) { + work := t.TempDir() + logPath := filepath.Join(work, "openshell.log") + binDir := t.TempDir() + script := `#!/bin/sh +echo "$@" >> '` + logPath + `' +if [ "$2" = "exec" ]; then + for last; do :; done + case "$last" in + if\ [\ -s\ *) printf '%s\n' '/sandbox/pi-config/subagents/usage.jsonl.read'; exit 0 ;; + find\ *) printf '%s\n' '/sandbox/pi-config/sessions/2026-08-29T10-00-00_parent.jsonl' '/sandbox/pi-config/sessions/agent-1/2026-08-29T10-01-00_kid.jsonl' '/sandbox/pi-config/sessions/agent-12/repo/2026-08-29T10-01-00_kid.jsonl'; exit 0 ;; + esac + exit 0 +fi +if [ "$2" = "download" ]; then + case "$4" in + *usage.jsonl) printf '{"seq":1,"model":"anthropic-vertex/claude-sonnet-4-6","usage":{"input":1,"output":2,"cost":0.1}}\n' > "$5/$(basename "$4")" ;; + *) printf '{"type":"message","message":{"role":"assistant","stopReason":"stop"}}\n' > "$5/$(basename "$4")" ;; + esac + exit 0 +fi +exit 0 +` + require.NoError(t, os.WriteFile(filepath.Join(binDir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + out := filepath.Join(work, "transcripts") + require.NoError(t, PiRuntime{}.ExtractTranscripts("sb", "review", out)) + entries, err := os.ReadDir(out) + require.NoError(t, err) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + assert.ElementsMatch(t, []string{ + "review-2026-08-29T10-00-00_parent.jsonl", + "review-sub1-2026-08-29T10-01-00_kid.jsonl", + "review-sub12-2026-08-29T10-01-00_kid.jsonl", + "review-subagents-usage.jsonl", + }, names, "children are named after the Agent call that spawned them, so equal basenames do not collide") + log, err := os.ReadFile(logPath) + require.NoError(t, err) + assert.Contains(t, string(log), "download sb /sandbox/pi-config/subagents/usage.jsonl.read", + "Run consumed the usage file by renaming it; extraction has to follow the rename") + assert.Empty(t, PiRuntime{}.ParseTranscriptErrors(out), "clean child sessions produce no error annotations") +} + +// TestPiRuntime_ExtractTranscripts_UsageFileIsContained checks the usage +// file's local name goes through the same os.Root containment as the +// transcripts. Both names are built from agentLabel, which is the caller's, +// so joining either onto outputDir unchecked would let a label write +// outside the artifact directory. +func TestPiRuntime_ExtractTranscripts_UsageFileIsContained(t *testing.T) { + work := t.TempDir() + binDir := t.TempDir() + script := `#!/bin/sh +if [ "$2" = "exec" ]; then + for last; do :; done + case "$last" in + if\ [\ -s\ *) printf '%s\n' '/sandbox/pi-config/subagents/usage.jsonl.read'; exit 0 ;; + find\ *) printf '%s\n' '/sandbox/pi-config/sessions/2026-08-29T10-00-00_parent.jsonl'; exit 0 ;; + esac + exit 0 +fi +if [ "$2" = "download" ]; then + printf 'downloaded\n' > "$5/$(basename "$4")" + exit 0 +fi +exit 0 +` + require.NoError(t, os.WriteFile(filepath.Join(binDir, "openshell"), []byte(script), 0o755)) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + out := filepath.Join(work, "artifacts", "transcripts") + require.NoError(t, PiRuntime{}.ExtractTranscripts("sb", "../../escaped", out), + "a rejected name is reported and skipped, not an extraction failure") + + // The usage file and a session transcript go through the same check, + // and both names are built from the label, so neither may land outside + // the output dir. + for _, name := range []string{"escaped-subagents-usage.jsonl", "escaped-2026-08-29T10-00-00_parent.jsonl"} { + assert.NoFileExists(t, filepath.Join(work, name), + "a label that traverses out of the output dir must not place %s there", name) + assert.NoFileExists(t, filepath.Join(work, "artifacts", name)) + } + entries, err := os.ReadDir(out) + require.NoError(t, err) + assert.Empty(t, entries, "the rejected names leave nothing behind either") +} + +func TestPiSubagentTranscriptName(t *testing.T) { + t.Parallel() + assert.Equal(t, "review-s.jsonl", piSubagentTranscriptName("review", "/sandbox/pi-config/sessions/s.jsonl")) + assert.Equal(t, "review-sub3-s.jsonl", piSubagentTranscriptName("review", "/sandbox/pi-config/sessions/agent-3/s.jsonl")) + assert.Equal(t, "review-sub3-s.jsonl", piSubagentTranscriptName("review", "/sandbox/pi-config/sessions/agent-3/nested/s.jsonl"), + "pi may nest a session under its working directory") + assert.Equal(t, "review-agent-x.jsonl", piSubagentTranscriptName("review", "/sandbox/pi-config/sessions/agent-x.jsonl"), + "only a directory named agent- marks a child") +} + // TestPiAgentTool_ManifestBlock covers the `agent` manifest block Bootstrap // writes for the fullsend-agent.js extension: enabled/disabled cases, the // model alias table, the extension list from the sandbox probe, and the @@ -567,7 +765,8 @@ func TestPiRuntimeClearIterationArtifacts(t *testing.T) { require.NoError(t, PiRuntime{}.ClearIterationArtifacts("sb")) log, err := os.ReadFile(logPath) require.NoError(t, err) - assert.Contains(t, string(log), "rm -rf '/sandbox/workspace'/output/* '/sandbox/pi-config/sessions'/* '/sandbox/workspace/pi-debug.log'") + assert.Contains(t, string(log), "rm -rf '/sandbox/workspace'/output/* '/sandbox/pi-config/sessions'/* '/sandbox/workspace/pi-debug.log' '/sandbox/pi-config/subagents/usage.jsonl'", + "the sessions glob takes the sub-agent session dirs; their usage file sits outside it and is named") } // TestPiDefaultTools_CoversToolMap keeps the settings.json default set and diff --git a/internal/runtime/pi_run.go b/internal/runtime/pi_run.go index e9576683fa..cdfa4b9098 100644 --- a/internal/runtime/pi_run.go +++ b/internal/runtime/pi_run.go @@ -734,6 +734,27 @@ func (r PiRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printe return exitCode, fmt.Errorf("a pi extension directory under %s is missing or was modified since Bootstrap uploaded it; refusing to load it (did the agent or the extension itself write there between iterations? extensions must not write into their own directory)", r.piExtensionsDir()) } + if m.Agent != nil && m.Agent.Enabled { + // Children are separate pi processes, so none of their tokens + // reached the stream just parsed; the extension's usage file is the + // only record of what they spent. A read failure is not fatal — + // losing the breakdown must not fail an iteration that succeeded. + if usage, _, _, uerr := sandbox.Exec(params.SandboxName, piSubagentUsageReadCommand(m.Agent.UsageFile), 10*time.Second); uerr != nil { + printer.StepWarn("Could not read the sub-agent usage file: " + sanitizeOutput(uerr.Error())) + } else { + // Unconditional: the parent's own entry belongs in the + // breakdown even when this iteration dispatched nothing, or + // per_model_usage stops summing to the totals across a retry. + n, skipped := foldPiSubagentUsage([]byte(usage), modelSpec, metrics) + if n > 0 { + printer.StepInfo(fmt.Sprintf("%d sub-agent call(s) folded into the run metrics", n)) + } + if skipped > 0 { + printer.StepWarn(fmt.Sprintf("%d unreadable line(s) in the sub-agent usage file were skipped; their cost is missing from the run metrics", skipped)) + } + } + } + if exitCode == 0 && lastResult != nil && lastResult.IsError { msg := lastResult.ErrorMessage if msg == "" { @@ -746,10 +767,14 @@ func (r PiRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printe } // ClearIterationArtifacts removes the previous iteration's outputs and -// sessions so transcripts and output files are per-iteration. +// sessions so transcripts and output files are per-iteration. The sessions +// glob also takes the sub-agent session dirs (sessions/agent-/); the +// Agent extension's usage file lives outside them and is named explicitly, +// or a retry would re-count the first iteration's children. func (r PiRuntime) ClearIterationArtifacts(sandboxName string) error { - clearCmd := fmt.Sprintf("rm -rf %s/output/* %s/* %s", - shellQuote(r.WorkspaceDir()), shellQuote(r.piSessionsDir()), shellQuote(r.WorkspaceDir()+"/"+piDebugLogFile)) + clearCmd := fmt.Sprintf("rm -rf %s/output/* %s/* %s %s %s", + shellQuote(r.WorkspaceDir()), shellQuote(r.piSessionsDir()), shellQuote(r.WorkspaceDir()+"/"+piDebugLogFile), + shellQuote(r.piAgentUsagePath()), shellQuote(r.piAgentUsagePath()+piSubagentUsageReadSuffix)) _, _, _, err := sandbox.Exec(sandboxName, clearCmd, 10*time.Second) return err } diff --git a/internal/runtime/pi_subagents.go b/internal/runtime/pi_subagents.go new file mode 100644 index 0000000000..48bbaefaed --- /dev/null +++ b/internal/runtime/pi_subagents.go @@ -0,0 +1,207 @@ +package runtime + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "math" + "strings" +) + +// Sub-agent accounting. fullsend-agent.js appends one JSON line per child +// to the manifest's usageFile; Run reads it after the iteration and folds +// it into RunMetrics so metrics.json carries the children's cost. The +// parent's own numbers come from the --mode json stream — a child's do not +// appear there at all, because each child is a separate pi process. Run +// folds on every Agent-enabled iteration, dispatching or not, so the +// parent's entry is always in the breakdown (see foldPiSubagentUsage). +// +// The fold covers exactly the fields a per-model entry has: input, +// output, cache-creation and cache-read tokens, and cost. Reasoning +// tokens are not among them — RunMetrics carries a run-level +// ReasoningTokens with no ModelUsage counterpart — so the +// breakdown-sums-to-the-totals invariant is about those five and not +// about every number in metrics.json. + +// piSubagentUnknownModel keys a usage record that carries no model spec: +// the line has a seq and a usage object but its "model" is empty or +// missing. Better a visible bucket than silently dropped cost. +const piSubagentUnknownModel = "unknown" + +// piSubagentUsageRecord is the line shape fullsend-agent.js writes. Fields +// the runner does not aggregate (description, stopReason, durationMs) are +// left out: the child session transcript carries them. +type piSubagentUsageRecord struct { + Model string `json:"model"` + Usage struct { + Input int `json:"input"` + Output int `json:"output"` + CacheRead int `json:"cacheRead"` + CacheWrite int `json:"cacheWrite"` + Cost float64 `json:"cost"` + } `json:"usage"` +} + +// piSubagentUsageMaxBytes bounds the usage file the runner reads back. The +// file is agent-reachable inside the sandbox (as everything under the +// config dir is), so an unbounded read is an unbounded allocation in the +// runner; the same reason piAgentProbe caps its probe output. A truncated +// tail becomes a skipped malformed line, which foldPiSubagentUsage reports. +const piSubagentUsageMaxBytes = 1 << 20 // 1 MiB, ~4k child records + +// piSubagentUsageReadSuffix marks a usage file the runner has already +// folded. The read renames before printing, so the fold is idempotent per +// iteration: a retry whose ClearIterationArtifacts failed finds no +// unconsumed file and cannot count the same children twice. +const piSubagentUsageReadSuffix = ".read" + +// piSubagentUsageReadCommand renders the in-sandbox read of the usage file: +// take it out of the way first, then print what it holds. A missing file is +// the ordinary case — the agent dispatched no sub-agent — so the command +// succeeds with empty output rather than failing the run. +func piSubagentUsageReadCommand(usageFile string) string { + f := shellQuote(usageFile) + r := shellQuote(usageFile + piSubagentUsageReadSuffix) + return fmt.Sprintf("if [ -f %s ]; then mv -f %s %s && head -c %d %s; fi 2>/dev/null || true", + f, f, r, piSubagentUsageMaxBytes, r) +} + +// foldPiSubagentUsage adds the children's usage to m's totals and builds +// m.PerModelUsage: one entry per child model spec plus the parent's own +// iteration under parentSpec, so the breakdown sums to the totals. It +// returns the number of child records folded and the number of lines that +// were not usage records. +// +// The parent entry is added whenever this runs, not only when children were +// found. Iterations are summed across a run (internal/cli aggregateRunMetrics), +// so an iteration that dispatched nothing must still contribute its own +// tokens to the breakdown — otherwise a retry run where only one iteration +// dispatched children has a per_model_usage that no longer sums to the +// totals. +// +// Malformed lines are skipped: the file is written by the extension inside +// the sandbox, so a truncated last line (a child killed mid-append, or the +// read hitting piSubagentUsageMaxBytes) must not cost the run its metrics. +func foldPiSubagentUsage(data []byte, parentSpec string, m *RunMetrics) (folded, skipped int) { + records, skipped := parsePiSubagentUsage(data) + per := m.PerModelUsage + if per == nil { + per = make(map[string]ModelUsage, len(records)+1) + } + addUsage := func(key string, u ModelUsage) { + entry := per[key] + entry.Add(u) + per[key] = entry + } + if parentSpec == "" { + parentSpec = piSubagentUnknownModel + } + addUsage(parentSpec, ModelUsage{ + Requests: 1, + InputTokens: m.InputTokens, + OutputTokens: m.OutputTokens, + CacheCreationInputTokens: m.CacheCreationInputTokens, + CacheReadInputTokens: m.CacheReadInputTokens, + CostUSD: m.TotalCostUSD, + }) + for _, rec := range records { + key := strings.TrimSpace(rec.Model) + if key == "" { + key = piSubagentUnknownModel + } + // Clamped at zero: the file is written inside the sandbox, so a + // negative count is either a truncated append reparsed as a number + // or an agent-authored line, and either way subtracting it would + // let the breakdown understate the run's real spend - the one + // number metrics.json exists to make auditable. + u := ModelUsage{ + Requests: 1, + InputTokens: nonNegative(rec.Usage.Input), + OutputTokens: nonNegative(rec.Usage.Output), + CacheCreationInputTokens: nonNegative(rec.Usage.CacheWrite), + CacheReadInputTokens: nonNegative(rec.Usage.CacheRead), + CostUSD: nonNegativeCost(rec.Usage.Cost), + } + addUsage(key, u) + m.InputTokens += u.InputTokens + m.OutputTokens += u.OutputTokens + m.CacheCreationInputTokens += u.CacheCreationInputTokens + m.CacheReadInputTokens += u.CacheReadInputTokens + m.TotalCostUSD += u.CostUSD + } + m.PerModelUsage = per + return len(records), skipped +} + +// nonNegative and nonNegativeCost clamp a usage figure at zero. See the +// call site in foldPiSubagentUsage. Only the float form has to consider the +// non-finite values; an int can hold neither NaN nor an infinity. +func nonNegative(v int) int { + if v < 0 { + return 0 + } + return v +} + +func nonNegativeCost(v float64) float64 { + // The inverted comparison catches NaN and -Inf; math.IsInf catches the + // one non-finite value it lets through. Either would poison every total + // it is added to for the rest of the run, and TotalCostUSD is the + // number metrics.json exists to make auditable. Neither reaches here + // through encoding/json today -- JSON has no NaN or Infinity literal + // and the decoder rejects a number that overflows float64 outright -- + // but this clamp is the only thing between an agent-writable file and + // the run totals, so it does not lean on that. + if !(v > 0) || math.IsInf(v, 1) { + return 0 + } + return v +} + +// parsePiSubagentUsage decodes the usage JSONL, dropping lines that are not +// a usage record and counting them. A record has to carry both a seq and a +// usage object: `{}` has neither, and a line with only one of them is a +// truncated or hand-written entry whose numbers cannot be trusted to be the +// whole of what a child spent. Both cases are counted as skipped rather +// than folded, so the gap shows up in the run log instead of as a silent +// hole in per_model_usage. +func parsePiSubagentUsage(data []byte) ([]piSubagentUsageRecord, int) { + var out []piSubagentUsageRecord + skipped := 0 + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(make([]byte, 0, 8*1024), maxTranscriptLineSize) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + // Two passes rather than one struct with pointer fields: an + // embedded record whose Usage is shadowed by a probe field is + // never filled by encoding/json (the shallower field wins). + var probe struct { + Usage *json.RawMessage `json:"usage"` + Seq *int `json:"seq"` + } + if err := json.Unmarshal(line, &probe); err != nil { + skipped++ + continue + } + if probe.Usage == nil || probe.Seq == nil { + skipped++ + continue + } + var rec piSubagentUsageRecord + if err := json.Unmarshal(line, &rec); err != nil { + skipped++ + continue + } + out = append(out, rec) + } + if err := scanner.Err(); err != nil { + // A line past maxTranscriptLineSize; the rest of the file is not + // read, so report it rather than silently losing the tail. + skipped++ + } + return out, skipped +} diff --git a/internal/runtime/pi_subagents_test.go b/internal/runtime/pi_subagents_test.go new file mode 100644 index 0000000000..11650279d0 --- /dev/null +++ b/internal/runtime/pi_subagents_test.go @@ -0,0 +1,148 @@ +package runtime + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestFoldPiSubagentUsage covers the usage file the Agent extension appends +// one line per sub-agent to: the per-model breakdown includes the parent's +// own iteration so it reconciles with the run totals, and the totals grow +// by what the children spent. +func TestFoldPiSubagentUsage(t *testing.T) { + t.Parallel() + const parent = "anthropic-vertex/claude-opus-4-6" + const child = "anthropic-vertex/claude-sonnet-4-6" + lines := `{"seq":1,"model":"` + child + `","provider":"anthropic-vertex","description":"security pass","durationMs":1000,"usage":{"input":100,"output":20,"cacheRead":5,"cacheWrite":3,"cost":0.25},"stopReason":"stop","isError":false} +not json +{"seq":2,"model":"` + child + `","usage":{"input":10,"output":2,"cost":0.05},"stopReason":"error","isError":true} +{"seq":3,"model":"","usage":{"input":7,"output":1,"cost":0.01}} +` + m := &RunMetrics{TotalCostUSD: 1.5, InputTokens: 1000, OutputTokens: 200, CacheReadInputTokens: 50, CacheCreationInputTokens: 30} + n, skipped := foldPiSubagentUsage([]byte(lines), parent, m) + + assert.Equal(t, 3, n, "every well-formed record counts, including the failed one and the one with no model") + assert.Equal(t, 1, skipped, "the line that is not JSON is reported, so its cost is not silently lost") + assert.InDelta(t, 1.5+0.25+0.05+0.01, m.TotalCostUSD, 1e-9, "child cost lands in the run total") + assert.Equal(t, 1000+100+10+7, m.InputTokens) + assert.Equal(t, 200+20+2+1, m.OutputTokens) + assert.Equal(t, 50+5, m.CacheReadInputTokens) + assert.Equal(t, 30+3, m.CacheCreationInputTokens) + + require := assert.New(t) + require.Equal(ModelUsage{Requests: 1, InputTokens: 1000, OutputTokens: 200, CacheReadInputTokens: 50, CacheCreationInputTokens: 30, CostUSD: 1.5}, + m.PerModelUsage[parent], "the parent's own iteration is one entry, keyed by its model spec") + require.Equal(ModelUsage{Requests: 2, InputTokens: 110, OutputTokens: 22, CacheReadInputTokens: 5, CacheCreationInputTokens: 3, CostUSD: 0.30}, + m.PerModelUsage[child]) + require.Equal(ModelUsage{Requests: 1, InputTokens: 7, OutputTokens: 1, CostUSD: 0.01}, + m.PerModelUsage[piSubagentUnknownModel], "a record without a model spec is still accounted for") +} + +// An empty or absent usage file is the common case (no sub-agent was +// dispatched). The totals must be untouched, but the parent's own entry +// still has to appear: iterations are summed, so an iteration missing from +// the breakdown makes the run's per_model_usage stop matching its totals. +func TestFoldPiSubagentUsage_NoChildren(t *testing.T) { + t.Parallel() + const parent = "anthropic-vertex/claude-opus-4-6" + for _, data := range []string{"", " \n\n"} { + m := &RunMetrics{TotalCostUSD: 2, InputTokens: 5} + n, skipped := foldPiSubagentUsage([]byte(data), parent, m) + assert.Zero(t, n, data) + assert.Zero(t, skipped, data) + assert.Equal(t, ModelUsage{Requests: 1, InputTokens: 5, CostUSD: 2}, m.PerModelUsage[parent], data) + assert.Len(t, m.PerModelUsage, 1, data) + assert.Equal(t, 2.0, m.TotalCostUSD, "the parent's own numbers are not double-counted into the totals") + assert.Equal(t, 5, m.InputTokens) + } + + // A `{}` line carries neither a seq nor a usage object, so it is not a + // sub-agent that ran — but it is not a record either, and saying so is + // how a truncated read surfaces. + m := &RunMetrics{TotalCostUSD: 2, InputTokens: 5} + n, skipped := foldPiSubagentUsage([]byte("{}\n"), parent, m) + assert.Zero(t, n) + assert.Equal(t, 1, skipped) + + // Half a record is not a record. A line with a seq but no usage object + // (a child killed between the two, or a hand-written entry) would fold + // as a zero-cost sub-agent and hide the missing spend; a usage object + // with no seq is not something the extension ever writes. + for _, line := range []string{ + `{"seq":4,"model":"m"}`, + `{"usage":{"input":10,"output":2,"cost":0.05},"model":"m"}`, + } { + m := &RunMetrics{TotalCostUSD: 2, InputTokens: 5} + n, skipped := foldPiSubagentUsage([]byte(line+"\n"), parent, m) + assert.Zero(t, n, line) + assert.Equal(t, 1, skipped, line) + assert.Equal(t, 2.0, m.TotalCostUSD, line) + assert.Len(t, m.PerModelUsage, 1, line) + } +} + +// TestFoldPiSubagentUsage_ClampsNegativeAndNonFinite covers the numbers in +// the usage file, which is written inside the sandbox: a negative, NaN or +// infinite figure must not be able to subtract from (or poison) the run +// totals that metrics.json reports. +func TestFoldPiSubagentUsage_ClampsNegativeAndNonFinite(t *testing.T) { + t.Parallel() + const parent = "anthropic-vertex/claude-opus-4-6" + const child = "anthropic-vertex/claude-sonnet-4-6" + lines := `{"seq":1,"model":"` + child + `","usage":{"input":-1000,"output":-5,"cacheRead":-7,"cacheWrite":-9,"cost":-100}} +{"seq":2,"model":"` + child + `","usage":{"input":10,"output":2,"cacheRead":1,"cacheWrite":1,"cost":0.05}} +` + m := &RunMetrics{TotalCostUSD: 2, InputTokens: 100, OutputTokens: 20, CacheReadInputTokens: 3, CacheCreationInputTokens: 4} + n, skipped := foldPiSubagentUsage([]byte(lines), parent, m) + assert.Equal(t, 2, n) + assert.Zero(t, skipped) + assert.InDelta(t, 2+0.05, m.TotalCostUSD, 1e-9, "a negative cost cannot reduce the run total") + assert.Equal(t, 110, m.InputTokens) + assert.Equal(t, 22, m.OutputTokens) + assert.Equal(t, 4, m.CacheReadInputTokens) + assert.Equal(t, 5, m.CacheCreationInputTokens) + assert.Equal(t, ModelUsage{Requests: 2, InputTokens: 10, OutputTokens: 2, CacheReadInputTokens: 1, CacheCreationInputTokens: 1, CostUSD: 0.05}, + m.PerModelUsage[child], "the clamped record still counts as a request, with zeroed figures") + + // JSON has no NaN or Infinity literal and the decoder rejects a number + // that overflows float64, so no single record reaches the fold + // non-finite today; the clamp is what keeps one out of the totals + // whatever produced it. + assert.Zero(t, nonNegativeCost(math.NaN())) + assert.Zero(t, nonNegativeCost(math.Inf(1)), "+Inf would poison total_cost_usd for the rest of the run") + assert.Zero(t, nonNegativeCost(math.Inf(-1))) + assert.Zero(t, nonNegativeCost(-0.0)) + assert.Equal(t, 1.5, nonNegativeCost(1.5)) + assert.Equal(t, math.MaxFloat64, nonNegativeCost(math.MaxFloat64), "a finite figure, however large, is still folded") + assert.Zero(t, nonNegative(-1)) + assert.Equal(t, 7, nonNegative(7)) +} + +// A truncated tail — the read hit piSubagentUsageMaxBytes, or a child was +// killed mid-append — must cost the run one skipped line, not its metrics. +func TestFoldPiSubagentUsage_TruncatedTail(t *testing.T) { + t.Parallel() + const parent = "anthropic-vertex/claude-opus-4-6" + const child = "anthropic-vertex/claude-sonnet-4-6" + data := `{"seq":1,"model":"` + child + `","usage":{"input":10,"output":2,"cost":0.05}} +{"seq":2,"model":"` + child + `","usage":{"input":10,"outp` + m := &RunMetrics{TotalCostUSD: 1, InputTokens: 100} + n, skipped := foldPiSubagentUsage([]byte(data), parent, m) + assert.Equal(t, 1, n) + assert.Equal(t, 1, skipped) + assert.InDelta(t, 1.05, m.TotalCostUSD, 1e-9) +} + +func TestPiSubagentUsageReadCommand(t *testing.T) { + t.Parallel() + const usage = "/sandbox/pi-config/subagents/usage.jsonl" + cmd := piSubagentUsageReadCommand(usage) + assert.Contains(t, cmd, "mv -f '"+usage+"' '"+usage+piSubagentUsageReadSuffix+"'", + "the file is consumed as it is read, so a failed ClearIterationArtifacts cannot double-count it") + assert.Contains(t, cmd, "head -c 1048576 '"+usage+piSubagentUsageReadSuffix+"'", + "the read is bounded: the file is agent-reachable inside the sandbox") + assert.NotContains(t, cmd, "cat ", "an unbounded read is an unbounded allocation in the runner") + assert.Contains(t, cmd, "|| true", "a missing file is the no-children case, not a failure") +} diff --git a/internal/runtime/pi_transcript.go b/internal/runtime/pi_transcript.go index 163955f0c9..64d7efbe47 100644 --- a/internal/runtime/pi_transcript.go +++ b/internal/runtime/pi_transcript.go @@ -8,16 +8,45 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "time" "github.com/fullsend-ai/fullsend/internal/sandbox" ) +// piSubagentSessionDir matches a sub-agent's session directory in a remote +// path: fullsend-agent.js gives child its own `--session-dir +// /agent-`, so the sequence number is recoverable from +// the path and the child transcript can be named after the call that made +// it rather than colliding with the parent's on basename alone. +var piSubagentSessionDir = regexp.MustCompile(`/agent-(\d+)/`) + +// piSubagentTranscriptName renders the local name of a session file found +// under the sessions dir: -sub- for a child, +// - for the parent's own session. +func piSubagentTranscriptName(agentLabel, remotePath string) string { + base := filepath.Base(remotePath) + if m := piSubagentSessionDir.FindStringSubmatch(remotePath); m != nil { + return fmt.Sprintf("%s-sub%s-%s", agentLabel, m[1], base) + } + return fmt.Sprintf("%s-%s", agentLabel, base) +} + +// piSubagentUsageLocalName is the extracted name of the Agent extension's +// usage file. It sits next to the transcripts so a run's sub-agent cost is +// auditable from the artifacts alone, not only through metrics.json. +func piSubagentUsageLocalName(agentLabel string) string { + return agentLabel + "-subagents-usage.jsonl" +} + // ExtractTranscripts downloads pi's session JSONL files (written under the // runner-owned sessions dir, possibly nested by working directory) into // outputDir as -, with the same path containment as -// the Claude handler. +// the Claude handler. Sub-agent sessions (sessions/agent-/, written by +// the children fullsend-agent.js spawns) are saved as +// -sub-, and the extension's usage file +// alongside them. func (r PiRuntime) ExtractTranscripts(sandboxName, agentLabel, outputDir string) error { if err := os.MkdirAll(outputDir, 0o755); err != nil { return fmt.Errorf("creating output dir: %w", err) @@ -28,6 +57,35 @@ func (r PiRuntime) ExtractTranscripts(sandboxName, agentLabel, outputDir string) } defer root.Close() + // The usage file lives beside the sessions dir, not inside it, so the + // find below never sees it. Its absence is the ordinary case (no + // sub-agent was dispatched, or the Agent tool is off). + // Run renames it to .read when it folds the cost into the metrics + // (piSubagentUsageReadCommand), so that is the usual name here; the + // un-renamed one is what is left when Run never got that far. + read := r.piAgentUsagePath() + piSubagentUsageReadSuffix + if remote, _, _, uerr := sandbox.Exec(sandboxName, + fmt.Sprintf("if [ -s %s ]; then echo %s; elif [ -s %s ]; then echo %s; fi", + shellQuote(read), shellQuote(read), shellQuote(r.piAgentUsagePath()), shellQuote(r.piAgentUsagePath())), + 10*time.Second, + ); uerr == nil && strings.TrimSpace(remote) != "" { + remotePath := strings.TrimSpace(remote) + localName := piSubagentUsageLocalName(agentLabel) + // Through the same containment as the transcripts: this name is + // built from agentLabel too, and agentLabel is the caller's. + localPath, createErr := reserveContainedFile(root, outputDir, localName) + switch { + case createErr != nil: + fmt.Fprintf(os.Stderr, " [%s] Skipping the sub-agent usage file (path rejected): %s: %v\n", agentLabel, localName, createErr) + default: + if dlErr := sandbox.DownloadFile(sandboxName, remotePath, localPath); dlErr != nil { + fmt.Fprintf(os.Stderr, " [%s] Failed to copy the sub-agent usage file: %v\n", agentLabel, dlErr) + } else { + fmt.Fprintf(os.Stderr, " [%s] Saved sub-agent usage: %s\n", agentLabel, localName) + } + } + } + stdout, _, _, err := sandbox.Exec(sandboxName, fmt.Sprintf("find %s -name '*.jsonl' 2>/dev/null || true", shellQuote(r.piSessionsDir())), 10*time.Second, @@ -45,15 +103,12 @@ func (r PiRuntime) ExtractTranscripts(sandboxName, agentLabel, outputDir string) if remotePath == "" { continue } - localName := fmt.Sprintf("%s-%s", agentLabel, filepath.Base(remotePath)) - f, createErr := root.Create(localName) + localName := piSubagentTranscriptName(agentLabel, remotePath) + localPath, createErr := reserveContainedFile(root, outputDir, localName) if createErr != nil { fmt.Fprintf(os.Stderr, " [%s] Skipping (path rejected): %s: %v\n", agentLabel, localName, createErr) continue } - f.Close() - localPath := filepath.Join(outputDir, localName) - os.Remove(localPath) if dlErr := sandbox.DownloadFile(sandboxName, remotePath, localPath); dlErr != nil { fmt.Fprintf(os.Stderr, " [%s] Failed to copy transcript: %v\n", agentLabel, dlErr) continue @@ -63,6 +118,25 @@ func (r PiRuntime) ExtractTranscripts(sandboxName, agentLabel, outputDir string) return nil } +// reserveContainedFile checks localName against an os.Root opened on +// outputDir and returns the path to download to. Every name this file +// builds embeds the caller-supplied agentLabel, so none of them may be +// joined onto outputDir unchecked: root.Create refuses a name that would +// traverse out of the directory or follow a symlink out of it, and +// sandbox.DownloadFile takes a plain path, so the check has to happen +// before the download rather than inside it. The reservation is removed +// again because the downloader wants to create the file itself. +func reserveContainedFile(root *os.Root, outputDir, localName string) (string, error) { + f, err := root.Create(localName) + if err != nil { + return "", err + } + f.Close() + localPath := filepath.Join(outputDir, localName) + os.Remove(localPath) + return localPath, nil +} + // ExtractDebugLog downloads pi's stderr capture written when --debug is set. func (r PiRuntime) ExtractDebugLog(sandboxName, localPath, debug string) error { if debug == "" { diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 9c6d28dffd..a4b83feb2b 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -20,6 +20,34 @@ type RunMetrics struct { CacheCreationInputTokens int `json:"cache_creation_input_tokens"` CacheReadInputTokens int `json:"cache_read_input_tokens"` Model string `json:"model"` + // PerModelUsage breaks the totals above down by the model spec that + // spent them. Runtimes that dispatch sub-agents (pi's Agent tool) fill + // it with one entry per child model plus the parent's own, so a run + // whose cost is dominated by children is legible in metrics.json; + // runtimes without sub-agents leave it nil and the totals stand alone. + PerModelUsage map[string]ModelUsage `json:"per_model_usage,omitempty"` +} + +// ModelUsage is one model's token and cost contribution to a run. Requests +// counts the agent invocations attributed to the model (one for the parent +// iteration, one per sub-agent call). +type ModelUsage struct { + Requests int `json:"requests"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + CostUSD float64 `json:"cost_usd"` +} + +// Add accumulates other into u. +func (u *ModelUsage) Add(other ModelUsage) { + u.Requests += other.Requests + u.InputTokens += other.InputTokens + u.OutputTokens += other.OutputTokens + u.CacheCreationInputTokens += other.CacheCreationInputTokens + u.CacheReadInputTokens += other.CacheReadInputTokens + u.CostUSD += other.CostUSD } // DefaultAgentPrompt is the prompt handed to the agent CLI when RunParams From 8b3f49bc8a8afd12c52bcec6a6448294ae2e330f Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 14:01:24 -0400 Subject: [PATCH 6/7] docs(pi): document the Agent tool, its manifest block and child artifacts The pi runtime pages still said sub-agents were not wired and that review/retro must stay on Claude Code, which is no longer true and would send readers to the single-context workaround. pi.md gains a Sub-agents section: the tool contract, what a child inherits (hooks, providers, trust off, tool allowlist), the model alias table and why an unservable model is rejected rather than passed through, the medium thinking default with its env override and the review-budget reason for it, and where child transcripts, the usage file and per_model_usage land. runtimes.md's matrix row and the "stay on claude for sub-agents" advice follow. runtime-implementation.md gets an Agent tool contract section with the manifest fields, the depth guard, the shared exit-97 integrity check and the process-group kill, plus the two new paths in the sandbox layout tree. Assisted-by: Claude Signed-off-by: Wayne Sun --- docs/cli/run.md | 24 +++ docs/contributing/runtime-implementation.md | 114 ++++++++++++-- .../getting-started/choosing-a-runtime.md | 4 +- docs/runtimes.md | 10 +- docs/runtimes/claude.md | 5 +- docs/runtimes/pi.md | 141 ++++++++++++++++-- 6 files changed, 265 insertions(+), 33 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index 205b4327e0..eb31a711d2 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -81,6 +81,30 @@ Each run produces artifacts in the output directory: | `total_cost_usd` | Total inference cost in USD, as reported by the runtime (raw floating-point aggregate across all iterations; no fullsend-side pricing-table fallback). See [Cost data contract](../guides/infrastructure/distributed-tracing.md#cost-data-contract) | | `num_turns` | Number of conversation turns | | `iterations` | Number of retry iterations | +| `per_model_usage` | Per-model-spec breakdown, present only when a runtime reports one (today: `pi` with the `Agent` tool enabled). See below | + +#### Per-model usage + +A map from pi model spec (`anthropic-vertex/claude-opus-4-6`) to +`{requests, input_tokens, output_tokens, cache_creation_input_tokens, cache_read_input_tokens, cost_usd}`. +It exists because a pi sub-agent is a separate `pi` process whose tokens never appear in the +parent's stream, so without it `total_cost_usd` would grow with no way to attribute it. + +- **What folds.** Tokens and cost, from both the parent and every child, summed across retry + iterations. Each iteration contributes one `requests` for the parent plus one per sub-agent call, + so `requests` counts inference *episodes*, not HTTP requests. +- **The invariant.** The breakdown sums to the run totals for the five fields an entry has: + `sum(cost_usd) == total_cost_usd`, and likewise for `input_tokens`, `output_tokens`, + `cache_creation_input_tokens` and `cache_read_input_tokens`. `reasoning_tokens` is a run-level + total with no per-model counterpart, so it is outside the invariant. The parent's entry is + recorded on every iteration of an `Agent`-enabled run, including ones that dispatched no + sub-agent, which is what keeps the invariant true across a retry. +- **What stays parent-only.** `num_turns` and `tool_calls` are read from the parent's stream and are + not broken down or added to per model — a child's turns and tool calls are recorded in its own + session transcript (`transcripts/-sub-*.jsonl`) instead. +- A model spec of `unknown` is a usage record that carries no model spec at all; its cost is + bucketed there rather than dropped. A dispatch rejected *before* a model was resolved writes + no record, so it never reaches the breakdown. ## OpenAI credentials on pi diff --git a/docs/contributing/runtime-implementation.md b/docs/contributing/runtime-implementation.md index d8f4a9f79f..d1c2671926 100644 --- a/docs/contributing/runtime-implementation.md +++ b/docs/contributing/runtime-implementation.md @@ -248,9 +248,9 @@ Net: after #6358 and #6357, both PreToolUse and PostToolUse halves of the contra | Binary | Pin | Re-check on bump | |--------|-----|------------------| | Claude Code | `ARG CLAUDE_CODE_VERSION` (npm, Renovate-tracked). The OpenShell base image ships its **own** unpinned Claude Code at `/usr/local/bin/claude` (whatever `curl claude.ai/install.sh` fetched when the base was built), and `/usr/local/bin` precedes npm's `/usr/bin` on the sandbox `PATH` — so the Containerfile replaces that file with a symlink to the npm install and fails the build unless `claude --version` equals the pin (#6612; before that fix the base image's 2.1.156 shadowed every pin). `TestSandboxImageClaudeCodePinWins` guards the step | the [tool-name vocabulary](#tool-name-vocabulary-608); the hook contract caveats above; the alias table — `opus`/`sonnet`/`haiku` resolve from the running version's built-in defaults on Vertex, and `ANTHROPIC_DEFAULT_*_MODEL` does not steer the request there, so a harness or `agents:` entry that needs a specific generation must name the id | -| pi | `ARG PI_VERSION` (npm, `--ignore-scripts`, Renovate-tracked; `TestSandboxImagePinsAreRenovateTracked`) | `parsePiStream` fixtures (`internal/runtime/testdata/pi/regen.sh`); the extension compatibility notes in [pi runtime internals](#pi-runtime-internals-6464) | +| pi | `ARG PI_VERSION` (npm, `--ignore-scripts`, Renovate-tracked; `TestSandboxImagePinsAreRenovateTracked`) | `parsePiStream` fixtures (`internal/runtime/testdata/pi/regen.sh`); the extension compatibility notes in [pi runtime internals](#pi-runtime-internals-6464); `piGoogleVertexModels` in `pi_bootstrap.go`, which is the bundled `google-vertex` catalog verbatim and is what the `Agent` tool accepts as a Gemini id | | `pi-anthropic-vertex` | `ARG PI_ANTHROPIC_VERTEX_VERSION` + tarball SHA256, under `/usr/local/share/pi-extensions/anthropic-vertex` | its `sync/compat.json` against the pinned pi; the `@anthropic-ai/sdk` override vs pi's `packages/ai/package.json` | -| `pi-xai-vertex` | `ARG PI_XAI_VERTEX_VERSION` + tarball SHA256, under `/usr/local/share/pi-extensions/xai-vertex` | its `peerDependencies` floor (it mirrors no pi internals) | +| `pi-xai-vertex` | `ARG PI_XAI_VERTEX_VERSION` + tarball SHA256, under `/usr/local/share/pi-extensions/xai-vertex` | its `peerDependencies` floor (it mirrors no pi internals); `piXaiVertexModels` in `pi_bootstrap.go`, the ids it registers and what the `Agent` tool accepts as a Grok id | The run log's `Agent: (vX.Y.Z)` line is the ground truth for which Claude Code version served a run; the Containerfile pin is a claim, the assertion at build time is what makes it true. @@ -267,8 +267,11 @@ The sandbox has two key directories that map to Claude Code's config levels (plu │ ├── extensions// Declared harness extensions (ADR 0094; loaded with -e, tree-hash preflight) │ ├── hooks/*.py Security hook scripts (same files as claude-config/hooks/) │ ├── fullsend-hooks.js Hook adapter extension (loaded with -e; --no-extensions otherwise) -│ ├── fullsend-manifest.json Agent tools/allowlist, HookPlan, pi version — read by Run and the extension +│ ├── fullsend-agent.js Agent/Task sub-agent extension (loaded with -e when the tool is enabled) +│ ├── fullsend-manifest.json Agent tools/allowlist, HookPlan, agent block, pi version — read by Run and the extensions +│ ├── subagents/usage.jsonl One line per sub-agent (model, usage, stop reason) — folded into metrics.json │ └── sessions/ PI_CODING_AGENT_SESSION_DIR (session JSONL → transcripts) +│ └── agent-/ One sub-agent's session (→ transcripts/-sub-…) │ ├── claude-config/ ← CLAUDE_CONFIG_DIR (personal level) │ ├── agents/ @@ -373,21 +376,24 @@ One iteration, end to end — the amber decision is what makes "hooks enabled" e ```mermaid flowchart TB B["Bootstrap (once per run)\nagent .md → APPEND_SYSTEM.md + --tools\nhook scripts + manifest + adapter\npi --version preflight"] - G{"shell guard, before .env (command -p):\nadapter present and SHA-256 = embedded copy?\nmanifest present?"} - X["exit 97\npi never starts unhooked\n(Run refuses earlier, exit -1,\nif the manifest has no hook plan)"] - E["source .env\nunset ANTHROPIC_*\npin GOOGLE_CLOUD_PROJECT"] - P["pi --print --mode json --no-approve\n--no-extensions [-e vertex, on Vertex] -e hooks\n--tools ... --model ... #lt;/dev/null"] + G{"shell guard, before .env (command -p):\nadapter present and SHA-256 = embedded copy?\nAgent extension SHA-256 = embedded copy?\nmanifest present and SHA-256 = the one Bootstrap wrote?"} + X["exit 97 (adapter)\nexit 94 (Agent extension)\nexit 95 (manifest)\npi never starts unhooked\n(Run refuses earlier, exit -1,\nif the manifest has no hook plan)"] + E["source .env\nunset ANTHROPIC_*\npin GOOGLE_CLOUD_PROJECT\nre-check manifest SHA-256"] + P["pi --print --mode json --no-approve\n--no-extensions [-e vertex, on Vertex] -e hooks [-e agent]\n--tools ... --model ... #lt;/dev/null"] + C["child pi per Agent call\nprompt on stdin · own session dir\nSIGTERM then SIGKILL\nusage.jsonl folded into metrics"] S["parsePiStream\nexactly one ResultEvent\nexit 0 + stream error ⇒ run fails"] - A["artifacts\noutput.jsonl · transcripts/\nmetrics.json (runtime: pi)"] + A["artifacts\noutput.jsonl · transcripts/ (incl. sub#lt;seq#gt;)\nmetrics.json (runtime: pi, per_model_usage)"] B --> G G -- no --> X G -- yes --> E --> P --> S --> A + P -. "Agent/Task tool" .-> C + C -. "final message" .-> P classDef guard fill:#fbf0d6,stroke:#d98e04,color:#1b2230; classDef bad fill:#f8e1de,stroke:#c0392b,color:#1b2230; classDef opt fill:#e3e9fb,stroke:#2d5be3,color:#1b2230; class G guard; class X bad; - class B,P,S opt; + class B,P,S,C opt; ``` ### Posture @@ -408,11 +414,11 @@ Parity with `claude -p --dangerously-skip-permissions`, verified against pi v0.8 - `--no-approve` sets the project-trust override, so the trust-gated project resources — `.pi/{settings.json,extensions,skills,prompts,themes,SYSTEM.md,APPEND_SYSTEM.md}` and `.agents/skills` (`core/trust-manager.ts`); `AGENTS.md` itself is still read as context — are ignored without a dialog (`cli/args.ts`, `main.ts`). `defaultProjectTrust: never` in the global settings covers the no-flag case (verified on the pinned build: a planted `.pi/extensions/evil.js` in the repo does not load under `--no-approve` and does under `--approve`). - First-run setup, theme selection, telemetry consent and the version check are interactive-only code paths (`PI_TELEMETRY=0`, `PI_SKIP_VERSION_CHECK=1`/`PI_OFFLINE=1` set anyway). - A missing credential raises `No API key found` and exits 1 — no `/login` prompt (`core/agent-session.ts`, `modes/print-mode.ts`); retries are bounded (`retry.maxRetries: 3`, 2/4/8 s) and compaction is automatic. -- **The one blocker found:** print mode reads a non-TTY stdin to EOF before the first prompt, even with a positional message (`main.ts` `readPipedStdin`), so an exec that keeps stdin open with no writer hangs pi — `Run` therefore appends ` --thinking '' >/sandbox/workspace/pi-debug.log]`; `settings.json` sets `defaultProjectTrust: never` (repo-owned `.pi/` never loaded) and `defaultTools: [read, bash, edit, write, grep, find, ls]` (pi alone activates only the first four; `--tools`, when emitted, replaces the set); `PI_OFFLINE=1`/`PI_TELEMETRY=0`/`PI_SKIP_VERSION_CHECK=1`/`JITI_FS_CACHE=false` come from `EnvExports`. The loader environment (`JITI_FS_CACHE`, the `JITI_*`/`NODE_*` `unset` after `.env`) is pinned for the same reason the extension tree is hashed — see [Pi extensions](#pi-extensions-adr-0094). Context files (`AGENTS.md`) and skills stay on — they are the harness's own inputs. `PI_CODING_AGENT_DIR/extensions/` is arbitrary TypeScript loaded at startup and the config dir is not a permission boundary, which is why only the explicit `-e` paths load (at most one vendored provider extension plus the hook adapter). Right after `.env` is sourced, `Run` re-exports `EnvExports()` (`PI_CODING_AGENT_DIR`, the session dir, the offline switches) so a rewritten `.env` cannot relocate pi's config directory. For the built-in `openai` provider (`openai/`, [ADR 0092](../ADRs/0092-openai-wif-credential-delivery.md)) no extension loads and no `--api-key` is passed; `Run` instead seeds `auth.json` under `PI_CODING_AGENT_DIR` with the placeholder the sandbox environment carries for `OPENAI_API_KEY` (`PiOpenAIAuthSeed`, before `.env` is sourced), because pi re-reads that file on every revision change and resolves the key per request — the runner re-runs the same seed through `sandbox exec` after each credential refresh, which is what lets a running iteration follow a refresh on OpenShell 0.0.115, where a revision-scoped placeholder stays pinned to its generation and the unrevisioned alias is refused (`--api-key` would outrank the file and pin the iteration). It unsets `OPENAI_BASE_URL`/`AZURE_OPENAI_API_KEY`/`OPENAI_API_KEY`/`NODE_OPTIONS`/`NODE_PATH` after `.env`, fails the iteration with exit 1 when the environment holds anything but a gateway placeholder at seed time (before `.env`), and runs a config-dir integrity guard that exits 98 when `models.json` exists or `auth.json` is anything but pi's own `{}` or exactly the seeded placeholder entry — `models.json` is the only way to move the provider's base URL, a redirect to another allowed REST host is the placeholder-leak path ADR 0025 describes, and pi itself writes an empty `auth.json` on every start so only its content counts. The guard runs before the agent-writable `.env` is sourced and again after it behind `unset -f test command grep tr sed printf`, whether or not hooks are enabled. +- **Hardening levers in use** — `Run` executes `pi --print --mode json --no-approve --no-extensions --no-prompt-templates --no-themes --session-dir /sandbox/pi-config/sessions [-e /usr/local/share/pi-extensions/anthropic-vertex | -e /usr/local/share/pi-extensions/xai-vertex] [-e /sandbox/pi-config/fullsend-hooks.js] [-e /sandbox/pi-config/fullsend-agent.js] [--tools ...] --model --thinking '' >/sandbox/workspace/pi-debug.log]`; `settings.json` sets `defaultProjectTrust: never` (repo-owned `.pi/` never loaded) and `defaultTools: [read, bash, edit, write, grep, find, ls]` (pi alone activates only the first four; `--tools`, when emitted, replaces the set); `PI_OFFLINE=1`/`PI_TELEMETRY=0`/`PI_SKIP_VERSION_CHECK=1`/`JITI_FS_CACHE=false` come from `EnvExports`. The loader environment (`JITI_FS_CACHE`, the `JITI_*`/`NODE_*` `unset` after `.env`) is pinned for the same reason the extension tree is hashed — see [Pi extensions](#pi-extensions-adr-0094). Context files (`AGENTS.md`) and skills stay on — they are the harness's own inputs. `PI_CODING_AGENT_DIR/extensions/` is arbitrary TypeScript loaded at startup and the config dir is not a permission boundary, which is why only the explicit `-e` paths load (at most one vendored provider extension plus the hook adapter). Right after `.env` is sourced, `Run` re-exports `EnvExports()` (`PI_CODING_AGENT_DIR`, the session dir, the offline switches) so a rewritten `.env` cannot relocate pi's config directory. For the built-in `openai` provider (`openai/`, [ADR 0092](../ADRs/0092-openai-wif-credential-delivery.md)) no extension loads and no `--api-key` is passed; `Run` instead seeds `auth.json` under `PI_CODING_AGENT_DIR` with the placeholder the sandbox environment carries for `OPENAI_API_KEY` (`PiOpenAIAuthSeed`, before `.env` is sourced), because pi re-reads that file on every revision change and resolves the key per request — the runner re-runs the same seed through `sandbox exec` after each credential refresh, which is what lets a running iteration follow a refresh on OpenShell 0.0.115, where a revision-scoped placeholder stays pinned to its generation and the unrevisioned alias is refused (`--api-key` would outrank the file and pin the iteration). It unsets `OPENAI_BASE_URL`/`AZURE_OPENAI_API_KEY`/`OPENAI_API_KEY`/`NODE_OPTIONS`/`NODE_PATH` after `.env`, fails the iteration with exit 1 when the environment holds anything but a gateway placeholder at seed time (before `.env`), and runs a config-dir integrity guard that exits 98 when `models.json` exists or `auth.json` is anything but pi's own `{}` or exactly the seeded placeholder entry — `models.json` is the only way to move the provider's base URL, a redirect to another allowed REST host is the placeholder-leak path ADR 0025 describes, and pi itself writes an empty `auth.json` on every start so only its content counts. The guard runs before the agent-writable `.env` is sourced and again after it behind `unset -f test command grep tr sed printf`, whether or not hooks are enabled. When the agent's `tools:` allow sub-agents, `-e /sandbox/pi-config/fullsend-agent.js` is appended after the hook adapter and two more pre-`.env` guards join the block: the Agent extension must be byte-identical to the embedded copy (exit 94, its own code so `Run` names that extension rather than the hook adapter), and `fullsend-manifest.json` must be byte-identical to the one `Bootstrap` wrote (exit 95, re-checked after `.env` behind `unset -f test [ command sha256sum cut` — `[` is in that list because the guard uses it, and the sandbox's `sh` is dash, which accepts `unset -f [`). The digest is then exported as `FULLSEND_PI_MANIFEST_SHA256` so `fullsend-hooks.js` can re-verify the manifest whenever it loads, including inside a sub-agent started later in the iteration — see [Agent tool contract](#agent-tool-contract). Children are launched by that extension, not by `Run`: same flag set, prompt on stdin, own session dir, no `Agent` tool of their own. - **`--mode json` exits 0 on model error** — only text mode maps `stopReason: error|aborted` to exit 1. `parsePiStream` is the intended detector (assistant `stopReason` on `message_end.message` / last `agent_end.messages` entry) for the runner's exit-0-override (#2786/#5361). `Run` tees the stream to `output.jsonl`, `ParseTranscriptFile` reads it, and `Run` itself returns 1 on a stream-reported error, so the override and the runtime agree. - **Exit code** — `Run` returns 1 when pi exited 0 but the stream's single `ResultEvent` reports an error (model error, incomplete stream), so the runner's exit-0 override and this agree; `ParseTranscriptFile` gives the same verdict from the tee'd `output.jsonl`. @@ -632,7 +638,91 @@ The pinned `pi` CLI and the vendored Vertex extensions ship in every sandbox ima - The Vertex model ids and the copied `compat` flags have not been exercised against Vertex — smoke an adaptive and a non-adaptive model first; override with `--model`/`FULLSEND_MODEL` if an id is rejected. - Parser fixtures are hand-authored to the v0.84.2 wire docs — re-record with `internal/runtime/testdata/pi/regen.sh` once a run exists; `extension_error` events are not mapped. - The behaviour scenario `features/runtime/pi.feature` (a real haiku run on Vertex of a minimal tool-using agent, asserting `metrics.json` `runtime: pi`, a `toolCall` in the pi session transcript and token usage) is gated on `BEHAVIOUR_CAPABILITIES=runtime-pi` until `fullsend-sandbox:latest` carries `PI_VERSION`; `features/triage/triage.feature` asserts the runtime selected from the repo config on every run. -- Pilot on a disposable org with `triage`/`prioritize` (no sub-agent assumptions) before `code`/`fix`. `review`/`retro` rely on Claude sub-agent rosters and are not supported: pi v0.84.2 has no sub-agent tool or `agents/*.md` concept in core — only the bundled example extension (`examples/extensions/subagent/`, spawns `pi -p --mode json` children without our hook adapter, Vertex provider, `--no-approve` or session dir) and the SDK route (`createAgentSession()` per child; parent extensions do not fire for children). A fullsend-owned sub-agent extension with the full child flag set is a follow-up tracked on #6527 (runtime parity backlog); until then `Bootstrap` appends a runtime note telling the agent no sub-agent tool exists and to execute sub-agent definitions itself, in order. +- Pilot on a disposable test repository with `triage`/`prioritize` before `code`/`fix` ([ADR 0044](../ADRs/0044-deprecate-per-org-installation-mode.md): per-repo is the only supported installation model). `review`/`retro` now run their real sub-agent roster through the runner-owned `Agent` tool (see [pi: Agent tool contract](#agent-tool-contract)), but that roster has been exercised locally, not on a fleet lifecycle run — watch the wall clock on the first one, and note that children default to `--thinking medium` for that reason. pi has no sub-agent tool or `agents/*.md` concept in core — fullsend supplies one as an extension; the bundled example extension (`examples/extensions/subagent/`) spawns children without the hook adapter, Vertex provider, `--no-approve` or a session dir, which is why fullsend ships its own. + +### Agent tool contract + +`Bootstrap` writes an `agent` block into `fullsend-manifest.json` and uploads the embedded +`fullsend-agent.js`, which registers Claude Code's `Agent` tool (and its legacy alias `Task`) so the +fleet's sub-agent skills dispatch unchanged. The block is written — and `Run` loads the extension +with `-e`, after the hook adapter — when the agent definition has no `tools:` frontmatter or lists +`Agent`/`Task`; otherwise the older "no sub-agent tool" runtime note is appended instead. + +| Manifest field | Meaning | +|---|---| +| `enabled` | The tool is registered. `Run` also gates the `-e` and the integrity guard on it | +| `piBin` | The `pi` binary children run, resolved in the sandbox at `Bootstrap` (`command -v pi`); `pi` when the probe found nothing | +| `sessionsDir` | Parent's session dir; child `` gets `--session-dir /agent-` | +| `extensions` | The `-e` list every child gets, in order: the vendored provider extensions actually present in the image (`test -d` at `Bootstrap`) then the hook adapter when security is on. Never `fullsend-agent.js` itself | +| `extensionDigests` | SHA-256 (hex) of each `extensions` entry `Bootstrap` itself wrote under the config dir — today only the hook adapter, and the same bytes the launch guard checks. Re-hashed before every dispatch. The vendored provider extensions are absent on purpose: root-owned and read-only in the image, outside anything the agent can write. Omitted when hooks are off, because then nothing in the list came from `Bootstrap` | +| `models` | `default` (the agent's model, translated) plus the Claude aliases on the Anthropic Vertex provider. The extension resolves a call's `model` through this table and **rejects** anything it cannot serve | +| `providerModels` | Per provider a run can serve with no `models` entry, the model ids it can serve: `google-vertex` verbatim from the catalog the pinned pi bundles (`@earendil-works/pi-ai` `dist/providers/data/google-vertex.json`), and `xai-vertex` from the ids the vendored extension registers (`xai/grok-4.6`, the publisher-qualified wire form). A `provider/id` call is accepted only when the full spec is in `models`, is the parent's own spec, or is listed here — a provider prefix alone is not enough, or an invented id would reach the API. A Grok spec is normalized to `xai-vertex/xai/` first, exactly as `normalizeXaiVertexModel` does for the parent, and then goes through this same set | +| `thinking` | Children's `--thinking`; `FULLSEND_PI_SUBAGENT_THINKING` (validated) else `medium` | +| `tools` / `exploreTools` | The child `--tools` allowlist: the parent's built-ins minus `Agent`/`Task`, or — for `subagent_type: Explore` — the read-only set **intersected with** the parent's, so a child never reaches past its parent. An empty result becomes `--no-builtin-tools`, as it does for the parent | +| `maxConcurrent`, `timeoutSeconds` | Semaphore and per-child deadline (4, 900 s) | +| `usageFile` | Where the extension appends one line per child; `Run` folds it into `RunMetrics` and `ExtractTranscripts` downloads it | + +- Children run with `FULLSEND_SUBAGENT_DEPTH=1` and the extension refuses to register when that + variable is already set, so recursion is impossible even if the `-e` list were tampered with. +- The extension's code is SHA-256-checked against the embedded copy in the same pre-`.env` guard + block as the hook adapter, under its own exit 94 so `Run` can name the artifact that actually + failed, and **the manifest it reads is checked the same way** + (exit 95, both before and after `.env` — `.env` is agent-writable and could rewrite it between + the two, the same reason `piOpenAIConfigGuard` runs twice). Those are launch checks, so the + manifest is re-verified twice more during the iteration: by the Agent extension against the + bytes it loaded, immediately before each dispatch, and by `fullsend-hooks.js` against + `FULLSEND_PI_MANIFEST_SHA256` whenever it loads — which in a sub-agent is long after the + launch guard ran. Both are security controls: the + manifest names the binary children run, their `-e` list, their tool allowlists and where their + usage is recorded, and the config dir is agent-writable between iterations — so an unchecked + manifest would let an agent with `Write` but no `Bash` (which cannot start `pi` itself) launch + children of its own choosing. +- **The hook adapter gets the same per-dispatch treatment**, through `agent.extensionDigests`. + The launch guard checks it once; a parent with `write` can replace it mid-iteration, and a + rewritten adapter simply drops its own `FULLSEND_PI_MANIFEST_SHA256` check, so verifying only + the manifest would leave the adapter unprotected and every later child unhooked. `Bootstrap` + therefore records the SHA-256 of each `agent.extensions` entry it wrote under the config dir + (today just the adapter, the same bytes `piHooksGuard` checks) into the manifest, and the Agent + extension re-hashes them alongside the manifest before every dispatch, inside the slot the + dispatch holds, refusing with `hook adapter changed since load; refusing to dispatch`. The map + is part of the manifest, so the manifest digest covers it. The vendored provider extensions + under `/usr/local/share/pi-extensions` need no digest — the image installs them root-owned and + read-only outside the config dir. +- **`Bootstrap` and `Run` must run in one process** for any of the manifest guards to exist. The + digest lives in a package-level map keyed by sandbox name (`piManifestHashes`), recorded by + `Bootstrap` and read by `Run`; the CLI's `run` path does both in one process. A caller that + bootstrapped elsewhere gets no entry, and then neither the shell guard nor the + `FULLSEND_PI_MANIFEST_SHA256` export is emitted — silently, because failing closed there would + break every such caller. The per-dispatch checks above are unaffected: they compare against + bytes the extension read itself. +- The hook adapter's `*` PreToolUse groups therefore see `Agent` calls, and the manifest's + `hooks.toolNames` carries `Agent`/`Task` verbatim (they are already Claude vocabulary). Claude + Code runs the same hooks on its own `Agent` tool. +- The prompt is delivered on the child's **stdin**, never as a positional argument. Unterminated, + pi's argv parser reads a leading `-` as an unknown option (a startup error), a leading `--` as an + unknown flag that swallows the next word and a leading `@` as a file argument. pi *does* honour a + `--` end-of-options terminator (`dist/cli/args.js`), but that is not a way out: after it an + `@`-prefixed positional is still taken as a file argument, and argv is capped by the kernel + either way (`spawn E2BIG` past ~128 KiB), which a context package exceeds. In `--print` mode pi + reads a non-TTY stdin to EOF and uses it as the initial message (`dist/main.js` + `readPipedStdin`, `dist/cli/initial-message.js` `buildInitialMessage`, 0.84.4). +- Children get `--append-system-prompt` with a short sub-agent role note. They share the parent's + `PI_CODING_AGENT_DIR`, and pi only discovers `APPEND_SYSTEM.md` there when no + `--append-system-prompt` was passed (`dist/core/resource-loader.js`), so without the flag every + child would inherit the parent's orchestrator persona — including its "make several `Agent` calls + in one message" dispatch note, for a tool it does not have. +- Children are spawned with the parent's process group (**not** `detached`), so a killed parent + cannot leave them spending tokens. A child that times out, is aborted, or is caught by + `session_shutdown` gets **`SIGTERM` first**, escalated to `SIGKILL` after a 3 s grace. `SIGTERM` + is what makes the child clean up after itself: pi's own bash tool spawns commands `detached` in + their own process groups and reaps them from its `SIGTERM` handler before exiting 143 + (`dist/modes/print-mode.js` → `killTrackedDetachedChildren`). Signalling the child's process group + with `SIGKILL` instead would leave those grandchildren running. +- The child's environment is rebuilt per resolved provider with the same rules `buildPiRunCommand` + applies to the parent (`ANTHROPIC_*` cleared and `GOOGLE_CLOUD_PROJECT` pinned for + `anthropic-vertex`; `XAI_API_KEY` cleared and `XAI_VERTEX_PROJECT_ID` pinned for `xai-vertex`). + The shell hygiene only ever matched the *parent's* provider, so without this a Claude child under + a Grok parent would run with a stray `ANTHROPIC_API_KEY`. ### OpenAI via Workload Identity Federation diff --git a/docs/guides/getting-started/choosing-a-runtime.md b/docs/guides/getting-started/choosing-a-runtime.md index 155a5ea454..515f157ce1 100644 --- a/docs/guides/getting-started/choosing-a-runtime.md +++ b/docs/guides/getting-started/choosing-a-runtime.md @@ -4,7 +4,7 @@ sidebar_label: Choose a Runtime # Choose an agent runtime -> **Claude Code is the stable default.** The fleet agents have run on Claude Code in production for a long time; it is what a new installation gets unless you ask for something else. **pi is in its enablement (experimental) phase** — it works end to end for `triage`, `prioritize`, `code` and `fix`, has no sub-agent tool yet (`review`/`retro` run in a single context), and its fleet pilot is still in progress. Unless you are taking part in that pilot, keep the default. +> **Claude Code is the stable default.** The fleet agents have run on Claude Code in production for a long time; it is what a new installation gets unless you ask for something else. **pi is in its enablement (experimental) phase** — it works end to end for `triage`, `prioritize`, `code` and `fix`, and `review`/`retro` now dispatch their real sub-agent roster through a fullsend-supplied `Agent`/`Task` tool, but its fleet pilot is still in progress. Unless you are taking part in that pilot, keep the default. This page explains what the choice means and where it is made. **You do not select anything on this page** — the selection happens in the next step, [Configuring GitHub](configuring-github.md), when `fullsend github setup` prompts for the runtime (press Enter for `claude`) or when you pass `--runtime`. @@ -15,7 +15,7 @@ Fullsend supports multiple agent runtimes. A runtime is the program that runs in | Runtime | Status | Description | When to use | |---------|--------|-------------|-------------| | `claude` | **Stable (default)** | Claude Code on Vertex AI | Every production deployment — mature, full sub-agent support for `review`/`retro` | -| `pi` | Experimental (enablement phase) | [Pi](https://github.com/earendil-works/pi) — Claude on Vertex by default; any provider pi supports by model name (e.g. Gemini on Vertex with the same credentials) | Opt-in pilots only; no sub-agent tool yet, so `review`/`retro` run single-context; see [Runtimes](../../runtimes.md) for known constraints | +| `pi` | Experimental (enablement phase) | [Pi](https://github.com/earendil-works/pi) — Claude on Vertex by default; any provider pi supports by model name (e.g. Gemini on Vertex with the same credentials) | Opt-in pilots only; `Agent`/`Task` sub-agents come from a fullsend extension (children are `pi` processes) rather than from pi itself; see [Runtimes](../../runtimes.md) for known constraints | ## When and how the runtime is selected diff --git a/docs/runtimes.md b/docs/runtimes.md index a7e29b5359..5a0861ddda 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -7,7 +7,7 @@ sandbox, the credentials, and the verdict. | Runtime | Use it for | Status | |---|---|---| | **[`claude`](runtimes/claude.md)** | Production agent runs (Claude Code) | Default | -| **[`pi`](runtimes/pi.md)** | Second runtime, opt-in per repo — Claude, Grok and Gemini on Vertex; GPT via OpenAI WIF (wired, not yet exercised live) | Supported for `triage`, `prioritize`, `code`, `fix` | +| **[`pi`](runtimes/pi.md)** | Second runtime, opt-in per repo — Claude, Grok and Gemini on Vertex; GPT via OpenAI WIF (wired, not yet exercised live) | Supported for all roles | | `dummy` | Behaviour tests — scripted ops, no inference | Internal | | `opencode` | Not yet functional | Stub | @@ -50,15 +50,15 @@ sequenceDiagram | | Claude Code | pi | |---|---|---| | Models | Anthropic on Vertex | Claude, **Grok** and **Gemini** on Vertex; **GPT** via OpenAI WIF (opt-in, [not yet exercised live](runtimes/pi.md#models-and-providers)) | -| Sub-agents | Native (`Agent` tool) | Not wired — agents execute sub-agent definitions inline ([#6527](https://github.com/fullsend-ai/fullsend/issues/6527)) | +| Sub-agents | Native (`Agent` tool) | `Agent`/`Task` via a fullsend extension — children are `pi` processes with the same hooks, providers and tool allowlist ([pi runtime § Sub-agents](runtimes/pi.md#sub-agents)) | | Fallback model chain | `FULLSEND_FALLBACK_MODELS`, tried in order | Ignored with a warning | -| Roles | All | `review`/`retro` stay on Claude Code — they rely on sub-agent rosters | +| Roles | All | All; `review`/`retro` run their real sub-agent roster, at `--thinking medium` by default | | Effort | `--effort low..max` | `--thinking`, same levels (`high` when unset) | | Security controls | Full matrix | Full matrix; stricter on failed-call sanitizing | Both run unattended in the same sandbox, on the same WIF credentials, behind the same egress -allowlist. Choose `pi` when you want a non-Anthropic model; stay on `claude` when you need -sub-agents or a fallback chain. +allowlist. Choose `pi` when you want a non-Anthropic model; stay on `claude` when you need a +fallback chain. ## Selecting a runtime and model diff --git a/docs/runtimes/claude.md b/docs/runtimes/claude.md index 017f4f379e..623027dfd5 100644 --- a/docs/runtimes/claude.md +++ b/docs/runtimes/claude.md @@ -28,7 +28,7 @@ unsupported and ignores it. | | | |---|---| -| Roles | All, including `review` and `retro` — they need sub-agents | +| Roles | All, including `review` and `retro` — they need sub-agents (pi covers these too, through a fullsend extension: [pi § Sub-agents](pi.md#sub-agents)) | | Credentials | WIF `external_account` + a refreshed OIDC token; `ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_BASE_URL` and `ANTHROPIC_VERTEX_BASE_URL` are unset so a stray key cannot redirect traffic | | Unattended | `--dangerously-skip-permissions`; hooks wired from the harness, never from agent-writable files | | Artifacts | `output.jsonl`, transcripts, `metrics.json` with `runtime: claude`, and `claude-debug.log` with `--debug` | @@ -41,8 +41,7 @@ These are the places Claude Code differs from pi — useful when comparing a run - **The agent definition *replaces* the system prompt.** `--agent` makes the agent `.md` body the system prompt outright. pi appends it to its own default instead, so an agent that relies on Claude Code's exact framing can read differently there. -- **Native sub-agents** via the `Agent` tool, which is why `review` and `retro` are Claude-only - today. +- **Native sub-agents** via the `Agent` tool. This is no longer Claude-only: pi serves the same `Agent`/`Task` contract from a runner-owned extension that runs each child as its own `pi` process ([pi § Sub-agents](pi.md#sub-agents)). What stays Claude-specific is that the sub-agents are *native* — no child process, no separate session dir, no per-child provider hygiene. - **A `CLAUDE.md` bridge is injected** when the repo has `AGENTS.md` but no `CLAUDE.md`, because Claude Code auto-loads only the former. pi reads `AGENTS.md` natively and needs no bridge. - **`tools:` is enforced unreliably** (≥ 2.1.119); pi enforces its `--tools` allowlist strictly. In diff --git a/docs/runtimes/pi.md b/docs/runtimes/pi.md index 52222d6bc7..48a7e06413 100644 --- a/docs/runtimes/pi.md +++ b/docs/runtimes/pi.md @@ -1,7 +1,6 @@ # Pi -[pi](https://github.com/earendil-works/pi) is fullsend's second agent runtime, opt-in per org or -repo. It reaches models Claude Code cannot — **Grok** and **Gemini** alongside Claude — through the +[pi](https://github.com/earendil-works/pi) is fullsend's second agent runtime, opt-in per repo. It reaches models Claude Code cannot — **Grok** and **Gemini** alongside Claude — through the same sandbox, credentials and egress policy. ```bash @@ -85,10 +84,11 @@ endpoints answer `FAILED_PRECONDITION` — so region variables are deliberately |---|---| | Credentials | Same WIF `external_account` + refreshed OIDC token as Claude Code for Vertex providers. `ANTHROPIC_*` unset on the Claude provider, `XAI_API_KEY` unset on the Grok one; `OPENAI_BASE_URL`/`AZURE_OPENAI_API_KEY` unset on the OpenAI one. OpenAI uses a runner-exchanged WIF token (ADR 0092) | | Unattended | No approval prompts, stdin closed, bounded retries; a missing credential exits 1 | -| Artifacts | `output.jsonl`, `transcripts/-_.jsonl`, `metrics.json` with `runtime: pi`, plus `pi-debug.log` with `--debug` | -| Extra knobs | `FULLSEND_PI_PROVIDER` (prefix for bare ids), `FULLSEND_PI_BASH_ALLOWLIST=enforce` | +| Artifacts | `output.jsonl`, `transcripts/-_.jsonl` (plus `-sub-…` per sub-agent and `-subagents-usage.jsonl`), `metrics.json` with `runtime: pi`, plus `pi-debug.log` with `--debug` | +| Extra knobs | `FULLSEND_PI_PROVIDER` (prefix for bare ids), `FULLSEND_PI_BASH_ALLOWLIST=enforce`, `FULLSEND_PI_SUBAGENT_THINKING` | | Extensions | Harness `extensions:` directories, uploaded and loaded with `-e` after a tree-hash preflight ([Extensions](#extensions)) | -| Not supported | Sub-agents, fallback chains, `plugins:`, Bedrock/Azure providers | +| Sub-agents | `Agent` (alias `Task`) via a fullsend extension: children are `pi` processes with the same hooks, providers and tool allowlist ([Sub-agents](#sub-agents)) | +| Not supported | Fallback chains, `plugins:`, Bedrock/Azure providers | ## Running it locally @@ -140,8 +140,9 @@ What a local pi run needs, beyond the guide: `podman pull ghcr.io/fullsend-ai/fullsend-sandbox:latest` fixes it. - **Platforms** — verified end to end on macOS Apple Silicon (podman machine, Homebrew `openshell`) and Fedora with rootless Podman; the guide's platform notes apply unchanged. -- **`review` and `retro`** complete with schema-valid results but in a single context — pi has no - sub-agent tool, so the parallel reviewer roster is not exercised (see [Not yet exercised](#not-yet-exercised)). +- **`review` and `retro`** run their real sub-agent roster through the `Agent` tool; the children + default to `--thinking medium`, which keeps the roster inside the 20-minute review budget (see + [Sub-agents](#sub-agents)). - **Knobs** — `FULLSEND_PI_PROVIDER` sets the provider for bare model ids (default `anthropic-vertex`); `FULLSEND_PI_BASH_ALLOWLIST=enforce` makes the Bash first-token allowlist block instead of warn. @@ -265,13 +266,131 @@ How the runner protects this path — the tree hash, the loader cache, the symli deny-list — is in [Runtime Implementation § Pi extensions](../contributing/runtime-implementation.md#pi-extensions-adr-0094). +## Sub-agents + +The `Agent` tool (registered under its legacy alias `Task` as well) is provided by a +runner-owned pi extension, `fullsend-agent.js`, so skills written for Claude Code's sub-agent +roster — `pr-review`, `retro-analysis` — dispatch unchanged. + +**Contract.** Same parameters as Claude Code's tool: + +| Parameter | Meaning | +|---|---| +| `prompt` (required) | The whole task. The child starts with no memory of the conversation, so the prompt must carry its context package | +| `description` | Short label; shows in the run log | +| `model` | `opus`, `sonnet`, `haiku`, a Claude id (`claude-sonnet-4-6@default` — the `@suffix` is dropped) or a `provider/id` spec this run can serve. That is a closed set, not a provider check: the specs in the run's model table, the parent's own spec, and the ids listed for a provider with no model-table entry (`google-vertex`, and the vendored `xai-vertex` extension's `xai/grok-4.6`). An id the model invented is rejected even under an allowed provider. Omitted → the parent's model. A `:` suffix (pi's `provider/id:high` shorthand) is dropped, so it cannot override the child's thinking level | +| `subagent_type` | `Explore` gives a read-only child (`read`, `grep`, `find`, `ls`, intersected with the parent's set — a child never reaches past its parent); anything else or omitted gives the parent's tool set | +| `run_in_background` | Accepted and ignored — a child always runs to completion inside the call | + +The result is the child's final assistant message, trimmed and capped at 64 KB with a +`[truncated]` marker. The call is an error when the child's stream reports `stopReason` +`error`/`aborted`, when it exits non-zero, times out (15 minutes) or never emits `agent_end`. + +A model the run cannot serve is **rejected with the accepted forms** rather than passed through: +an invented Claude id would otherwise reach pi's built-in `anthropic` provider, which has no +credentials in the sandbox, and the dispatch would be lost to a confusing auth error. The same +applies inside a provider the run *can* reach — `anthropic-vertex/claude-sonnet-4-20250514`, +`google-vertex/gemini-9` or `xai/grok-9` is refused rather than sent on as an unknown model. A +Grok spec is normalized to the three-segment `xai-vertex/xai/` first (as the runner does for +the parent) and then checked against that same closed set, so the short `xai/` form is neither +more nor less permissive than the long one. + +**Parallel dispatch** is several `Agent` calls in one assistant message — pi runs sibling tool +calls from one message concurrently. At most four children run at once; the rest queue. The +runtime note appended to the agent's system prompt says so, so a skill that asks for "dispatch +these in parallel" gets it. + +**What a child inherits.** Each child is a `pi --print --mode json` process started with the same +posture as the parent: `--no-approve`, `--no-extensions` with an explicit `-e` list, no prompt +templates or themes, its own `--session-dir`, and a `--tools` allowlist (or `--no-builtin-tools` +when that allowlist is empty). The `-e` list is the +vendored provider extensions present in the image plus the hook adapter when security is enabled, +so **PreToolUse/PostToolUse hooks and the Bash allowlist apply inside sub-agents too** — as they do +on Claude Code, where the same hooks run on `Agent` calls. Children never receive the Agent +extension itself and refuse to register it if they somehow did (`FULLSEND_SUBAGENT_DEPTH`), so +there is no recursion. + +What a child does **not** inherit: + +- **Harness `extensions:`.** The declared extension directories are loaded for the parent only. A + child's `-e` list is fixed at `Bootstrap` (providers plus the hook adapter), so a tool a harness + extension registers is not available inside a sub-agent. +- **The parent's system prompt.** Children get `--append-system-prompt` with a short sub-agent role + note instead of the parent's `APPEND_SYSTEM.md`, which is the orchestrator persona and tells it to + make several `Agent` calls in one message — advice a child cannot act on. +- **Provider credentials it does not use.** The child's environment is rebuilt per resolved + provider with the same rules the runner applies to the parent, so a Claude child under a Grok + parent does not carry a stray `ANTHROPIC_API_KEY` or an ambient `GOOGLE_CLOUD_PROJECT`. + +**Prompt delivery.** The prompt goes over the child's stdin, not on its command line. Unterminated, +pi reads a leading `-` as an unknown option, a leading `--` as an unknown flag and a leading `@` +as a file argument. pi *does* honour a `--` end-of-options terminator, but that is not a way out: +after it an `@`-prefixed positional is still taken as a file argument, and argv is capped by the +kernel well below the size of a real context package either way. In `--print` mode pi reads a +non-TTY stdin to EOF and uses it as the initial message, whatever it starts with and however +long it is. + +**Stopping a child.** A child that times out (15 minutes), is aborted, or is caught by +`session_shutdown` gets `SIGTERM`, escalated to `SIGKILL` after a 3-second grace. `SIGTERM` is +deliberate: pi's `bash` tool runs commands in their own process groups and reaps them from its own +`SIGTERM` handler, so an immediate `SIGKILL` would orphan them. Children are not `detached`, so a +process-group kill or a `SIGTERM` to the parent takes them with it — a bare `SIGKILL` of the +parent alone does not, since nothing is left to run the stop sequence. + +**`.env` is the same trust class here as everywhere else.** A rewritten `/sandbox/workspace/.env` +that exports `FULLSEND_SUBAGENT_DEPTH` disables the tool for that iteration — the extension reads +it as "this process is a child" and registers nothing. That is the same exposure as any other +`.env`-settable knob; what `.env` cannot do is change the manifest, the Agent extension or the hook +adapter. All three are SHA-256-checked before `.env` is sourced and the manifest again after it, +and because those checks all happen at launch they are repeated while the iteration runs. + +Immediately before **every** dispatch, and while holding the concurrency slot the dispatch will +run in, the Agent extension re-hashes two things: the manifest, against the bytes it loaded +(`manifest changed since load; refusing to dispatch`), and every `agent.extensions` entry that +lives in the config dir — today the hook adapter — against the digest `Bootstrap` recorded for it +in `agent.extensionDigests` (`hook adapter changed since load; refusing to dispatch`). The adapter +needs its own digest because nothing else re-verifies it after launch: a parent with `write` can +replace the file mid-iteration, and a rewritten adapter simply omits the manifest check below, so +every child dispatched afterwards would come up unhooked. The digest map travels inside the +manifest, so the manifest's own digest covers it. The vendored provider extensions under +`/usr/local/share/pi-extensions` carry no digest — the image installs them root-owned and +read-only outside the config dir, so there is nothing there to rewrite. + +The hook adapter in turn verifies the manifest against the digest the runner exported +(`FULLSEND_PI_MANIFEST_SHA256`) whenever it loads — which for a sub-agent is minutes in, so a +parent that rewrites `hooks.groups` mid-iteration cannot dispatch a child whose adapter comes up +with no hooks in it. + +**Thinking level.** Children run at `--thinking medium`, not the parent's `high`: a full +`pr-review` roster at `high` overran the 20-minute review budget. Override with +`FULLSEND_PI_SUBAGENT_THINKING=`; an unrecognised value +warns and falls back to `medium`. + +**Where the output lands.** A child's session file is extracted as +`transcripts/-sub-.jsonl` (the sequence number is the call's, so children +with the same session basename do not collide), and one JSON line per child — model, usage, stop +reason, duration — as `transcripts/-subagents-usage.jsonl`. The runner folds that file into +`metrics.json`: the totals include the children, and `per_model_usage` attributes them per model +spec, with the parent's own iteration as one entry so the breakdown sums to the totals +([`fullsend run` § metrics.json](../cli/run.md#per-model-usage)). The parent entry is recorded on every +iteration that ran with the `Agent` tool enabled, dispatching or not, because iterations are +summed. A record with no model spec is bucketed under `unknown`. The usage file is consumed as it is +read (renamed to `usage.jsonl.read`), so a retry cannot count the same children twice. The run log +carries a `[fullsend-agent] # start ""` / `done ` pair +per child. + +**Turning it off.** The tool is enabled when the agent's definition has no `tools:` frontmatter (the +default set, as under Claude Code) or lists `Agent`/`Task`. An agent that lists tools without them +gets no Agent tool and the older runtime note telling it to execute sub-agent definitions itself, +in order. + ## Not yet exercised `runtime: pi` is selectable and has been run end to end, but no **fleet lifecycle** run on Vertex is -recorded yet. Pilot on a disposable repo with `triage`/`prioritize` before `code`/`fix`. `review` and -`retro` run to schema-valid results, but in a **single context**: pi has no sub-agent tool, so the -parallel persona roster and its per-persona models are never exercised — treat them as unsupported -for that purpose. `extension_error` events are not mapped. +recorded yet. Pilot on a disposable repo with `triage`/`prioritize` before `code`/`fix`. The +sub-agent roster of `review`/`retro` has been exercised locally, not yet on a fleet lifecycle run — +watch the wall clock on the first one (see [Sub-agents](#sub-agents)). `extension_error` events are +not mapped. ## Troubleshooting From ca12f63c0674c9448ca305fbe2924cb5c3feca1b Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Sat, 29 Aug 2026 17:16:21 -0400 Subject: [PATCH 7/7] docs(pi): make the sub-agent docs a walkthrough and move internals to contributing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same split as the extensions section: the `Sub-agents` section on the pi runtime page was written as security design notes — "Prompt delivery", "Stopping a child", "`.env` is the same trust class here as everywhere else", the manifest/adapter digest re-hash sequencing — so an agent author who just wants to use `Agent` could not read it straight through. - `docs/runtimes/pi.md` § Sub-agents is now a walkthrough: a parameter table, choosing a model (four accepted forms and the one rule that rejects the rest), running children in parallel, what a child does and does not inherit, thinking level, where the output lands, turning it off, and a symptom/cause/fix table. 118 lines down to 102. - `docs/contributing/runtime-implementation.md` § Agent tool contract is renamed "Pi sub-agents: the Agent tool contract" so it sits beside the Pi extensions subsection, and gains the two facts the user page no longer carries: the usage file is consumed as it is read (`.read`, capped at 1 MiB) so a retry cannot double-count children, and the `.env` trust discussion around `FULLSEND_SUBAGENT_DEPTH`. Everything else the user page dropped — stdin prompt delivery and the argv hazards, the SIGTERM-then-SIGKILL stop sequence and process groups, the per-dispatch digest re-checks — was already documented there and is now linked rather than restated. `docs/cli/run.md` § Per-model usage already read cleanly and is unchanged. Assisted-by: Claude Signed-off-by: Wayne Sun --- docs/contributing/runtime-implementation.md | 21 +- docs/runtimes/pi.md | 208 +++++++++----------- 2 files changed, 113 insertions(+), 116 deletions(-) diff --git a/docs/contributing/runtime-implementation.md b/docs/contributing/runtime-implementation.md index d1c2671926..6648b7949b 100644 --- a/docs/contributing/runtime-implementation.md +++ b/docs/contributing/runtime-implementation.md @@ -414,11 +414,11 @@ Parity with `claude -p --dangerously-skip-permissions`, verified against pi v0.8 - `--no-approve` sets the project-trust override, so the trust-gated project resources — `.pi/{settings.json,extensions,skills,prompts,themes,SYSTEM.md,APPEND_SYSTEM.md}` and `.agents/skills` (`core/trust-manager.ts`); `AGENTS.md` itself is still read as context — are ignored without a dialog (`cli/args.ts`, `main.ts`). `defaultProjectTrust: never` in the global settings covers the no-flag case (verified on the pinned build: a planted `.pi/extensions/evil.js` in the repo does not load under `--no-approve` and does under `--approve`). - First-run setup, theme selection, telemetry consent and the version check are interactive-only code paths (`PI_TELEMETRY=0`, `PI_SKIP_VERSION_CHECK=1`/`PI_OFFLINE=1` set anyway). - A missing credential raises `No API key found` and exits 1 — no `/login` prompt (`core/agent-session.ts`, `modes/print-mode.ts`); retries are bounded (`retry.maxRetries: 3`, 2/4/8 s) and compaction is automatic. -- **The one blocker found:** print mode reads a non-TTY stdin to EOF before the first prompt, even with a positional message (`main.ts` `readPipedStdin`), so an exec that keeps stdin open with no writer hangs pi — `Run` therefore appends ` --thinking '' >/sandbox/workspace/pi-debug.log]`; `settings.json` sets `defaultProjectTrust: never` (repo-owned `.pi/` never loaded) and `defaultTools: [read, bash, edit, write, grep, find, ls]` (pi alone activates only the first four; `--tools`, when emitted, replaces the set); `PI_OFFLINE=1`/`PI_TELEMETRY=0`/`PI_SKIP_VERSION_CHECK=1`/`JITI_FS_CACHE=false` come from `EnvExports`. The loader environment (`JITI_FS_CACHE`, the `JITI_*`/`NODE_*` `unset` after `.env`) is pinned for the same reason the extension tree is hashed — see [Pi extensions](#pi-extensions-adr-0094). Context files (`AGENTS.md`) and skills stay on — they are the harness's own inputs. `PI_CODING_AGENT_DIR/extensions/` is arbitrary TypeScript loaded at startup and the config dir is not a permission boundary, which is why only the explicit `-e` paths load (at most one vendored provider extension plus the hook adapter). Right after `.env` is sourced, `Run` re-exports `EnvExports()` (`PI_CODING_AGENT_DIR`, the session dir, the offline switches) so a rewritten `.env` cannot relocate pi's config directory. For the built-in `openai` provider (`openai/`, [ADR 0092](../ADRs/0092-openai-wif-credential-delivery.md)) no extension loads and no `--api-key` is passed; `Run` instead seeds `auth.json` under `PI_CODING_AGENT_DIR` with the placeholder the sandbox environment carries for `OPENAI_API_KEY` (`PiOpenAIAuthSeed`, before `.env` is sourced), because pi re-reads that file on every revision change and resolves the key per request — the runner re-runs the same seed through `sandbox exec` after each credential refresh, which is what lets a running iteration follow a refresh on OpenShell 0.0.115, where a revision-scoped placeholder stays pinned to its generation and the unrevisioned alias is refused (`--api-key` would outrank the file and pin the iteration). It unsets `OPENAI_BASE_URL`/`AZURE_OPENAI_API_KEY`/`OPENAI_API_KEY`/`NODE_OPTIONS`/`NODE_PATH` after `.env`, fails the iteration with exit 1 when the environment holds anything but a gateway placeholder at seed time (before `.env`), and runs a config-dir integrity guard that exits 98 when `models.json` exists or `auth.json` is anything but pi's own `{}` or exactly the seeded placeholder entry — `models.json` is the only way to move the provider's base URL, a redirect to another allowed REST host is the placeholder-leak path ADR 0025 describes, and pi itself writes an empty `auth.json` on every start so only its content counts. The guard runs before the agent-writable `.env` is sourced and again after it behind `unset -f test command grep tr sed printf`, whether or not hooks are enabled. When the agent's `tools:` allow sub-agents, `-e /sandbox/pi-config/fullsend-agent.js` is appended after the hook adapter and two more pre-`.env` guards join the block: the Agent extension must be byte-identical to the embedded copy (exit 94, its own code so `Run` names that extension rather than the hook adapter), and `fullsend-manifest.json` must be byte-identical to the one `Bootstrap` wrote (exit 95, re-checked after `.env` behind `unset -f test [ command sha256sum cut` — `[` is in that list because the guard uses it, and the sandbox's `sh` is dash, which accepts `unset -f [`). The digest is then exported as `FULLSEND_PI_MANIFEST_SHA256` so `fullsend-hooks.js` can re-verify the manifest whenever it loads, including inside a sub-agent started later in the iteration — see [Agent tool contract](#agent-tool-contract). Children are launched by that extension, not by `Run`: same flag set, prompt on stdin, own session dir, no `Agent` tool of their own. +- **Hardening levers in use** — `Run` executes `pi --print --mode json --no-approve --no-extensions --no-prompt-templates --no-themes --session-dir /sandbox/pi-config/sessions [-e /usr/local/share/pi-extensions/anthropic-vertex | -e /usr/local/share/pi-extensions/xai-vertex] [-e /sandbox/pi-config/fullsend-hooks.js] [-e /sandbox/pi-config/fullsend-agent.js] [--tools ...] --model --thinking '' >/sandbox/workspace/pi-debug.log]`; `settings.json` sets `defaultProjectTrust: never` (repo-owned `.pi/` never loaded) and `defaultTools: [read, bash, edit, write, grep, find, ls]` (pi alone activates only the first four; `--tools`, when emitted, replaces the set); `PI_OFFLINE=1`/`PI_TELEMETRY=0`/`PI_SKIP_VERSION_CHECK=1`/`JITI_FS_CACHE=false` come from `EnvExports`. The loader environment (`JITI_FS_CACHE`, the `JITI_*`/`NODE_*` `unset` after `.env`) is pinned for the same reason the extension tree is hashed — see [Pi extensions](#pi-extensions-adr-0094). Context files (`AGENTS.md`) and skills stay on — they are the harness's own inputs. `PI_CODING_AGENT_DIR/extensions/` is arbitrary TypeScript loaded at startup and the config dir is not a permission boundary, which is why only the explicit `-e` paths load (at most one vendored provider extension plus the hook adapter). Right after `.env` is sourced, `Run` re-exports `EnvExports()` (`PI_CODING_AGENT_DIR`, the session dir, the offline switches) so a rewritten `.env` cannot relocate pi's config directory. For the built-in `openai` provider (`openai/`, [ADR 0092](../ADRs/0092-openai-wif-credential-delivery.md)) no extension loads and no `--api-key` is passed; `Run` instead seeds `auth.json` under `PI_CODING_AGENT_DIR` with the placeholder the sandbox environment carries for `OPENAI_API_KEY` (`PiOpenAIAuthSeed`, before `.env` is sourced), because pi re-reads that file on every revision change and resolves the key per request — the runner re-runs the same seed through `sandbox exec` after each credential refresh, which is what lets a running iteration follow a refresh on OpenShell 0.0.115, where a revision-scoped placeholder stays pinned to its generation and the unrevisioned alias is refused (`--api-key` would outrank the file and pin the iteration). It unsets `OPENAI_BASE_URL`/`AZURE_OPENAI_API_KEY`/`OPENAI_API_KEY`/`NODE_OPTIONS`/`NODE_PATH` after `.env`, fails the iteration with exit 1 when the environment holds anything but a gateway placeholder at seed time (before `.env`), and runs a config-dir integrity guard that exits 98 when `models.json` exists or `auth.json` is anything but pi's own `{}` or exactly the seeded placeholder entry — `models.json` is the only way to move the provider's base URL, a redirect to another allowed REST host is the placeholder-leak path ADR 0025 describes, and pi itself writes an empty `auth.json` on every start so only its content counts. The guard runs before the agent-writable `.env` is sourced and again after it behind `unset -f test command grep tr sed printf`, whether or not hooks are enabled. When the agent's `tools:` allow sub-agents, `-e /sandbox/pi-config/fullsend-agent.js` is appended after the hook adapter and two more pre-`.env` guards join the block: the Agent extension must be byte-identical to the embedded copy (exit 94, its own code so `Run` names that extension rather than the hook adapter), and `fullsend-manifest.json` must be byte-identical to the one `Bootstrap` wrote (exit 95, re-checked after `.env` behind `unset -f test [ command sha256sum cut` — `[` is in that list because the guard uses it, and the sandbox's `sh` is dash, which accepts `unset -f [`). The digest is then exported as `FULLSEND_PI_MANIFEST_SHA256` so `fullsend-hooks.js` can re-verify the manifest whenever it loads, including inside a sub-agent started later in the iteration — see [Pi sub-agents](#pi-sub-agents-the-agent-tool-contract). Children are launched by that extension, not by `Run`: same flag set, prompt on stdin, own session dir, no `Agent` tool of their own. - **`--mode json` exits 0 on model error** — only text mode maps `stopReason: error|aborted` to exit 1. `parsePiStream` is the intended detector (assistant `stopReason` on `message_end.message` / last `agent_end.messages` entry) for the runner's exit-0-override (#2786/#5361). `Run` tees the stream to `output.jsonl`, `ParseTranscriptFile` reads it, and `Run` itself returns 1 on a stream-reported error, so the override and the runtime agree. - **Exit code** — `Run` returns 1 when pi exited 0 but the stream's single `ResultEvent` reports an error (model error, incomplete stream), so the runner's exit-0 override and this agree; `ParseTranscriptFile` gives the same verdict from the tee'd `output.jsonl`. @@ -638,9 +638,9 @@ The pinned `pi` CLI and the vendored Vertex extensions ship in every sandbox ima - The Vertex model ids and the copied `compat` flags have not been exercised against Vertex — smoke an adaptive and a non-adaptive model first; override with `--model`/`FULLSEND_MODEL` if an id is rejected. - Parser fixtures are hand-authored to the v0.84.2 wire docs — re-record with `internal/runtime/testdata/pi/regen.sh` once a run exists; `extension_error` events are not mapped. - The behaviour scenario `features/runtime/pi.feature` (a real haiku run on Vertex of a minimal tool-using agent, asserting `metrics.json` `runtime: pi`, a `toolCall` in the pi session transcript and token usage) is gated on `BEHAVIOUR_CAPABILITIES=runtime-pi` until `fullsend-sandbox:latest` carries `PI_VERSION`; `features/triage/triage.feature` asserts the runtime selected from the repo config on every run. -- Pilot on a disposable test repository with `triage`/`prioritize` before `code`/`fix` ([ADR 0044](../ADRs/0044-deprecate-per-org-installation-mode.md): per-repo is the only supported installation model). `review`/`retro` now run their real sub-agent roster through the runner-owned `Agent` tool (see [pi: Agent tool contract](#agent-tool-contract)), but that roster has been exercised locally, not on a fleet lifecycle run — watch the wall clock on the first one, and note that children default to `--thinking medium` for that reason. pi has no sub-agent tool or `agents/*.md` concept in core — fullsend supplies one as an extension; the bundled example extension (`examples/extensions/subagent/`) spawns children without the hook adapter, Vertex provider, `--no-approve` or a session dir, which is why fullsend ships its own. +- Pilot on a disposable test repository with `triage`/`prioritize` before `code`/`fix` ([ADR 0044](../ADRs/0044-deprecate-per-org-installation-mode.md): per-repo is the only supported installation model). `review`/`retro` now run their real sub-agent roster through the runner-owned `Agent` tool (see [pi: Pi sub-agents](#pi-sub-agents-the-agent-tool-contract)), but that roster has been exercised locally, not on a fleet lifecycle run — watch the wall clock on the first one, and note that children default to `--thinking medium` for that reason. pi has no sub-agent tool or `agents/*.md` concept in core — fullsend supplies one as an extension; the bundled example extension (`examples/extensions/subagent/`) spawns children without the hook adapter, Vertex provider, `--no-approve` or a session dir, which is why fullsend ships its own. -### Agent tool contract +### Pi sub-agents: the Agent tool contract `Bootstrap` writes an `agent` block into `fullsend-manifest.json` and uploads the embedded `fullsend-agent.js`, which registers Claude Code's `Agent` tool (and its legacy alias `Task`) so the @@ -723,6 +723,19 @@ with `-e`, after the hook adapter — when the agent definition has no `tools:` `anthropic-vertex`; `XAI_API_KEY` cleared and `XAI_VERTEX_PROJECT_ID` pinned for `xai-vertex`). The shell hygiene only ever matched the *parent's* provider, so without this a Claude child under a Grok parent would run with a stray `ANTHROPIC_API_KEY`. +- The usage file is **consumed as it is read**: the in-sandbox command renames it to + `.read` before printing it (`piSubagentUsageReadCommand`), so folding it into + `RunMetrics` is idempotent per iteration and a retry whose `ClearIterationArtifacts` failed cannot + count the same children twice. The read is capped at 1 MiB (~4k child records) because the file + sits under the agent-reachable config dir, and a truncated tail surfaces as a skipped malformed + line rather than an unbounded allocation in the runner. +- **`.env` is the same trust class here as everywhere else.** A rewritten + `/sandbox/workspace/.env` that exports `FULLSEND_SUBAGENT_DEPTH` disables the tool for that + iteration: the extension reads it as "this process is a child" and registers nothing. That is the + same exposure as any other `.env`-settable knob, and it fails in the safe direction. What `.env` + cannot do is change the manifest, the Agent extension or the hook adapter — all three are + SHA-256-checked before `.env` is sourced, the manifest again after it, and the manifest and the + adapter again before every dispatch. ### OpenAI via Workload Identity Federation diff --git a/docs/runtimes/pi.md b/docs/runtimes/pi.md index 48a7e06413..4b4fa6ee38 100644 --- a/docs/runtimes/pi.md +++ b/docs/runtimes/pi.md @@ -268,121 +268,105 @@ deny-list — is in ## Sub-agents -The `Agent` tool (registered under its legacy alias `Task` as well) is provided by a -runner-owned pi extension, `fullsend-agent.js`, so skills written for Claude Code's sub-agent -roster — `pr-review`, `retro-analysis` — dispatch unchanged. - -**Contract.** Same parameters as Claude Code's tool: +The `Agent` tool (registered under its legacy alias `Task` as well) comes from a runner-owned pi +extension, `fullsend-agent.js`, so skills written for Claude Code's sub-agent roster — `pr-review`, +`retro-analysis` — dispatch unchanged. Each child is its own `pi --print` process. | Parameter | Meaning | |---|---| -| `prompt` (required) | The whole task. The child starts with no memory of the conversation, so the prompt must carry its context package | +| `prompt` (required) | The whole task. The child starts with no memory of the conversation, so the prompt must carry its own context package | | `description` | Short label; shows in the run log | -| `model` | `opus`, `sonnet`, `haiku`, a Claude id (`claude-sonnet-4-6@default` — the `@suffix` is dropped) or a `provider/id` spec this run can serve. That is a closed set, not a provider check: the specs in the run's model table, the parent's own spec, and the ids listed for a provider with no model-table entry (`google-vertex`, and the vendored `xai-vertex` extension's `xai/grok-4.6`). An id the model invented is rejected even under an allowed provider. Omitted → the parent's model. A `:` suffix (pi's `provider/id:high` shorthand) is dropped, so it cannot override the child's thinking level | -| `subagent_type` | `Explore` gives a read-only child (`read`, `grep`, `find`, `ls`, intersected with the parent's set — a child never reaches past its parent); anything else or omitted gives the parent's tool set | +| `model` | A model this run can serve (see below). Omitted → the parent's model | +| `subagent_type` | `Explore` gives a read-only child (`read`, `grep`, `find`, `ls`, intersected with the parent's set); anything else, or omitted, gives the parent's tool set | | `run_in_background` | Accepted and ignored — a child always runs to completion inside the call | -The result is the child's final assistant message, trimmed and capped at 64 KB with a -`[truncated]` marker. The call is an error when the child's stream reports `stopReason` -`error`/`aborted`, when it exits non-zero, times out (15 minutes) or never emits `agent_end`. - -A model the run cannot serve is **rejected with the accepted forms** rather than passed through: -an invented Claude id would otherwise reach pi's built-in `anthropic` provider, which has no -credentials in the sandbox, and the dispatch would be lost to a confusing auth error. The same -applies inside a provider the run *can* reach — `anthropic-vertex/claude-sonnet-4-20250514`, -`google-vertex/gemini-9` or `xai/grok-9` is refused rather than sent on as an unknown model. A -Grok spec is normalized to the three-segment `xai-vertex/xai/` first (as the runner does for -the parent) and then checked against that same closed set, so the short `xai/` form is neither -more nor less permissive than the long one. - -**Parallel dispatch** is several `Agent` calls in one assistant message — pi runs sibling tool -calls from one message concurrently. At most four children run at once; the rest queue. The -runtime note appended to the agent's system prompt says so, so a skill that asks for "dispatch -these in parallel" gets it. - -**What a child inherits.** Each child is a `pi --print --mode json` process started with the same -posture as the parent: `--no-approve`, `--no-extensions` with an explicit `-e` list, no prompt -templates or themes, its own `--session-dir`, and a `--tools` allowlist (or `--no-builtin-tools` -when that allowlist is empty). The `-e` list is the -vendored provider extensions present in the image plus the hook adapter when security is enabled, -so **PreToolUse/PostToolUse hooks and the Bash allowlist apply inside sub-agents too** — as they do -on Claude Code, where the same hooks run on `Agent` calls. Children never receive the Agent -extension itself and refuse to register it if they somehow did (`FULLSEND_SUBAGENT_DEPTH`), so -there is no recursion. - -What a child does **not** inherit: - -- **Harness `extensions:`.** The declared extension directories are loaded for the parent only. A - child's `-e` list is fixed at `Bootstrap` (providers plus the hook adapter), so a tool a harness - extension registers is not available inside a sub-agent. -- **The parent's system prompt.** Children get `--append-system-prompt` with a short sub-agent role - note instead of the parent's `APPEND_SYSTEM.md`, which is the orchestrator persona and tells it to - make several `Agent` calls in one message — advice a child cannot act on. -- **Provider credentials it does not use.** The child's environment is rebuilt per resolved - provider with the same rules the runner applies to the parent, so a Claude child under a Grok - parent does not carry a stray `ANTHROPIC_API_KEY` or an ambient `GOOGLE_CLOUD_PROJECT`. - -**Prompt delivery.** The prompt goes over the child's stdin, not on its command line. Unterminated, -pi reads a leading `-` as an unknown option, a leading `--` as an unknown flag and a leading `@` -as a file argument. pi *does* honour a `--` end-of-options terminator, but that is not a way out: -after it an `@`-prefixed positional is still taken as a file argument, and argv is capped by the -kernel well below the size of a real context package either way. In `--print` mode pi reads a -non-TTY stdin to EOF and uses it as the initial message, whatever it starts with and however -long it is. - -**Stopping a child.** A child that times out (15 minutes), is aborted, or is caught by -`session_shutdown` gets `SIGTERM`, escalated to `SIGKILL` after a 3-second grace. `SIGTERM` is -deliberate: pi's `bash` tool runs commands in their own process groups and reaps them from its own -`SIGTERM` handler, so an immediate `SIGKILL` would orphan them. Children are not `detached`, so a -process-group kill or a `SIGTERM` to the parent takes them with it — a bare `SIGKILL` of the -parent alone does not, since nothing is left to run the stop sequence. - -**`.env` is the same trust class here as everywhere else.** A rewritten `/sandbox/workspace/.env` -that exports `FULLSEND_SUBAGENT_DEPTH` disables the tool for that iteration — the extension reads -it as "this process is a child" and registers nothing. That is the same exposure as any other -`.env`-settable knob; what `.env` cannot do is change the manifest, the Agent extension or the hook -adapter. All three are SHA-256-checked before `.env` is sourced and the manifest again after it, -and because those checks all happen at launch they are repeated while the iteration runs. - -Immediately before **every** dispatch, and while holding the concurrency slot the dispatch will -run in, the Agent extension re-hashes two things: the manifest, against the bytes it loaded -(`manifest changed since load; refusing to dispatch`), and every `agent.extensions` entry that -lives in the config dir — today the hook adapter — against the digest `Bootstrap` recorded for it -in `agent.extensionDigests` (`hook adapter changed since load; refusing to dispatch`). The adapter -needs its own digest because nothing else re-verifies it after launch: a parent with `write` can -replace the file mid-iteration, and a rewritten adapter simply omits the manifest check below, so -every child dispatched afterwards would come up unhooked. The digest map travels inside the -manifest, so the manifest's own digest covers it. The vendored provider extensions under -`/usr/local/share/pi-extensions` carry no digest — the image installs them root-owned and -read-only outside the config dir, so there is nothing there to rewrite. - -The hook adapter in turn verifies the manifest against the digest the runner exported -(`FULLSEND_PI_MANIFEST_SHA256`) whenever it loads — which for a sub-agent is minutes in, so a -parent that rewrites `hooks.groups` mid-iteration cannot dispatch a child whose adapter comes up -with no hooks in it. - -**Thinking level.** Children run at `--thinking medium`, not the parent's `high`: a full -`pr-review` roster at `high` overran the 20-minute review budget. Override with -`FULLSEND_PI_SUBAGENT_THINKING=`; an unrecognised value -warns and falls back to `medium`. - -**Where the output lands.** A child's session file is extracted as -`transcripts/-sub-.jsonl` (the sequence number is the call's, so children -with the same session basename do not collide), and one JSON line per child — model, usage, stop -reason, duration — as `transcripts/-subagents-usage.jsonl`. The runner folds that file into -`metrics.json`: the totals include the children, and `per_model_usage` attributes them per model -spec, with the parent's own iteration as one entry so the breakdown sums to the totals -([`fullsend run` § metrics.json](../cli/run.md#per-model-usage)). The parent entry is recorded on every -iteration that ran with the `Agent` tool enabled, dispatching or not, because iterations are -summed. A record with no model spec is bucketed under `unknown`. The usage file is consumed as it is -read (renamed to `usage.jsonl.read`), so a retry cannot count the same children twice. The run log -carries a `[fullsend-agent] # start ""` / `done ` pair -per child. - -**Turning it off.** The tool is enabled when the agent's definition has no `tools:` frontmatter (the -default set, as under Claude Code) or lists `Agent`/`Task`. An agent that lists tools without them -gets no Agent tool and the older runtime note telling it to execute sub-agent definitions itself, -in order. +The call returns the child's final assistant message, trimmed and capped at 64 KB with a +`[truncated]` marker. + +### Choosing a model + +For a run that reaches all three Vertex providers: + +| `model` | The child runs on | +|---|---| +| `sonnet` (also `opus`, `haiku`) | Claude on Vertex, whatever provider the parent runs on | +| `claude-sonnet-4-6` | the same — a bare Claude id resolves through that alias table, and a persona-style `@default` suffix is dropped | +| `google-vertex/gemini-3.7-flash` | Gemini, on pi's built-in provider | +| `xai/grok-4.6` | Grok on Vertex — normalized to `xai-vertex/xai/grok-4.6`, as the runner does for the parent | + +Anything else is **rejected, with the accepted forms listed in the error**, so the orchestrator can +correct itself instead of losing the dispatch. The accepted set is closed — the run's model table, +the parent's own spec, and the ids registered for a provider that has no table entry — rather than a +provider-prefix check, so an id the model invented (`google-vertex/gemini-9`, +`anthropic-vertex/claude-sonnet-4-20250514`) is refused even under a provider the run can reach. A +trailing `:` is dropped rather than passed through. + +### Running children in parallel + +Put several `Agent` calls in one assistant message: pi runs sibling tool calls from one message +concurrently. At most four children run at once and the rest queue. The runtime note appended to the +agent's system prompt says so, so a skill that asks for "dispatch these in parallel" gets it. + +### What a child inherits, and what it does not + +A child starts with the parent's posture — `--no-approve`, `--no-extensions` with an explicit `-e` +list, no prompt templates or themes, its own session dir, and a `--tools` allowlist +(`--no-builtin-tools` when that allowlist is empty). It inherits: + +- **The sandbox hooks.** Its `-e` list carries the vendored provider extensions and the hook adapter, + so PreToolUse/PostToolUse hooks and the Bash allowlist apply inside sub-agents too — as they do on + Claude Code, where the same hooks run on `Agent` calls. +- **The parent's tool set**, minus `Agent`/`Task`: a child cannot dispatch children of its own. + +It does not inherit: + +- **Harness `extensions:`.** A child's `-e` list is fixed at bootstrap, so a tool one of your + declared extensions registers is not available inside a sub-agent. +- **The parent's system prompt.** Children get a short sub-agent role note instead of the + orchestrator persona, whose "make several `Agent` calls in one message" advice a child cannot act + on. +- **Provider credentials it does not use.** The environment is rebuilt for the provider the child + resolved to, so a Claude child under a Grok parent carries no stray `ANTHROPIC_API_KEY`. + +### Thinking level + +Children run at `--thinking medium`, not the parent's `high`: a full `pr-review` roster at `high` +overran the 20-minute review budget. Override with +`FULLSEND_PI_SUBAGENT_THINKING=`; an unrecognised value warns +and falls back to `medium`. + +### Where the output lands + +- **Transcripts** — `transcripts/-sub-.jsonl`, one per child. The sequence + number is the call's, so children sharing a session basename do not collide. +- **Usage** — one JSON line per child (model, usage, stop reason, duration) in + `transcripts/-subagents-usage.jsonl`. +- **`metrics.json`** — the totals include the children, and `per_model_usage` attributes them per + model spec, with the parent's own iteration as one entry so the breakdown sums to the totals + ([`fullsend run` § metrics.json](../cli/run.md#per-model-usage)). A record with no model spec is + bucketed under `unknown`. +- **Run log** — `[fullsend-agent] # start ""` and + `[fullsend-agent] # done ms ` per child. + +### Turning it off + +The tool is enabled when the agent's definition has no `tools:` frontmatter (the default set, as +under Claude Code) or lists `Agent`/`Task`. An agent that lists tools without them gets no `Agent` +tool, and the runtime note telling it to execute sub-agent definitions itself, in order. + +### Troubleshooting sub-agents + +| Symptom | Cause | Fix | +|---|---|---| +| `model "": ...; use opus, sonnet, haiku, or one of ...` | The `model` argument is not one this run can serve | Use one of the forms the message lists, or omit `model` to inherit the parent's | +| `manifest changed since load; refusing to dispatch` | `fullsend-manifest.json` changed after the extension read it | Runner-owned config was rewritten inside the sandbox — treat it as tampering, not a transient | +| `hook adapter changed since load; refusing to dispatch` | `fullsend-hooks.js` changed after bootstrap recorded its digest | The same: the child would otherwise have come up unhooked | +| A child call fails after 15 minutes | The per-child deadline; the child is signalled and reaped | Narrow the child's prompt, or split the task across more children | +| A child call reports `error` or `aborted` | The child's own run failed — model error, non-zero exit, or no `agent_end` | Read that child's transcript under `transcripts/-sub-*.jsonl` | + +How children are launched and kept honest — prompt delivery, the stop sequence, the per-dispatch +digest re-checks — is in +[Runtime Implementation § Pi sub-agents](../contributing/runtime-implementation.md#pi-sub-agents-the-agent-tool-contract). ## Not yet exercised @@ -395,9 +379,9 @@ not mapped. ## Troubleshooting **The model is not found, or the provider is missing.** A pi provider comes from an extension loaded -with `-e`, so an extension that did not load takes its provider with it. The table in -[Extensions § Troubleshooting extensions](#troubleshooting-extensions) separates the two ways that happens — the loud -one (`Failed to load extension`, exit 1) and the silent one (pi exits 0 having loaded nothing). +with `-e`, so an extension that did not load takes its provider with it. The table under +[Extensions](#troubleshooting-extensions) separates the two ways that happens — the loud one +(`Failed to load extension`, exit 1) and the silent one (pi exits 0 having loaded nothing). **`No API key found for `.** The provider is registered but its credentials did not resolve. For Vertex providers that means ADC — check the project variable for *that* provider in the