diff --git a/src/deployment/AGENTS.md b/src/deployment/AGENTS.md index d9149d3..75667e2 100644 --- a/src/deployment/AGENTS.md +++ b/src/deployment/AGENTS.md @@ -31,6 +31,7 @@ src/deployment/ ├── dockerLabels.ts # Docker label construction for managed units ├── dockerInspect.ts # Bounded Docker container inspection for status --live ├── dockerLogs.ts # Bounded Docker log collection with redaction for status --logs +├── candidateDiagnostics.ts # Private startup evidence captured before failed-image rollback ├── dockerProbeGateway.ts # Manager-mediated artifact reads and ephemeral same-image network-namespace health probes ├── dockerRecover.ts # Docker label recovery for context-backed remote status ├── buildImageCacheStore.ts # Strict, atomic Spawnfile-home cache for verified Docker image builds @@ -55,6 +56,11 @@ src/deployment/ - Keep deployment records free of secrets. Paths are allowed only for local operator metadata. - Docker labels must contain identifiers only, never local paths or secret-bearing values. - Records are written only after a detached deployment has successfully started. +- Failed image candidates save bounded, sanitized stdout/stderr and health + evidence under the home deployment's `diagnostics/` before rollback removes + the container. Files are 0600 inside an owned 0700 directory; collection or + storage failure must never prevent rollback. Logs are still private operator + data: redaction cannot recognize every application-specific secret. - Build-image cache entries live under the Spawnfile home, are strict-schema parsed, mode 0600, and best-effort only: cache corruption or I/O failure must never fail a compile/build. diff --git a/src/deployment/candidateDiagnostics.test.ts b/src/deployment/candidateDiagnostics.test.ts new file mode 100644 index 0000000..cb749dc --- /dev/null +++ b/src/deployment/candidateDiagnostics.test.ts @@ -0,0 +1,137 @@ +import os from "node:os"; +import path from "node:path"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { captureCandidateDiagnostics, sanitizeCandidateDiagnostic } from "./candidateDiagnostics.js"; +import type { DockerCommandRunner } from "../distribution/dockerRunner.js"; + +const candidateId = "c".repeat(64); +let home: string; +let deployment: string; +const previousHome = process.env.SPAWNFILE_HOME; + +beforeEach(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "candidate-diagnostics-")); + process.env.SPAWNFILE_HOME = home; + deployment = path.join(home, "deployments", "candidate"); + await mkdir(deployment, { recursive: true }); +}); +afterEach(async () => { + if (previousHome === undefined) delete process.env.SPAWNFILE_HOME; + else process.env.SPAWNFILE_HOME = previousHome; + await rm(home, { recursive: true, force: true }); +}); + +const runner = (state: unknown = { Status: "running", Health: { Status: "unhealthy" }, ExitCode: 0 }): DockerCommandRunner => + async args => Buffer.from(args[0] === "logs" ? "Error: required module not found\nknown-secret" : JSON.stringify(state)); +const capture = (runDocker: DockerCommandRunner = runner()) => captureCandidateDiagnostics({ + candidateId, deploymentName: "candidate", runDocker, secretValues: ["known-secret"] +}); + +describe("failed candidate diagnostics", () => { + it("writes private unique evidence with allowlisted health fields and bounded sanitized logs", async () => { + const runDocker = vi.fn(runner({ + Running: false, Status: "exited", ExitCode: 137, OOMKilled: true, + Error: "module failed known-secret", ignored: "unknown-private-value", + Health: { Status: "unhealthy", Log: [ + { Output: "first old health output" }, { Output: "known-secret" }, { Output: "not listening" }, + { Output: "a".repeat(5_000) } + ] } + })); + const first = await capture(runDocker), second = await capture(runDocker); + expect(first.path).not.toBe(second.path); + expect(first.summary).toBe("state=exited, health=unhealthy, exit=137, oom-killed"); + const text = await readFile(first.path!, "utf8"), evidence = JSON.parse(text); + expect(text).toContain("required module not found"); + expect(text).not.toMatch(/known-secret|unknown-private-value|first old health output/u); + expect(evidence.health.healthOutput).toHaveLength(3); + expect(evidence.health.healthOutput[2]).toHaveLength(4_096); + expect((await stat(first.path!)).mode & 0o777).toBe(0o600); + expect((await stat(path.dirname(first.path!))).mode & 0o777).toBe(0o700); + for (const [args, options] of runDocker.mock.calls) { + expect(args.at(-1)).toBe(candidateId); + expect(options).toEqual({ captureStderr: true, maxOutputBytes: 65_536, timeoutMs: 5_000 }); + expect(args).not.toContain("--follow"); + } + }); + + it("preserves useful health metadata even when short env values redact log text", async () => { + const result = await captureCandidateDiagnostics({ candidateId, deploymentName: "candidate", + runDocker: runner({ Status: "exited", ExitCode: 1 }), secretValues: ["x", "1"] }); + expect(result.summary).toBe("state=exited, exit=1"); + }); + + it.each([null, "malformed", { Status: "injected-secret", Health: { Status: "private-health" } }])( + "never surfaces arbitrary state metadata: %j", async value => { + const result = await capture(runner(value)); + expect(result.summary).toMatch(/^state (?:unavailable)$|^state=unknown$/u); + expect(await readFile(result.path!, "utf8")).not.toMatch(/injected-secret|private-health/u); + }); + + it("records collection failure without persisting raw Docker errors", async () => { + const result = await capture(async () => { throw new Error("transport arbitrary-private-value"); }); + const text = await readFile(result.path!, "utf8"); + expect(text).not.toContain("arbitrary-private-value"); + expect(JSON.parse(text).logs.available).toBe(false); + expect(result.summary).toBe("state unavailable"); + }); + + it("rejects oversized output from injectable command runners", async () => { + const result = await capture(async () => Buffer.alloc(65_537, "x")); + const evidence = JSON.parse(await readFile(result.path!, "utf8")); + expect(evidence.logs.available).toBe(false); + expect(evidence.health.available).toBe(false); + }); + + it("reports absent diagnostics when storage is unavailable, without throwing", async () => { + await writeFile(path.join(deployment, "diagnostics"), "occupied"); + await expect(capture()).resolves.toEqual({ path: null, summary: "state=running, health=unhealthy, exit=0" }); + }); + + it("refuses a public or symlinked diagnostic directory", async () => { + const directory = path.join(deployment, "diagnostics"); + await mkdir(directory, { mode: 0o755 }); + expect((await capture()).path).toBeNull(); + await rm(directory, { recursive: true }); + const target = path.join(home, "unrelated"); + await mkdir(target, { mode: 0o700 }); + await symlink(target, directory); + expect((await capture()).path).toBeNull(); + expect(await readdir(target)).toEqual([]); + }); + + it("refuses writable deployment parents and invalid candidate identities", async () => { + await chmod(deployment, 0o777); + expect((await capture()).path).toBeNull(); + const runDocker = vi.fn(runner()); + expect(await captureCandidateDiagnostics({ candidateId: "ambiguous", deploymentName: "candidate", runDocker, secretValues: [] })) + .toEqual({ path: null, summary: "invalid candidate identity" }); + expect(runDocker).not.toHaveBeenCalled(); + }); + + it("sanitizes known values, token shapes, URLs, keys and terminal control characters", () => { + const original = [ + "known-secret", "Bearer abcdefghijklmnop", "sk-proj-abcdefghijklmnopqrstuvwxyz", + "TOKEN=value", '{"refreshToken":"refresh-value"}', "Authorization: Basic basic-credential", + '{"Authorization":"Basic hidden-json-basic"}', "{'refreshToken':'single-quoted-secret'}", + "https://user:pass@example.org/private?token=foo", "eyJhbGciOiJIUzI1NiJ9.payload.signature", + "-----BEGIN PRIVATE KEY-----\nprivate-key-bytes\n-----END PRIVATE KEY-----", + "\u001b[31mError: missing module\u001b[0m" + ].join("\n"); + const clean = sanitizeCandidateDiagnostic(original, ["known-secret"]); + expect(clean).not.toMatch(/known-secret|abcdefghijklmnop|refresh-value|basic-credential|hidden-json-basic|single-quoted-secret|user:pass|eyJhbGci|private-key-bytes|\u001b/u); + expect(clean).toContain("Error: missing module"); + }); + + it("redacts overlapping secrets longest first without leaving suffixes", () => { + const secrets = ["abc", "abcdef1"]; + expect(sanitizeCandidateDiagnostic("before abcdef1 after abc", secrets)).toBe("before [REDACTED] after [REDACTED]"); + expect(secrets).toEqual(["abc", "abcdef1"]); + }); + + it("normalizes terminal escapes before matching known secrets", () => { + expect(sanitizeCandidateDiagnostic("Error: sec\u001b[0mret value\nsec\rret", ["secret"])) + .toBe("Error: [REDACTED] value\n[REDACTED]"); + }); +}); diff --git a/src/deployment/candidateDiagnostics.ts b/src/deployment/candidateDiagnostics.ts new file mode 100644 index 0000000..1073343 --- /dev/null +++ b/src/deployment/candidateDiagnostics.ts @@ -0,0 +1,98 @@ +import { randomUUID } from "node:crypto"; +import { lstat, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { resolveSpawnfileHome } from "../auth/index.js"; +import type { DockerCommandRunner } from "../distribution/dockerRunner.js"; +import { redactDockerLogText } from "./dockerLogs.js"; +import { resolveHomeDeploymentDirectory } from "./homeStore.js"; + +const outputLimit = 64 * 1024; +const commandOptions = { captureStderr: true, maxOutputBytes: outputLimit, timeoutMs: 5_000 }; +const containerIdPattern = /^[a-f0-9]{64}$/u; + +export interface CandidateDiagnosticsInput { + candidateId: string; + deploymentName: string; + runDocker: DockerCommandRunner; + secretValues: string[]; +} + +export interface CandidateDiagnosticsResult { + path: string | null; + summary: string; +} + +/** Redaction is best effort; diagnostic artifacts remain private operator data. */ +export const sanitizeCandidateDiagnostic = (text: string, secrets: string[]): string => + redactDockerLogText(text, secrets) + .replace(/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?(?:-----END [^-]*PRIVATE KEY-----|$)/gu, "[REDACTED PRIVATE KEY]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, "[REDACTED JWT]") + .replace(/https?:\/\/[^\s<>"']+/giu, "[REDACTED URL]") + .replace(/\b([a-z0-9_-]*(?:authorization|cookie|password|secret|token|api[_-]?key)[a-z0-9_-]*)["']?\s*[:=]\s*[^\r\n]+/giu, "$1: [REDACTED]"); + +const collect = async (runDocker: DockerCommandRunner, args: string[]) => { + try { + const bytes = await runDocker(args, commandOptions); + // Injectable runners must obey the same limit as the default Docker runner. + if (bytes.length > outputLimit) throw new Error("limit"); + return { available: true, text: bytes.toString("utf8") }; + } catch { + // Docker transport errors can contain command/env details; never persist them. + return { available: false, text: "Diagnostic command failed, timed out, or exceeded its byte limit" }; + } +}; + +const summarizeState = (raw: string, secrets: string[]) => { + try { + const state = JSON.parse(raw) as Record; + const health = (state.Health as { Status?: unknown } | undefined)?.Status; + const status = ["created", "running", "paused", "restarting", "removing", "exited", "dead"].includes(String(state.Status)) + ? state.Status : "unknown"; + const parts = [`state=${status}`]; + if (["starting", "healthy", "unhealthy"].includes(String(health))) parts.push(`health=${health}`); + if (Number.isSafeInteger(state.ExitCode)) parts.push(`exit=${state.ExitCode}`); + if (state.OOMKilled === true) parts.push("oom-killed"); + const healthState = state.Health as { Log?: unknown } | undefined; + const text = (value: unknown) => typeof value === "string" + ? sanitizeCandidateDiagnostic(value, secrets).slice(0, 4_096) : ""; + return { available: true, summary: parts.join(", "), error: text(state.Error), + healthOutput: Array.isArray(healthState?.Log) ? healthState.Log.slice(-3).map((entry: unknown) => + text(entry && typeof entry === "object" ? (entry as { Output?: unknown }).Output : null)) : [] }; + } catch { return { available: false, summary: "state unavailable", error: "", healthOutput: [] }; } +}; + +/** Collect before rollback deletes the only container-local startup evidence. */ +export const captureCandidateDiagnostics = async ( + input: CandidateDiagnosticsInput +): Promise => { + if (!containerIdPattern.test(input.candidateId)) return { path: null, summary: "invalid candidate identity" }; + const [state, logs] = await Promise.all([ + collect(input.runDocker, ["container", "inspect", "--format", "{{json .State}}", input.candidateId]), + collect(input.runDocker, ["logs", "--tail", "100", input.candidateId]) + ]); + const health = summarizeState(state.available ? state.text : "", input.secretValues); + const summary = health.summary; + try { + const deploymentDirectory = resolveHomeDeploymentDirectory(input.deploymentName); + for (const parent of [resolveSpawnfileHome(), path.dirname(deploymentDirectory), deploymentDirectory]) { + const info = await lstat(parent); + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o022) !== 0 + || (process.getuid && info.uid !== process.getuid())) throw new Error("unsafe diagnostics parent"); + } + const directory = path.join(deploymentDirectory, "diagnostics"); + await mkdir(directory, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + const info = await lstat(directory); + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o777) !== 0o700 + || (process.getuid && info.uid !== process.getuid())) throw new Error("unsafe diagnostics directory"); + const destination = path.join(directory, `candidate-${Date.now()}-${randomUUID()}.json`); + await writeFile(destination, `${JSON.stringify({ + version: "spawnfile.candidate-diagnostics.v1", capturedAt: new Date().toISOString(), + candidateId: input.candidateId, deployment: input.deploymentName, summary, health, + logs: { available: logs.available, text: sanitizeCandidateDiagnostic(logs.text, input.secretValues) } + }, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + return { path: destination, summary }; + } catch { return { path: null, summary }; } +}; diff --git a/src/deployment/dockerLogs.test.ts b/src/deployment/dockerLogs.test.ts index 28dee64..e8c62a6 100644 --- a/src/deployment/dockerLogs.test.ts +++ b/src/deployment/dockerLogs.test.ts @@ -146,6 +146,24 @@ describe("docker deployment logs", () => { }); describe("redactDockerLogText", () => { + it("masks longer overlapping secrets before their prefixes without mutating the supplied values", () => { + const secrets = ["abc", "abcdef1"]; + expect(redactDockerLogText("before abcdef1 after abc", secrets)).toBe("before [REDACTED] after [REDACTED]"); + expect(secrets).toEqual(["abc", "abcdef1"]); + }); + + it("strips terminal controls before matching known secrets", () => { + expect(redactDockerLogText("Error: sec\u001b[0mret value\nsec\rret", ["secret"])) + .toBe("Error: [REDACTED] value\n[REDACTED]"); + }); + + it("normalizes secret values identically and ignores values that normalize to empty", () => { + expect(redactDockerLogText("value token-abc\r\n", ["token-abc\r", "\u001b[0m"])) + .toBe("value [REDACTED]\n"); + expect(redactDockerLogText("value abc\u001b[0mdef", ["abc", "abc\u001b[0mdef"])) + .toBe("value [REDACTED]"); + }); + it("redacts direct secret values without touching unrelated text", () => { expect(redactDockerLogText("before token-value after", ["token-value"])).toBe( "before [REDACTED] after" diff --git a/src/deployment/dockerLogs.ts b/src/deployment/dockerLogs.ts index b7f591a..2d54607 100644 --- a/src/deployment/dockerLogs.ts +++ b/src/deployment/dockerLogs.ts @@ -60,15 +60,22 @@ const withDockerTarget = ( const normalizeTail = (tail: number | undefined): number => typeof tail === "number" && Number.isInteger(tail) && tail > 0 ? tail : 100; +const stripTerminalControls = (value: string): string => + value.replace(/\x1b\[[0-?]*[ -/]*[@-~]|[\x00-\x08\x0b-\x1f\x7f]/gu, ""); + const redactKnownSecrets = (text: string, secretValues: string[]): string => secretValues + .map(stripTerminalControls) .filter((secret) => secret.length > 0) + .sort((left, right) => right.length - left.length) .reduce((redacted, secret) => redacted.replaceAll(secret, "[REDACTED]"), text); export const redactDockerLogText = ( text: string, secretValues: string[] = [] -): string => redactSensitiveText(redactKnownSecrets(text, secretValues)); +): string => redactSensitiveText(redactKnownSecrets( + stripTerminalControls(text), secretValues +)); const combineLogStreams = (stdout: string, stderr: string): string => { if (!stdout) { diff --git a/src/deployment/index.ts b/src/deployment/index.ts index 4ba7746..33b5deb 100644 --- a/src/deployment/index.ts +++ b/src/deployment/index.ts @@ -4,6 +4,7 @@ export * from "./artifactsExportPlan.js"; export * from "./artifactsExportTypes.js"; export * from "./privateArtifactsExport.js"; export * from "./buildImageCacheStore.js"; +export * from "./candidateDiagnostics.js"; export * from "./dockerLabels.js"; export * from "./dockerInspect.js"; export * from "./dockerLogs.js"; diff --git a/src/distribution/AGENTS.md b/src/distribution/AGENTS.md index 0baf4b2..c077f6f 100644 --- a/src/distribution/AGENTS.md +++ b/src/distribution/AGENTS.md @@ -31,3 +31,7 @@ src/distribution/ - This folder is pure: no filesystem access, no compiler imports. The compiler projects its plan/report data into the structural input types defined here. - The fingerprint is computed over the report body excluding `generated_at`, so identical compiles fingerprint identically across machines and times. - Frozen contract values (`spawnfile.distribution-report.v1`, `spawnfile.image.v1`, the in-image report path) live in `types.ts` and change only with a contract version bump. +- Candidate startup gets the full 120-second readiness budget even if Docker + marks a running process unhealthy during initialization. Only running and + healthy (or a container without a health check) passes; terminal or unknown + health state fails. Deployment diagnostics must finish before candidate removal. diff --git a/src/distribution/consumeImage.test.ts b/src/distribution/consumeImage.test.ts index 403ddfa..37a14ef 100644 --- a/src/distribution/consumeImage.test.ts +++ b/src/distribution/consumeImage.test.ts @@ -1,6 +1,6 @@ import os from "node:os"; import path from "node:path"; -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, writeFile } from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -581,18 +581,33 @@ describe("consumeImageUp", () => { it("restores the previous container when candidate readiness inspection fails", async () => { const calls: string[][] = []; const base = createFakeDocker({ calls: [] }, undefined, { liveExists: true }); + const diagnosticsDirectory = path.join(homeDirectory, "deployments", "readiness-rollback", "diagnostics"); + let evidenceBeforeRemoval = false; const runDocker = async (args: string[]): Promise => { calls.push(args); + if (args[0] === "logs") return Buffer.from("Error: missing startup module\nsecret-value\n"); + if (args.includes("{{json .State}}")) return Buffer.from(JSON.stringify({ + Running: false, Status: "exited", ExitCode: 1, Health: { Status: "unhealthy" } + })); if (args[0] === "container" && args.some((arg) => arg.includes("{{json .State}}"))) { throw new Error("readiness transport failed"); } + if (args[0] === "rm" && args[2] === candidateContainerId) { + const files = await readdir(diagnosticsDirectory); + const evidence = await readFile(path.join(diagnosticsDirectory, files[0]!), "utf8"); + expect(evidence).toContain("missing startup module"); + expect(evidence).not.toContain("secret-value"); + expect(JSON.parse(evidence).summary).toBe("state=exited, health=unhealthy, exit=1"); + evidenceBeforeRemoval = true; + } return base(args); }; await expect(consumeImageUp("you/org:1.0.0", { - authValues: { ANTHROPIC_API_KEY: "sk", DIST_REQUIRED_TOKEN: "x" }, + authValues: { ANTHROPIC_API_KEY: "secret-value", DIST_REQUIRED_TOKEN: "x" }, deploymentName: "readiness-rollback", runDocker })).rejects.toThrow(/readiness transport failed/u); + expect(evidenceBeforeRemoval).toBe(true); const live = "spawnfile-readiness-rollback"; const backup = calls.find((call) => call[0] === "rename" && call[1] === live)?.[2]; diff --git a/src/distribution/consumeImage.ts b/src/distribution/consumeImage.ts index 8ab4760..dc66155 100644 --- a/src/distribution/consumeImage.ts +++ b/src/distribution/consumeImage.ts @@ -4,11 +4,13 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { acquireHomeDeploymentLock, + captureCandidateDiagnostics, createDockerDeploymentLabels, homeDeploymentExists, normalizeDeploymentName, readHomeDeploymentRecord, resolveDockerDeploymentTarget, + sanitizeCandidateDiagnostic, verifyDockerDeploymentTarget, writeHomeDeployment, type DeploymentRecord, @@ -333,14 +335,21 @@ const consumeImageUpLocked = async ( recordPath: written.recordPath }; } catch (error) { + const diagnostics = candidateId ? await captureCandidateDiagnostics({ + candidateId, deploymentName, runDocker, secretValues: Object.values(env) + }) : null; + const detail = diagnostics ? ` (${diagnostics.summary}); ${diagnostics.path + ? `private diagnostics: ${diagnostics.path}` : "private diagnostics unavailable"}` : ""; + const failure = (cause: unknown): SpawnfileError => new SpawnfileError("runtime_error", + sanitizeCandidateDiagnostic(cause instanceof Error ? cause.message : String(cause), Object.values(env)) + detail); try { await rollbackCandidateContainer( runDocker, candidateId, containerName, previousContainer, backupName ); } catch (rollbackError) { - throw rollbackError; + throw failure(rollbackError); } - throw error; + throw failure(error); } } finally { try { await volumeReservation?.release(); } diff --git a/src/distribution/consumeImageLifecycle.test.ts b/src/distribution/consumeImageLifecycle.test.ts index 1e83850..86a633b 100644 --- a/src/distribution/consumeImageLifecycle.test.ts +++ b/src/distribution/consumeImageLifecycle.test.ts @@ -87,6 +87,45 @@ describe("image deployment lifecycle", () => { await expect(readiness).resolves.toBeUndefined(); }); + it("allows unhealthy startup at 65 seconds to recover inside the existing 120 second budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + let settled = false; + const readiness = assertCandidateContainerReady(async () => ready(candidateId, "candidate", { + Running: true, Status: "running", + Health: { Status: Date.now() < 65_000 ? "starting" : Date.now() < 80_000 ? "unhealthy" : "healthy" } + }), candidateId, "candidate"); + const outcome = readiness.then(() => { settled = true; return "ready"; }, () => { settled = true; return "failed"; }); + await vi.advanceTimersByTimeAsync(79_000); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + await expect(outcome).resolves.toBe("ready"); + }); + + it("never accepts an unhealthy candidate and fails at the existing deadline", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + let settled = false; + const readiness = assertCandidateContainerReady(async () => ready(candidateId, "candidate", { + Running: true, Status: "running", Health: { Status: "unhealthy" } + }), candidateId, "candidate"); + const rejected = expect(readiness.finally(() => { settled = true; })).rejects.toThrow(/did not become ready/u); + await vi.advanceTimersByTimeAsync(119_000); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + await rejected; + expect(Date.now()).toBe(120_000); + }); + + it.each([ + { Running: false, Status: "exited", Health: { Status: "unhealthy" } }, + { Running: true, Status: "running", Health: { Status: "unknown" } } + ])("rejects terminal or unrecognized state without polling: %j", async (state) => { + const runDocker = vi.fn(async () => ready(candidateId, "candidate", state)); + await expect(assertCandidateContainerReady(runDocker, candidateId, "candidate")).rejects.toThrow(/did not become ready/u); + expect(runDocker).toHaveBeenCalledTimes(1); + }); + it("fails a candidate that remains starting through the bounded readiness deadline", async () => { vi.useFakeTimers(); vi.setSystemTime(0); diff --git a/src/distribution/consumeImageLifecycle.ts b/src/distribution/consumeImageLifecycle.ts index 3917a57..dffda2e 100644 --- a/src/distribution/consumeImageLifecycle.ts +++ b/src/distribution/consumeImageLifecycle.ts @@ -214,7 +214,9 @@ export const assertCandidateContainerReady = async ( } const health = state.Health?.Status; if (state.Running === true && (health === undefined || health === "healthy")) return; - if (state.Running !== true || (health !== undefined && health !== "starting")) break; + // Docker may exhaust its health retries while a live process is still + // initializing. Honor our full readiness budget; only healthy can pass. + if (state.Running !== true || (health !== "starting" && health !== "unhealthy")) break; const remainingMs = deadline - Date.now(); if (remainingMs <= 0) break; await delay(Math.min(candidateReadinessPollMs, remainingMs)); diff --git a/src/distribution/dockerRunner.test.ts b/src/distribution/dockerRunner.test.ts index e2874ce..d9bbca3 100644 --- a/src/distribution/dockerRunner.test.ts +++ b/src/distribution/dockerRunner.test.ts @@ -23,4 +23,25 @@ describe("createConsumerDockerRunner", () => { ); await expect(run(["anything"])).rejects.toThrow(); }); + + it("includes application stderr only when diagnostic capture requests it", async () => { + const run = createConsumerDockerRunner(process.execPath, ["-e"]); + const command = ["process.stdout.write('out'); process.stderr.write('err')"]; + expect((await run(command)).toString()).toBe("out"); + const captured = (await run(command, { captureStderr: true, maxOutputBytes: 100, timeoutMs: 1_000 })).toString(); + expect(captured).toContain("out"); + expect(captured).toContain("err"); + }); + + it.each(["stdout", "stderr"])("bounds diagnostic %s before buffering unbounded output", async (stream) => { + const run = createConsumerDockerRunner(process.execPath, ["-e"]); + await expect(run([`process.${stream}.write('x'.repeat(100_000))`], { + captureStderr: true, maxOutputBytes: 100, timeoutMs: 1_000 + })).rejects.toThrow("byte limit"); + }); + + it("kills a hanging diagnostic command within its time budget", async () => { + const run = createConsumerDockerRunner(process.execPath, ["-e"]); + await expect(run(["setInterval(() => {}, 1000)"], { timeoutMs: 50 })).rejects.toThrow("timed out"); + }); }); diff --git a/src/distribution/dockerRunner.ts b/src/distribution/dockerRunner.ts index 1826eda..657965b 100644 --- a/src/distribution/dockerRunner.ts +++ b/src/distribution/dockerRunner.ts @@ -4,14 +4,20 @@ import { SpawnfileError } from "../shared/index.js"; export interface DockerCommandRunner { /** Runs docker with args; resolves stdout as a Buffer. Rejects on non-zero exit. */ - (args: string[]): Promise; + (args: string[], options?: DockerCommandOptions): Promise; +} + +export interface DockerCommandOptions { + captureStderr?: boolean; + maxOutputBytes?: number; + timeoutMs?: number; } export const createConsumerDockerRunner = ( dockerCommand: string, baseArgs: string[] ): DockerCommandRunner => - async (args: string[]): Promise => + async (args: string[], options: DockerCommandOptions = {}): Promise => new Promise((resolve, reject) => { const finalArgs = [...baseArgs, ...args]; const child = spawn(dockerCommand, finalArgs, { @@ -20,10 +26,32 @@ export const createConsumerDockerRunner = ( }); const stdout: Buffer[] = []; const stderr: string[] = []; - child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); - child.stderr.on("data", (chunk: Buffer | string) => stderr.push(String(chunk))); - child.on("error", reject); + let bytes = 0; + let stopped = false; + const stop = (reason: string): void => { + stopped = true; + child.kill("SIGKILL"); + reject(new SpawnfileError("runtime_error", reason)); + }; + const timer = options.timeoutMs === undefined ? undefined + : setTimeout(() => stop("Docker diagnostic command timed out"), options.timeoutMs); + const clearTimer = (): void => { if (timer) clearTimeout(timer); }; + const collect = (chunk: Buffer, isStderr: boolean): void => { + if (stopped) return; + bytes += chunk.length; + if (options.maxOutputBytes !== undefined && bytes > options.maxOutputBytes) { + stop("Docker diagnostic output exceeded its byte limit"); + return; + } + if (isStderr) stderr.push(chunk.toString("utf8")); + if (!isStderr || options.captureStderr) stdout.push(chunk); + }; + child.stdout.on("data", (chunk: Buffer) => collect(chunk, false)); + child.stderr.on("data", (chunk: Buffer) => collect(chunk, true)); + child.on("error", (error) => { clearTimer(); reject(error); }); child.on("close", (code) => { + clearTimer(); + if (stopped) return; if (code === 0) { resolve(Buffer.concat(stdout)); return;