fix: preserve the readiness window and failed candidate diagnostics

src/deployment/AGENTS.md

@@ -31,6 +31,7 @@ src/deployment/
 ├── dockerLabels.ts  # Docker label construction for managed units
 ├── dockerInspect.ts # Bounded Docker container inspection for status --live
 ├── dockerLogs.ts    # Bounded Docker log collection with redaction for status --logs
+├── candidateDiagnostics.ts # Private startup evidence captured before failed-image rollback
 ├── dockerProbeGateway.ts # Manager-mediated artifact reads and ephemeral same-image network-namespace health probes
 ├── dockerRecover.ts # Docker label recovery for context-backed remote status
 ├── buildImageCacheStore.ts # Strict, atomic Spawnfile-home cache for verified Docker image builds
@@ -55,6 +56,11 @@ src/deployment/
 - Keep deployment records free of secrets. Paths are allowed only for local operator metadata.
 - Docker labels must contain identifiers only, never local paths or secret-bearing values.
 - Records are written only after a detached deployment has successfully started.
+- Failed image candidates save bounded, sanitized stdout/stderr and health
+  evidence under the home deployment's `diagnostics/` before rollback removes
+  the container. Files are 0600 inside an owned 0700 directory; collection or
+  storage failure must never prevent rollback. Logs are still private operator
+  data: redaction cannot recognize every application-specific secret.
 - Build-image cache entries live under the Spawnfile home, are strict-schema
   parsed, mode 0600, and best-effort only: cache corruption or I/O failure must
   never fail a compile/build.

src/deployment/candidateDiagnostics.test.ts

@@ -0,0 +1,137 @@
+import os from "node:os";
+import path from "node:path";
+import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { captureCandidateDiagnostics, sanitizeCandidateDiagnostic } from "./candidateDiagnostics.js";
+import type { DockerCommandRunner } from "../distribution/dockerRunner.js";
+
+const candidateId = "c".repeat(64);
+let home: string;
+let deployment: string;
+const previousHome = process.env.SPAWNFILE_HOME;
+
+beforeEach(async () => {
+  home = await mkdtemp(path.join(os.tmpdir(), "candidate-diagnostics-"));
+  process.env.SPAWNFILE_HOME = home;
+  deployment = path.join(home, "deployments", "candidate");
+  await mkdir(deployment, { recursive: true });
+});
+afterEach(async () => {
+  if (previousHome === undefined) delete process.env.SPAWNFILE_HOME;
+  else process.env.SPAWNFILE_HOME = previousHome;
+  await rm(home, { recursive: true, force: true });
+});
+
+const runner = (state: unknown = { Status: "running", Health: { Status: "unhealthy" }, ExitCode: 0 }): DockerCommandRunner =>
+  async args => Buffer.from(args[0] === "logs" ? "Error: required module not found\nknown-secret" : JSON.stringify(state));
+const capture = (runDocker: DockerCommandRunner = runner()) => captureCandidateDiagnostics({
+  candidateId, deploymentName: "candidate", runDocker, secretValues: ["known-secret"]
+});
+
+describe("failed candidate diagnostics", () => {
+  it("writes private unique evidence with allowlisted health fields and bounded sanitized logs", async () => {
+    const runDocker = vi.fn(runner({
+      Running: false, Status: "exited", ExitCode: 137, OOMKilled: true,
+      Error: "module failed known-secret", ignored: "unknown-private-value",
+      Health: { Status: "unhealthy", Log: [
+        { Output: "first old health output" }, { Output: "known-secret" }, { Output: "not listening" },
+        { Output: "a".repeat(5_000) }
+      ] }
+    }));
+    const first = await capture(runDocker), second = await capture(runDocker);
+    expect(first.path).not.toBe(second.path);
+    expect(first.summary).toBe("state=exited, health=unhealthy, exit=137, oom-killed");
+    const text = await readFile(first.path!, "utf8"), evidence = JSON.parse(text);
+    expect(text).toContain("required module not found");
+    expect(text).not.toMatch(/known-secret|unknown-private-value|first old health output/u);
+    expect(evidence.health.healthOutput).toHaveLength(3);
+    expect(evidence.health.healthOutput[2]).toHaveLength(4_096);
+    expect((await stat(first.path!)).mode & 0o777).toBe(0o600);
+    expect((await stat(path.dirname(first.path!))).mode & 0o777).toBe(0o700);
+    for (const [args, options] of runDocker.mock.calls) {
+      expect(args.at(-1)).toBe(candidateId);
+      expect(options).toEqual({ captureStderr: true, maxOutputBytes: 65_536, timeoutMs: 5_000 });
+      expect(args).not.toContain("--follow");
+    }
+  });
+
+  it("preserves useful health metadata even when short env values redact log text", async () => {
+    const result = await captureCandidateDiagnostics({ candidateId, deploymentName: "candidate",
+      runDocker: runner({ Status: "exited", ExitCode: 1 }), secretValues: ["x", "1"] });
+    expect(result.summary).toBe("state=exited, exit=1");
+  });
+
+  it.each([null, "malformed", { Status: "injected-secret", Health: { Status: "private-health" } }])(
+    "never surfaces arbitrary state metadata: %j", async value => {
+      const result = await capture(runner(value));
+      expect(result.summary).toMatch(/^state (?:unavailable)$|^state=unknown$/u);
+      expect(await readFile(result.path!, "utf8")).not.toMatch(/injected-secret|private-health/u);
+    });
+
+  it("records collection failure without persisting raw Docker errors", async () => {
+    const result = await capture(async () => { throw new Error("transport arbitrary-private-value"); });
+    const text = await readFile(result.path!, "utf8");
+    expect(text).not.toContain("arbitrary-private-value");
+    expect(JSON.parse(text).logs.available).toBe(false);
+    expect(result.summary).toBe("state unavailable");
+  });
+
+  it("rejects oversized output from injectable command runners", async () => {
+    const result = await capture(async () => Buffer.alloc(65_537, "x"));
+    const evidence = JSON.parse(await readFile(result.path!, "utf8"));
+    expect(evidence.logs.available).toBe(false);
+    expect(evidence.health.available).toBe(false);
+  });
+
+  it("reports absent diagnostics when storage is unavailable, without throwing", async () => {
+    await writeFile(path.join(deployment, "diagnostics"), "occupied");
+    await expect(capture()).resolves.toEqual({ path: null, summary: "state=running, health=unhealthy, exit=0" });
+  });
+
+  it("refuses a public or symlinked diagnostic directory", async () => {
+    const directory = path.join(deployment, "diagnostics");
+    await mkdir(directory, { mode: 0o755 });
+    expect((await capture()).path).toBeNull();
+    await rm(directory, { recursive: true });
+    const target = path.join(home, "unrelated");
+    await mkdir(target, { mode: 0o700 });
+    await symlink(target, directory);
+    expect((await capture()).path).toBeNull();
+    expect(await readdir(target)).toEqual([]);
+  });
+
+  it("refuses writable deployment parents and invalid candidate identities", async () => {
+    await chmod(deployment, 0o777);
+    expect((await capture()).path).toBeNull();
+    const runDocker = vi.fn(runner());
+    expect(await captureCandidateDiagnostics({ candidateId: "ambiguous", deploymentName: "candidate", runDocker, secretValues: [] }))
+      .toEqual({ path: null, summary: "invalid candidate identity" });
+    expect(runDocker).not.toHaveBeenCalled();
+  });
+
+  it("sanitizes known values, token shapes, URLs, keys and terminal control characters", () => {
+    const original = [
+      "known-secret", "Bearer abcdefghijklmnop", "sk-proj-abcdefghijklmnopqrstuvwxyz",
+      "TOKEN=value", '{"refreshToken":"refresh-value"}', "Authorization: Basic basic-credential",
+      '{"Authorization":"Basic hidden-json-basic"}', "{'refreshToken':'single-quoted-secret'}",
+      "https://user:pass@example.org/private?token=foo", "eyJhbGciOiJIUzI1NiJ9.payload.signature",
+      "-----BEGIN PRIVATE KEY-----\nprivate-key-bytes\n-----END PRIVATE KEY-----",
+      "\u001b[31mError: missing module\u001b[0m"
+    ].join("\n");
+    const clean = sanitizeCandidateDiagnostic(original, ["known-secret"]);
+    expect(clean).not.toMatch(/known-secret|abcdefghijklmnop|refresh-value|basic-credential|hidden-json-basic|single-quoted-secret|user:pass|eyJhbGci|private-key-bytes|\u001b/u);
+    expect(clean).toContain("Error: missing module");
+  });
+
+  it("redacts overlapping secrets longest first without leaving suffixes", () => {
+    const secrets = ["abc", "abcdef1"];
+    expect(sanitizeCandidateDiagnostic("before abcdef1 after abc", secrets)).toBe("before [REDACTED] after [REDACTED]");
+    expect(secrets).toEqual(["abc", "abcdef1"]);
+  });
+
+  it("normalizes terminal escapes before matching known secrets", () => {
+    expect(sanitizeCandidateDiagnostic("Error: sec\u001b[0mret value\nsec\rret", ["secret"]))
+      .toBe("Error: [REDACTED] value\n[REDACTED]");
+  });
+});

src/deployment/candidateDiagnostics.ts

@@ -0,0 +1,98 @@
+import { randomUUID } from "node:crypto";
+import { lstat, mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { resolveSpawnfileHome } from "../auth/index.js";
+import type { DockerCommandRunner } from "../distribution/dockerRunner.js";
+import { redactDockerLogText } from "./dockerLogs.js";
+import { resolveHomeDeploymentDirectory } from "./homeStore.js";
+
+const outputLimit = 64 * 1024;
+const commandOptions = { captureStderr: true, maxOutputBytes: outputLimit, timeoutMs: 5_000 };
+const containerIdPattern = /^[a-f0-9]{64}$/u;
+
+export interface CandidateDiagnosticsInput {
+  candidateId: string;
+  deploymentName: string;
+  runDocker: DockerCommandRunner;
+  secretValues: string[];
+}
+
+export interface CandidateDiagnosticsResult {
+  path: string | null;
+  summary: string;
+}
+
+/** Redaction is best effort; diagnostic artifacts remain private operator data. */
+export const sanitizeCandidateDiagnostic = (text: string, secrets: string[]): string =>
+  redactDockerLogText(text, secrets)
+    .replace(/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?(?:-----END [^-]*PRIVATE KEY-----|$)/gu, "[REDACTED PRIVATE KEY]")
+    .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, "[REDACTED JWT]")
+    .replace(/https?:\/\/[^\s<>"']+/giu, "[REDACTED URL]")
+    .replace(/\b([a-z0-9_-]*(?:authorization|cookie|password|secret|token|api[_-]?key)[a-z0-9_-]*)["']?\s*[:=]\s*[^\r\n]+/giu, "$1: [REDACTED]");
+
+const collect = async (runDocker: DockerCommandRunner, args: string[]) => {
+  try {
+    const bytes = await runDocker(args, commandOptions);
+    // Injectable runners must obey the same limit as the default Docker runner.
+    if (bytes.length > outputLimit) throw new Error("limit");
+    return { available: true, text: bytes.toString("utf8") };
+  } catch {
+    // Docker transport errors can contain command/env details; never persist them.
+    return { available: false, text: "Diagnostic command failed, timed out, or exceeded its byte limit" };
+  }
+};
+
+const summarizeState = (raw: string, secrets: string[]) => {
+  try {
+    const state = JSON.parse(raw) as Record<string, unknown>;
+    const health = (state.Health as { Status?: unknown } | undefined)?.Status;
+    const status = ["created", "running", "paused", "restarting", "removing", "exited", "dead"].includes(String(state.Status))
+      ? state.Status : "unknown";
+    const parts = [`state=${status}`];
+    if (["starting", "healthy", "unhealthy"].includes(String(health))) parts.push(`health=${health}`);
+    if (Number.isSafeInteger(state.ExitCode)) parts.push(`exit=${state.ExitCode}`);
+    if (state.OOMKilled === true) parts.push("oom-killed");
+    const healthState = state.Health as { Log?: unknown } | undefined;
+    const text = (value: unknown) => typeof value === "string"
+      ? sanitizeCandidateDiagnostic(value, secrets).slice(0, 4_096) : "";
+    return { available: true, summary: parts.join(", "), error: text(state.Error),
+      healthOutput: Array.isArray(healthState?.Log) ? healthState.Log.slice(-3).map((entry: unknown) =>
+        text(entry && typeof entry === "object" ? (entry as { Output?: unknown }).Output : null)) : [] };
+  } catch { return { available: false, summary: "state unavailable", error: "", healthOutput: [] }; }
+};
+
+/** Collect before rollback deletes the only container-local startup evidence. */
+export const captureCandidateDiagnostics = async (
+  input: CandidateDiagnosticsInput
+): Promise<CandidateDiagnosticsResult> => {
+  if (!containerIdPattern.test(input.candidateId)) return { path: null, summary: "invalid candidate identity" };
+  const [state, logs] = await Promise.all([
+    collect(input.runDocker, ["container", "inspect", "--format", "{{json .State}}", input.candidateId]),
+    collect(input.runDocker, ["logs", "--tail", "100", input.candidateId])
+  ]);
+  const health = summarizeState(state.available ? state.text : "", input.secretValues);
+  const summary = health.summary;
+  try {
+    const deploymentDirectory = resolveHomeDeploymentDirectory(input.deploymentName);
+    for (const parent of [resolveSpawnfileHome(), path.dirname(deploymentDirectory), deploymentDirectory]) {
+      const info = await lstat(parent);
+      if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o022) !== 0
+        || (process.getuid && info.uid !== process.getuid())) throw new Error("unsafe diagnostics parent");
+    }
+    const directory = path.join(deploymentDirectory, "diagnostics");
+    await mkdir(directory, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => {
+      if (error.code !== "EEXIST") throw error;
+    });
+    const info = await lstat(directory);
+    if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o777) !== 0o700
+      || (process.getuid && info.uid !== process.getuid())) throw new Error("unsafe diagnostics directory");
+    const destination = path.join(directory, `candidate-${Date.now()}-${randomUUID()}.json`);
+    await writeFile(destination, `${JSON.stringify({
+      version: "spawnfile.candidate-diagnostics.v1", capturedAt: new Date().toISOString(),
+      candidateId: input.candidateId, deployment: input.deploymentName, summary, health,
+      logs: { available: logs.available, text: sanitizeCandidateDiagnostic(logs.text, input.secretValues) }
+    }, null, 2)}\n`, { flag: "wx", mode: 0o600 });
+    return { path: destination, summary };
+  } catch { return { path: null, summary }; }
+};

src/deployment/dockerLogs.test.ts

@@ -146,6 +146,24 @@ describe("docker deployment logs", () => {
 });
 
 describe("redactDockerLogText", () => {
+  it("masks longer overlapping secrets before their prefixes without mutating the supplied values", () => {
+    const secrets = ["abc", "abcdef1"];
+    expect(redactDockerLogText("before abcdef1 after abc", secrets)).toBe("before [REDACTED] after [REDACTED]");
+    expect(secrets).toEqual(["abc", "abcdef1"]);
+  });
+
+  it("strips terminal controls before matching known secrets", () => {
+    expect(redactDockerLogText("Error: sec\u001b[0mret value\nsec\rret", ["secret"]))
+      .toBe("Error: [REDACTED] value\n[REDACTED]");
+  });
+
+  it("normalizes secret values identically and ignores values that normalize to empty", () => {
+    expect(redactDockerLogText("value token-abc\r\n", ["token-abc\r", "\u001b[0m"]))
+      .toBe("value [REDACTED]\n");
+    expect(redactDockerLogText("value abc\u001b[0mdef", ["abc", "abc\u001b[0mdef"]))
+      .toBe("value [REDACTED]");
+  });
+
   it("redacts direct secret values without touching unrelated text", () => {
     expect(redactDockerLogText("before token-value after", ["token-value"])).toBe(
       "before [REDACTED] after"

src/deployment/dockerLogs.ts

@@ -60,15 +60,22 @@ const withDockerTarget = (
 const normalizeTail = (tail: number | undefined): number =>
   typeof tail === "number" && Number.isInteger(tail) && tail > 0 ? tail : 100;
 
+const stripTerminalControls = (value: string): string =>
+  value.replace(/\x1b\[[0-?]*[ -/]*[@-~]|[\x00-\x08\x0b-\x1f\x7f]/gu, "");
+
 const redactKnownSecrets = (text: string, secretValues: string[]): string =>
   secretValues
+    .map(stripTerminalControls)
     .filter((secret) => secret.length > 0)
+    .sort((left, right) => right.length - left.length)
     .reduce((redacted, secret) => redacted.replaceAll(secret, "[REDACTED]"), text);
 
 export const redactDockerLogText = (
   text: string,
   secretValues: string[] = []
-): string => redactSensitiveText(redactKnownSecrets(text, secretValues));
+): string => redactSensitiveText(redactKnownSecrets(
+  stripTerminalControls(text), secretValues
+));
 
 const combineLogStreams = (stdout: string, stderr: string): string => {
   if (!stdout) {

src/deployment/index.ts

@@ -4,6 +4,7 @@ export * from "./artifactsExportPlan.js";
 export * from "./artifactsExportTypes.js";
 export * from "./privateArtifactsExport.js";
 export * from "./buildImageCacheStore.js";
+export * from "./candidateDiagnostics.js";
 export * from "./dockerLabels.js";
 export * from "./dockerInspect.js";
 export * from "./dockerLogs.js";

src/distribution/AGENTS.md

@@ -31,3 +31,7 @@ src/distribution/
 - This folder is pure: no filesystem access, no compiler imports. The compiler projects its plan/report data into the structural input types defined here.
 - The fingerprint is computed over the report body excluding `generated_at`, so identical compiles fingerprint identically across machines and times.
 - Frozen contract values (`spawnfile.distribution-report.v1`, `spawnfile.image.v1`, the in-image report path) live in `types.ts` and change only with a contract version bump.
+- Candidate startup gets the full 120-second readiness budget even if Docker
+  marks a running process unhealthy during initialization. Only running and
+  healthy (or a container without a health check) passes; terminal or unknown
+  health state fails. Deployment diagnostics must finish before candidate removal.

src/distribution/consumeImage.test.ts

@@ -1,6 +1,6 @@
 import os from "node:os";
 import path from "node:path";
-import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, readdir, writeFile } from "node:fs/promises";
 
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
 
@@ -581,18 +581,33 @@ describe("consumeImageUp", () => {
   it("restores the previous container when candidate readiness inspection fails", async () => {
     const calls: string[][] = [];
     const base = createFakeDocker({ calls: [] }, undefined, { liveExists: true });
+    const diagnosticsDirectory = path.join(homeDirectory, "deployments", "readiness-rollback", "diagnostics");
+    let evidenceBeforeRemoval = false;
     const runDocker = async (args: string[]): Promise<Buffer> => {
       calls.push(args);
+      if (args[0] === "logs") return Buffer.from("Error: missing startup module\nsecret-value\n");
+      if (args.includes("{{json .State}}")) return Buffer.from(JSON.stringify({
+        Running: false, Status: "exited", ExitCode: 1, Health: { Status: "unhealthy" }
+      }));
       if (args[0] === "container" && args.some((arg) => arg.includes("{{json .State}}"))) {
         throw new Error("readiness transport failed");
       }
+      if (args[0] === "rm" && args[2] === candidateContainerId) {
+        const files = await readdir(diagnosticsDirectory);
+        const evidence = await readFile(path.join(diagnosticsDirectory, files[0]!), "utf8");
+        expect(evidence).toContain("missing startup module");
+        expect(evidence).not.toContain("secret-value");
+        expect(JSON.parse(evidence).summary).toBe("state=exited, health=unhealthy, exit=1");
+        evidenceBeforeRemoval = true;
+      }
       return base(args);
     };
     await expect(consumeImageUp("you/org:1.0.0", {
-      authValues: { ANTHROPIC_API_KEY: "sk", DIST_REQUIRED_TOKEN: "x" },
+      authValues: { ANTHROPIC_API_KEY: "secret-value", DIST_REQUIRED_TOKEN: "x" },
       deploymentName: "readiness-rollback",
       runDocker
     })).rejects.toThrow(/readiness transport failed/u);
+    expect(evidenceBeforeRemoval).toBe(true);
 
     const live = "spawnfile-readiness-rollback";
     const backup = calls.find((call) => call[0] === "rename" && call[1] === live)?.[2];