diff --git a/src/common/constants/compactionCancellation.ts b/src/common/constants/compactionCancellation.ts new file mode 100644 index 00000000000..9a4c6789b44 --- /dev/null +++ b/src/common/constants/compactionCancellation.ts @@ -0,0 +1 @@ +export const COMPACTION_CANCELLATION_FILE = "compaction-cancellation.json"; diff --git a/src/common/orpc/schemas/continuousCompaction.ts b/src/common/orpc/schemas/continuousCompaction.ts index fcc8c46250e..e139140ddc1 100644 --- a/src/common/orpc/schemas/continuousCompaction.ts +++ b/src/common/orpc/schemas/continuousCompaction.ts @@ -75,6 +75,8 @@ const attachment: z.ZodType = z.discriminatedUnion("ty export const ContinuousCompactionJournalSchema = z .object({ version: z.literal(1), + // Legacy absence is valid only before the first durable repair generation. + publicationGeneration: z.string().min(1).optional(), boundary: row, staticCopies: z.array(row), liveTailCopySpec: z.object({ diff --git a/src/common/orpc/schemas/message.ts b/src/common/orpc/schemas/message.ts index 15513f36a59..5c0af1d75e1 100644 --- a/src/common/orpc/schemas/message.ts +++ b/src/common/orpc/schemas/message.ts @@ -161,6 +161,7 @@ export const MuxMessageSchema = z.object({ metadata: z .object({ historySequence: z.number().optional(), + compactionCancellationNonce: z.string().optional().catch(undefined), // Step cuts are an optimization; malformed legacy metadata must not block chat replay. stepStartPartIndices: z.array(z.number()).optional().catch(undefined), timestamp: z.number().optional(), diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 587be9673b5..351b3b376a6 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -939,6 +939,8 @@ export interface ContextBudgetRejectedMessage { // Our custom metadata type export interface MuxMetadata { + /** Durable explicit replacement of a stopped compaction intent, safe across sidecar retirement crashes. */ + compactionCancellationNonce?: string; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/constants/continuousCompaction.ts b/src/constants/continuousCompaction.ts index 3312bd56dcc..33a074701f9 100644 --- a/src/constants/continuousCompaction.ts +++ b/src/constants/continuousCompaction.ts @@ -4,3 +4,6 @@ export const TAIL_MIN_TOKENS = 4_000; export const TAIL_MAX_TOKENS = 60_000; export const MIN_HEAD_TOKENS = 8_000; export const SUMMARIZER_INPUT_FRACTION = 0.7; + +export const CONTINUOUS_COMPACTION_JOURNAL_FILE = "continuous-compaction.json"; +export const CONTINUOUS_COMPACTION_GENERATION_FILE = "continuous-compaction-generation.json"; diff --git a/src/node/services/agentSession.admissionGates.test.ts b/src/node/services/agentSession.admissionGates.test.ts index 523f113e27d..3c905923c04 100644 --- a/src/node/services/agentSession.admissionGates.test.ts +++ b/src/node/services/agentSession.admissionGates.test.ts @@ -211,6 +211,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { expect(result).toEqual({ success: false, error: { type: "unknown", raw: CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE }, + superseded: true, }); // Accepted, then notified so delivered-state bookkeeping can revert // (terminal-attention outbox contract, r41) — and the stale snapshot diff --git a/src/node/services/agentSession.compactionAcceptance.test.ts b/src/node/services/agentSession.compactionAcceptance.test.ts new file mode 100644 index 00000000000..0632fdb4d46 --- /dev/null +++ b/src/node/services/agentSession.compactionAcceptance.test.ts @@ -0,0 +1,316 @@ +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { createMuxMessage } from "@/common/types/message"; +import { Err } from "@/common/types/result"; +import type { TurnCoordinator } from "./turnCoordinator"; +import * as branchSummary from "./branchSummary"; +import { createAgentSessionHarness } from "./agentSession.testHarness"; + +afterEach(() => mock.restore()); + +test.each([ + ["before", false], + ["after", false], + ["before", true], + ["after", true], +] as const)( + "Stop %s handoff append commits clears only its own pending continuation (successor=%s)", + async (commitPoint, hasSuccessor) => { + const workspaceId = "stopped-appending-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + const summary = createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }); + await h.historyService.appendToHistory(workspaceId, summary); + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + dispatchPendingFollowUp(): Promise; + }; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementation(async (...args) => { + const isHandoff = + args[1].role === "user" && + args[1].parts.some((part) => part.type === "text" && part.text === "Continue"); + if (!isHandoff) return append(...args); + if (commitPoint === "before") { + entered.resolve(); + await release.promise; + return append(...args); + } + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = internals.dispatchPendingFollowUp(); + let restarted: Awaited> | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + let successor: ReturnType; + if (hasSuccessor) { + expect((await h.session.sendMessage("manual replacement", options)).success).toBe(true); + successor = internals.coordinator.beginCompactionObservation("continuous"); + expect(successor).toBeDefined(); + // A's late rollback must not retire B's compaction work or erase a + // replacement summary, even when its history row reuses the source ID. + await h.historyService.updateHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Replacement boundary", { + ...summary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Replacement follow-up", ...options }, + }, + }) + ); + } + release.resolve(); + expect(await pending).toBe(false); + expect(stream).toHaveBeenCalledTimes(hasSuccessor ? 1 : 0); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + if (!history.success) throw new Error(history.error); + expect( + history.data + .filter((message) => message.role === "user") + .map((message) => + message.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("") + ) + ).toEqual(hasSuccessor ? ["manual replacement"] : []); + const source = history.data.find((message) => message.id === summary.id); + if (hasSuccessor) { + if (!successor) throw new Error("Expected successor compaction"); + expect(internals.coordinator.isCurrentCompaction(successor)).toBe(true); + expect(source?.metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "Replacement follow-up" + ); + } else { + expect(source?.metadata?.muxMetadata).not.toHaveProperty("pendingFollowUp"); + await h.session.dispose(); + restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: h.historyService, + }); + const recovered = restarted.session as unknown as { + dispatchPendingFollowUp(): Promise; + }; + const resumed = spyOn(restarted.aiService, "streamMessage"); + expect(await recovered.dispatchPendingFollowUp()).toBe(false); + expect(resumed).not.toHaveBeenCalled(); + } + } finally { + release.resolve(); + await pending.catch(() => undefined); + await restarted?.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["before preparation", "during provider startup"] as const)( + "an accepted handoff stays continued when replaced %s before send returns an error", + async (replacementPoint) => { + const workspaceId = "replaced-accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + dispatchPendingFollowUp(): Promise; + }; + const send = h.session.sendMessage.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + if (replacementPoint === "before preparation") { + spyOn(h.session, "sendMessage").mockImplementationOnce((message, sendOptions, internal) => + send(message, sendOptions, { + ...internal, + onAccepted: async () => { + await internal?.onAccepted?.(); + entered.resolve(); + await release.promise; + }, + }) + ); + } else { + stream.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return Err({ type: "runtime_start_failed", message: "retired startup failed" }); + }); + } + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }) + ); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + // A blocked startup can be retired before its engine call returns; its + // durable continuation still belongs to the predecessor's completed handoff. + if (replacementPoint === "during provider startup") + internals.coordinator.preemptPreparation(); + expect((await send("manual replacement", options)).success).toBe(true); + release.resolve(); + expect(await pending).toBe(true); + expect(stream).toHaveBeenCalledTimes(replacementPoint === "before preparation" ? 1 : 2); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && + history.data.filter((message) => message.role === "user").map((message) => message.parts) + ).toMatchObject([ + [{ type: "text", text: "Continue" }], + [{ type: "text", text: "manual replacement" }], + ]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "an accepted handoff still reports a current provider startup failure (later replacement=%s)", + async (replaceAfterFailure) => { + const workspaceId = "failed-accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Err({ type: "runtime_start_failed", message: "provider startup failed" }) + ); + if (replaceAfterFailure) { + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementationOnce(async (...args) => { + const result = await send(...args); + expect(result.success).toBe(false); + expect( + (await send("manual replacement", { model: "openai:gpt-4o", agentId: "exec" })).success + ).toBe(true); + return result; + }); + } + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + try { + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + const failure = await dispatch.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).toHaveProperty("message", expect.stringContaining("provider startup failed")); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && history.data.find((message) => message.role === "user")?.parts + ).toMatchObject([{ type: "text", text: "Continue" }]); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("disposal during real pre-acceptance preparation does not report a continued handoff", async () => { + const workspaceId = "unaccepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + spyOn(branchSummary, "awaitPendingBranchSummary").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return null; + }); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + const pending = dispatch.dispatchPendingFollowUp(); + let disposal: Promise | undefined; + try { + await entered.promise; + disposal = h.session.dispose(); + release.resolve(); + expect(await pending).toBe(false); + await disposal; + expect(stream).not.toHaveBeenCalled(); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((message) => message.role === "user")).toBe(false); + } finally { + release.resolve(); + await pending; + await disposal; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("an accepted real handoff stays continued after a subsequent manual replacement", async () => { + const workspaceId = "accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }) + ); + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementationOnce(async (...args) => { + const result = await send(...args); + expect(result.success).toBe(true); + await h.session.interruptStream(); + expect((await send("manual replacement", options)).success).toBe(true); + return result; + }); + try { + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + expect(await dispatch.dispatchPendingFollowUp()).toBe(true); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && + history.data.filter((message) => message.role === "user").map((message) => message.parts) + ).toMatchObject([ + [{ type: "text", text: "Continue" }], + [{ type: "text", text: "manual replacement" }], + ]); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts new file mode 100644 index 00000000000..919fb453721 --- /dev/null +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -0,0 +1,3675 @@ +import { makeTestEffectRunner } from "./di/testEffectRunner"; +import { calculateBackoffDelay } from "@/common/utils/messages/retryState"; +import * as fs from "node:fs/promises"; +import { readFile, readdir, writeFile } from "node:fs/promises"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import { CompactionCancellation } from "./compactionCancellation"; +import type { CompactionMonitor } from "./compactionMonitor"; +import type { ContinuousCompactor } from "./continuousCompactor"; +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { createMuxMessage } from "@/common/types/message"; +import { Err, Ok } from "@/common/types/result"; +import type { TurnCompletion } from "./streamManager"; +import type { TurnCoordinator } from "./turnCoordinator"; +import type { CompactionHandler } from "./compactionHandler"; +import { HistoryService } from "./historyService"; +import { log } from "./log"; +import { ExtensionMetadataService } from "./ExtensionMetadataService"; +import { WorkspaceGoalService } from "./workspaceGoalService"; +import { createTestHistoryService } from "./testHistoryService"; +import { createAgentSessionHarness, createStreamLifecycleMocks } from "./agentSession.testHarness"; + +const workspaceId = "compaction-shutdown"; +const options = { model: "openai:gpt-4o", agentId: "exec" }; +const summary = () => + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }); + +async function setup() { + const history = await createTestHistoryService(); + const goalService = new WorkspaceGoalService( + history.config, + history.historyService, + new ExtensionMetadataService(`${history.config.rootDir}/extension.json`), + { recordGoalLifecycleEvent: mock(() => undefined) } + ); + const h = await createAgentSessionHarness({ + ...history, + workspaceId, + workspaceGoalService: goalService, + }); + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + activeCompactionRequest?: { id: string; modelString: string }; + compactionHandler: CompactionHandler; + dispatchPendingFollowUp(summaryId?: string, cancelResume?: () => boolean): Promise; + }; + return { ...h, cleanup: history.cleanup, goalService, internals }; +} + +afterEach(() => mock.restore()); + +test.each(["Stop", "replacement", "shutdown", "dispose"] as const)( + "a durable handoff survives %s during goal synchronization", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const sync = h.goalService.syncGoalModeWithChatTail.bind(h.goalService); + spyOn(h.goalService, "syncGoalModeWithChatTail").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return sync(...args); + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + let closing: Promise | undefined; + try { + await entered.promise; + if (action === "Stop") await h.session.interruptStream({ abandonPartial: true }); + if (action === "replacement") + expect((await h.session.sendMessage("manual replacement", options)).success).toBe(true); + if (action === "shutdown") closing = h.session.finishShutdown(); + if (action === "dispose") closing = h.session.dispose(); + release.resolve(); + expect(await pending).toBe(true); + await closing; + expect(stream).toHaveBeenCalledTimes(action === "replacement" ? 1 : 0); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && history.data.find((row) => row.role === "user")?.parts + ).toMatchObject([{ type: "text", text: "Continue" }]); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "a real goal-sync failure retains the durable handoff and reports failure (Stop=%s)", + async (stop) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + spyOn(h.goalService, "syncGoalModeWithChatTail").mockImplementationOnce(async () => { + if (stop) await h.session.interruptStream({ abandonPartial: true }); + throw new Error("goal reconciliation failed"); + }); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).toHaveProperty( + "message", + expect.stringContaining("goal reconciliation failed") + ); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((row) => row.role === "user")).toBe(true); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["Stop", "replacement"] as const)( + "a synchronous row listener preserves the durable handoff on %s", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + let replacement: Promise | undefined; + const detach = h.session.onChatEvent(({ message }) => { + if (message.type !== "message" || message.role !== "user") return; + detach(); + if (action === "Stop") replacement = h.session.interruptStream({ abandonPartial: true }); + else { + // Simulate reentrant manual admission in the same synchronous event publication. + expect( + h.internals.coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: h.internals.coordinator.turnId, + }).status + ).toBe("admitted"); + } + }); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + await replacement; + expect(stream).not.toHaveBeenCalled(); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + } finally { + detach(); + await replacement; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["shutdown", "dispose"] as const)( + "Stop cleanup retains its exact guarded commit while %s joins it", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const history = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const write = history.writeGuardedHistory.bind(history); + spyOn(history, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stream = spyOn(h.aiService, "streamMessage"); + // A ready owner is already clearing a canceled resume when Stop and shutdown arrive. + const pending = h.internals.dispatchPendingFollowUp(undefined, () => true); + let closing: Promise | undefined; + try { + await entered.promise; + const stopping = h.session.interruptStream({ abandonPartial: true }); + let closed = false; + closing = (action === "shutdown" ? h.session.finishShutdown() : h.session.dispose()).then( + () => { + closed = true; + } + ); + await Promise.resolve(); + expect(closed).toBe(false); + release.resolve(); + await stopping; + expect(await pending).toBe(false); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + await pending; + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +async function expectNoRecovery( + config: Awaited>["config"], + historyService: HistoryService +) { + const restarted = await createAgentSessionHarness({ workspaceId, config, historyService }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + const recovered = restarted.session as unknown as { + dispatchPendingFollowUp(): Promise; + }; + expect(await recovered.dispatchPendingFollowUp()).toBe(false); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } +} + +test.each(["success", "goal rejection", "boundary publication rejection"] as const)( + "Stop then shutdown before terminal cleanup removes its durable intent (%s)", + async (outcome) => { + const h = await setup(); + const completion = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Ok({ messageId: "assistant", completion: completion.promise }) + ); + spyOn(h.internals.compactionHandler, "handleCompletion").mockImplementationOnce(async () => { + await h.historyService.appendToHistory(workspaceId, summary()); + if (outcome === "boundary publication rejection") { + entered.resolve(); + await release.promise; + throw new Error("boundary publication failed"); + } + return true; + }); + spyOn(h.goalService, "applyPendingAfterStreamEnd").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (outcome === "goal rejection") throw new Error("pending goal application failed"); + return null; + }); + const errors = spyOn(log, "error"); + const consumer = spyOn(h.internals.coordinator, "consumeCompletion"); + let closing: Promise | undefined; + try { + expect((await h.session.sendMessage("original", options)).success).toBe(true); + h.internals.activeCompactionRequest = { id: "compact-request", modelString: options.model }; + completion.resolve({ + status: "completed", + streamEnd: { + type: "stream-end", + workspaceId, + metadata: { model: options.model }, + parts: [], + }, + }); + await entered.promise; + const stopping = h.session.interruptStream({ abandonPartial: true }); + let closed = false; + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await Promise.resolve(); + expect(closed).toBe(false); + release.resolve(); + await stopping; + const policy = consumer.mock.results.at(-1); + if (policy?.type !== "return") throw new Error("Expected terminal consumer"); + await policy.value; + await closing; + expect(stream).toHaveBeenCalledTimes(1); + if (outcome !== "success") + expect(errors).toHaveBeenCalledWith("stream-end cleanup failed", { + workspaceId, + error: + outcome === "goal rejection" + ? "pending goal application failed" + : "boundary publication failed", + }); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + completion.resolve({ status: "aborted", abortReason: "user" }); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "Stop during goal redispatch admission finishes cleanup through shutdown (rejection=%s)", + async (reject) => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + ...options, + goalKind: "goal_continuation", + goalId: "00000000-0000-4000-8000-000000000001", + }, + }, + }, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + spyOn(h.goalService, "buildGoalRedispatchAdmission").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (reject) throw new Error("goal admission read failed"); + return { admissible: true, admissionStale: () => false }; + }); + const pending = h.internals.dispatchPendingFollowUp(); + let closing: Promise | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + closing = h.session.finishShutdown(); + release.resolve(); + const result = await pending.catch((error: unknown) => error); + if (reject) expect(result).toHaveProperty("message", "goal admission read failed"); + else expect(result).toBe(false); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("a failed Stop rollback acknowledges the retained durable continuation", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const stream = spyOn(h.aiService, "streamMessage"); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await h.session.interruptStream({ abandonPartial: true }); + return result; + }); + spyOn(h.historyService, "deleteMessages").mockResolvedValueOnce(Err("disk unavailable")); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(true); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each([ + [false, false, false], + [true, false, false], + [false, true, false], + [true, true, false], + [false, false, true], + [true, false, true], +] as const)( + "a failed initial follow-up read retries only abandoned ownership (targeted=%s, replacement=%s, result error=%s)", + async (targeted, replacement, resultError) => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, boundary); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = targeted ? "getHistoryFromLatestBoundary" : "getLastMessages"; + spyOn(h.historyService, read).mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (resultError) return Err("initial history read failed"); + throw new Error("initial history read failed"); + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(targeted ? boundary.id : undefined); + let closing: Promise | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + if (replacement) { + using _mutation = h.session.holdTurnAdmission(); + await h.historyService.updateHistory(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "replacement", ...options }, + }, + }, + }); + await h.session.contextMutationCommitted(); + } + closing = h.session.finishShutdown(); + release.resolve(); + const failure = await pending.catch((error: unknown) => error); + expect(failure).toHaveProperty( + "message", + expect.stringContaining("initial history read failed") + ); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + if (replacement) + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement" + ); + else { + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["fields", "pending request", "message ID", "sequence"] as const)( + "abandoned cleanup matches handoff identity and preserves newer summary fields (%s)", + async (changed) => { + const h = await setup(); + const boundary = summary(); + boundary.metadata = { + ...boundary.metadata, + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }; + await h.historyService.appendToHistory(workspaceId, boundary); + await h.session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + const rewritten = { + ...boundary, + id: changed === "message ID" ? "replacement-summary" : boundary.id, + parts: [{ type: "text" as const, text: "finalized summary" }], + metadata: { + ...boundary.metadata, + model: "openai:gpt-4o", + duration: 42, + muxMetadata: { + type: "compaction-summary" as const, + pendingFollowUp: { + text: changed === "pending request" ? "replacement" : "Continue", + ...options, + }, + }, + }, + }; + if (changed === "sequence") { + await h.historyService.deleteMessages(workspaceId, [boundary.id]); + await h.historyService.appendToHistory(workspaceId, { + ...rewritten, + metadata: { ...rewritten.metadata, historySequence: undefined }, + }); + } else if (changed === "fields") { + // Real late finalization omits compaction metadata; HistoryService must + // preserve the handoff before cleanup merges only its pending field away. + await update(workspaceId, { + ...rewritten, + metadata: { + historySequence: boundary.metadata?.historySequence, + model: "openai:gpt-4o", + duration: 42, + }, + }); + } else await update(workspaceId, rewritten); + release.resolve(); + expect(await pending).toBe(false); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "finalized summary" }, + ]); + expect(rows.success && rows.data[0].metadata).toHaveProperty("duration", 42); + expect(rows.success && rows.data[0].metadata).toHaveProperty("compactionBoundary", true); + if (changed !== "fields") + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + else { + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("context replacement after Stop dispatches B while A's held cleanup is retired", async () => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, boundary); + await h.session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const stale = h.internals.dispatchPendingFollowUp(); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await entered.promise; + { + using _mutation = h.session.holdTurnAdmission(); + await update(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "replacement", ...options }, + }, + }, + }); + await h.session.contextMutationCommitted(); + } + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + release.resolve(); + expect(await stale).toBe(false); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.find((row) => row.role === "user")?.parts).toMatchObject([ + { type: "text", text: "replacement" }, + ]); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement" + ); + } finally { + release.resolve(); + await stale; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("a persistently unreadable abandoned follow-up retries once and preserves the initial error", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages") + .mockRejectedValueOnce(new Error("original read failure")) + .mockRejectedValueOnce(new Error("cleanup read failure")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "original read failure"); + expect(read).toHaveBeenCalledTimes(2); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["recovery", "shutdown", "dispose", "teardown read recovers"] as const)( + "failed abandoned cleanup keeps ownership until %s retries it", + async (retry) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages") + .mockRejectedValueOnce(new Error("initial read unavailable")) + .mockRejectedValueOnce(new Error("cleanup read unavailable")); + if (retry === "teardown read recovers") + read.mockRejectedValueOnce(new Error("teardown first read unavailable")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "initial read unavailable"); + const owner = h.internals.coordinator.compactionIntent.followUp; + expect(owner).toBeDefined(); + if (!owner) throw new Error("Expected retained cleanup owner"); + expect(h.internals.coordinator.canClearCompactionFollowUp(owner)).toBe(true); + if (retry === "recovery") expect(await h.internals.dispatchPendingFollowUp()).toBe(false); + if (retry === "shutdown") await h.session.finishShutdown(); + else await h.session.dispose(); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } finally { + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } + } +); + +test("permanent abandoned cleanup failure is bounded and fails shutdown after releasing resources", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages").mockRejectedValue( + new Error("history unavailable") + ); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + const failure = await h.session.finishShutdown().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(read).toHaveBeenCalledTimes(4); + expect(h.aiEmitter.listenerCount("stream-start")).toBe(0); + expect(h.aiEmitter.listenerCount("stream-end")).toBe(0); + expect(h.internals.coordinator.compactionIntent.followUp).toBeDefined(); + } finally { + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("retrying retired cleanup never dispatches the replacement handoff", async () => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, boundary); + await h.session.interruptStream({ abandonPartial: true }); + spyOn(h.historyService, "getLastMessages") + .mockRejectedValueOnce(new Error("initial read unavailable")) + .mockRejectedValueOnce(new Error("cleanup read unavailable")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + { + using _mutation = h.session.holdTurnAdmission(); + await h.historyService.updateHistory(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "replacement", ...options }, + }, + }, + }); + await h.session.contextMutationCommitted(); + } + await h.session.retryPendingCompactionCleanup(); + expect(stream).not.toHaveBeenCalled(); + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + expect(stream).toHaveBeenCalledTimes(1); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("stopped follow-up stays canceled after failed cleanup and a fresh process session", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages").mockRejectedValue( + new Error("history unavailable") + ); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + await h.session.finishShutdown().catch(() => undefined); + read.mockRestore(); + const freshHistory = new HistoryService(h.config); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } + } finally { + read.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("an initial follow-up waits for admission release without consuming recovery", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const hold = h.session.holdTurnAdmission(); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const pending = h.internals.dispatchPendingFollowUp(); + await Promise.resolve(); + expect(h.internals.coordinator.compactionIntent.followUp).toBeUndefined(); + expect(stream).not.toHaveBeenCalled(); + hold[Symbol.dispose](); + expect(await pending).toBe(true); + expect(stream).toHaveBeenCalledTimes(1); + } finally { + hold[Symbol.dispose](); + await h.session.dispose(); + await h.cleanup(); + } +}, 1000); + +test.each([ + "release", + "nested release", + "Stop", + "replacement", + "shutdown", + "hold during read", + "hold during prepare", +] as const)( + "startup follow-up admission wait handles %s without another recovery trigger", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + let hold = + action === "hold during read" || action === "hold during prepare" + ? undefined + : h.session.holdTurnAdmission(); + const second = action === "nested release" ? h.session.holdTurnAdmission() : undefined; + const waiting = Promise.withResolvers(); + const wait = h.internals.coordinator.waitForAdmissionRelease.bind(h.internals.coordinator); + spyOn(h.internals.coordinator, "waitForAdmissionRelease").mockImplementation(() => { + waiting.resolve(); + return wait(); + }); + if (action === "hold during read") { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await read(...args); + hold = h.session.holdTurnAdmission(); + return result; + }); + } + if (action === "hold during prepare") { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + hold = h.session.holdTurnAdmission(); + return result; + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const recovery = h.session.runStartupRecovery(); + try { + await waiting.promise; + // No physical lease is waiting for a policy hold: shutdown can drain it. + await h.internals.coordinator.drain(); + expect(stream).not.toHaveBeenCalled(); + if (action === "Stop") await h.session.interruptStream({ abandonPartial: true }); + if (action === "replacement") { + await h.historyService.clearHistory(workspaceId); + await h.session.contextMutationCommitted(); + } + if (action === "shutdown") await h.session.finishShutdown(); + hold?.[Symbol.dispose](); + if (second) { + await Promise.resolve(); + expect(stream).not.toHaveBeenCalled(); + second[Symbol.dispose](); + } + await recovery; + expect(stream).toHaveBeenCalledTimes( + action === "release" || + action === "nested release" || + action === "hold during read" || + action === "hold during prepare" + ? 1 + : 0 + ); + } finally { + hold?.[Symbol.dispose](); + second?.[Symbol.dispose](); + await recovery; + await h.session.dispose(); + await h.cleanup(); + } + }, + 2000 +); + +test("a delayed cancellation write survives failed manual preparation", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + try { + const stopping = h.session.interruptStream({ abandonPartial: true }); + await entered.promise; + spyOn(h.historyService, "appendToHistory").mockResolvedValueOnce( + Err("replacement append failed") + ); + const replacement = h.session.sendMessage("replacement", options); + release.resolve(); + expect((await replacement).success).toBe(false); + await stopping; + await h.session.dispose(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + await expectNoRecovery(h.config, freshHistory); + } finally { + release.resolve(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test.each([false, true])( + "durable replacement retirement failure preserves acceptance or its genuine goal error (goal error=%s)", + async (goalError) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const nonce = await h.session.getCompactionCancellationNonce(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("cancellation unlink failed"); + return write(...args); + } + ); + const stream = spyOn(h.aiService, "streamMessage"); + const accepted = mock(() => undefined); + const visibleRows: string[] = []; + const detach = h.session.onChatEvent(({ message }) => { + if (message.type === "message" && message.role === "user") visibleRows.push(message.id); + }); + if (goalError) + spyOn(h.goalService, "syncGoalModeWithChatTail").mockRejectedValueOnce( + new Error("genuine goal failure") + ); + try { + const result = await h.session + .sendMessage("replacement", options, { onAccepted: accepted }) + .catch((error: unknown) => error); + if (goalError) expect(result).toHaveProperty("message", "genuine goal failure"); + else expect(result).toHaveProperty("success", true); + expect(stream).toHaveBeenCalledTimes(goalError ? 0 : 1); + expect(accepted).toHaveBeenCalledTimes(goalError ? 0 : 1); + expect(visibleRows).toHaveLength(goalError ? 0 : 1); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + expect(h.session.hasBlockingCompactionCleanup).toBe(false); + const freshHistory = new HistoryService(h.config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && + rows.data.find((row) => row.role === "user")?.metadata?.compactionCancellationNonce + ).toBe(nonce); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + try { + await restarted.session.runStartupRecovery(); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + const after = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + after.success && after.data.filter((row) => row.role === "user").map((row) => row.parts) + ).toMatchObject([[{ type: "text", text: "replacement" }]]); + } finally { + await restarted.session.dispose(); + } + } finally { + detach(); + writes.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } + } +); + +test("cancellation persistence failures fail teardown until an explicit durable retry", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValue( + new Error("cancellation disk unavailable") + ); + try { + await h.session.interruptStream({ abandonPartial: true }); + const failure = await h.session.dispose().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "cancellation disk unavailable"); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + expect(h.aiEmitter.listenerCount("stream-start")).toBe(0); + write.mockRestore(); + await h.session.retryPendingCompactionCleanup(); + expect(h.session.hasPendingCompactionCleanup).toBe(false); + await expectNoRecovery(h.config, new HistoryService(h.config)); + } finally { + write.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("an exact cancellation survives failed summary writes across a fresh service", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const historyWrites = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const update = spyOn(historyWrites, "writeGuardedHistory").mockRejectedValue( + new Error("history rewrite unavailable") + ); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + await h.session.finishShutdown().catch(() => undefined); + const freshHistory = new HistoryService(h.config); + expect((await freshHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + await expectNoRecovery(h.config, freshHistory); + } finally { + update.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test.each(["{", JSON.stringify({ version: 1, nonce: "broken", scope: {} })])( + "malformed cancellation %s self-heals before automatic startup recovery", + async (malformed) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const sessionDir = `${h.config.sessionsDir}/${workspaceId}`; + await writeFile(`${sessionDir}/${COMPACTION_CANCELLATION_FILE}`, malformed); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + await writeFile(journal.path, "untrusted interrupted journal"); + const compactor = (h.session as unknown as { continuousCompactor: ContinuousCompactor }) + .continuousCompactor; + const recoverJournal = compactor.recover.bind(compactor); + const recover = spyOn(compactor, "recover").mockImplementation(async () => { + // The old journal must be gone before normal recovery can inspect it. + expect(await journal.exists()).toBe(false); + return recoverJournal(); + }); + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recoverGoal).toHaveBeenCalledTimes(1); + expect(recover).toHaveBeenCalledTimes(1); + expect(stream).not.toHaveBeenCalled(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(await freshHistory.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + const files = await readdir(sessionDir); + const preserved = await Promise.all( + files.map((file) => readFile(`${sessionDir}/${file}`, "utf8").catch(() => "")) + ); + expect(preserved).toContain(malformed); + await expectNoRecovery(h.config, freshHistory); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([ + [false, false], + [true, false], + [false, true], + [true, true], +] as const)( + "direct follow-up dispatch discards its pre-repair summary (targeted=%s, during history=%s)", + async (targeted, duringHistory) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + if (duringHistory) { + if (targeted) { + const read = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const captured = await read(...args); + await h.session.getCompactionCancellationNonce(); + return captured; + } + ); + } else { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const captured = await read(...args); + await h.session.getCompactionCancellationNonce(); + return captured; + }); + } + } + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp(targeted ? "summary" : undefined)).toBe( + false + ); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["goal admission", "preparation", "row persistence"] as const)( + "late repair during %s invalidates an unaccepted follow-up", + async (stage) => { + const h = await setup(); + const boundary = summary(); + if (stage === "goal admission") { + boundary.metadata = { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + ...options, + goalKind: "goal_continuation", + goalId: "00000000-0000-4000-8000-000000000001", + }, + }, + }; + } + await h.historyService.appendToHistory(workspaceId, boundary); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "goal admission") { + spyOn(h.goalService, "buildGoalRedispatchAdmission").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return { admissible: true, admissionStale: () => false }; + }); + } else if (stage === "preparation") { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + } + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await h.session.runStartupRecovery(); + release.resolve(); + expect(await pending).toBe(false); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["summary read", "preparation", "auto compaction preparation"] as const)( + "a foreign repair during %s prevents the captured follow-up from committing", + async (stage) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + if (stage === "auto compaction preparation") { + const monitor = (h.session as unknown as { compactionMonitor: CompactionMonitor }) + .compactionMonitor; + spyOn(monitor, "getThreshold").mockReturnValue(0.85); + spyOn(monitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + } + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "summary read") { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + }); + } else { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const foreignStream = spyOn(foreign.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.session.runStartupRecovery(); + expect(await foreignHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(foreignStream).not.toHaveBeenCalled(); + release.resolve(); + expect(await pending).toBe(false); + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( + "a skipped follow-up never accepts preparation snapshots (%s)", + async (failureMode) => { + const cleanupFails = failureMode === "cleanup failure"; + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const ownSnapshot = createMuxMessage("own-preparation", "assistant", "Snapshot", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: ownSnapshot, + materializedTokens: [], + fileStates: [], + }); + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const append = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { + if (failureMode === "guard I/O failure") { + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValueOnce( + new Error("guard storage unavailable") + ); + } else { + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.session.runStartupRecovery(); + } + return append(...args); + }); + if (cleanupFails) + spyOn(h.historyService, "deleteMessages").mockResolvedValueOnce( + Err("snapshot cleanup unavailable") + ); + let durableReceipts = 0; + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementation((message, sendOptions, internal) => + send(message, sendOptions, { + ...internal, + onRowsDurable: () => { + durableReceipts++; + internal?.onRowsDurable?.(); + }, + }) + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + if (failureMode === "guard I/O failure") + expect(result).toHaveProperty( + "message", + "Failed to append to history: guard storage unavailable" + ); + else expect(result).toBe(false); + expect(durableReceipts).toBe(0); + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data.some((row) => row.id === ownSnapshot.id)).toBe(false); + } finally { + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("cancellation storage-access failures still block automatic startup recovery", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValue( + Object.assign(new Error("cancellation inaccessible"), { code: "EACCES" }) + ); + const compactor = (h.session as unknown as { continuousCompactor: ContinuousCompactor }) + .continuousCompactor; + const recover = spyOn(compactor, "recover"); + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recover).not.toHaveBeenCalled(); + expect(recoverGoal).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toHaveProperty("code", "EACCES"); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["quarantine", "journal", "history", "unlink"] as const)( + "failed corrupt cancellation repair at %s retains the fence and retries safely", + async (step) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const cancellationPath = `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`; + await writeFile(cancellationPath, "{"); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + await writeFile(journal.path, "interrupted journal"); + const failure = new Error(`${step} unavailable`); + if (step === "quarantine") { + const write = fs.writeFile; + let failed = false; + spyOn(fs, "writeFile").mockImplementation(async (...args) => { + if (!failed && typeof args[0] === "string" && args[0].includes(".corrupt.")) { + failed = true; + throw failure; + } + return write(...args); + }); + } else if (step === "journal") { + spyOn(journal, "invalidateUnderHistoryLock").mockRejectedValueOnce(failure); + } else if (step === "unlink") { + const remove = fs.rm; + let failed = false; + spyOn(fs, "rm").mockImplementation(async (...args) => { + if (!failed && typeof args[0] === "string" && args[0] === cancellationPath) { + failed = true; + throw failure; + } + return remove(...args); + }); + } else { + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, contents: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce(failure); + } + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recoverGoal).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + expect(await readFile(cancellationPath, "utf8")).toBe("{"); + expect(await journal.exists()).toBe(step === "quarantine" || step === "journal"); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + const metadata = rows.success && rows.data[0].metadata?.muxMetadata; + if (step === "unlink") expect(metadata).not.toHaveProperty("pendingFollowUp"); + else expect(metadata).toHaveProperty("pendingFollowUp"); + await h.session.runStartupRecovery(); + expect(recoverGoal).toHaveBeenCalledTimes(1); + expect(stream).not.toHaveBeenCalled(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(await freshHistory.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + await expectNoRecovery(h.config, freshHistory); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("ordinary manual sends do not write cancellation state or attach a witness", async () => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation"); + try { + expect((await h.session.sendMessage("ordinary user", options)).success).toBe(true); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.find((row) => row.role === "user")?.metadata + ).not.toHaveProperty("compactionCancellationNonce"); + expect(write).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("Stop during cancellation loading retains exact summary cleanup", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = h.historyService.readCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "readCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return read(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + release.resolve(); + expect(await pending).toBe(false); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["manual", "context replacement"] as const)( + "explicit %s repairs a corrupt cancellation without permitting old automatic recovery", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + if (action === "manual") { + expect((await h.session.sendMessage("explicit repair", options)).success).toBe(true); + } else { + using _hold = h.session.holdTurnAdmission(); + await h.historyService.clearHistory(workspaceId); + await h.session.contextMutationCommitted(); + } + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user").length).toBe( + action === "manual" ? 1 : 0 + ); + expect(stream).toHaveBeenCalledTimes(action === "manual" ? 1 : 0); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("explicit resume durably supersedes an earlier Stop without appending another user row", async () => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "original request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).not.toBeNull(); + try { + expect((await h.session.resumeStream(options)).success).toBe(true); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user").length).toBe(1); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("a second Stop after the resume witness commits cannot be retired by the first nonce", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "request")); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const firstNonce = await h.session.getCompactionCancellationNonce(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const history = h.historyService as unknown as { + writeGuardedHistory( + path: string, + serialized: string, + guard: () => boolean, + committed?: () => void + ): Promise; + }; + const write = history.writeGuardedHistory.bind(history); + spyOn(history, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + const committed = await write(...args); + entered.resolve(); + await release.promise; + return committed; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const resume = h.session.resumeStream(options); + try { + await entered.promise; + const stopping = h.session.interruptStream({ abandonPartial: true }); + const secondNonce = ( + await ( + h.session as unknown as { compactionCancellation: CompactionCancellation } + ).compactionCancellation.read() + )?.nonce; + expect(secondNonce).not.toBe(firstNonce); + release.resolve(); + await stopping; + expect(await resume).toEqual(Ok({ started: false })); + await h.session.retryPendingCompactionCleanup(); + expect(stream).not.toHaveBeenCalled(); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(secondNonce); + } finally { + release.resolve(); + await resume; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["history failure", "witness no-op", "automatic"] as const)( + "resume keeps cancellation before explicit durable acceptance (%s)", + async (outcome) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const nonce = await h.session.getCompactionCancellationNonce(); + if (outcome === "history failure") + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockResolvedValueOnce( + Err("resume history unavailable") + ); + if (outcome === "witness no-op") + spyOn(h.historyService, "acceptResumeCancellation").mockResolvedValueOnce(Ok(undefined)); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.resumeStream(options, { automatic: outcome === "automatic" }); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(nonce); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["user row", "summary only", "failed unlink"] as const)( + "accepted explicit resume allows B's follow-up across restart (%s)", + async (shape) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + shape === "summary only" ? summary() : createMuxMessage("user", "user", "request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (shape === "failed unlink" && args[1] === null) throw new Error("resume unlink failed"); + return write(...args); + } + ); + try { + await h.session.resumeStream(options).catch(() => undefined); + // A new process first reconciles an already-durable Retry receipt if unlink failed. + const freshHistory = new HistoryService(h.config); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + try { + await restarted.session.runStartupRecovery(); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await restarted.session.dispose(); + } + await freshHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + const next = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const stream = spyOn(next.aiService, "streamMessage"); + try { + await next.session.runStartupRecovery(); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "Continue B" }, + ]); + } finally { + await next.session.dispose(); + } + } finally { + writes.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } + } +); + +test.each(["cached absence", "changed nonce"] as const)( + "a live backend observes another backend's durable Stop after %s", + async (cached) => { + const a = await setup(); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const bDispatch = ( + b.session as unknown as { dispatchPendingFollowUp(): Promise } + ).dispatchPendingFollowUp.bind(b.session); + const bWrites = bHistory as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const failedCleanup = spyOn(bWrites, "writeGuardedHistory").mockRejectedValue( + new Error("cleanup unavailable in backend B") + ); + const stream = spyOn(a.aiService, "streamMessage"); + try { + await a.historyService.appendToHistory(workspaceId, summary()); + if (cached === "changed nonce") { + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + await bDispatch().catch(() => undefined); + expect((await bHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + } + const cachedNonce = await a.session.getCompactionCancellationNonce(); + if (cached === "changed nonce") { + await bHistory.clearHistory(workspaceId); + await b.session.contextMutationCommitted(); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + } + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + await bDispatch().catch(() => undefined); + expect( + (await new HistoryService(a.config).readCompactionCancellation(workspaceId))?.nonce + ).not.toBe(cachedNonce); + expect(await a.internals.dispatchPendingFollowUp()).toBe(false); + expect(stream).not.toHaveBeenCalled(); + } finally { + failedCleanup.mockRestore(); + await a.session.dispose().catch(() => undefined); + await b.session.dispose().catch(() => undefined); + await a.cleanup(); + } + } +); + +test("stale cross-backend summary cleanup cannot narrow or retire a successor Stop", async () => { + const a = await setup(); + await a.historyService.appendToHistory(workspaceId, summary()); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = a.historyService.readCompactionCancellation.bind(a.historyService); + spyOn(a.historyService, "readCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return read(...args); + }); + const stale = a.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await bHistory.clearHistory(workspaceId); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + const canceledB = await bHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await stale).toBe(false); + expect(await new HistoryService(a.config).readCompactionCancellation(workspaceId)).toEqual( + canceledB + ); + await expectNoRecovery(a.config, new HistoryService(a.config)); + } finally { + release.resolve(); + await stale; + await a.session.dispose(); + await b.session.dispose(); + await a.cleanup(); + } +}); + +test("failed cleanup cannot narrow a foreign replacement's newer Stop and preserves its original error", async () => { + const a = await setup(); + await a.historyService.appendToHistory(workspaceId, summary()); + await a.session.interruptStream({ abandonPartial: true }); + await a.session.retryPendingCompactionCleanup(); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const writes = a.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce( + new Error("original guarded rewrite failed") + ); + const update = a.historyService.updateHistory.bind(a.historyService); + spyOn(a.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + const result = await update(...args); + entered.resolve(); + await release.promise; + return result; + }); + const cleanup = a.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + try { + await entered.promise; + await bHistory.clearHistory(workspaceId); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + const canceledB = await bHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await cleanup).toHaveProperty( + "message", + expect.stringContaining("original guarded rewrite failed") + ); + expect(await new HistoryService(a.config).readCompactionCancellation(workspaceId)).toEqual( + canceledB + ); + await expectNoRecovery(a.config, new HistoryService(a.config)); + } finally { + release.resolve(); + await cleanup; + await a.session.dispose(); + await b.session.dispose(); + await a.cleanup(); + } +}); + +test.each([ + ["legacy", false], + ["failed apply", false], + ["failed apply compact", false], + ["legacy", true], + ["failed apply", true], + ["failed apply compact", true], +] as const)( + "direct %s handoff honors another backend's durable Stop during preparation (narrowed=%s)", + async (kind, narrowed) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + narrowed ? { ...summary(), id: "original" } : createMuxMessage("original", "user", "Work") + ); + const context = { modelString: options.model, options, providersConfig: null }; + const direct = h.session as unknown as { + activeStreamContext: typeof context; + interruptForCompaction(): Promise; + finishContinuousCompaction( + applied: boolean, + capturedContext: typeof context, + token: NonNullable> + ): Promise; + compactionMonitor: CompactionMonitor; + }; + direct.activeStreamContext = context; + if (kind === "failed apply compact") + spyOn(direct.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const token = + kind === "legacy" + ? undefined + : h.internals.coordinator.beginCompactionObservation("continuous"); + if (token) h.internals.coordinator.setCompactionStage(token, "stopped"); + const pending = + kind === "legacy" + ? direct.interruptForCompaction() + : direct.finishContinuousCompaction(false, context, token!); + const stream = spyOn(h.aiService, "streamMessage"); + let cleanupForeign: (() => Promise) | undefined; + try { + await entered.promise; + const foreignHistory = new HistoryService(h.config); + await new CompactionCancellation(foreignHistory, workspaceId).cancel(); + if (narrowed) { + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const writes = foreignHistory as unknown as { + writeGuardedHistory( + path: string, + serialized: string, + guard: () => boolean + ): Promise; + }; + const failure = spyOn(writes, "writeGuardedHistory").mockRejectedValue( + new Error("foreign summary rewrite failed") + ); + cleanupForeign = async () => { + failure.mockRestore(); + await foreign.session.dispose(); + }; + await foreign.session.runStartupRecovery(); + expect((await foreignHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + } + release.resolve(); + await pending; + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual(["original"]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await cleanupForeign?.(); + if (token) h.internals.coordinator.finishCompactionObservation(token); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("idle Stop reports cancellation publication failure instead of success", async () => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValue( + new Error("Stop publication failed") + ); + try { + const stopped = await h.session.interruptStream({ abandonPartial: true }); + expect(stopped).toEqual(Err("Stop publication failed")); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + } finally { + write.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("idle Stop cannot acknowledge before its held publication commits", async () => { + const h = await setup(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let committed = false; + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + await write(...args); + committed = true; + }); + const stopping = h.session + .interruptStream({ abandonPartial: true }) + .then((result) => { + expect(committed).toBe(true); + return result; + }) + .catch((error: unknown) => error); + try { + await entered.promise; + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + release.resolve(); + expect(await stopping).toEqual(Ok(undefined)); + } finally { + release.resolve(); + await stopping; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("overlapping Stops wait for the newest publication when the older write becomes a no-op", async () => { + const h = await setup(); + const firstEntered = Promise.withResolvers(); + const firstRelease = Promise.withResolvers(); + const secondEntered = Promise.withResolvers(); + const secondRelease = Promise.withResolvers(); + let secondCommitted = false; + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation") + .mockImplementationOnce(async (...args) => { + firstEntered.resolve(); + await firstRelease.promise; + return write(...args); + }) + .mockImplementationOnce(async (...args) => { + secondEntered.resolve(); + await secondRelease.promise; + await write(...args); + secondCommitted = true; + }); + const first = h.session + .interruptStream({ abandonPartial: true }) + .then((result) => { + expect(secondCommitted).toBe(true); + return result; + }) + .catch((error: unknown) => error); + let second: ReturnType | undefined; + try { + await firstEntered.promise; + second = h.session.interruptStream({ abandonPartial: true }); + firstRelease.resolve(); + await secondEntered.promise; + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + secondRelease.resolve(); + expect(await first).toEqual(Ok(undefined)); + expect(await second).toEqual(Ok(undefined)); + } finally { + firstRelease.resolve(); + secondRelease.resolve(); + await first; + await second; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each([false, true])( + "explicit Retry rejects a foreign Stop after capture (initial Stop=%s)", + async (initialStop) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "Work")); + if (initialStop) await h.session.interruptStream({ abandonPartial: true }); + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + entered.resolve(); + await release.promise; + return nonce; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.session.resumeStream(options); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const stopped = await foreign.read(); + release.resolve(); + expect(await pending).toEqual(Ok({ started: false })); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata).not.toHaveProperty( + "compactionCancellationNonce" + ); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([ + "absent", + "witnessed debt", + "retirement failure", + "shared read failure", + "witness write failure", +] as const)("explicit Retry locked acceptance handles %s", async (state) => { + const h = await setup(); + const user = createMuxMessage("user", "user", "Work", { + requestPreludeMessageIds: ["owned-prelude"], + }); + await h.historyService.appendToHistory(workspaceId, user); + const cancellation = (h.session as unknown as { compactionCancellation: CompactionCancellation }) + .compactionCancellation; + const debt = state === "witnessed debt" || state === "retirement failure"; + let restoreWrites: (() => void) | undefined; + if (debt) { + await h.session.interruptStream({ abandonPartial: true }); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("unlink unavailable"); + return write(...args); + } + ); + restoreWrites = () => writes.mockRestore(); + if (state === "witnessed debt") { + await h.historyService.updateHistory(workspaceId, { + ...user, + metadata: { ...user.metadata, compactionCancellationNonce: record.nonce }, + }); + await cancellation.retireReplacement(record.nonce).catch(() => undefined); + expect(await cancellation.read()).toBeNull(); + } + } + if (state === "shared read failure") { + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValueOnce( + new Error("shared storage unavailable") + ); + return nonce; + }); + } + if (state === "witness write failure") { + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce( + new Error("witness storage unavailable") + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.session.resumeStream(options); + const storageFailure = state === "shared read failure" || state === "witness write failure"; + if (storageFailure) { + expect(result.success).toBe(false); + expect(JSON.stringify(result)).toContain("storage unavailable"); + } else expect(result).toEqual(Ok({ started: true })); + expect(stream).toHaveBeenCalledTimes(storageFailure ? 0 : 1); + expect(cancellation.needsPersistence).toBe(debt); + if (debt) expect(h.session.hasBlockingCompactionCleanup).toBe(false); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.filter((row) => row.role === "user").map((row) => row.id) + ).toEqual([user.id]); + expect(rows.success && rows.data[0].metadata?.requestPreludeMessageIds).toEqual([ + "owned-prelude", + ]); + } finally { + restoreWrites?.(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each( + [false, true].flatMap((tokenBudget) => + ["accepted", "foreign Stop", "raw reset", "append failure"].map((action) => ({ + tokenBudget, + action, + })) + ) +)( + "handoff batch preserves locked admission ($action, tokenBudget=$tokenBudget)", + async ({ action, tokenBudget }) => { + const h = await setup(); + const source = summary(); + source.metadata = { + ...source.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options, experiments: { tokenBudget } }, + }, + }; + await h.historyService.appendToHistory(workspaceId, source); + const snapshot = createMuxMessage("owned-snapshot", "assistant", "Expanded context", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: snapshot, + materializedTokens: [], + fileStates: [], + }); + const append = h.historyService.appendManyToHistory.bind(h.historyService); + const batch = spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce( + async (...args) => { + if (action === "foreign Stop") + await new CompactionCancellation(new HistoryService(h.config), workspaceId).cancel(); + if (action === "raw reset") + await fs.appendFile( + `${h.config.sessionsDir}/${workspaceId}/chat.jsonl`, + '{"metadata":{"contextBoundaryKind":"reset"},broken\n' + ); + if (action === "append failure") return Err("batch storage unavailable"); + return append(...args); + } + ); + const send = h.session.sendMessage.bind(h.session); + const durable = mock(() => undefined); + spyOn(h.session, "sendMessage").mockImplementation((message, sendOptions, internal) => + send(message, sendOptions, { + ...internal, + onRowsDurable: () => { + durable(); + internal?.onRowsDurable?.(); + }, + }) + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + if (action === "append failure") + expect(result).toHaveProperty("message", "batch storage unavailable"); + else expect(result).toBe(action === "accepted"); + expect(batch).toHaveBeenCalledTimes(1); + expect(batch.mock.calls[0][1]).toHaveLength(2); + expect(durable).toHaveBeenCalledTimes(action === "accepted" ? 1 : 0); + expect(stream).toHaveBeenCalledTimes(action === "accepted" ? 1 : 0); + const rows = await h.historyService.getLastMessages(workspaceId, 10); + expect(rows.success && rows.data.some((row) => row.id === snapshot.id)).toBe( + action === "accepted" + ); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["empty", "raw reset", "journal only"])( + "startup retains a settled Stop on %s history against a late foreign summary", + async (state) => { + const h = await setup(); + try { + await h.session.interruptStream({ abandonPartial: true }); + if (state === "raw reset") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("private", "user", "Old context") + ); + await fs.appendFile( + `${h.config.sessionsDir}/${workspaceId}/chat.jsonl`, + '{"metadata":{"contextBoundaryKind":"reset"},broken\n' + ); + } + if (state === "journal only") + await writeFile( + h.historyService.getContinuousCompactionJournal(workspaceId).path, + "old journal" + ); + const stopped = await h.historyService.readCompactionCancellation(workspaceId); + await h.session.runStartupRecovery(); + const foreign = new HistoryService(h.config); + expect(await foreign.readCompactionCancellation(workspaceId)).toEqual(stopped); + expect(h.session.hasBlockingCompactionCleanup).toBe(false); + await foreign.appendToHistory(workspaceId, summary()); + await expectNoRecovery(h.config, foreign); + expect((await h.session.sendMessage("Explicit replacement", options)).success).toBe(true); + expect(await foreign.readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("failed legacy compaction preflight releases idle waiters without a stream terminal", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("original", "user", "Work")); + const direct = h.session as unknown as { + activeStreamContext: { modelString: string; options: typeof options; providersConfig: null }; + interruptForCompaction(): Promise; + }; + direct.activeStreamContext = { modelString: options.model, options, providersConfig: null }; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + throw new Error("legacy preparation unavailable"); + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = direct.interruptForCompaction().catch((error: unknown) => error); + try { + await entered.promise; + let settled = false; + const waiting = h.session.waitForMidStreamCompactionSettled().then(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + release.resolve(); + await pending; + await waiting; + expect(settled).toBe(true); + expect(stream).not.toHaveBeenCalled(); + expect(h.session.hasActiveOrPendingTurnWork()).toBe(false); + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("cancelable acceptance precedes blocked cancellation retirement and runs once", async () => { + const h = await setup(); + await h.session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementation(async (...args) => { + if (args[1] === null) { + entered.resolve(); + await release.promise; + } + return write(...args); + }); + const accepted = mock(() => undefined); + const controller = new AbortController(); + const pending = h.session.sendMessage("Explicit replacement", options, { + cancelSignal: controller.signal, + onAccepted: accepted, + }); + try { + await entered.promise; + expect(accepted).toHaveBeenCalledTimes(1); + release.resolve(); + expect((await pending).success).toBe(true); + expect(accepted).toHaveBeenCalledTimes(1); + expect(await h.historyService.readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each( + [false, true].flatMap((initialStop) => + [ + "single", + "token-budget single", + "token-budget batch", + "pre-turn batch", + "on-send compaction", + ].map((branch) => ({ initialStop, branch })) + ) +)( + "manual $branch rejects a foreign Stop after capture (initial Stop=$initialStop)", + async ({ initialStop, branch }) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original", "user", "Earlier work") + ); + if (initialStop) await h.session.interruptStream({ abandonPartial: true }); + if (branch === "on-send compaction") { + const monitor = (h.session as unknown as { compactionMonitor: CompactionMonitor }) + .compactionMonitor; + spyOn(monitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + } + if (branch === "token-budget batch") { + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: createMuxMessage("snapshot", "assistant", "Context", { synthetic: true }), + materializedTokens: [], + fileStates: [], + }); + } + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + entered.resolve(); + await release.promise; + return nonce; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const durable = mock(() => undefined); + const pending = h.session.sendMessage( + "Manual replacement", + { + ...options, + ...(branch.startsWith("token-budget") ? { experiments: { tokenBudget: true } } : {}), + }, + { + onRowsDurable: durable, + ...(branch === "pre-turn batch" + ? { + preTurnMessages: [ + createMuxMessage("payload", "assistant", "Payload", { synthetic: true }), + ], + } + : {}), + } + ); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const stopped = await foreign.read(); + release.resolve(); + expect((await pending).success).toBe(false); + expect(stream).not.toHaveBeenCalled(); + expect(durable).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual(["original"]); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "live dispatch reconciles a foreign replacement witness (archived=%s)", + async (archived) => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const stop = new CompactionCancellation(foreign, workspaceId); + await stop.cancel(); + const record = await stop.read(); + if (!record) throw new Error("Expected Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Fresh work", { + compactionCancellationNonce: record.nonce, + }) + ); + spyOn(foreign, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + expect( + await stop.retireReplacement(record.nonce).catch((error: unknown) => error) + ).toHaveProperty("message", "unlink unavailable"); + expect(await foreign.readCompactionCancellation(workspaceId)).toEqual(record); + const boundary = summary(); + if (archived) + boundary.metadata = { + ...boundary.metadata, + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }; + await foreign.appendToHistory(workspaceId, boundary); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].role).toBe("user"); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("interrupted handoff snapshot persistence leaves a fresh service able to recover", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const snapshot = createMuxMessage("interrupted-snapshot", "user", "Expanded context", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeAgentSkillSnapshots(): Promise>>; + materializeMcpPromptSnapshots(): Promise>>; + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: snapshot, + materializedTokens: [], + fileStates: [], + }); + spyOn(materializer, "materializeAgentSkillSnapshots").mockResolvedValue([ + createMuxMessage("skill-prelude", "user", "Skill context", { synthetic: true }), + ]); + spyOn(materializer, "materializeMcpPromptSnapshots").mockResolvedValue([ + createMuxMessage("mcp-prelude", "user", "MCP context", { synthetic: true }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + const batch = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { + // Simulate process loss before publication by observing disk from an independent service. + entered.resolve(); + await release.promise; + return batch(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + let fresh: Awaited> | undefined; + try { + await entered.promise; + fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const recoveredStream = spyOn(fresh.aiService, "streamMessage"); + await fresh.session.runStartupRecovery(); + expect(recoveredStream).toHaveBeenCalledTimes(1); + const recovered = await fresh.historyService.getLastMessages(workspaceId, 1); + expect(recovered.success && recovered.data[0].parts).toMatchObject([ + { type: "text", text: "Continue" }, + ]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await fresh?.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("Stop survives failed rollback of a committed handoff snapshot batch across restart", async () => { + const h = await setup(); + const clock = makeTestEffectRunner(); + await h.historyService.appendToHistory(workspaceId, summary()); + const snapshot = createMuxMessage("retained-snapshot", "user", "Expanded context", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: snapshot, + materializedTokens: [], + fileStates: [], + }); + const append = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await h.session.interruptStream({ abandonPartial: true }); + return result; + }); + spyOn(h.historyService, "deleteMessages").mockResolvedValue(Err("rollback unavailable")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.internals.dispatchPendingFollowUp(); + expect(stream).not.toHaveBeenCalled(); + const foreign = new HistoryService(h.config); + expect(await foreign.readCompactionCancellation(workspaceId)).not.toBeNull(); + const rows = await foreign.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user")).toHaveLength(2); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreign, + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, + }); + try { + const recovered = spyOn(fresh.aiService, "streamMessage"); + const retries = fresh.session as unknown as { retryActiveStream(): Promise }; + const deliver = spyOn(retries, "retryActiveStream"); + await fresh.session.runStartupRecovery(); + await clock.adjust(calculateBackoffDelay(6) * 2); + // Deliver any armed retry and join its actual callback before checking provider entry. + await Promise.all(deliver.mock.results.map((result) => result.value)); + expect(deliver).not.toHaveBeenCalled(); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + expect(recovered).not.toHaveBeenCalled(); + expect(await foreign.readCompactionCancellation(workspaceId)).not.toBeNull(); + } finally { + await fresh.session.dispose(); + } + } finally { + await h.session.dispose(); + await h.cleanup(); + await clock.dispose(); + } +}); + +test.each(["witness read", "retirement"] as const)( + "live witness reconciliation preserves newer Stop during %s", + async (stage) => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Work", { compactionCancellationNonce: record.nonce }) + ); + await foreign.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "witness read") { + const read = h.historyService.hasCompactionReplacementWitness.bind(h.historyService); + spyOn(h.historyService, "hasCompactionReplacementWitness").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await cancellation.cancel(); + const newer = await cancellation.read(); + // Keep the newer cancellation observable through a failed durable cleanup. + const writes = h.historyService as unknown as { + writeGuardedHistory( + path: string, + serialized: string | Buffer, + guard: () => boolean + ): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce(new Error("cleanup unavailable")); + release.resolve(); + expect(await pending.catch((error: unknown) => error)).toHaveProperty( + "message", + "Failed to clear skipped pending follow-up: Failed to update history: cleanup unavailable" + ); + expect(stream).not.toHaveBeenCalled(); + expect((await foreign.readCompactionCancellation(workspaceId))?.nonce).toBe(newer?.nonce); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("cancellation cleanup rechecks a replacement witness under the summary write lock", async () => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const user = createMuxMessage("replacement", "user", "Work"); + await foreign.appendToHistory(workspaceId, user); + const boundary = summary(); + await foreign.appendToHistory(workspaceId, boundary); + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + await foreign.updateHistory(workspaceId, { + ...user, + metadata: { ...user.metadata, compactionCancellationNonce: record.nonce }, + }); + return update(...args); + }); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(false); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["absence", "witnessed debt", "Stop after commit"] as const)( + "manual locked acceptance handles %s", + async (state) => { + const h = await setup(); + const cancellation = ( + h.session as unknown as { compactionCancellation: CompactionCancellation } + ).compactionCancellation; + let captured: string | undefined; + let newer: string | undefined; + let restoreWrite: (() => void) | undefined; + if (state !== "absence") { + await h.session.interruptStream({ abandonPartial: true }); + captured = await h.session.getCompactionCancellationNonce(); + if (!captured) throw new Error("Expected Stop"); + if (state === "witnessed debt") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("accepted-earlier", "user", "Work", { + compactionCancellationNonce: captured, + }) + ); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const spy = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + (...args) => + args[1] === null ? Promise.reject(new Error("unlink unavailable")) : write(...args) + ); + restoreWrite = () => spy.mockRestore(); + await cancellation.retireReplacement(captured).catch(() => undefined); + expect(await h.session.getCompactionCancellationNonce()).toBeUndefined(); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + newer = (await foreign.read())?.nonce; + return result; + }); + } + } + const durable = mock(() => undefined); + const accepted = mock(() => undefined); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect( + ( + await h.session.sendMessage("Accepted work", options, { + onRowsDurable: durable, + onAccepted: accepted, + }) + ).success + ).toBe(true); + expect(durable).toHaveBeenCalledTimes(1); + expect(accepted).toHaveBeenCalledTimes(1); + expect(stream).toHaveBeenCalledTimes(1); + const foreign = new HistoryService(h.config); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.compactionCancellationNonce).toBe( + state === "Stop after commit" ? captured : undefined + ); + if (state === "Stop after commit") + expect((await foreign.readCompactionCancellation(workspaceId))?.nonce).toBe(newer); + if (state === "witnessed debt") expect(cancellation.blocksRecovery).toBe(false); + } finally { + restoreWrite?.(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "manual replacement retries a failed Stop publication (storage still fails=%s)", + async (persistent) => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("Stop disk unavailable") + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect((await h.session.interruptStream({ abandonPartial: true })).success).toBe(false); + if (persistent) write.mockRejectedValue(new Error("Stop disk unavailable")); + const result = await h.session + .sendMessage("Fresh explicit work", options) + .catch((error: unknown) => error); + if (persistent) { + expect(result).toHaveProperty("message", "Stop disk unavailable"); + expect(stream).not.toHaveBeenCalled(); + expect(h.session.hasBlockingCompactionCleanup).toBe(true); + } else { + expect(result).toEqual(Ok(undefined)); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await new HistoryService(h.config).getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "Fresh explicit work" }, + ]); + } + } finally { + write.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each( + ["legacy", "failed apply", "failed apply compact"].flatMap((kind) => + [false, true].map((unlinkDebt) => ({ kind, unlinkDebt })) + ) +)( + "captured source-less $kind cannot follow a foreign replacement (unlink debt=$unlinkDebt)", + async ({ kind, unlinkDebt }) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original", "user", "Old work") + ); + const context = { + modelString: options.model, + options, + providersConfig: null, + compactionPublication: { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }, + }; + const direct = h.session as unknown as { + activeStreamContext: typeof context; + interruptForCompaction(): Promise; + finishContinuousCompaction( + applied: boolean, + streamContext: typeof context, + token: NonNullable> + ): Promise; + compactionMonitor: CompactionMonitor; + }; + direct.activeStreamContext = context; + if (kind === "failed apply compact") + spyOn(direct.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const token = + kind === "legacy" + ? undefined + : h.internals.coordinator.beginCompactionObservation("continuous"); + if (token) h.internals.coordinator.setCompactionStage(token, "stopped"); + const pending = + kind === "legacy" + ? direct.interruptForCompaction() + : direct.finishContinuousCompaction(false, context, token!); + const stream = spyOn(h.aiService, "streamMessage"); + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + let restoreWrite: (() => void) | undefined; + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + if (unlinkDebt) { + const write = foreignHistory.writeCompactionCancellation.bind(foreignHistory); + const spy = spyOn(foreignHistory, "writeCompactionCancellation").mockImplementation( + (...args) => + args[1] === null ? Promise.reject(new Error("unlink unavailable")) : write(...args) + ); + restoreWrite = () => spy.mockRestore(); + } + expect((await foreign.session.sendMessage("New accepted work", options)).success).toBe(true); + release.resolve(); + await pending; + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getLastMessages(workspaceId, 10); + expect( + rows.success && rows.data.filter((row) => row.role === "user").map((row) => row.parts) + ).toMatchObject([[{ text: "Old work" }], [{ text: "New accepted work" }]]); + if (token) h.internals.coordinator.finishCompactionObservation(token); + direct.activeStreamContext = { + ...context, + compactionPublication: { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }, + }; + await direct.interruptForCompaction(); + expect(stream).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await pending.catch(() => undefined); + restoreWrite?.(); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["unresolved", "absent", "witnessed", "Stop during backoff", "Stop during preparation"])( + "startup retry actually delivers retained user rows only without unresolved Stop (%s)", + async (state) => { + const h = await setup(); + const clock = makeTestEffectRunner(); + const foreign = new HistoryService(h.config); + const cancellation = new CompactionCancellation(foreign, workspaceId); + if (state === "unresolved" || state === "witnessed") await cancellation.cancel(); + const record = await cancellation.read(); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("retained-trigger", "user", "Retained explicit work", { + synthetic: true, + uiVisible: true, + retrySendOptions: options, + ...(state === "witnessed" ? { compactionCancellationNonce: record?.nonce } : {}), + }) + ); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreign, + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, + }); + const deliver = spyOn( + fresh.session as unknown as { retryActiveStream(): Promise }, + "retryActiveStream" + ); + const stream = spyOn(fresh.aiService, "streamMessage"); + try { + await fresh.session.runStartupRecovery(); + if (state === "Stop during backoff") await cancellation.cancel(); + if (state === "Stop during preparation") { + const read = foreign.getHistoryFromLatestBoundary.bind(foreign); + spyOn(foreign, "getHistoryFromLatestBoundary").mockImplementationOnce(async (...args) => { + const result = await read(...args); + await cancellation.cancel(); + return result; + }); + } + await clock.adjust(calculateBackoffDelay(6) * 2); + await Promise.all(deliver.mock.results.map((result) => result.value)); + expect(deliver).toHaveBeenCalledTimes(state === "unresolved" ? 0 : 1); + expect(stream).toHaveBeenCalledTimes(state === "absent" || state === "witnessed" ? 1 : 0); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe( + state !== "unresolved" + ); + } finally { + await fresh.session.dispose(); + await h.session.dispose(); + await clock.dispose(); + await h.cleanup(); + } + } +); + +test.each(["before capture", "after capture"])( + "a real stream pins handoff publication when foreign Stop wins %s", + async (stage) => { + const h = await setup(); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + if (stage === "before capture") { + entered.resolve(); + await release.promise; + } + const generation = await capture(); + if (stage === "after capture") { + entered.resolve(); + await release.promise; + } + return generation; + }); + const completed = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Ok({ messageId: "original-live-stream", completion: completed.promise }) + ); + spyOn(h.aiService, "stopStream").mockImplementation(() => { + completed.resolve({ status: "aborted", abortReason: "system" }); + return Promise.resolve(Ok(undefined)); + }); + const sending = h.session.sendMessage("Original work", options); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("Replacement work", options)).success).toBe(true); + release.resolve(); + expect((await sending).success).toBe(true); + const direct = h.session as unknown as { interruptForCompaction(): Promise }; + await direct.interruptForCompaction(); + // Capturing after Stop precedes this stream's first context read/provider call; + // capturing before it keeps the older epoch even through delayed provider entry. + expect(stream).toHaveBeenCalledTimes(stage === "before capture" ? 2 : 1); + } finally { + release.resolve(); + completed.resolve({ status: "aborted", abortReason: "system" }); + await sending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["replacement", "dispose"])( + "held stream generation capture cannot mutate its %s successor", + async (action) => { + const h = await setup(); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + const generation = await capture(); + entered.resolve(); + await release.promise; + return generation; + }); + const pending = h.session.sendMessage("Old held preparation", options); + let closing: Promise | undefined; + try { + await entered.promise; + if (action === "replacement") { + await h.session.interruptStream({ abandonPartial: true }); + // This fixture has no engine startup-abort event; finish its logical turn + // while the old physical capture remains held, then admit the successor. + h.internals.coordinator.finishTurn(h.internals.coordinator.turnId); + expect(await h.session.sendMessage("New successor", options)).toEqual(Ok(undefined)); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([{ text: "New successor" }]); + } else closing = h.session.dispose(); + const state = h.session as unknown as { + activeStreamContext: unknown; + activeStreamUserMessageId: string | undefined; + }; + const context = state.activeStreamContext; + const userId = state.activeStreamUserMessageId; + await h.historyService.writePartial( + workspaceId, + createMuxMessage("successor-partial", "assistant", "Successor owned partial") + ); + const commit = spyOn(h.historyService, "commitPartial"); + release.resolve(); + await pending; + await closing; + expect(commit).not.toHaveBeenCalled(); + expect(state.activeStreamContext).toBe(context); + expect(state.activeStreamUserMessageId).toBe(userId); + expect((await h.historyService.readPartial(workspaceId))?.id).toBe("successor-partial"); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["heartbeat", "legacy"])( + "invalidated %s publication preserves a successor partial and pending context", + async (producer) => { + const h = await setup(); + const followUp = { text: "Continue old work", ...options }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + const generation = await capture(); + entered.resolve(); + await release.promise; + return generation; + }); + const pending = + producer === "heartbeat" + ? h.internals.compactionHandler.appendHeartbeatContextResetBoundary({ + boundaryText: "Old context", + pendingFollowUp: followUp, + }) + : h.internals.compactionHandler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Old context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("B replacement", options)).success).toBe(true); + await foreign.historyService.writePartial( + workspaceId, + createMuxMessage("b-partial", "assistant", "B live partial") + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + await writeFile(pendingPath, "foreign B pending state"); + release.resolve(); + await pending; + expect((await foreign.historyService.readPartial(workspaceId))?.id).toBe("b-partial"); + expect(await readFile(pendingPath, "utf8")).toBe("foreign B pending state"); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each( + ["heartbeat", "legacy"].flatMap((producer) => + ["prepare write", "failed cleanup", "invalid read cleanup"].map((stage) => ({ + producer, + stage, + })) + ) +)( + "retired $producer $stage cannot change successor pending-state bytes", + async ({ producer, stage }) => { + const h = await setup(); + const followUp = { text: "Continue old work", ...options }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + if (producer === "legacy") + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-stream", "assistant", "Streamed old summary") + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + if (stage === "invalid read cleanup") await writeFile(pendingPath, "malformed old state"); + const handler = h.internals.compactionHandler; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage !== "failed cleanup") { + const writes = handler as unknown as { + enqueuePendingStateWrite(write: () => Promise): Promise; + }; + const enqueue = writes.enqueuePendingStateWrite.bind(writes); + spyOn(writes, "enqueuePendingStateWrite").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return enqueue(...args); + }); + } else if (producer === "heartbeat") { + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return Err("held heartbeat append failure"); + }); + } else { + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + } + const pending = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Old context", + pendingFollowUp: followUp, + }) + : handler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Old context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const successor = JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/foreign/successor.ts"], + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("B replacement", options)).success).toBe(true); + await writeFile(pendingPath, successor); + release.resolve(); + await pending; + expect(await readFile(pendingPath, "utf8").catch(() => "missing")).toBe(successor); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each( + ["heartbeat", "legacy append", "legacy update"].flatMap((producer) => + ["restart before cleanup", "cleanup", "reset", "edit", "successor"].map((stage) => ({ + producer, + stage, + })) + ) +)("uncommitted $producer pending state is owned across $stage", async ({ producer, stage }) => { + const h = await setup(); + const readMessage = createMuxMessage("read", "assistant", ""); + readMessage.parts = [ + { + type: "dynamic-tool", + toolCallId: "read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/same-logical-read.ts" }, + output: { success: true }, + }, + ]; + await h.historyService.appendToHistory(workspaceId, readMessage); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: { text: "Continue", ...options } }, + }, + }) + ); + if (producer === "legacy update") + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-stream", "assistant", "Uncommitted summary") + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (producer === "legacy update") { + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return append(...args); + }); + } + const handler = h.internals.compactionHandler; + const pending = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Captured context", + pendingFollowUp: { text: "Continue", ...options }, + }) + : handler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Captured context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const foreignHandler = (foreign.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + try { + await entered.promise; + expect(JSON.parse(await readFile(pendingPath, "utf8"))).toHaveProperty("readFiles", [ + "/same-logical-read.ts", + ]); + if (stage === "reset") { + await foreign.historyService.appendToHistory( + workspaceId, + createMuxMessage("reset", "assistant", "", { contextBoundaryKind: "reset" }) + ); + } else if (stage === "edit") { + await foreign.historyService.truncateAfterMessage(workspaceId, "read"); + } else { + await foreign.session.interruptStream({ abandonPartial: true }); + } + if (stage === "restart before cleanup") + expect(await foreignHandler.peekPendingState()).toBeNull(); + let successor: string | undefined; + if (stage === "successor") { + await foreignHandler.preparePendingStateFromMessages([readMessage]); + successor = await readFile(pendingPath, "utf8"); + } + release.resolve(); + await pending; + if (successor !== undefined) { + expect(await readFile(pendingPath, "utf8")).toBe(successor); + } else { + expect(await foreignHandler.peekPendingState()).toBeNull(); + expect(await readFile(pendingPath, "utf8").catch(() => null)).toBeNull(); + } + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each([ + "same-generation successor", + "new-generation successor", + "write rejected", + "write acknowledgement lost", + "cleanup failure", + "current prior state", +])("provisional pending receipt handles %s without claiming successor bytes", async (scenario) => { + const h = await setup(); + const handler = h.internals.compactionHandler; + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const publication = { generation: await journal.captureGeneration() }; + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + const cleanup = ( + handler as unknown as { + cleanupProvisionalPendingState( + receipt: Awaited>, + capturedPublication: typeof publication + ): Promise; + } + ).cleanupProvisionalPendingState.bind(handler); + let prior: string | undefined; + if (scenario === "write rejected" || scenario === "current prior state") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("committed", "assistant", "Prior context", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + prior = JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/prior.ts"], + boundaryMessageId: "committed", + }); + await writeFile(pendingPath, prior); + } + if (scenario === "write rejected" || scenario === "write acknowledgement lost") { + const write = h.historyService.withCompactionPublicationWrite.bind(h.historyService); + spyOn(h.historyService, "withCompactionPublicationWrite").mockImplementationOnce( + async (...args) => { + if (scenario === "write acknowledgement lost") await write(...args); + throw new Error("pending write unavailable"); + } + ); + } + const now = spyOn(Date, "now").mockReturnValue(1234); + try { + const read = createMuxMessage("a-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "a-read", + toolName: "file_read", + state: "output-available", + input: { path: "/provisional.ts" }, + output: { success: true }, + }, + ]; + const receipt = await handler.preparePendingStateFromMessages( + [read], + "provisional", + undefined, + publication + ); + if (scenario === "write rejected") expect(receipt.write).toBeUndefined(); + else expect(receipt.write).toBeDefined(); + if ( + scenario === "new-generation successor" || + scenario === "write acknowledgement lost" || + scenario === "cleanup failure" + ) { + await new CompactionCancellation(new HistoryService(h.config), workspaceId).cancel(); + } + if (scenario === "new-generation successor") { + expect((await h.session.sendMessage("Fresh successor", options)).success).toBe(true); + } + let successor: string | undefined; + if (scenario === "same-generation successor" || scenario === "new-generation successor") { + await handler.preparePendingStateFromMessages([read], "provisional", undefined, { + generation: await journal.captureGeneration(), + }); + successor = await readFile(pendingPath, "utf8"); + expect(successor).not.toBe(receipt.write?.serialized); + } + if (scenario === "cleanup failure") { + const unlink = fs.unlink; + let failed = false; + spyOn(fs, "unlink").mockImplementation(async (...args) => { + if (!failed && args[0] === pendingPath) { + failed = true; + throw new Error("cleanup unavailable"); + } + return unlink(...args); + }); + } + await cleanup(receipt, publication); + const expected = successor ?? prior; + if (expected !== undefined) { + expect(await readFile(pendingPath, "utf8")).toBe(expected); + if (prior !== undefined) + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/prior.ts"]); + } else { + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + expect( + await ( + fresh.session as unknown as { compactionHandler: CompactionHandler } + ).compactionHandler.peekPendingState() + ).toBeNull(); + expect(await readFile(pendingPath, "utf8").catch(() => null)).toBeNull(); + } finally { + await fresh.session.dispose(); + } + } + } finally { + now.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("provisional cleanup cannot clear memory prepared by a successor waiting for its lock", async () => { + const h = await setup(); + const handler = h.internals.compactionHandler; + const publication = { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }; + const receipt = await handler.preparePendingStateFromMessages( + [], + "a-boundary", + undefined, + publication + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const locked = h.historyService.withCompactionPublicationLock.bind(h.historyService); + spyOn(h.historyService, "withCompactionPublicationLock").mockImplementationOnce( + (id, captured, operation) => + locked(id, captured, async (current) => { + entered.resolve(); + await release.promise; + await operation(current); + }) + ); + const cleanup = ( + handler as unknown as { + cleanupProvisionalPendingState( + capturedReceipt: typeof receipt, + captured: typeof publication + ): Promise; + } + ).cleanupProvisionalPendingState(receipt, publication); + await entered.promise; + const queued = Promise.withResolvers(); + const writes = handler as unknown as { + enqueuePendingStateWrite(write: () => Promise): Promise; + }; + const enqueue = writes.enqueuePendingStateWrite.bind(writes); + spyOn(writes, "enqueuePendingStateWrite").mockImplementationOnce((...args) => { + queued.resolve(); + return enqueue(...args); + }); + const read = createMuxMessage("b-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "b-read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/b.ts" }, + output: { success: true }, + }, + ]; + const preparation = handler.preparePendingStateFromMessages( + [read], + "b-boundary", + undefined, + publication + ); + try { + await queued.promise; + release.resolve(); + await cleanup; + await preparation; + expect( + await handler.persistContinuousCompaction({ + boundaryMessageId: "b-boundary", + messages: [read], + text: "B summary", + model: options.model, + tail: [], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + publication, + }) + ).toBe(true); + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/b.ts"]); + expect(handler as unknown as { pendingStateBoundaryMessageId: string }).toHaveProperty( + "pendingStateBoundaryMessageId", + "b-boundary" + ); + } finally { + release.resolve(); + await cleanup; + await preparation; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("held pending-state load cannot overwrite a newer preparation", async () => { + const h = await setup(); + const handler = h.internals.compactionHandler; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "assistant", "Prior context", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + await writeFile( + pendingPath, + JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/old.ts"], + boundaryMessageId: "prior", + }) + ); + const publication = { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const readHistory = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const history = await readHistory(...args); + entered.resolve(); + await release.promise; + return history; + } + ); + const oldPreparation = handler.preparePendingStateFromMessages( + [], + "a-boundary", + undefined, + publication + ); + try { + await entered.promise; + const read = createMuxMessage("b-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "b-read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/b.ts" }, + output: { success: true }, + }, + ]; + const successor = await handler.preparePendingStateFromMessages( + [read], + "b-boundary", + undefined, + publication + ); + const serialized = successor.write?.serialized; + if (serialized === undefined) throw new Error("Expected successor pending write"); + release.resolve(); + expect((await oldPreparation).write).toBeUndefined(); + expect(await fs.readFile(pendingPath, "utf8")).toBe(serialized); + expect( + await handler.persistContinuousCompaction({ + boundaryMessageId: "b-boundary", + messages: [read], + text: "B summary", + model: options.model, + tail: [], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + publication, + }) + ).toBe(true); + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/b.ts"]); + expect(handler as unknown as { pendingStateBoundaryMessageId: string }).toHaveProperty( + "pendingStateBoundaryMessageId", + "b-boundary" + ); + } finally { + release.resolve(); + await oldPreparation; + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 6390fc0178a..7e9f9fd3a73 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -1,5 +1,5 @@ import type { TurnCoordinator } from "./turnCoordinator"; -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; @@ -26,16 +26,31 @@ interface AutoRetryResumeRequest { } interface SessionInternals { - dispatchPendingFollowUp: () => Promise; + coordinator: TurnCoordinator; + dispatchPendingFollowUp: (summaryId?: string) => Promise; sendMessage: ( message: string, options?: SendOptions, - internal?: { synthetic?: boolean; agentInitiated?: boolean } + internal?: { + synthetic?: boolean; + agentInitiated?: boolean; + onAccepted?: () => void | Promise; + } ) => Promise; runStartupRecovery: () => Promise; lastAutoRetryResumeRequest?: AutoRetryResumeRequest; } +function mockAcceptedSend( + send: SessionInternals["sendMessage"] = () => Promise.resolve({ success: true }) +) { + return mock(async (...args: Parameters) => { + const result = await send(...args); + if (result.success) await args[2]?.onAccepted?.(); + return result; + }); +} + const idleFollowUp = (): CompactionFollowUpRequest => ({ text: "heartbeat follow-up", model: "openai:gpt-4o", @@ -187,7 +202,7 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-1", legacyFollowUp), ]); - internals.sendMessage = mock( + internals.sendMessage = mockAcceptedSend( ( message: string, options?: SendOptions, @@ -207,6 +222,245 @@ describe("AgentSession continue-message agentId fallback", () => { expect(dispatchedInternal?.synthetic).toBe(true); }); + test.each([false, true])( + "a completed manual replacement retires a held follow-up read (targeted=%s)", + async (targeted) => { + const { internals, historyService } = await createSession([ + compactionSummaryMessage("held-summary", { + text: "old request", + agentId: "exec", + model: "openai:gpt-4o", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = targeted ? "getHistoryFromLatestBoundary" : "getLastMessages"; + const original = historyService[read].bind(historyService); + spyOn(historyService, read).mockImplementationOnce(async (workspaceId: string) => { + const result = await original(workspaceId, 1); + entered.resolve(); + await release.promise; + return result; + }); + const send = mockAcceptedSend(() => Promise.resolve({ success: true as const })); + internals.sendMessage = send; + const pending = internals.dispatchPendingFollowUp(targeted ? "held-summary" : undefined); + try { + await entered.promise; + const admitted = internals.coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: internals.coordinator.turnId, + }); + expect(admitted.status).toBe("admitted"); + if (admitted.status !== "admitted") throw new Error("Expected replacement"); + await historyService.appendToHistory("ws", createMuxMessage("manual", "user", "new work")); + internals.coordinator.finishTurn(admitted.turnId); + release.resolve(); + expect(await pending).toBe(false); + expect(send).not.toHaveBeenCalled(); + expect(internals.lastAutoRetryResumeRequest).toBeUndefined(); + } finally { + release.resolve(); + await pending; + } + } + ); + + test("concurrent durable dispatch has one owner and preserves ordinary request priority", async () => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("held-summary", { + text: "interrupted request", + agentId: "exec", + model: "openai:gpt-4o", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const send = mockAcceptedSend(async () => { + entered.resolve(); + await release.promise; + await historyService.appendToHistory( + "ws", + createMuxMessage("accepted", "user", "interrupted request") + ); + return { success: true as const }; + }); + internals.sendMessage = send; + session.queueMessage("later manual work", { agentId: "exec", model: "openai:gpt-4o" }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + expect(await internals.dispatchPendingFollowUp()).toBe(false); + release.resolve(); + expect(await pending).toBe(true); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(send).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await pending; + } + }); + + test.each([false, true])( + "a delayed follow-up cleanup cannot overwrite a replacement summary (abandoned=%s)", + async (abandoned) => { + const summary = compactionSummaryMessage("summary", { + text: "obsolete goal", + goalKind: "goal_continuation", + agentId: "exec", + model: "openai:gpt-4o", + }); + const { session, internals, historyService } = await createSession([summary]); + if (abandoned) await session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = historyService.updateHistory.bind(historyService); + spyOn(historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + using _mutation = session.holdTurnAdmission(); + expect( + ( + await update("ws", { + ...summary, + parts: [{ type: "text", text: "replacement summary" }], + metadata: { + ...summary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "replacement request", + agentId: "exec", + model: "openai:gpt-4o", + }, + }, + }, + }) + ).success + ).toBe(true); + await session.contextMutationCommitted(); + release.resolve(); + expect(await pending).toBe(false); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].parts).toEqual([ + { type: "text", text: "replacement summary" }, + ]); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement request" + ); + } finally { + release.resolve(); + await pending; + } + } + ); + + test("a delayed heartbeat rollback cannot delete a replacement context", async () => { + const summary = heartbeatBoundaryMessage(); + const { session, internals, historyService } = await createSession([summary]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const remove = historyService.deleteMessage.bind(historyService); + spyOn(historyService, "deleteMessage").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return remove(...args); + }); + session.queueMessage("manual replacement", { agentId: "exec", model: "openai:gpt-4o" }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + using _mutation = session.holdTurnAdmission(); + expect( + ( + await historyService.updateHistory("ws", { + ...summary, + parts: [{ type: "text", text: "new context" }], + }) + ).success + ).toBe(true); + await session.contextMutationCommitted(); + release.resolve(); + expect(await pending).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary("ws"); + expect(history.success && history.data[0].parts).toEqual([ + { type: "text", text: "new context" }, + ]); + } finally { + release.resolve(); + await pending; + } + }); + + test.each([false, true])( + "Stop before a follow-up claim clears its durable handoff (targeted=%s)", + async (targeted) => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("summary", { + text: "resume", + model: "openai:gpt-4o", + agentId: "exec", + }), + ]); + const send = mockAcceptedSend(); + internals.sendMessage = send; + await session.interruptStream({ abandonPartial: true }); + expect(await internals.dispatchPendingFollowUp(targeted ? "summary" : undefined)).toBe(false); + expect(send).not.toHaveBeenCalled(); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + } + ); + + test("Stop during a held summary read clears the canceled durable handoff", async () => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("summary", { + text: "resume", + model: "openai:gpt-4o", + agentId: "exec", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = historyService.getHistoryFromLatestBoundary.bind(historyService); + spyOn(historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const send = mockAcceptedSend(() => Promise.resolve({ success: true as const })); + internals.sendMessage = send; + const pending = internals.dispatchPendingFollowUp("summary"); + try { + await entered.promise; + await session.interruptStream({ abandonPartial: true }); + release.resolve(); + expect(await pending).toBe(false); + expect(send).not.toHaveBeenCalled(); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + } finally { + release.resolve(); + await pending; + } + }); + test("dispatchPendingFollowUp aliases legacy exclusive-PTC experiments", async () => { // An older build can persist {programmaticToolCalling: false, // programmaticToolCallingExclusive: true}; dispatch copies raw persisted @@ -227,7 +481,7 @@ describe("AgentSession continue-message agentId fallback", () => { }, }), ]); - internals.sendMessage = mock((_message: string, options?: SendOptions) => { + internals.sendMessage = mockAcceptedSend((_message: string, options?: SendOptions) => { dispatchedOptions = options; return Promise.resolve({ success: true as const }); }); @@ -248,7 +502,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentInitiated: true, }), ]); - internals.sendMessage = mock( + internals.sendMessage = mockAcceptedSend( ( _message: string, _options?: SendOptions, @@ -275,7 +529,7 @@ describe("AgentSession continue-message agentId fallback", () => { strictAgentResolution: true, }), ]); - internals.sendMessage = mock((_message: string, options?: SendOptions) => { + internals.sendMessage = mockAcceptedSend((_message: string, options?: SendOptions) => { dispatchedOptions = options; return Promise.resolve({ success: true as const }); }); @@ -312,7 +566,7 @@ describe("AgentSession continue-message agentId fallback", () => { ], archivedConfig ); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); const dispatched = await internals.dispatchPendingFollowUp(); @@ -330,7 +584,7 @@ describe("AgentSession continue-message agentId fallback", () => { const { session, historyService, internals } = await createSession([ compactionSummaryMessage("summary-idle-only", idleFollowUp()), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); session.queueMessage( "user returned", { model: "openai:gpt-4o", agentId: "exec" }, @@ -356,7 +610,7 @@ describe("AgentSession continue-message agentId fallback", () => { earlierMessage, heartbeatBoundaryMessage(), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); session.queueMessage( "user returned", { model: "openai:gpt-4o", agentId: "exec" }, @@ -386,7 +640,7 @@ describe("AgentSession continue-message agentId fallback", () => { earlierMessage, heartbeatBoundaryMessage(), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = () => true; @@ -408,7 +662,7 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-active-turn", idleFollowUp()), ]); const busyInternals = internals as SessionInternals & { isBusy: () => boolean }; - busyInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + busyInternals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); busyInternals.isBusy = () => true; const dispatched = await busyInternals.dispatchPendingFollowUp(); @@ -427,7 +681,7 @@ describe("AgentSession continue-message agentId fallback", () => { test("dispatchPendingFollowUp keeps heartbeat reset boundaries once a non-idle turn has started", async () => { const { historyService, internals } = await createSession([heartbeatBoundaryMessage()]); const busyInternals = internals as SessionInternals & { isBusy: () => boolean }; - busyInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + busyInternals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); busyInternals.isBusy = () => true; const dispatched = await busyInternals.dispatchPendingFollowUp(); @@ -449,7 +703,9 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-completing-turn", idleFollowUp()), ]); const completingInternals = internals as SessionInternals & { coordinator: TurnCoordinator }; - completingInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + completingInternals.sendMessage = mockAcceptedSend(() => + Promise.resolve({ success: true as const }) + ); completingInternals.coordinator.beginPolicy(completingInternals.coordinator.turnId); const dispatched = await completingInternals.dispatchPendingFollowUp(); @@ -478,7 +734,7 @@ describe("AgentSession continue-message agentId fallback", () => { }, agentInitiated: true, }; - internals.sendMessage = mock(() => + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: false as const, error: { type: "runtime_start_failed", message: "startup failed" }, @@ -544,7 +800,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ]); - internals.sendMessage = mock(() => { + internals.sendMessage = mockAcceptedSend(() => { sendCount += 1; return Promise.resolve({ success: true as const }); }); @@ -563,7 +819,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ]); - internals.sendMessage = mock(() => { + internals.sendMessage = mockAcceptedSend(() => { sendCount += 1; if (sendCount === 1) { return Promise.resolve({ @@ -596,7 +852,7 @@ describe("AgentSession continue-message agentId fallback", () => { preservedTailCopy("tail-copy-1", "user", "original user message"), preservedTailCopy("tail-copy-2", "assistant", "original assistant reply"), ]); - internals.sendMessage = mock((message: string) => { + internals.sendMessage = mockAcceptedSend((message: string) => { dispatchedMessage = message; return Promise.resolve({ success: true as const }); }); @@ -621,7 +877,7 @@ describe("AgentSession continue-message agentId fallback", () => { createMuxMessage("post-compaction-turn", "assistant", "new turn after compaction"), preservedTailCopy("tail-copy-2", "assistant", "trailing copy"), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); const dispatched = await internals.dispatchPendingFollowUp(); diff --git a/src/node/services/agentSession.continuousCompaction.test.ts b/src/node/services/agentSession.continuousCompaction.test.ts index 1cfd9552915..d34613ad34f 100644 --- a/src/node/services/agentSession.continuousCompaction.test.ts +++ b/src/node/services/agentSession.continuousCompaction.test.ts @@ -24,6 +24,7 @@ import { type AgentSessionHarness, } from "./agentSession.testHarness"; import type { ContinuousCompactor } from "./continuousCompactor"; +import type { CompactionToken, TurnCoordinator } from "./turnCoordinator"; const workspaceId = "continuous-session"; const model = "openai:gpt-4o"; @@ -34,6 +35,10 @@ const sendOptions: SendMessageOptions = { }; interface SessionInternals { + coordinator: TurnCoordinator; + runContinuousCompactionObservation( + observe: (token: CompactionToken) => Promise + ): Promise; continuousCompactor: ContinuousCompactor; activeStreamContext?: { modelString: string; @@ -42,7 +47,8 @@ interface SessionInternals { }; finishContinuousCompaction: ( applied: boolean, - context: NonNullable + context: NonNullable, + token: CompactionToken ) => Promise; interruptForContinuousCompaction: ( apply: (followUp?: CompactionFollowUpRequest) => Promise @@ -60,9 +66,13 @@ async function applyThenFinish( const state = internals(session); const context = state.activeStreamContext; if (!context) throw new Error("Expected active stream context"); - const applied = await state.interruptForContinuousCompaction(apply); - await state.finishContinuousCompaction(applied, context); - return applied; + return ( + (await state.runContinuousCompactionObservation(async (token) => { + const applied = await state.interruptForContinuousCompaction(apply); + await state.finishContinuousCompaction(applied, context, token); + return applied; + })) ?? false + ); } function deferred() { @@ -244,16 +254,22 @@ describe("AgentSession continuous compaction wiring", () => { await h.historyService.writePartial(workspaceId, source); streaming = false; if (mode === "failed-consumed-apply") { - Reflect.set(h.session, "continuousCompactionStopped", true); + const token = internals(h.session).coordinator.beginCompactionObservation("continuous"); + assert(token, "Expected observation"); + internals(h.session).coordinator.setCompactionStage(token, "stopped"); const reset = spyOn(compactor, "reset"); - await internals(h.session).finishContinuousCompaction(false, { - modelString: model, - options: sendOptions, - providersConfig: null, - }); + await internals(h.session).finishContinuousCompaction( + false, + { + modelString: model, + options: sendOptions, + providersConfig: null, + }, + token + ); expect(reset).not.toHaveBeenCalled(); expect(await store.read()).not.toBeNull(); - Reflect.set(h.session, "continuousCompactionStopped", false); + internals(h.session).coordinator.finishCompactionObservation(token); } if (mode !== "startup") { const terminal = Reflect.get(h.session, "observeContinuousCompactionAtStreamEnd") as ( @@ -499,6 +515,36 @@ describe("AgentSession continuous compaction wiring", () => { }); }); + test("a late durable completion cannot replace a successor's summary target", async () => { + const h = await setup(); + const handler = (h.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const tail = createMuxMessage("tail", "user", "Retain this request"); + await h.historyService.appendToHistory(workspaceId, tail); + const history = await rows(h); + const persist = h.historyService.persistBoundaryWithTailCopies.bind(h.historyService); + spyOn(h.historyService, "persistBoundaryWithTailCopies").mockImplementationOnce( + async (...args) => { + const result = await persist(...args); + internals(h.session).coordinator.invalidateCompaction(); + internals(h.session).coordinator.recordCompactionSummary("successor-summary"); + return result; + } + ); + expect( + await handler.persistContinuousCompaction({ + messages: history, + text: "Old summary", + model, + tail: [tail], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + }) + ).toBe(true); + expect(internals(h.session).coordinator.compactionIntent.summaryId).toBe("successor-summary"); + }); + test("invalidation contains an initial wait rejection and safely stops the blocked stream", async () => { const h = await setup(); const compactor = internals(h.session).continuousCompactor; @@ -574,10 +620,10 @@ describe("AgentSession continuous compaction wiring", () => { const invoked = deferred(); const drain = spyOn(h.session, "drainQueuedMessagesIfIdle").mockImplementation(() => undefined); const run = Reflect.get(h.session, "runContinuousCompactionObservation") as ( - observe: () => Promise + observe: (token: CompactionToken) => Promise ) => Promise; - const first = run.call(h.session, async () => { - Reflect.set(h.session, "midStreamCompactionPending", true); + const first = run.call(h.session, async (token) => { + internals(h.session).coordinator.setCompactionStage(token, "stopping"); await release.promise; }); const observe = spyOn(internals(h.session).continuousCompactor, "observe").mockImplementation( @@ -811,7 +857,6 @@ describe("AgentSession continuous compaction wiring", () => { spyOn(internals(h.session).continuousCompactor, "waitForIdle").mockReturnValueOnce( observationFinished.promise ); - Reflect.set(h.session, "continuousCompactionObserving", true); } h.aiEmitter.emit(eventType, { type: eventType, @@ -821,7 +866,6 @@ describe("AgentSession continuous compaction wiring", () => { }); if (eventType === "prefix-swap-invalidated") { expect(order).toEqual([]); - Reflect.set(h.session, "continuousCompactionObserving", false); observationFinished.resolve(); } await resumed.promise; @@ -1218,6 +1262,8 @@ describe("AgentSession continuous compaction wiring", () => { const h = await setup(); const reset = spyOn(internals(h.session).continuousCompactor, "reset"); using _admission = h.session.holdTurnAdmission(); + expect(reset).not.toHaveBeenCalled(); + await h.session.contextMutationCommitted(); expect(reset).toHaveBeenCalled(); reset.mockClear(); await h.session.discardAutoRetryForContextMutation(); diff --git a/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts b/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts index 1a1e7306458..5532e1b8e67 100644 --- a/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts +++ b/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts @@ -92,7 +92,9 @@ describe("AgentSession.drainQueuedMessagesIfIdle", () => { [ "a pending mid-stream compaction owns the next dispatch", (s) => { - s.midStreamCompactionPending = true; + const token = s.coordinator.beginCompactionObservation("legacy"); + if (!token) throw new Error("Expected compaction owner"); + s.coordinator.setCompactionStage(token, "stopping"); }, 0, ], diff --git a/src/node/services/agentSession.goalAutoPause.test.ts b/src/node/services/agentSession.goalAutoPause.test.ts index 2d57ba1f01a..3b103bff3a3 100644 --- a/src/node/services/agentSession.goalAutoPause.test.ts +++ b/src/node/services/agentSession.goalAutoPause.test.ts @@ -532,65 +532,71 @@ describe("AgentSession goal safety hooks", () => { await session.dispose(); }); - test("a recovered budget wrap-up follow-up installs its missing reservation", async () => { - // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): a crash between wrap-up send - // acceptance and tryMarkBudgetLimitInjected leaves the goal unmarked. - // The redispatched follow-up must install the reservation or the - // recovered stream's end arms a second wrap-up. - const workspaceId = "compaction-followup-wrapup-reservation"; - const { session, goalService, historyService, cleanup } = - await createSessionHarness(workspaceId); - cleanups.push(cleanup); - const created = await setGoalOk(goalService, { - workspaceId, - objective: "Recover the owed wrap-up", - budgetCents: 100, - }); - await goalService.recordStreamAccounting({ - workspaceId, - costUsd: 1.25, - streamStartedAtMs: created.createdAtMs + 1, - streamOriginKind: "goal_continuation", - }); - expect(await goalService.getGoal(workspaceId)).toMatchObject({ - status: "budget_limited", - budgetLimitInjectedForGoalId: null, - }); - const summary = createMuxMessage( - `summary-${crypto.randomUUID()}`, - "assistant", - "Compacted conversation.", - { - muxMetadata: { - type: "compaction-summary", - pendingFollowUp: { - text: "Wrap up the budget-limited goal.", - agentId: "exec", - model: "openai:gpt-4o", - goalKind: GOAL_BUDGET_LIMIT_KIND, - goalId: created.goalId, + test.each([true, false])( + "a recovered wrap-up reserves only an accepted handoff (accepted=%s)", + async (accepted) => { + // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): a crash between wrap-up send + // acceptance and tryMarkBudgetLimitInjected leaves the goal unmarked. + // The redispatched follow-up must install the reservation or the + // recovered stream's end arms a second wrap-up. + const workspaceId = "compaction-followup-wrapup-reservation"; + const { session, goalService, historyService, cleanup } = + await createSessionHarness(workspaceId); + cleanups.push(cleanup); + const created = await setGoalOk(goalService, { + workspaceId, + objective: "Recover the owed wrap-up", + budgetCents: 100, + }); + await goalService.recordStreamAccounting({ + workspaceId, + costUsd: 1.25, + streamStartedAtMs: created.createdAtMs + 1, + streamOriginKind: "goal_continuation", + }); + expect(await goalService.getGoal(workspaceId)).toMatchObject({ + status: "budget_limited", + budgetLimitInjectedForGoalId: null, + }); + const summary = createMuxMessage( + `summary-${crypto.randomUUID()}`, + "assistant", + "Compacted conversation.", + { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Wrap up the budget-limited goal.", + agentId: "exec", + model: "openai:gpt-4o", + goalKind: GOAL_BUDGET_LIMIT_KIND, + goalId: created.goalId, + }, }, - }, - } - ); - expect((await historyService.appendToHistory(workspaceId, summary)).success).toBe(true); - const sendSpy = spyOn(session, "sendMessage").mockImplementation(() => - Promise.resolve(Ok(undefined)) - ); + } + ); + expect((await historyService.appendToHistory(workspaceId, summary)).success).toBe(true); + const sendSpy = spyOn(session, "sendMessage").mockImplementation( + async (_message, _options, internal) => { + if (accepted) await internal?.onAccepted?.(); + return Ok(undefined); + } + ); - const dispatched = await ( - session as unknown as { dispatchPendingFollowUp: (id?: string) => Promise } - ).dispatchPendingFollowUp(); + const dispatched = await ( + session as unknown as { dispatchPendingFollowUp: (id?: string) => Promise } + ).dispatchPendingFollowUp(); - expect(dispatched).toBe(true); - expect(sendSpy).toHaveBeenCalledTimes(1); - expect(await goalService.getGoal(workspaceId)).toMatchObject({ - status: "budget_limited", - budgetLimitInjectedForGoalId: created.goalId, - }); - sendSpy.mockRestore(); - await session.dispose(); - }); + expect(dispatched).toBe(accepted); + expect(sendSpy).toHaveBeenCalledTimes(1); + expect(await goalService.getGoal(workspaceId)).toMatchObject({ + status: "budget_limited", + budgetLimitInjectedForGoalId: accepted ? created.goalId : null, + }); + sendSpy.mockRestore(); + await session.dispose(); + } + ); test("malformed persisted follow-up goal IDs are discarded during recovery", async () => { const workspaceId = "compaction-followup-malformed-goal-id"; diff --git a/src/node/services/agentSession.scopedLifetimes.test.ts b/src/node/services/agentSession.scopedLifetimes.test.ts index fb3ee16a324..2c8cf75320e 100644 --- a/src/node/services/agentSession.scopedLifetimes.test.ts +++ b/src/node/services/agentSession.scopedLifetimes.test.ts @@ -12,11 +12,127 @@ import { Effect, Exit, Scope } from "effect"; import { Err, Ok } from "@/common/types/result"; import { defaultEffectRunner as runner } from "./di/effectRunner"; import { createAgentSessionHarness } from "./agentSession.testHarness"; +import type { ContinuousCompactor } from "./continuousCompactor"; const workspaceId = "scoped-turn"; const options = { model: "openai:gpt-4o", agentId: "exec" }; describe("AgentSession scoped turn lifetimes", () => { + test("a canceled non-cooperative summary stays joined while its replacement applies", async () => { + const h = await createAgentSessionHarness({ workspaceId }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const { continuousCompactor: compactor } = h.session as unknown as { + continuousCompactor: ContinuousCompactor; + }; + const { deps } = compactor as unknown as { + deps: ConstructorParameters[0]; + }; + const summarize = spyOn(deps, "summarize") + .mockImplementationOnce(async (_head, signal) => { + entered.resolve(); + await release.promise; + expect(signal.aborted).toBe(true); + return { text: "obsolete summary", model: options.model }; + }) + .mockResolvedValue({ text: "replacement summary", model: options.model }); + const context = { + enabled: true, + model: options.model, + contextWindowTokens: 100_000, + thresholdPercent: 80, + phase: "on-send" as const, + }; + let closing: Promise | undefined; + try { + for (const row of [ + createMuxMessage("old-user", "user", "Investigate"), + createMuxMessage("old-answer", "assistant", "earlier investigation ".repeat(4_000)), + createMuxMessage("recent-user", "user", "Implement"), + createMuxMessage("recent-answer", "assistant", "Done"), + ]) + expect((await h.historyService.appendToHistory(workspaceId, row)).success).toBe(true); + await compactor.observe(75, context); + await entered.promise; + { + using _mutation = h.session.holdTurnAdmission(); + await h.session.contextMutationCommitted(); + } + expect(h.session.isBusy()).toBe(false); + await compactor.observe(75, context); + const replacement = (compactor as unknown as { job: { done: Promise } }).job; + await replacement.done; + expect(await compactor.observe(80, context)).toBe("applied"); + expect(summarize).toHaveBeenCalledTimes(2); + let closed = false; + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await runner.runPromise(Effect.yieldNow); + expect(closed).toBe(false); + release.resolve(); + await closing; + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data[0].parts).toMatchObject([ + { type: "text", text: "replacement summary" }, + ]); + } finally { + release.resolve(); + await closing; + await h.session.dispose(); + await h.cleanup(); + mock.restore(); + } + }); + + test("shutdown joins an eager job whose prepare synchronously closes the session", async () => { + const h = await createAgentSessionHarness({ workspaceId }); + const stream = spyOn(h.aiService, "streamMessage"); + const release = Promise.withResolvers(); + const prepareEntered = Promise.withResolvers(); + const { continuousCompactor } = h.session as unknown as { + continuousCompactor: ContinuousCompactor; + }; + const { deps } = continuousCompactor as unknown as { + deps: { prepare(): Promise }; + }; + spyOn(deps, "prepare").mockImplementation(() => { + h.session.beginShutdown(); + prepareEntered.resolve(); + return release.promise; + }); + let closed = false; + let closing: Promise | undefined; + try { + await continuousCompactor.observe(75, { + enabled: true, + model: options.model, + contextWindowTokens: 100_000, + thresholdPercent: 80, + phase: "on-send", + }); + await prepareEntered.promise; + // Reset detaches the job immediately; its original preparation still owns I/O. + expect(h.session.closingSignal.aborted).toBe(true); + expect(h.session.isBusy()).toBe(false); + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await runner.runPromise(Effect.yieldNow); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(closed).toBe(true); + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await closing; + await h.session.dispose(); + await h.cleanup(); + mock.restore(); + } + }); + test("queue clear publication cannot outrun its cancellation refund", async () => { const appFiberScope = Scope.makeUnsafe("parallel"); const entered = Promise.withResolvers(); diff --git a/src/node/services/agentSession.tokenBudget.test.ts b/src/node/services/agentSession.tokenBudget.test.ts index 9a81d2f5440..96fce3d9584 100644 --- a/src/node/services/agentSession.tokenBudget.test.ts +++ b/src/node/services/agentSession.tokenBudget.test.ts @@ -16,6 +16,7 @@ import { prepareProviderRequestMessages } from "./turnContextAssembler"; import { MuxMessageSchema } from "@/common/orpc/schemas/message"; import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; +import type { TurnCoordinator } from "./turnCoordinator"; import type { AgentSessionAIService } from "./agentSession"; import { createAgentSessionHarness, type AgentSessionHarness } from "./agentSession.testHarness"; import { createTurnCompletionController, type SettledStepBudget } from "./streamManager"; @@ -769,61 +770,80 @@ describe("AgentSession token-budget lifecycle", () => { } ); - test("on-send rollover appends reset, hidden lead-in, skill snapshot and the original user together", async () => { - const h = await setup(); - await seedHistory(h, 110_000); - const skillDir = path.join(h.config.rootDir, ".xum", "skills", "budget-test"); - await fs.mkdir(skillDir, { recursive: true }); - await fs.writeFile( - path.join(skillDir, "SKILL.md"), - "---\nname: budget-test\ndescription: Test skill\n---\n\nPreserve this instruction.\n" - ); - spyOn(h.aiService, "getWorkspaceMetadata").mockResolvedValue( - Ok({ - id: workspaceId, - name: "budget", - projectName: "project", - projectPath: h.config.rootDir, - namedWorkspacePath: h.config.rootDir, - runtimeConfig: { type: "local" }, - } as FrontendWorkspaceMetadata) - ); - const append = spyOn(h.historyService, "appendManyToHistory"); - const result = await h.session.sendMessage("Do the requested work", { - ...options, - muxMetadata: { - type: "agent-skill", - rawCommand: "/budget-test Do the requested work", - skillName: "budget-test", - scope: "project", - }, - }); - expect(result.success).toBe(true); - const rows = await allRows(h); - expect(rows.slice(0, 3).map((row) => row.id)).toEqual([ - "old-user", - "first-answer", - "old-answer", - ]); - const boundaryIndex = rows.findIndex((row) => rolloverRows([row]).length > 0); - expect(boundaryIndex).toBe(3); - const [boundary, leadIn, snapshot, user] = rows.slice(boundaryIndex); - expect(boundary.metadata?.contextBoundaryKind).toBe("reset"); - expect(leadIn.metadata).toMatchObject({ synthetic: true, uiVisible: false }); - expect(snapshot.metadata?.agentSkillSnapshot?.skillName).toBe("budget-test"); - expect(text(user)).toBe("Do the requested work"); - expect(user.metadata?.muxMetadata?.type).toBe("agent-skill"); - expect(append.mock.calls).toHaveLength(1); - expect(append.mock.calls[0][1].map((row) => row.id)).toEqual( - rows.slice(boundaryIndex).map((row) => row.id) - ); - expect(h.requests).toHaveLength(1); - const providerRows = sliceMessagesForProviderFromLatestContextBoundary(h.requests[0].messages); - expect(providerRows.map((row) => row.id)).toEqual([leadIn.id, snapshot.id, user.id]); - expect(rows.some((row) => row.metadata?.muxMetadata?.type === "compaction-request")).toBe( - false - ); - }); + test.each([false, true])( + "on-send rollover preserves the atomic request and its own handoff (%s)", + async (handoff) => { + const h = await setup(); + await seedHistory(h, 110_000); + const coordinator = (h.session as unknown as { coordinator: TurnCoordinator }).coordinator; + const oldObservation = coordinator.beginCompactionObservation("continuous"); + expect(oldObservation).toBeDefined(); + const skillDir = path.join(h.config.rootDir, ".xum", "skills", "budget-test"); + await fs.mkdir(skillDir, { recursive: true }); + await fs.writeFile( + path.join(skillDir, "SKILL.md"), + "---\nname: budget-test\ndescription: Test skill\n---\n\nPreserve this instruction.\n" + ); + spyOn(h.aiService, "getWorkspaceMetadata").mockResolvedValue( + Ok({ + id: workspaceId, + name: "budget", + projectName: "project", + projectPath: h.config.rootDir, + namedWorkspacePath: h.config.rootDir, + runtimeConfig: { type: "local" }, + } as FrontendWorkspaceMetadata) + ); + const append = spyOn(h.historyService, "appendManyToHistory"); + const result = await h.session.sendMessage( + "Do the requested work", + { + ...options, + muxMetadata: { + type: "agent-skill", + rawCommand: "/budget-test Do the requested work", + skillName: "budget-test", + scope: "project", + }, + }, + handoff + ? { + synthetic: true, + compactionHandoff: oldObservation!, + admissionStale: () => !coordinator.isCurrentCompaction(oldObservation!), + } + : undefined + ); + expect(result.success).toBe(true); + expect(coordinator.isCurrentCompaction(oldObservation!)).toBe(handoff); + const rows = await allRows(h); + expect(rows.slice(0, 3).map((row) => row.id)).toEqual([ + "old-user", + "first-answer", + "old-answer", + ]); + const boundaryIndex = rows.findIndex((row) => rolloverRows([row]).length > 0); + expect(boundaryIndex).toBe(3); + const [boundary, leadIn, snapshot, user] = rows.slice(boundaryIndex); + expect(boundary.metadata?.contextBoundaryKind).toBe("reset"); + expect(leadIn.metadata).toMatchObject({ synthetic: true, uiVisible: false }); + expect(snapshot.metadata?.agentSkillSnapshot?.skillName).toBe("budget-test"); + expect(text(user)).toBe("Do the requested work"); + expect(user.metadata?.muxMetadata?.type).toBe("agent-skill"); + expect(append.mock.calls).toHaveLength(1); + expect(append.mock.calls[0][1].map((row) => row.id)).toEqual( + rows.slice(boundaryIndex).map((row) => row.id) + ); + expect(h.requests).toHaveLength(1); + const providerRows = sliceMessagesForProviderFromLatestContextBoundary( + h.requests[0].messages + ); + expect(providerRows.map((row) => row.id)).toEqual([leadIn.id, snapshot.id, user.id]); + expect(rows.some((row) => row.metadata?.muxMetadata?.type === "compaction-request")).toBe( + false + ); + } + ); test("on-send usage below the force buffer preserves history while warning permissions are unknown", async () => { const h = await setup(); @@ -1518,6 +1538,22 @@ describe("AgentSession token-budget lifecycle", () => { } ); + test("explicit Retry consumes its Stop receipt once across emergency budget rollover", async () => { + const h = await setup({ failure: (attempt) => (attempt === 1 ? exceeded : undefined) }); + await seedHistory(h, 20_000); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("retry-user", "user", "Resume work") + ); + await h.session.interruptStream({ abandonPartial: true }); + const accept = spyOn(h.historyService, "acceptResumeCancellation"); + expect(await h.session.resumeStream(options)).toEqual(Ok({ started: true })); + expect(accept).toHaveBeenCalledTimes(1); + expect(h.requests).toHaveLength(2); + expect(rolloverRows(await allRows(h))).toHaveLength(1); + expect(await h.historyService.readCompactionCancellation(workspaceId)).toBeNull(); + }); + test("a primary on-send rollover followed by fresh preflight overflow is blocked without a second reset", async () => { const h = await setup({ failure: () => exceeded }); await seedHistory(h, 110_000); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e00e7cde006..611530f9975 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1,3 +1,4 @@ +import type { ContinuousCompactionPublication } from "./continuousCompactionJournal"; import type { AIService } from "./aiService"; import { AsyncLocalStorage } from "node:async_hooks"; import type { GoalRecordV1 } from "@/common/types/goal"; @@ -34,12 +35,17 @@ import * as path from "path"; import assert from "@/common/utils/assert"; import { EventEmitter } from "events"; import { Effect, Fiber } from "effect"; +import { + CompactionCancellation, + type CompactionCancellationRecord, +} from "./compactionCancellation"; import { StartupRecovery, type StartupRecoveryOutcome } from "./startupRecovery"; import { mkdir, readdir, readFile, unlink, writeFile } from "fs/promises"; import type { Dirent } from "fs"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import { PlatformPaths } from "@/common/utils/paths"; import { log } from "@/node/services/log"; +import { isDeepStrictEqual } from "node:util"; import { eventSpine } from "@/node/services/events/eventSpine"; import { ProvidersConfigStore, type Config } from "@/node/config"; import { @@ -48,6 +54,7 @@ import { type TurnId, type QueueDrainTrigger, type OperationId, + type CompactionToken, type StreamErrorRecoveryOutcome, } from "./turnCoordinator"; export type { StreamErrorRecoveryOutcome } from "./turnCoordinator"; @@ -258,10 +265,18 @@ type SessionCompactionContext = ContinuousCompactionContext & { * Result shape for turn-starting session methods. failureHandled marks errors * whose retry/abandon bookkeeping already ran inside streamWithHistory, so * callers must not re-handle them (would double-increment backoff attempts). + * superseded records ownership loss at the failure source, before cleanup awaits; + * an accepted durable handoff remains acknowledged even though its startup retired. */ type AgentSessionResult = | { success: true; data: T } - | { success: false; error: SendMessageError; failureHandled?: true }; + | { + success: false; + error: SendMessageError; + failureHandled?: true; + superseded?: true; + admissionDeferred?: true; + }; /** * Tracked file state for detecting external edits. @@ -746,7 +761,16 @@ interface CachedMemoryContext { hotSetEnabled: boolean; } +interface CompactionFollowUpDispatch { + summary?: MuxMessage; + accepted: boolean; +} + interface SendMessageInternalOptions { + compactionHandoff?: CompactionToken; + compactionHandoffPublication?: ContinuousCompactionPublication; + /** Durable-summary handoffs revalidate their source inside the history append lock. */ + compactionHandoffSource?: { summary: MuxMessage; onSkipped: () => void }; preparation?: PreparationAttempt; /** A dequeued send keeps its admission owner through acceptance and startup failure. */ turnReservation?: TurnId; @@ -757,6 +781,8 @@ interface SendMessageInternalOptions { /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ goalId?: string; startStreamInBackground?: boolean; + /** Synchronous receipt at the rollback frontier, before goal sync or event observers. */ + onRowsDurable?: () => void; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -844,6 +870,8 @@ interface PreparationAttempt { failureAttempts?: number; failure?: SendMessageError; onFailure?: (error: SendMessageError) => Promise | void; + resumeCancellation?: { nonce: string | null; epoch: number }; + automaticResume?: boolean; } export class AgentSession { @@ -854,6 +882,7 @@ export class AgentSession { private readonly workspaceId: string; private readonly config: Config; private readonly historyService: HistoryService; + private readonly compactionCancellation: CompactionCancellation; private readonly aiService: AgentSessionAIService; private readonly streamManager: AgentSessionStreamManager; private readonly mcpServerManager?: MCPServerManager; @@ -872,7 +901,6 @@ export class AgentSession { []; private readonly coordinator = new TurnCoordinator({ streamStarted: (payload) => { - this.continuousCompactionAbandoned = false; this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; @@ -934,10 +962,16 @@ export class AgentSession { // Persisted compaction follow-ups precede goal continuation recovery. Each successful // step is checkpointed, so retrying a later read cannot replay an earlier side effect. steps: [ + () => this.runStartupRecoveryStep(() => this.reconcileCompactionCancellation()), () => this.runStartupRecoveryStep(() => this.requireGoalAcknowledgmentForCrashRecoveredPartial()), - () => this.runStartupRecoveryStep(() => this.continuousCompactor.recover()), - () => this.runStartupRecoveryStep(() => this.dispatchPendingFollowUp()), + () => + this.runStartupRecoveryStep(async () => { + // An unresolved Stop also excludes a journal that has not published its summary yet. + if ((await this.compactionCancellation.read())?.scope.kind !== "unresolved") + await this.continuousCompactor.recover(); + }), + () => this.dispatchPendingFollowUp(), () => this.runStartupRecoveryStep(() => this.workspaceGoalService?.recoverPendingDispatchAfterRestart(this.workspaceId) @@ -972,12 +1006,16 @@ export class AgentSession { private lastSystemMessageTokens?: number; /** Prevent duplicate mid-stream compaction interrupts while we are already transitioning. */ - private midStreamCompactionPending = false; - private midStreamCompactionSettledWaiters: Array<() => void> = []; - private continuousCompactionAbandoned = false; - private continuousCompactionStopped = false; - private continuousCompactionObserving = false; - private continuousCompactionObservation: Promise | null = null; + private readonly compactionObservations = new Map>(); + private get midStreamCompactionPending(): boolean { + return this.coordinator.midStreamCompactionPending; + } + private get continuousCompactionAbandoned(): boolean { + return this.coordinator.compactionIntent.status === "abandoned"; + } + private get continuousCompactionObserving(): boolean { + return this.coordinator.compactionIntent.observation?.kind === "continuous"; + } /** Tracks file state for detecting external edits. */ private readonly fileChangeTracker = new FileChangeTracker(); @@ -1093,6 +1131,7 @@ export class AgentSession { /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { + compactionPublication?: ContinuousCompactionPublication; modelString: string; contextBudgetRetried?: boolean; requestAssemblySnapshot?: RequestAssemblySnapshot; @@ -1120,7 +1159,9 @@ export class AgentSession { * dispatch must target it by ID; null for default (RLM-off) compactions so * their "last message is the summary" staleness guard stays byte-identical. */ - private pendingCompactionFollowUpSummaryId: string | null = null; + private get pendingCompactionFollowUpSummaryId(): string | null { + return this.coordinator.compactionIntent.summaryId; + } constructor(options: AgentSessionOptions) { assert(options, "AgentSession requires options"); @@ -1152,6 +1193,7 @@ export class AgentSession { this.workspaceId = trimmedWorkspaceId; this.config = config; this.historyService = historyService; + this.compactionCancellation = new CompactionCancellation(historyService, trimmedWorkspaceId); this.aiService = aiService; const streamManagerCandidate = streamManager ?? aiService; assert( @@ -1174,6 +1216,11 @@ export class AgentSession { this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, + getCompactionPublication: () => this.activeStreamContext?.compactionPublication, + captureCompletionGuard: () => { + const epoch = this.coordinator.compactionIntent.epoch; + return () => !this.coordinator.closing && this.coordinator.compactionIntent.epoch === epoch; + }, historyService: this.historyService, sessionDir: path.join(this.config.sessionsDir, this.workspaceId), telemetryService, @@ -1184,8 +1231,9 @@ export class AgentSession { // follow-up dispatch can target it directly. // Reset on every completion: a resumeless continuous fold may precede a // legacy compaction whose current follow-up is on its final summary row. - this.pendingCompactionFollowUpSummaryId = - (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null; + this.coordinator.recordCompactionSummary( + (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null + ); onCompactionComplete?.(metadata); }, onIdleCompactionOutcome, @@ -1198,6 +1246,7 @@ export class AgentSession { this.continuousCompactor = new ContinuousCompactor({ workspaceId: this.workspaceId, + enterExecution: () => this.coordinator.enterExecution(), historyService: this.historyService, compactionHandler: this.compactionHandler, streamManager: { @@ -1324,6 +1373,71 @@ export class AgentSession { } } + // Failed I/O keeps its exact semantic owner in the coordinator. This payload + // schedules a later explicit recovery/teardown attempt, never a busy physical lease. + private deferredCompactionCleanup?: { + token: CompactionToken; + dispatch: CompactionFollowUpDispatch; + summaryMessageId?: string; + }; + + private compactionCleanupRetry?: Promise; + + private hasDeferredCompactionCleanup(token: CompactionToken): boolean { + return this.deferredCompactionCleanup?.token.id === token.id; + } + + get hasPendingCompactionCleanup(): boolean { + return ( + this.compactionCancellation.needsPersistence || + this.compactionCleanupRetry != null || + (this.deferredCompactionCleanup != null && + this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token)) + ); + } + + get hasBlockingCompactionCleanup(): boolean { + return ( + this.compactionCancellation.blocksRecovery || + this.compactionCleanupRetry != null || + (this.deferredCompactionCleanup != null && + this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token)) + ); + } + + retryPendingCompactionCleanup(): Promise { + if (this.compactionCleanupRetry) return this.compactionCleanupRetry; + if (this.compactionCancellation.needsPersistence) + return this.compactionCancellation.retry().then(() => this.retryPendingCompactionCleanup()); + const deferred = this.deferredCompactionCleanup; + if (!deferred) return Promise.resolve(); + if (!this.coordinator.canClearCompactionFollowUp(deferred.token)) { + this.deferredCompactionCleanup = undefined; + return Promise.resolve(); + } + const settled = Promise.withResolvers(); + this.compactionCleanupRetry = settled.promise; + this.dispatchPendingFollowUp().then( + () => { + this.compactionCleanupRetry = undefined; + settled.resolve(); + }, + (error: unknown) => { + this.compactionCleanupRetry = undefined; + // Dispatch preserves its original error even when bounded cleanup succeeds. + if ( + this.hasDeferredCompactionCleanup(deferred.token) && + this.coordinator.canClearCompactionFollowUp(deferred.token) + ) + settled.reject( + new Error(`Stopped compaction cleanup remains pending: ${getErrorMessage(error)}`) + ); + else settled.resolve(); + } + ); + return settled.promise; + } + private disposePromise?: Promise; private disposalMessageId?: string; @@ -1377,10 +1491,20 @@ export class AgentSession { : cleanup("background processes", () => this.backgroundProcessManager.cleanup(this.workspaceId) ); + let unresolvedCompactionCleanup: Error | undefined; Promise.all([stopped, invalidated, compactionStopped, retryStopped, backgroundStopped]) .then(async () => { cleanupExecution[Symbol.dispose](); await cleanup("drain", () => this.coordinator.drain()); + // A failed producer may defer cleanup while drain is running. Retry only + // after its lease settles; retaining its token must never self-join drain. + try { + await this.compactionCancellation.retry(); + await this.retryPendingCompactionCleanup(); + } catch (error) { + unresolvedCompactionCleanup = + error instanceof Error ? error : new Error(getErrorMessage(error)); + } // Raw bridges stay attached through the attempt fence. Destructive disposal suppresses // recovery policy, but still presents its captured terminal exactly once below. for (const { event, handler } of this.aiListeners) this.aiService.off(event, handler); @@ -1392,7 +1516,10 @@ export class AgentSession { eventSpine.emit("session.end", { workspaceId: this.workspaceId }); }) .catch((error: unknown) => log.debug("dispose: final cleanup failed", { error })) - .finally(() => disposed.resolve()); + .finally(() => { + if (unresolvedCompactionCleanup) disposed.reject(unresolvedCompactionCleanup); + else disposed.resolve(); + }); return disposed.promise; } @@ -1641,6 +1768,14 @@ export class AgentSession { ): Promise { if (this.coordinator.closing || !isCurrent()) return; using _execution = this.coordinator.enterExecution(); + // A foreign Stop can arrive while backoff sleeps. Automatic delivery never + // replaces that intent; only an accepted explicit row can witness replacement. + if ( + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || + !isCurrent() || + this.coordinator.closing + ) + return; const request = this.lastAutoRetryResumeRequest; if (!request) { this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "missing_retry_options" }); @@ -1658,6 +1793,7 @@ export class AgentSession { goalKind: request.goalKind, goalId: request.goalId, retrySignal: signal, + automatic: true, requestAssemblySnapshot: request.requestAssemblySnapshot, }); // Interrupting the scheduling fiber cannot cancel resumeStream's original Promise. @@ -1665,6 +1801,11 @@ export class AgentSession { if (this.coordinator.closing || !isCurrent()) return; if (result.success) { if (!result.data.started) { + if ( + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || + !isCurrent() + ) + return; // resumeStream can defer when a turn is still PREPARING/COMPLETING. // Treat this as retriable so auto-retry keeps progressing instead of // stalling after the "auto-retry-starting" status event. @@ -2600,6 +2741,40 @@ export class AgentSession { return retryRequest?.model ?? null; } + private async reconcileCompactionCancellation(): Promise { + let cancellation = await this.compactionCancellation.read(); + // Empty history cannot retire Stop: only an exact active/archive row witness can. + // Live dispatch uses the same reconciliation as startup because another backend + // can commit a replacement while its ancillary sidecar unlink keeps failing. + while ( + cancellation && + (await this.historyService.hasCompactionReplacementWitness( + this.workspaceId, + cancellation.nonce + )) + ) { + await this.retireWitnessedCompactionCancellation(cancellation.nonce); + // Re-read shared identity after every await; retiring witnessed A cannot hide B. + cancellation = await this.compactionCancellation.read(); + } + return cancellation; + } + + private async retireWitnessedCompactionCancellation(nonce: string): Promise { + try { + await this.compactionCancellation.retireReplacement(nonce); + } catch (error) { + log.warn("Deferred witnessed compaction cancellation cleanup", { + workspaceId: this.workspaceId, + error, + }); + } + } + + getCompactionCancellationNonce(): Promise { + return this.compactionCancellation.readForReplacement().then((record) => record?.nonce); + } + private async runStartupRecoveryStep(step: () => unknown): Promise { if (this.coordinator.closing) return; using _execution = this.coordinator.enterExecution(); @@ -2715,6 +2890,10 @@ export class AgentSession { }); return "completed"; } + // A retained handoff row may look interrupted after failed rollback. The + // unresolved Stop still owns it even when no live session remembers the abort. + if ((await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || !isCurrent()) + return "completed"; await this.handleStreamFailureForAutoRetry( { type: "unknown", @@ -3411,6 +3590,7 @@ export class AgentSession { // Single admission-staleness predicate for all three turn-admission gates below. const isAdmissionStale = () => + compactionAppendSkipped || internal?.admissionEpochStale?.() === true || internal?.admissionStale?.() === true || !this.coordinator.isCurrentTurn(attempt.owner ?? attempt.expectedTurn) || @@ -3418,6 +3598,7 @@ export class AgentSession { const cancelSignal = internal?.cancelSignal; const persistedCancelableMessageIds: string[] = []; + let compactionAppendSkipped = false; // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** @@ -3429,7 +3610,16 @@ export class AgentSession { */ const rollbackPersistedTurnRows = async (): Promise => { if (persistedCancelableMessageIds.length === 0) return true; - this.continuousCompactor.reset("delete-messages"); + const handoff = internal?.compactionHandoff; + const ownsCleanup = handoff != null && this.coordinator.canClearCompactionFollowUp(handoff); + // Stop retains this handoff's summary cleanup even while its user-row append + // unwinds. A replaced handoff may delete its own rows, but cannot retire its successor. + if (handoff == null || ownsCleanup || this.coordinator.isCurrentCompaction(handoff)) { + this.coordinator.invalidateCompaction( + this.coordinator.compactionIntent.status === "abandoned" + ); + this.continuousCompactor.reset("delete-messages"); + } const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, persistedCancelableMessageIds @@ -3449,9 +3639,25 @@ export class AgentSession { ) ); }; + const refuseSkippedCompactionAppend = async (): Promise> => { + // Snapshot leftovers are not acceptance of a user/request row that never + // committed. Preserve the cleanup failure without publishing a durable receipt. + if (!(await rollbackPersistedTurnRows())) + throw new Error( + "Failed to roll back preparation rows after compaction follow-up became stale" + ); + return Err( + createUnknownSendMessageError( + isManualUserMessage + ? "Send superseded by a newer Stop before append" + : "Compaction follow-up source became stale before append" + ) + ); + }; const markRowsDurable = (): void => { if (attempt.durability !== "rollback-eligible") return; attempt.durability = "durable"; + internal?.onRowsDurable?.(); if ((internal?.preTurnMessages?.length ?? 0) > 0) internal?.onPreTurnRowsPersisted?.(); }; const accept = async (): Promise => { @@ -3459,12 +3665,39 @@ export class AgentSession { await internal?.onAccepted?.(); attempt.durability = "accepted"; }; + // Capture known admission refusal where it occurs; later token changes cannot + // turn a genuine goal/provider failure into a successful historical handoff. + const refuseAdmission = (error: SendMessageError): AgentSessionResult => ({ + success: false, + error, + ...(attempt.durability !== "rollback-eligible" ? { superseded: true as const } : {}), + }); const refuseBeforeAcceptance = async ( error: SendMessageError ): Promise> => { if (attempt.durability === "rollback-eligible" && !(await rollbackPersistedTurnRows())) markRowsDurable(); - return Err(error); + return refuseAdmission(error); + }; + const refuseContextMutation = (): Promise> => { + // Only an intact, pre-persistence handoff can wait and retry. Capture the + // refusal here; later state or error text cannot reclassify real failures. + if ( + this.coordinator.admissionBlocked && + !isAdmissionStale() && + internal?.compactionHandoff != null && + this.coordinator.isCurrentCompaction(internal.compactionHandoff) && + attempt.durability === "rollback-eligible" && + persistedCancelableMessageIds.length === 0 + ) + return Promise.resolve({ + success: false, + error: createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE), + admissionDeferred: true, + }); + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); }; let cancellationHandled = false; let cancellationDisabled = false; @@ -3722,6 +3955,7 @@ export class AgentSession { return refuseBeforeAcceptance( createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) ); + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("edit"); // Ensure no in-flight completion code can append after we truncate. if (this.isBusy()) { @@ -3776,9 +4010,7 @@ export class AgentSession { // other. The epoch probe (r41) also refuses edits whose target rows a // completed mutation already discarded. if (this.coordinator.admissionBlocked || isAdmissionStale()) { - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + return refuseContextMutation(); } if (this.coordinator.closing) { return refuseBeforeAcceptance( @@ -3889,6 +4121,26 @@ export class AgentSession { ? await this.withKeepRecentTailStamp(typedMuxMetadata, optionsForStream) : typedMuxMetadata; + const compactionCancellationNonce = isManualUserMessage + ? await this.getCompactionCancellationNonce() + : undefined; + const compactionAppendCondition = + isManualUserMessage || internal?.compactionHandoff + ? { + ...(isManualUserMessage + ? { replacementNonce: compactionCancellationNonce ?? null } + : {}), + summary: internal?.compactionHandoffSource?.summary, + publication: internal?.compactionHandoffPublication, + allowedTailMessageIds: persistedCancelableMessageIds, + isCurrent: () => + !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, + onSkipped: () => { + compactionAppendSkipped = true; + internal?.compactionHandoffSource?.onSkipped(); + }, + } + : undefined; const userMessage = createMuxMessage( messageId, "user", @@ -3899,6 +4151,7 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, retrySendOptions: pickStartupRetrySendOptions(optionsForStream, agentInitiated, goalKind), muxMetadata: stampedMuxMetadata, // Pass through frontend metadata as black-box + ...(compactionCancellationNonce ? { compactionCancellationNonce } : {}), ...(acpPromptId != null ? { acpPromptId } : {}), ...(goalKind != null ? { kind: goalKind } : {}), // Scope goal-loop rows to their goal so a replaced goal's continuation @@ -3994,7 +4247,13 @@ export class AgentSession { const providersConfigForCompaction = this.getProvidersConfigSafe(); // Recover before measuring pressure so the old pre-swap usage cannot force another fold. - if (await this.continuousCompactor.recover()) this.clearUsageState(); + if (await this.continuousCompactor.recover()) { + if (isAdmissionStale() || this.coordinator.closing) + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); + this.clearUsageState(); + } const compactionResult = this.compactionMonitor.checkBeforeSend({ model: modelForStream, usage: this.getUsageState(), @@ -4018,6 +4277,10 @@ export class AgentSession { phase: "on-send", }) : "none"; + if (isAdmissionStale() || this.coordinator.closing) + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); if (continuousResult === "applied") this.clearUsageState(); if (await cancelBeforeAcceptance()) return Ok(undefined); @@ -4101,15 +4364,13 @@ export class AgentSession { autoCompactionRequest.agentInitiated ), muxMetadata: autoCompactionRequest.metadata, + ...(compactionCancellationNonce ? { compactionCancellationNonce } : {}), synthetic: true, uiVisible: true, } ); - if (this.coordinator.admissionBlocked || isAdmissionStale()) - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + if (this.coordinator.admissionBlocked || isAdmissionStale()) return refuseContextMutation(); if (this.coordinator.closing) return refuseBeforeAcceptance( createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE) @@ -4118,11 +4379,14 @@ export class AgentSession { // Persist compaction request (NOT the user message — it's the follow-up) const appendCompactionResult = await this.historyService.appendToHistory( this.workspaceId, - autoCompactionMessage + autoCompactionMessage, + compactionAppendCondition ); if (!appendCompactionResult.success) { + if (internal?.compactionHandoff) throw new Error(appendCompactionResult.error); return Err(createUnknownSendMessageError(appendCompactionResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push(autoCompactionMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); @@ -4153,9 +4417,7 @@ export class AgentSession { // after a mutation commits; this check and the PREPARING gate remain // backstops for entry-accounting bypasses. if (this.coordinator.admissionBlocked || isAdmissionStale()) { - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + return refuseContextMutation(); } // Still pre-persist: a row appended now would read as a dispatched turn on the next startup // while streamWithHistory's own latch check keeps its stream from ever running. @@ -4168,6 +4430,9 @@ export class AgentSession { // Persist snapshots only when this turn will be sent immediately. // On on-send compaction paths, snapshots are deferred with the follow-up turn. const shouldPersistTurnSnapshots = autoCompactionMessage === null; + // A handoff snapshot without its trigger strands pendingFollowUp after a crash. + // Commit their exact materialized rows together, including with token budgets disabled. + const batchTurnSnapshots = tokenBudgetActive || internal?.compactionHandoff != null; let skillSnapshotMessages: MuxMessage[] = []; let mcpPromptSnapshotMessages: MuxMessage[] = []; @@ -4191,7 +4456,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && snapshotResult?.snapshotMessage) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && snapshotResult?.snapshotMessage) { const snapshotAppendResult = await this.historyService.appendToHistory( this.workspaceId, snapshotResult.snapshotMessage @@ -4205,7 +4470,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && skillSnapshotMessages.length > 0) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && skillSnapshotMessages.length > 0) { for (const snapshotMessage of skillSnapshotMessages) { const skillSnapshotAppendResult = await this.historyService.appendToHistory( this.workspaceId, @@ -4222,7 +4487,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && mcpPromptSnapshotMessages.length > 0) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && mcpPromptSnapshotMessages.length > 0) { for (const snapshotMessage of mcpPromptSnapshotMessages) { const appendResult = await this.historyService.appendToHistory( this.workspaceId, @@ -4255,7 +4520,7 @@ export class AgentSession { "sendMessage: preTurnMessages must be synthetic assistant rows" ); } - if (tokenBudgetActive) { + if (batchTurnSnapshots && !autoCompactionMessage) { const requestPrelude = [ ...(snapshotResult?.snapshotMessage ? [snapshotResult.snapshotMessage] : []), ...skillSnapshotMessages, @@ -4266,17 +4531,19 @@ export class AgentSession { // before clearing context state or publishing a reset. Reuse these rows below: // skill directives and MCP prompt expansion must not execute a second time. if (requestPrelude.length > 0) { - const freshBudget = await this.checkFreshContextBudget( - userMessage, - optionsForStream.model, - optionsForStream, - [...contextBudgetPrefix, ...requestPrelude] - ); - if (await cancelBeforeAcceptance()) return Ok(undefined); - if (isAdmissionStale() || this.coordinator.admissionBlocked || this.coordinator.closing) { - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + if (tokenBudgetActive) { + const freshBudget = await this.checkFreshContextBudget( + userMessage, + optionsForStream.model, + optionsForStream, + [...contextBudgetPrefix, ...requestPrelude] + ); + if (await cancelBeforeAcceptance()) return Ok(undefined); + if (isAdmissionStale() || this.coordinator.admissionBlocked || this.coordinator.closing) { + return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + } + if (!freshBudget.success) return await rejectBudgetSend(freshBudget.error); } - if (!freshBudget.success) return await rejectBudgetSend(freshBudget.error); userMessage.metadata = { ...userMessage.metadata, requestPreludeMessageIds: requestPrelude.map((row) => row.id), @@ -4325,12 +4592,25 @@ export class AgentSession { // Ordinary sends stay append-only; only coupled snapshots/boundaries need an atomic batch. const appended = batch.length === 1 - ? await this.historyService.appendToHistory(this.workspaceId, userMessage) - : await this.historyService.appendManyToHistory(this.workspaceId, batch); - if (!appended.success) return Err(createUnknownSendMessageError(appended.error)); + ? await this.historyService.appendToHistory( + this.workspaceId, + userMessage, + compactionAppendCondition + ) + : await this.historyService.appendManyToHistory( + this.workspaceId, + batch, + compactionAppendCondition + ); + if (!appended.success) { + if (internal?.compactionHandoff) throw new Error(appended.error); + return Err(createUnknownSendMessageError(appended.error)); + } } catch (error) { + if (internal?.compactionHandoff) throw error; return Err(createUnknownSendMessageError(getErrorMessage(error))); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push(...batch.map((row) => row.id)); if (contextRollover) { const sequences = [batch[0], batch[1], userMessage].map( @@ -4347,14 +4627,16 @@ export class AgentSession { } if (await cancelBeforeAcceptance()) return Ok(undefined); } else if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { - const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ - ...internal.preTurnMessages, - userMessage, - ]); + const batchAppendResult = await this.historyService.appendManyToHistory( + this.workspaceId, + [...internal.preTurnMessages, userMessage], + compactionAppendCondition + ); if (!batchAppendResult.success) { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(batchAppendResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push( ...internal.preTurnMessages.map((message) => message.id), userMessage.id @@ -4366,11 +4648,19 @@ export class AgentSession { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist // normally. - const appendResult = await this.historyService.appendToHistory(this.workspaceId, userMessage); + const appendResult = await this.historyService.appendToHistory( + this.workspaceId, + userMessage, + compactionAppendCondition + ); if (!appendResult.success) { await rollbackPersistedTurnRows(); + // Rollback can retire the local token; that must not disguise a real + // locked-read/write failure as an ordinary stale-source skip. + if (internal?.compactionHandoff) throw new Error(appendResult.error); return Err(createUnknownSendMessageError(appendResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push(userMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); @@ -4404,7 +4694,7 @@ export class AgentSession { // refunds keep the charge — refunding here would leave provider-visible rows uncharged. markRowsDurable(); } - return Err( + return refuseAdmission( createUnknownSendMessageError( "Send refused: the caller's admission became stale before the turn was accepted." ) @@ -4415,6 +4705,9 @@ export class AgentSession { // Branch summaries must remain discoverable if the append/rollback failed. Only // discard their registration once the new window has crossed the rollback horizon. this.clearContextBudgetState(); + // A committed reset retires old observations. The handoff publishing this + // very window still needs its token through the remaining acceptance gates. + if (!internal?.compactionHandoff) this.coordinator.invalidateCompaction(false); (internal?.onContextWindowRollover ?? this.onContextWindowRollover)?.(); await clearPendingBranchSummary(this.workspaceId); } else if (tokenBudgetActive) { @@ -4467,7 +4760,7 @@ export class AgentSession { // turn that made the admission stale consumes as context. const refuseStaleDurableSend = async (): Promise> => { await abandonWithdrawnSend(); - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAdmission(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); }; // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or @@ -4486,6 +4779,10 @@ export class AgentSession { return Err(createUnknownSendMessageError(getErrorMessage(error))); } } + // The committed row witnesses replacement even if ancillary unlink fails. + // Continue acceptance so a visible, durable user send is never reported lost. + if (compactionCancellationNonce) + await this.retireWitnessedCompactionCancellation(compactionCancellationNonce); try { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { @@ -4613,7 +4910,7 @@ export class AgentSession { // that bookkeeping (r41). await this.settlePreparationFailure(attempt, error); await abandonWithdrawnSend(); - return Err(error); + return { success: false, error, superseded: true }; } // A withdrawn send must not claim PREPARING (see abandonWithdrawnSend); it resolves Ok without // a stream, like cancelBeforeAcceptance and the disposed path above. @@ -4634,6 +4931,7 @@ export class AgentSession { intent: "direct", expectedTurnId: attempt.expectedTurn, editReservation: attempt.editReservation?.id, + compactionHandoff: internal?.compactionHandoff, }, preparedTurnAbortController, (turnId) => { @@ -4644,13 +4942,15 @@ export class AgentSession { } ); if (admission.status !== "admitted") { - return Err( - createUnknownSendMessageError( + return { + success: false, + error: createUnknownSendMessageError( admission.status === "rejected" && admission.reason === "closing" ? SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE : CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE - ) - ); + ), + superseded: true, + }; } const preparedTurn = admission.turnId; @@ -4738,6 +5038,7 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; goalId?: string; retrySignal?: AbortSignal; + automatic?: boolean; requestAssemblySnapshot?: RequestAssemblySnapshot; } ): Promise> { @@ -4794,6 +5095,7 @@ export class AgentSession { } const attempt: PreparationAttempt = { + automaticResume: internal?.automatic === true, expectedTurn: expectedTurnId, outcome: "preparing", durability: "accepted", @@ -4813,6 +5115,17 @@ export class AgentSession { ); if (admission.status !== "admitted") return Ok({ started: false }); const preparedTurn = admission.turnId; + if (!internal?.automatic && internal?.agentInitiated !== true) { + const epoch = this.coordinator.compactionIntent.epoch; + const nonce = await this.getCompactionCancellationNonce(); + if ( + !this.coordinator.isCurrentTurn(preparedTurn) || + this.coordinator.compactionIntent.epoch !== epoch || + this.coordinator.closing + ) + return Ok({ started: false }); + attempt.resumeCancellation = { nonce: nonce ?? null, epoch }; + } this.setAutoRetryResumeState( optionsForStream, internal?.agentInitiated, @@ -5330,6 +5643,9 @@ export class AgentSession { copiedFileBaseline?.tracking.restore(); this.acceptedFileSnapshotBaseline = copiedFileBaseline; this.clearContextBudgetState(); + // The accepted request owns this rollover; old compaction observations + // cannot carry work from the sealed window into its replacement. + this.coordinator.invalidateCompaction(false); this.onContextWindowRollover?.(); await clearPendingBranchSummary(this.workspaceId); if (!this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation)) @@ -6211,6 +6527,7 @@ export class AgentSession { this.deferQueuedFlushUntilAfterEdit ) return; + const epoch = this.coordinator.compactionIntent.epoch; try { const context = this.getContinuousCompactionContext(model, options); if (!context.enabled && !this.continuousCompactor.hasConsumedSwap()) { @@ -6231,7 +6548,12 @@ export class AgentSession { ...context, phase: "stream-end", }); - if (result === "applied") this.clearUsageState(); + if ( + result === "applied" && + epoch === this.coordinator.compactionIntent.epoch && + !this.coordinator.closing + ) + this.clearUsageState(); } catch (error) { await this.recoverContinuousCompactionFailure(error); } @@ -6264,46 +6586,40 @@ export class AgentSession { } private async waitForContinuousCompactionObservation(): Promise { - while (this.continuousCompactionObservation) await this.continuousCompactionObservation; + const observation = this.coordinator.compactionIntent.observation; + if (observation) await this.compactionObservations.get(observation.id); } private async runContinuousCompactionObservation( - observe: () => Promise + observe: (token: CompactionToken) => Promise ): Promise { if (this.coordinator.closing) return undefined; - // Own the actual apply and its continuation; the compactor's detached eager summary job - // retains its existing cancellation contract until the later compaction migration. using _execution = this.coordinator.enterExecution(); - if (this.continuousCompactionObservation) { - await this.continuousCompactionObservation; + const token = this.coordinator.beginCompactionObservation("continuous"); + if (!token) { + await this.waitForContinuousCompactionObservation(); return undefined; } - let finish!: () => void; - const observation = new Promise((resolve) => { - finish = resolve; - }); - this.continuousCompactionObservation = observation; - this.continuousCompactionObserving = true; + const observation = Promise.withResolvers(); + this.compactionObservations.set(token.id, observation.promise); try { - return await observe(); + return await observe(token); } catch (error) { - await this.recoverContinuousCompactionFailure(error, true); + if (this.coordinator.isCurrentCompaction(token)) + await this.recoverContinuousCompactionFailure(error, true); return undefined; } finally { - // Reserve through dispatch and cleanup, not just the compactor's apply latch. - // Waiters/duplicate invalidations never own or clear these flags. - if (this.continuousCompactionObservation === observation) { - this.settleMidStreamCompaction(); - this.continuousCompactionStopped = false; - this.continuousCompactionObserving = false; - this.continuousCompactionObservation = null; + // Semantic ownership can move to B while A's Promise is still finishing. + // Always release A's waiter; only its current owner may unblock queue dispatch. + this.compactionObservations.delete(token.id); + if (this.coordinator.finishCompactionObservation(token)) { try { this.drainQueuedMessagesIfIdle(); } catch (error) { log.warn("[continuous-compaction] queued drain failed", error); } } - finish(); + observation.resolve(); } } @@ -6311,7 +6627,10 @@ export class AgentSession { apply: (pendingFollowUp?: CompactionFollowUpRequest) => Promise ): Promise { const context = this.activeStreamContext; + const token = this.coordinator.compactionIntent.observation; if ( + !token || + !this.coordinator.isCurrentCompaction(token) || this.midStreamCompactionPending || !context?.options || this.coordinator.disposed || @@ -6319,17 +6638,17 @@ export class AgentSession { ) { return false; } - this.midStreamCompactionPending = true; + this.coordinator.setCompactionStage(token, "stopping"); const stopped = await this.streamManager.stopStream(this.workspaceId, { abortReason: "system", }); - if (!stopped.success) return false; - this.continuousCompactionStopped = true; + if (!stopped.success || !this.coordinator.isCurrentCompaction(token)) return false; + this.coordinator.setCompactionStage(token, "stopped"); await this.waitForIdle(); if ( this.coordinator.disposed || this.coordinator.closing || - this.continuousCompactionAbandoned || + !this.coordinator.isCurrentCompaction(token) || this.isWorkspaceArchivedOnDisk() ) return false; @@ -6358,13 +6677,19 @@ export class AgentSession { private async finishContinuousCompaction( applied: boolean, - context: NonNullable + context: NonNullable, + token: CompactionToken ): Promise { assert( !this.continuousCompactor.isApplying(), "Continue must dispatch after the apply latch clears" ); - if (!this.continuousCompactionStopped || !context.options) return; + if ( + this.coordinator.compactionIntent.observation?.id !== token.id || + this.coordinator.compactionIntent.observation.stage !== "stopped" || + !context.options + ) + return; // A consumed journal is an outstanding durable obligation, not a failed // speculative summary. Leave it retryable instead of resetting into legacy compaction. if (!applied && this.continuousCompactor.hasConsumedSwap()) return; @@ -6374,7 +6699,7 @@ export class AgentSession { // We already interrupted the turn, so recover using its captured context // rather than relying on activeStreamContext (cleared by stream-abort). if ( - this.continuousCompactionAbandoned || + !this.coordinator.isCurrentCompaction(token) || this.coordinator.disposed || this.coordinator.closing || this.isWorkspaceArchivedOnDisk() || @@ -6397,6 +6722,7 @@ export class AgentSession { reason: "mid-stream", }); } + if (!this.coordinator.isCurrentCompaction(token)) return; const fallback = pressure.shouldForceCompact ? this.buildAutoCompactionRequest({ baseOptions: context.options, @@ -6405,30 +6731,36 @@ export class AgentSession { }) : undefined; this.continuousCompactor.reset("failed-fast-apply"); + this.coordinator.setCompactionStage(token, "dispatching"); const sent = await this.sendMessage( fallback?.messageText ?? followUp.text, fallback ? { ...fallback.sendOptions, muxMetadata: fallback.metadata } : context.options, { synthetic: true, + compactionHandoff: token, + compactionHandoffPublication: context.compactionPublication ?? { generation: undefined }, agentInitiated: fallback?.agentInitiated ?? context.agentInitiated, goalKind: fallback ? undefined : context.goalKind, goalId: fallback ? undefined : context.goalId, - admissionStale: () => this.continuousCompactionAbandoned, + admissionStale: () => !this.coordinator.isCurrentCompaction(token), } ); - if (!sent.success && !sent.failureHandled && !this.continuousCompactionAbandoned) { + if (!sent.success && !sent.failureHandled && this.coordinator.isCurrentCompaction(token)) { this.emitChatEvent(createStreamErrorMessage(buildStreamErrorEventData(sent.error))); } return; } - this.lastUsageState = undefined; + if (this.coordinator.isCurrentCompaction(token)) this.lastUsageState = undefined; const summaryId = this.pendingCompactionFollowUpSummaryId; await this.dispatchPendingFollowUp( summaryId ?? undefined, - () => this.continuousCompactionAbandoned + () => !this.coordinator.isCurrentCompaction(token) ); - if (this.pendingCompactionFollowUpSummaryId === summaryId) - this.pendingCompactionFollowUpSummaryId = null; + if ( + this.coordinator.isCurrentCompaction(token) && + this.pendingCompactionFollowUpSummaryId === summaryId + ) + this.coordinator.recordCompactionSummary(null); } private async interruptForCompaction(): Promise { @@ -6442,10 +6774,12 @@ export class AgentSession { return; } + const token = this.coordinator.beginCompactionObservation("legacy"); + if (!token) return; const interruptedUserMessageId = this.activeStreamUserMessageId; this.continuousCompactor.reset("legacy-fallback"); - this.midStreamCompactionPending = true; + this.coordinator.setCompactionStage(token, "stopping"); try { const stopResult = await this.streamManager.stopStream(this.workspaceId, { abortReason: "system", @@ -6459,7 +6793,7 @@ export class AgentSession { } await this.waitForIdle(); - if (this.coordinator.disposed) { + if (!this.coordinator.isCurrentCompaction(token)) { return; } @@ -6480,6 +6814,8 @@ export class AgentSession { reason: "mid-stream", }); + if (!this.coordinator.isCurrentCompaction(token)) return; + this.coordinator.setCompactionStage(token, "dispatching"); const autoCompactionRequest = this.buildAutoCompactionRequest({ followUpContent, baseOptions: streamContext.options, @@ -6492,9 +6828,17 @@ export class AgentSession { ...autoCompactionRequest.sendOptions, muxMetadata: autoCompactionRequest.metadata, }, - { synthetic: true, agentInitiated: autoCompactionRequest.agentInitiated } + { + synthetic: true, + agentInitiated: autoCompactionRequest.agentInitiated, + compactionHandoff: token, + compactionHandoffPublication: streamContext.compactionPublication ?? { + generation: undefined, + }, + admissionStale: () => !this.coordinator.isCurrentCompaction(token), + } ); - if (!sendResult.success) { + if (!sendResult.success && this.coordinator.isCurrentCompaction(token)) { log.warn("Failed to dispatch mid-stream compaction request", { workspaceId: this.workspaceId, error: sendResult.error, @@ -6526,10 +6870,9 @@ export class AgentSession { } } } finally { - this.settleMidStreamCompaction(); // Preflight drains deferred to this pending compaction have no other retry: if the // compaction request never became a turn, release the queue now (no-op when it did). - this.drainQueuedMessagesIfIdle(); + if (this.coordinator.finishCompactionObservation(token)) this.drainQueuedMessagesIfIdle(); } } @@ -6560,8 +6903,22 @@ export class AgentSession { // a future boundary, so neither joins policy here. const interruptedPolicy = this.coordinator.captureInterruptSettlement(options?.soft); this.clearContextBudgetState(); - if (options?.abandonPartial || this.midStreamCompactionPending) { - this.continuousCompactionAbandoned = true; + const publishesCancellation = + options?.abandonPartial === true || this.midStreamCompactionPending; + if (publishesCancellation) { + this.coordinator.invalidateCompaction(true); + // Register durable cancellation before any Stop await. Its independent nonce + // survives a later manual preparation that fails before committing its row. + const execution = this.coordinator.enterExecution(); + this.compactionCancellation + .cancel() + .finally(() => execution[Symbol.dispose]()) + .catch((error: unknown) => + log.warn("Failed to persist compaction cancellation", { + workspaceId: this.workspaceId, + error, + }) + ); this.continuousCompactor.reset("user-interrupt"); } @@ -6582,6 +6939,13 @@ export class AgentSession { } await interruptedPolicy; + if (publishesCancellation) { + try { + await this.compactionCancellation.flush(); + } catch (error) { + return Err(getErrorMessage(error)); + } + } return Ok(undefined); } @@ -6593,6 +6957,8 @@ export class AgentSession { preStartErrors?: StreamErrorPayload[] | null, preparation?: PreparationAttempt ): Promise> { + const superseded = + !this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation); if (preparation) { await this.settlePreparationFailure(preparation, error); // Recovery may synchronously claim a follow-up. Its predecessor's callback has @@ -6607,7 +6973,12 @@ export class AgentSession { for (const payload of preStartErrors ?? []) { this.coordinator.resolveErrorDecision(payload.messageId, "terminal"); } - return { success: false, error, failureHandled: true }; + return { + success: false, + error, + failureHandled: true, + superseded: superseded ? true : undefined, + }; } // Collected pre-start error events own this failure; the branches below @@ -6708,6 +7079,15 @@ export class AgentSession { const operation = this.coordinator.registerOperation(turn); let completionTransferred = false; try { + // Capture before the original provider can produce old-context work. + // Publish no shared startup state if this await outlives its operation. + const compactionPublication = { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; + if (isStreamStartAborted() || !this.coordinator.isCurrentOperation(operation)) + return Ok(undefined); // Reset per-stream flags (used for retries / crash-safe bookkeeping). this.compactionMonitor.resetForNewStream(); this.clearLiveUsageState(); @@ -6716,6 +7096,7 @@ export class AgentSession { this.activeStreamHadPostCompactionInjection = false; const providersConfig = this.getProvidersConfigSafe(); this.activeStreamContext = { + compactionPublication, modelString, contextBudgetRetried, requestAssemblySnapshot, @@ -6917,6 +7298,53 @@ export class AgentSession { // collect them so the Err path resolves each exactly once. const preStartErrors: StreamErrorPayload[] = []; this.coordinator.configureOperation(operation, this.activeCompactionRequest != null); + if ( + preparation?.automaticResume && + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" + ) + return Ok(undefined); + if (isStreamStartAborted()) return Ok(undefined); + const resumedCancellation = preparation?.resumeCancellation; + if (resumedCancellation) { + const resumedUser = + lastUserMessage ?? requestMessages.findLast((row) => row.role === "user"); + if (!resumedUser) + return await fail( + createUnknownSendMessageError("Cannot accept resume without a user row") + ); + let committed = false; + // Retry is explicit user intent but reuses its existing row. Persist that + // acceptance before engine entry; failed preflight or a guarded no-op keeps Stop. + const witnessed = await this.historyService.acceptResumeCancellation( + this.workspaceId, + resumedUser, + resumedCancellation.nonce, + (current) => + !isStreamStartAborted() && + this.coordinator.compactionIntent.epoch === resumedCancellation.epoch && + current.id === resumedUser.id && + current.role === "user" && + current.metadata?.historySequence === resumedUser.metadata?.historySequence, + () => { + committed = true; + } + ); + if (committed) { + // A budget rollover re-enters streamWithHistory with this same attempt. + // Its accepted Retry receipt must not be replayed against the retired nonce. + if (preparation) preparation.resumeCancellation = undefined; + if (resumedCancellation.nonce !== null) + await this.retireWitnessedCompactionCancellation(resumedCancellation.nonce); + } + if (!witnessed.success) return await fail(createUnknownSendMessageError(witnessed.error)); + if ( + !committed || + isStreamStartAborted() || + this.coordinator.compactionIntent.epoch !== resumedCancellation.epoch + ) + return Ok(undefined); + } + const startRequest = preparedRequest ? preparedRequest.start.bind(preparedRequest) : this.aiService.streamMessage.bind(this.aiService); @@ -6976,7 +7404,12 @@ export class AgentSession { for (const payload of preStartErrors) { this.coordinator.resolveErrorDecision(payload.messageId, "terminal"); } - return { success: false, error: streamResult.error, failureHandled: true }; + return { + success: false, + error: streamResult.error, + failureHandled: true, + superseded: true, + }; } if ( streamResult.error.type === "context_budget_exceeded" && @@ -7101,6 +7534,7 @@ export class AgentSession { } private async clearFailedAssistantMessage(messageId: string, reason: string): Promise { + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("delete-message"); const [partialResult, deleteMessageResult] = await Promise.all([ this.historyService.deletePartial(this.workspaceId), @@ -7951,6 +8385,8 @@ export class AgentSession { let emittedStreamEnd = false; const completedCompactionRequest = this.activeCompactionRequest; let continuedAfterCompaction = false; + let handledCompaction = false; + let followUpDispatchStarted = false; try { this.activeCompactionRequest = undefined; @@ -7970,6 +8406,7 @@ export class AgentSession { streamEndPayload, completedCompactionRequest?.id ); + handledCompaction = handled; if (!this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation)) return; @@ -8047,7 +8484,8 @@ export class AgentSession { // RLM keep-recent floor: when tail copies were appended the summary is // not the last row, so target it by ID (stashed in onCompactionComplete). const rlmSummaryId = this.pendingCompactionFollowUpSummaryId; - this.pendingCompactionFollowUpSummaryId = null; + this.coordinator.recordCompactionSummary(null); + followUpDispatchStarted = true; continuedAfterCompaction = await this.dispatchPendingFollowUp(rlmSummaryId ?? undefined); if ( !this.coordinator.isCurrentTurn(turn) || @@ -8124,6 +8562,27 @@ export class AgentSession { // Best-effort; don't mask the original error. } } + + // Goal/accounting or boundary-publication failures can bypass dispatch + // after the boundary commits. History revalidation also makes an uncommitted + // compaction request a no-op here. + // Stop still owns that cleanup; the terminal lease joins it before shutdown ends. + // Once dispatch began, its own receipt protects any irrevocable continuation row. + if ( + (handledCompaction || completedCompactionRequest != null) && + !followUpDispatchStarted && + this.coordinator.isCurrentOperation(operation) && + this.coordinator.compactionIntent.status === "abandoned" + ) { + try { + await this.dispatchPendingFollowUp(this.pendingCompactionFollowUpSummaryId ?? undefined); + } catch (cleanupError) { + log.warn("Abandoned compaction follow-up cleanup failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(cleanupError), + }); + } + } } finally { if (completedCompactionRequest != null) { this.coordinator.resolveCompactionDecision( @@ -8286,14 +8745,12 @@ export class AgentSession { !this.streamManager.isStreaming(this.workspaceId) ) return; - await this.runContinuousCompactionObservation(async () => { + await this.runContinuousCompactionObservation(async (token) => { const result = await this.continuousCompactor.observe(0, { ...this.getContinuousCompactionContext(context.modelString, context.options), phase: "mid-stream", }); - // The observation's finally settles the pending window only after this dispatches the - // continuation; settling earlier would let an idle waiter race the follow-up send. - await this.finishContinuousCompaction(result === "applied", context); + await this.finishContinuousCompaction(result === "applied", context, token); }); } catch (error) { await this.recoverContinuousCompactionFailure(error); @@ -8358,16 +8815,17 @@ export class AgentSession { if (continuousContext.enabled || consumedSwapPending) { // One usage handler owns the eventual resume; observe itself shares its // latch result, which must not dispatch the continuation twice. - const observed = await this.runContinuousCompactionObservation(async () => { + const observed = await this.runContinuousCompactionObservation(async (token) => { const result = await this.continuousCompactor.observe(usagePercent, { ...continuousContext, phase: "mid-stream", }); if (this.midStreamCompactionPending) { - await this.finishContinuousCompaction(result === "applied", streamContext); + await this.finishContinuousCompaction(result === "applied", streamContext, token); return undefined; } - if (result === "applied") this.clearUsageState(); + if (result === "applied" && this.coordinator.isCurrentCompaction(token)) + this.clearUsageState(); return result; }); if (observed === undefined) return; @@ -8550,13 +9008,7 @@ export class AgentSession { * signal rather than the stream lifecycle. */ waitForMidStreamCompactionSettled(): Promise { - if (!this.midStreamCompactionPending) return Promise.resolve(); - return new Promise((resolve) => this.midStreamCompactionSettledWaiters.push(resolve)); - } - - private settleMidStreamCompaction(): void { - this.midStreamCompactionPending = false; - for (const resolve of this.midStreamCompactionSettledWaiters.splice(0)) resolve(); + return this.coordinator.waitForMidStreamCompactionSettled(); } /** @@ -8607,10 +9059,32 @@ export class AgentSession { * check. */ holdTurnAdmission(): Disposable { - this.continuousCompactor.reset("context-mutation"); + // Refine/archive and failed mutation acquisition share this temporary gate. + // Only committed replacement history may retire Stop's exact cleanup owner. return this.coordinator.reserve("admission"); } + async fenceCompactionForContextClear(): Promise { + // Empty history is no replacement witness: foreign legacy/heartbeat writers + // can still publish captured summaries after the clear has finished. + await this.compactionCancellation.cancel({ retainUntilReplacement: true }); + } + + async contextMutationCommitted(cancellationNonce?: string | null): Promise { + this.coordinator.invalidateCompaction(false); + this.continuousCompactor.reset("context-mutation"); + await this.retireCompactionCancellation(cancellationNonce); + } + + async retireCompactionCancellation(cancellationNonce?: string | null): Promise { + // null captures absence before a mutation; undefined preserves explicit repair callers. + const nonce = + cancellationNonce === undefined + ? (await this.compactionCancellation.readForReplacement())?.nonce + : cancellationNonce; + if (nonce) await this.compactionCancellation.retire(nonce); + } + /** * Mid-turn thinking change: request that the active turn's next model step * uses `level`. Returns accepted:false when no turn is active — the caller @@ -9258,11 +9732,101 @@ export class AgentSession { summaryMessageId?: string, cancelResume?: () => boolean ): Promise { - if (this.coordinator.disposed || this.coordinator.closing) { + const epoch = this.coordinator.compactionIntent.epoch; + for (;;) { + // Keep the recovery checkpoint pending while a temporary hold excludes sends. + // Wait outside the I/O lease so shutdown can cancel it without joining itself. + while (this.coordinator.admissionBlocked && !this.coordinator.closing) { + await this.coordinator.waitForAdmissionRelease(); + if (this.coordinator.compactionIntent.epoch !== epoch) return false; + } + const result = await this.tryDispatchPendingFollowUp(summaryMessageId, cancelResume); + if (result !== "deferred") return result; + } + } + + private async tryDispatchPendingFollowUp( + summaryMessageId?: string, + cancelResume?: () => boolean + ): Promise { + if ( + this.deferredCompactionCleanup && + !this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token) + ) { + this.deferredCompactionCleanup = undefined; + } + const deferred = this.deferredCompactionCleanup; + if ( + (this.coordinator.disposed && !deferred) || + (this.coordinator.closing && this.coordinator.compactionIntent.status !== "abandoned") + ) { return false; } + this.deferredCompactionCleanup = undefined; + const targetSummaryId = deferred ? deferred.summaryMessageId : summaryMessageId; + // A leased terminal producer can reach its abandoned boundary only after + // shutdown begins. Join its cleanup while continuing to forbid new sends. using _execution = this.coordinator.enterExecution(); + // Claim before history I/O: a send admitted and completed during that read + // must not make an obsolete summary look like a fresh idle continuation. + const token = + deferred?.token ?? + this.coordinator.claimCompactionFollowUp() ?? + this.coordinator.claimCompactionFollowUpCleanup(); + if (!token) return false; + const dispatch: CompactionFollowUpDispatch = deferred?.dispatch ?? { accepted: false }; + try { + await this.compactionCancellation.flush(); + if (deferred && dispatch.summary) { + await this.clearPendingFollowUpFromSummary(dispatch.summary, token); + return false; + } + return await this.dispatchOwnedCompactionFollowUp( + token, + dispatch, + targetSummaryId, + cancelResume + ); + } catch (error) { + // Rejections must settle the same abandoned obligation as normal refusals. + // Keep the captured payload guard and durable receipt: a replacement summary + // or already-persisted Continue must never lose its recovery marker here. + if ( + !dispatch.accepted && + this.coordinator.compactionIntent.status === "abandoned" && + this.coordinator.canClearCompactionFollowUp(token) + ) { + try { + if (dispatch.summary) await this.clearPendingFollowUpFromSummary(dispatch.summary, token); + else { + // The initial read can fail before capturing a summary. Retry once under + // the same abandoned owner; this path cannot admit a send or reclaim B. + await this.dispatchOwnedCompactionFollowUp(token, dispatch, targetSummaryId); + } + } catch (cleanupError) { + if (this.coordinator.canClearCompactionFollowUp(token)) { + this.deferredCompactionCleanup = { token, dispatch, summaryMessageId: targetSummaryId }; + } + log.warn("Abandoned compaction follow-up cleanup failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(cleanupError), + }); + } + } + throw error; + } finally { + if (!this.hasDeferredCompactionCleanup(token)) + this.coordinator.finishCompactionFollowUp(token); + } + } + private async dispatchOwnedCompactionFollowUp( + token: CompactionToken, + dispatch: CompactionFollowUpDispatch, + summaryMessageId?: string, + cancelResume?: () => boolean + ): Promise { + const repairRevision = this.compactionCancellation.repairRevision; let summaryMessage: MuxMessage | undefined; if (summaryMessageId) { const historyResult = await this.historyService.getHistoryFromLatestBoundary( @@ -9340,11 +9904,34 @@ export class AgentSession { if (!isCompactionSummaryMetadata(muxMeta) || !muxMeta.pendingFollowUp) { return false; } + dispatch.summary = lastMessage; + + if (!this.coordinator.isCurrentCompaction(token)) { + // Stop cancels dispatch while retaining ownership of its durable cleanup. + // A replacement/edit drops that ownership, so it cannot erase the new intent. + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + + const cancellation = await this.reconcileCompactionCancellation(); + if (!this.coordinator.isCurrentCompaction(token)) { + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + // A repairing read may have removed this captured follow-up from history. + // Its now-absent fence cannot authorize the stale request we read before repair. + if (repairRevision !== this.compactionCancellation.repairRevision) return false; + if (cancellation && this.compactionCancellation.matches(cancellation, lastMessage)) { + await this.clearPendingFollowUpFromSummary(lastMessage, token, cancellation.nonce); + return false; + } // A user can abandon after the boundary commits but before its continuation // dispatches. Keep the fold, but remove the crash-recoverable resume intent. if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -9368,7 +9955,7 @@ export class AgentSession { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -9385,7 +9972,7 @@ export class AgentSession { summaryMessageId: lastMessage.id, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -9414,7 +10001,8 @@ export class AgentSession { await this.skipIdleRuleFollowUp( lastMessage, hasQueuedMessages || hasExternalPreflightSend, - hasActiveNonCompletingTurn + hasActiveNonCompletingTurn, + token ); return false; } @@ -9449,12 +10037,18 @@ export class AgentSession { workspaceId: this.workspaceId, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } goalAdmissionStale = admission.admissionStale; } + if (!this.coordinator.isCurrentCompaction(token)) { + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + // Codex P1 (PRRT_kwDOPxxmWM6cQt3j): the queue/busy sample above ages // across the awaited goal read and the send's own preflight. Re-evaluate // the idle rule through the send-admission gates — all of them run before @@ -9467,13 +10061,16 @@ export class AgentSession { this.hasExternalSendPreflight?.() === true || (this.isBusy() && this.coordinator.phase !== "completing") : undefined; - const followUpAdmissionStale = - idleRuleStale != null || goalAdmissionStale != null || cancelResume != null - ? () => - idleRuleStale?.() === true || - goalAdmissionStale?.() === true || - cancelResume?.() === true - : undefined; + // Ordinary durable follow-ups reconstruct the interrupted request and precede + // queued input. Optional/goal/requireIdle continuations yield to manual work. + let sourceAppendSkipped = false; + const followUpAdmissionStale = () => + sourceAppendSkipped || + repairRevision !== this.compactionCancellation.repairRevision || + !this.coordinator.isCurrentCompaction(token) || + idleRuleStale?.() === true || + goalAdmissionStale?.() === true || + cancelResume?.() === true; log.debug("Dispatching pending follow-up from compaction summary", { workspaceId: this.workspaceId, @@ -9554,13 +10151,14 @@ export class AgentSession { } if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures // before stream startup do not fall back to the already-completed compact turn. + if (!this.coordinator.isCurrentCompaction(token)) return false; this.setAutoRetryResumeState( options, followUp.agentInitiated, @@ -9568,6 +10166,10 @@ export class AgentSession { persistedGoalId ); + // A hold may arrive during history/goal reads. Release this claim and wait + // at the outer checkpoint rather than turning temporary exclusion into failure. + if (this.coordinator.admissionBlocked) return "deferred"; + // Await sendMessage to ensure the follow-up is persisted before returning. // This guarantees ordering: the follow-up message is written to history // before sendQueuedMessages() runs, preventing race conditions. @@ -9575,6 +10177,21 @@ export class AgentSession { // re-enable auto-retry after a user explicitly opted out. const sendResult = await this.sendMessage(finalText, options, { synthetic: true, + compactionHandoff: token, + compactionHandoffSource: { + summary: lastMessage, + onSkipped: () => { + sourceAppendSkipped = true; + }, + }, + // Goal sync and synchronous message observers run after rollback becomes + // forbidden but before onAccepted. Stop cannot erase that durable handoff. + onRowsDurable: () => { + dispatch.accepted = true; + }, + onAccepted: () => { + dispatch.accepted = true; + }, agentInitiated: followUp.agentInitiated, goalKind: persistedGoalKind, // Keep the re-dispatched continuation row goal-scoped so a replaced @@ -9586,15 +10203,16 @@ export class AgentSession { // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, }); - if (!sendResult.success) { - if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + if (!sendResult.success && sendResult.admissionDeferred) return "deferred"; + if (!sendResult.success && !(dispatch.accepted && sendResult.superseded)) { + if (!dispatch.accepted && cancelResume?.()) { + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. - if (followUpAdmissionStale?.() === true) { + if (!dispatch.accepted && followUpAdmissionStale?.() === true) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, @@ -9602,7 +10220,8 @@ export class AgentSession { await this.skipIdleRuleFollowUp( lastMessage, this.hasPendingManualFollowUp() || this.hasExternalSendPreflight?.() === true, - this.isBusy() && this.coordinator.phase !== "completing" + this.isBusy() && this.coordinator.phase !== "completing", + token ); return false; } @@ -9610,6 +10229,11 @@ export class AgentSession { throw new Error(`Failed to dispatch pending follow-up: ${message}`); } + // Success can be a pre-acceptance no-op. Conversely, an accepted send may + // return a captured supersession error before startup; that cannot undo its + // durable handoff. Genuine startup errors still follow the failure path above. + if (!dispatch.accepted) return false; + // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): if the original wrap-up dispatcher // crashed between send acceptance and its tryMarkBudgetLimitInjected // commit, this redispatched follow-up owns the wrap-up now — install the @@ -9647,25 +10271,32 @@ export class AgentSession { private async skipIdleRuleFollowUp( summaryMessage: MuxMessage, hasUserContention: boolean, - hasActiveNonCompletingTurn: boolean + hasActiveNonCompletingTurn: boolean, + token: CompactionToken ): Promise { if ( summaryMessage.metadata?.compacted === "heartbeat" && hasUserContention && !hasActiveNonCompletingTurn ) { - const rollbackResult = - await this.compactionHandler.rollbackHeartbeatContextResetBoundary(summaryMessage); + const rollbackResult = await this.compactionHandler.rollbackHeartbeatContextResetBoundary( + summaryMessage, + () => this.coordinator.canClearCompactionFollowUp(token), + this.onPostCompactionStateChange + ); if (!rollbackResult.success) { throw new Error(`Failed to rollback heartbeat reset boundary: ${rollbackResult.error}`); } - this.onPostCompactionStateChange?.(); } else { - await this.clearPendingFollowUpFromSummary(summaryMessage); + await this.clearPendingFollowUpFromSummary(summaryMessage, token); } } - private async clearPendingFollowUpFromSummary(summaryMessage: MuxMessage): Promise { + private async clearPendingFollowUpFromSummary( + summaryMessage: MuxMessage, + token: CompactionToken, + cancellationNonce?: string + ): Promise { assert( summaryMessage.role === "assistant", "clearPendingFollowUpFromSummary requires an assistant summary message" @@ -9681,17 +10312,63 @@ export class AgentSession { return; } - const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; - const updateResult = await this.historyService.updateHistory(this.workspaceId, { - ...summaryMessage, - metadata: { - ...(summaryMessage.metadata ?? {}), - muxMetadata: muxMetadataWithoutFollowUp, + if (!this.coordinator.canClearCompactionFollowUp(token)) return; + const cancellation = await this.reconcileCompactionCancellation(); + if (!this.coordinator.canClearCompactionFollowUp(token)) return; + const canceled = + cancellation != null && this.compactionCancellation.matches(cancellation, summaryMessage); + if (cancellationNonce !== undefined && (!canceled || cancellation?.nonce !== cancellationNonce)) + return; + let matched = false; + let committed = false; + const updateResult = await this.historyService.updateHistory( + this.workspaceId, + summaryMessage, + (current) => { + const currentMeta = current.metadata?.muxMetadata; + // Summary finalization may change content/usage without replacing the + // handoff. Only its durable identity and pending request authorize cleanup. + return ( + this.coordinator.canClearCompactionFollowUp(token) && + current.id === summaryMessage.id && + current.metadata?.historySequence === summaryMessage.metadata?.historySequence && + current.role === "assistant" && + isCompactionSummaryMetadata(currentMeta) && + isDeepStrictEqual(currentMeta.pendingFollowUp, muxMeta.pendingFollowUp) + ); }, - }); + (current) => { + matched = true; + const currentMeta = current.metadata?.muxMetadata; + assert(isCompactionSummaryMetadata(currentMeta), "Cleanup requires the guarded summary"); + const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = currentMeta; + return { + ...current, + metadata: { ...current.metadata, muxMetadata: muxMetadataWithoutFollowUp }, + }; + }, + () => { + committed = true; + }, + cancellationNonce + ); if (!updateResult.success) { + // Only a target verified under the write lock may narrow an unresolved Stop. + // A conditional no-op for obsolete A must leave a foreign successor C intact. + if (canceled && matched && this.coordinator.canClearCompactionFollowUp(token)) { + try { + await this.compactionCancellation.narrow(cancellation.nonce, summaryMessage); + } catch (error) { + log.warn("Failed to narrow canceled compaction cleanup", { + workspaceId: this.workspaceId, + error, + }); + } + } throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } + if (committed && canceled && this.coordinator.canClearCompactionFollowUp(token)) + await this.compactionCancellation.retire(cancellation.nonce); } /** diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts new file mode 100644 index 00000000000..04b788c5cda --- /dev/null +++ b/src/node/services/compactionCancellation.test.ts @@ -0,0 +1,790 @@ +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { CompactionCancellation } from "./compactionCancellation"; +import { createTestHistoryService } from "./testHistoryService"; +import { HistoryService } from "./historyService"; +import { createMuxMessage } from "@/common/types/message"; + +afterEach(() => mock.restore()); + +test("retired cancellation writes and clears cannot overwrite the successor nonce", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-order"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stoppedA = state.cancel(); + try { + await entered.promise; + const a = await state.read(); + if (!a) throw new Error("Expected cancellation A"); + const retired = state.retire(a.nonce); + const stoppedB = state.cancel(); + const b = await state.read(); + expect(b?.nonce).not.toBe(a.nonce); + release.resolve(); + await Promise.all([stoppedA, retired, stoppedB]); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(b?.nonce); + } finally { + release.resolve(); + await stoppedA; + await state.flush(); + await h.cleanup(); + } +}); + +test("late exact narrowing cannot replace another service's newer cancellation", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-foreign"; + const a = new CompactionCancellation(h.historyService, workspaceId); + await a.cancel(); + const record = await a.read(); + if (!record) throw new Error("Expected cancellation"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const narrowing = a.narrow( + record.nonce, + createMuxMessage("summary-a", "assistant", "summary", { + historySequence: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + try { + await entered.promise; + const b = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await b.cancel(); + const successor = await b.read(); + release.resolve(); + await narrowing; + await a.retire(record.nonce); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(successor?.nonce); + } finally { + release.resolve(); + await narrowing; + await h.cleanup(); + } +}); + +test("failed retirement retries deletion while retaining conservative exclusion", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-retire"; + const state = new CompactionCancellation(h.historyService, workspaceId); + try { + await state.cancel(); + const record = await state.read(); + if (!record) throw new Error("Expected cancellation"); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink failed") + ); + const failure = await state.retire(record.nonce).catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "unlink failed"); + expect(state.needsPersistence).toBe(true); + expect((await state.read())?.nonce).toBe(record.nonce); + await state.retry(); + expect(state.needsPersistence).toBe(false); + expect(await state.read()).toBeNull(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await h.cleanup(); + } +}); + +test("failed initial publication cannot be acknowledged by exact narrowing", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-first-write"; + const state = new CompactionCancellation(h.historyService, workspaceId); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("first write failed") + ); + try { + const failure = await state.cancel().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "first write failed"); + const record = await state.read(); + if (!record) throw new Error("Expected retained cancellation"); + const narrowed = await state + .narrow( + record.nonce, + createMuxMessage("summary", "assistant", "summary", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ) + .catch((error: unknown) => error); + expect(narrowed).toHaveProperty("message", "first write failed"); + expect(state.needsPersistence).toBe(true); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + await state.retry(); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(record.nonce); + } finally { + await h.cleanup(); + } +}); + +test.each([ + ["cancel", false], + ["cancel", true], + ["narrow", false], + ["narrow", true], + ["retire", false], + ["retire", true], +] as const)( + "a held shared read cannot overwrite local %s (read error=%s)", + async (action, reject) => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-refresh-race"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const initial = await state.read(); + if (!initial) throw new Error("Expected initial cancellation"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = h.historyService.readCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "readCompactionCancellation").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + if (reject) throw new Error("obsolete shared read failed"); + return result; + } + ); + const reading = state.readForReplacement(); + try { + await entered.promise; + if (action === "cancel") await state.cancel(); + if (action === "retire") await state.retire(initial.nonce); + if (action === "narrow") + await state.narrow( + initial.nonce, + createMuxMessage("summary", "assistant", "summary", { + historySequence: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + const committed = await new HistoryService(h.config).readCompactionCancellation(workspaceId); + release.resolve(); + expect(await reading).toEqual(committed); + expect(await state.read()).toEqual(committed); + expect(state.needsPersistence).toBe(false); + } finally { + release.resolve(); + await reading; + await h.cleanup(); + } + } +); + +test("refreshing a foreign nonce does not reactivate settled mutation debt", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-refresh-debt"; + const a = new CompactionCancellation(h.historyService, workspaceId); + const b = new CompactionCancellation(new HistoryService(h.config), workspaceId); + try { + await a.cancel(); + const old = await a.read(); + if (!old) throw new Error("Expected old cancellation"); + await b.cancel(); + const current = await b.read(); + expect(await a.read()).toEqual(current); + await a.retry(); + await a.retire(old.nonce); + await a.narrow( + old.nonce, + createMuxMessage("old", "assistant", "old", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + expect(a.needsPersistence).toBe(false); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + current + ); + } finally { + await h.cleanup(); + } +}); + +test("corrupt-read repair preserves a newer valid foreign cancellation", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-race"; + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "work")); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const state = new CompactionCancellation(h.historyService, workspaceId); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const repair = h.historyService.repairCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "repairCompactionCancellation").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return repair(...args); + } + ); + const pending = state.read(); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const newer = await foreign.read(); + release.resolve(); + expect(await pending).toEqual(newer); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + newer + ); + } finally { + release.resolve(); + await pending; + await h.cleanup(); + } +}); + +test.each(["publication", "retirement"] as const)( + "corrupt shared bytes cannot discard local cancellation %s debt", + async (mutation) => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-debt"; + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "work")); + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected cancellation"); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("write unavailable") + ); + await (mutation === "publication" ? state.cancel() : state.retire(original.nonce)).catch( + () => undefined + ); + const owned = await state.read(); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const repair = spyOn(h.historyService, "repairCompactionCancellation"); + try { + expect(await state.read()).toEqual(owned); + expect(state.needsPersistence).toBe(true); + expect(repair).not.toHaveBeenCalled(); + } finally { + await h.cleanup(); + } + } +); + +test("a local Stop during corrupt repair prevents its obsolete history commit", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-local"; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const state = new CompactionCancellation(h.historyService, workspaceId); + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, contents: string, guard: () => boolean): Promise; + }; + const write = writes.writeGuardedHistory.bind(writes); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(writes, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const pending = state.read(); + try { + await entered.promise; + const stop = state.cancel(); + const newer = await state.read(); + release.resolve(); + expect(await pending).toEqual(newer); + await stop; + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toMatchObject(newer!); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + } finally { + release.resolve(); + await pending; + await state.flush(); + await h.cleanup(); + } +}); + +test.each( + [ + "unchanged", + "summary text", + "pending payload", + "summary ID", + "summary sequence", + "foreign synthetic tail", + "preserved tail", + "own snapshot", + "unresolved Stop", + "exact Stop", + "other exact Stop", + "malformed cancellation", + "malformed reset", + ].flatMap((change) => [false, true].map((batch) => [change, batch] as const)) +)("locked follow-up append revalidates %s (batch=%s)", async (change, batch) => { + const h = await createTestHistoryService(); + const workspaceId = "locked-follow-up"; + const source = createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + await h.historyService.appendToHistory(workspaceId, source); + const captured = structuredClone(source); + const foreign = new HistoryService(h.config); + const allowedTailMessageIds: string[] = []; + if (change === "summary text") { + await foreign.updateHistory(workspaceId, { + ...source, + parts: [{ type: "text", text: "Refined work" }], + }); + } else if (change === "pending payload") { + await foreign.updateHistory(workspaceId, { + ...source, + metadata: { + ...source.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Different request", model: "openai:gpt-4o", agentId: "exec" }, + }, + }, + }); + } else if (change === "summary ID") { + await foreign.updateHistory(workspaceId, { ...source, id: "replacement" }); + } else if (change === "summary sequence") { + await foreign.clearHistory(workspaceId); + await foreign.appendToHistory(workspaceId, createMuxMessage("earlier", "user", "Earlier")); + await foreign.appendToHistory(workspaceId, { + ...source, + metadata: { ...source.metadata, historySequence: undefined }, + }); + } else if ( + change === "foreign synthetic tail" || + change === "preserved tail" || + change === "own snapshot" + ) { + const tail = createMuxMessage("tail", "assistant", "Tail", { + synthetic: true, + ...(change === "preserved tail" ? { rlmPreservedTailCopy: true } : {}), + }); + await foreign.appendToHistory(workspaceId, tail); + if (change === "own snapshot") allowedTailMessageIds.push(tail.id); + } else if ( + change === "unresolved Stop" || + change === "exact Stop" || + change === "other exact Stop" + ) { + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected cancellation"); + if (change !== "unresolved Stop") + await cancellation.narrow( + record.nonce, + change === "exact Stop" ? source : { ...source, id: "other-summary" } + ); + } else if (change === "malformed reset") { + await appendFile( + `${h.config.sessionsDir}/${workspaceId}/chat.jsonl`, + '{"metadata":{"contextBoundaryKind":"reset"},broken\n' + ); + } else if (change === "malformed cancellation") { + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + } + const skipped = mock(() => undefined); + const candidate = createMuxMessage("follow-up", "user", "Continue", { synthetic: true }); + const allowed = [ + "unchanged", + "summary text", + "preserved tail", + "own snapshot", + "other exact Stop", + ].includes(change); + try { + const condition = { + summary: captured, + allowedTailMessageIds, + isCurrent: () => true, + onSkipped: skipped, + }; + const prelude = createMuxMessage("prelude", "assistant", "Expanded context", { + synthetic: true, + }); + const result = batch + ? await h.historyService.appendManyToHistory(workspaceId, [prelude, candidate], condition) + : await h.historyService.appendToHistory(workspaceId, candidate, condition); + if (!allowed) { + expect(candidate.metadata?.historySequence).toBeUndefined(); + expect(prelude.metadata?.historySequence).toBeUndefined(); + } + expect(result.success).toBe(change !== "malformed cancellation"); + expect(skipped).toHaveBeenCalledTimes(allowed || change === "malformed cancellation" ? 0 : 1); + const rows = await foreign.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.id === candidate.id)).toBe(allowed); + if (change === "summary text") + expect(rows.success && rows.data[0].parts).toEqual([{ type: "text", text: "Refined work" }]); + } finally { + await h.cleanup(); + } +}); + +test("witnessed deletion debt refreshes a foreign Stop without losing its own retry identity", async () => { + const h = await createTestHistoryService(); + const workspaceId = "witnessed-debt"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected Stop"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Replacement", { + compactionCancellationNonce: original.nonce, + }) + ); + const failure = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + await state.retireReplacement(original.nonce).catch(() => undefined); + try { + expect(await state.read()).toBeNull(); + await state.flush(); + expect(state.needsPersistence).toBe(true); + expect(state.blocksRecovery).toBe(false); + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const current = await foreign.read(); + expect(await state.read()).toEqual(current); + expect(state.needsPersistence).toBe(true); + failure.mockRestore(); + await state.retry(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + current + ); + } finally { + await h.cleanup(); + } +}); + +test("an archived replacement witness authorizes B without masking a newer Stop", async () => { + const h = await createTestHistoryService(); + const workspaceId = "archived-witness"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected Stop"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Replacement", { + compactionCancellationNonce: original.nonce, + }) + ); + const source = createMuxMessage("summary-b", "assistant", "B", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + await h.historyService.appendToHistory(workspaceId, source); + const foreign = new HistoryService(h.config); + const skipped = mock(() => undefined); + const condition = { + summary: source, + allowedTailMessageIds: [] as string[], + isCurrent: () => true, + onSkipped: skipped, + }; + try { + expect(await foreign.hasCompactionReplacementWitness(workspaceId, original.nonce)).toBe(true); + expect( + ( + await foreign.appendToHistory( + workspaceId, + createMuxMessage("b-user", "user", "Continue B"), + condition + ) + ).success + ).toBe(true); + expect(skipped).not.toHaveBeenCalled(); + await foreign.deleteMessage(workspaceId, "b-user"); + await new CompactionCancellation(foreign, workspaceId).cancel(); + expect( + ( + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("stopped", "user", "Continue B"), + condition + ) + ).success + ).toBe(true); + expect(skipped).toHaveBeenCalledTimes(1); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].id).toBe(source.id); + } finally { + await h.cleanup(); + } +}); + +test("a stale replacement receipt cannot make newer witnessed deletion debt blocking", async () => { + const h = await createTestHistoryService(); + const workspaceId = "stale-receipt"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const a = await state.read(); + await state.cancel(); + const b = await state.read(); + if (!a || !b) throw new Error("Expected distinct Stops"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement-b", "user", "B", { compactionCancellationNonce: b.nonce }) + ); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + await state.retireReplacement(b.nonce).catch(() => undefined); + try { + await state.retireReplacement(a.nonce); + expect(await state.read()).toBeNull(); + expect(state.blocksRecovery).toBe(false); + expect(state.needsPersistence).toBe(true); + await state.flush(); + } finally { + await h.cleanup(); + } +}); + +test("cancellation repair preserves raw reset privacy and invalidates an earlier scan cursor", async () => { + const h = await createTestHistoryService(); + const workspaceId = "repair-raw-floor"; + const source = createMuxMessage("summary", "assistant", "Private summary", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + const chatPath = `${h.config.sessionsDir}/${workspaceId}/chat.jsonl`; + const reset = '{"metadata":{"contextBoundaryKind":"reset"},broken\n'; + try { + await h.historyService.appendToHistory(workspaceId, source); + await appendFile(chatPath, reset); + await h.historyService.appendManyToHistory(workspaceId, [ + createMuxMessage("public-one", "user", "Public context"), + createMuxMessage("public-two", "assistant", "Public answer"), + ]); + const scan = await h.historyService.scanHistoryBounded(workspaceId, { visit: () => false }); + expect(scan.cursor).toBeDefined(); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const cancellation = new CompactionCancellation(h.historyService, workspaceId); + expect(await cancellation.read()).toBeNull(); + expect((await readFile(chatPath, "utf8")).includes(reset)).toBe(true); + const provider = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(provider.success && provider.data.map((row) => row.id)).toEqual([ + "public-one", + "public-two", + ]); + const history = await h.historyService.getLastMessages(workspaceId, 10); + expect(history.success && history.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + const resumed = await h.historyService + .scanHistoryBounded(workspaceId, { + cursor: scan.cursor, + visit: () => true, + }) + .catch((error: unknown) => error); + expect(resumed).toHaveProperty("message", "stale_cursor"); + } finally { + await h.cleanup(); + } +}); + +test.each([false, true])( + "concurrent replacement readers share the latest Stop publication (initial failure=%s)", + async (failed) => { + const h = await createTestHistoryService(); + const workspaceId = "shared-replacement-read"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const writes = spyOn(h.historyService, "writeCompactionCancellation"); + if (failed) writes.mockRejectedValueOnce(new Error("initial publication failed")); + writes.mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stopped = state.cancel().catch(() => undefined); + if (failed) await stopped; + const a = state.readForReplacement(); + const b = state.readForReplacement(); + try { + await entered.promise; + release.resolve(); + const [first, second] = await Promise.all([a, b]); + expect(first).toEqual(second); + expect(first).not.toBeNull(); + expect(writes).toHaveBeenCalledTimes(failed ? 2 : 1); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + first + ); + } finally { + release.resolve(); + await Promise.all([stopped, a.catch(() => undefined), b.catch(() => undefined)]); + await h.cleanup(); + } + } +); + +test.each([ + "before lock", + "witnessed before lock", + "before advance", + "after advance", + "foreign Stop", + "foreign repair", + "witnessed acknowledgement failure", +])("failed Stop retry preserves monotonic publication ownership (%s)", async (failure) => { + const h = await createTestHistoryService(); + const workspaceId = "failed-publication-generation"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const initial = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + if (failure === "before lock" || failure === "witnessed before lock") + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("publication unavailable before lock") + ); + else if (failure === "before advance") + spyOn(journal, "invalidateUnderHistoryLock").mockRejectedValueOnce( + new Error("generation unavailable") + ); + else if (failure !== "witnessed acknowledgement failure") + spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce(async (...args) => { + await invalidate(...args); + throw new Error("sidecar unavailable after advancement"); + }); + else { + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce( + async (...args) => { + await write(...args); + throw new Error("acknowledgement unavailable"); + } + ); + } + try { + expect( + await state.cancel().then( + () => false, + () => true + ) + ).toBe(true); + const attempted = await state.read(); + const advanced = await journal.captureGeneration(); + expect(advanced === initial).toBe( + failure === "before advance" || + failure === "before lock" || + failure === "witnessed before lock" + ); + const foreign = new HistoryService(h.config); + let successor = await foreign.readCompactionCancellation(workspaceId); + if ( + failure === "foreign Stop" || + failure === "before lock" || + failure === "witnessed before lock" + ) { + await new CompactionCancellation(foreign, workspaceId).cancel(); + successor = await foreign.readCompactionCancellation(workspaceId); + if (failure === "witnessed before lock") { + const replacement = new CompactionCancellation(foreign, workspaceId); + const stopped = await replacement.read(); + if (!stopped) throw new Error("Expected foreign Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("b-accepted", "user", "B accepted replacement", { + compactionCancellationNonce: stopped.nonce, + }) + ); + await replacement.retireReplacement(stopped.nonce); + successor = null; + } + } else if (failure === "foreign repair") { + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.repairCompactionCancellation( + workspaceId, + () => true, + () => undefined + ); + successor = null; + } else if (failure === "witnessed acknowledgement failure") { + await foreign.appendToHistory( + workspaceId, + createMuxMessage("accepted", "user", "New accepted work", { + compactionCancellationNonce: attempted?.nonce, + }) + ); + } + const epochBeforeRetry = await journal.captureGeneration(); + const preserveForeign = + failure.startsWith("foreign") || + failure === "witnessed acknowledgement failure" || + failure === "before lock" || + failure === "witnessed before lock"; + if (preserveForeign) await writeFile(journal.path, "newer journal must survive stale cleanup"); + const result = await state.readForReplacement(); + if (preserveForeign) { + expect(result).toEqual(successor); + expect(await journal.captureGeneration()).toBe(epochBeforeRetry); + expect(await readFile(journal.path, "utf8")).toBe("newer journal must survive stale cleanup"); + } else { + expect(result?.nonce).toBe(attempted?.nonce); + expect(await journal.captureGeneration()).not.toBe(advanced); + } + expect(state.needsPersistence).toBe(false); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts new file mode 100644 index 00000000000..01725b35150 --- /dev/null +++ b/src/node/services/compactionCancellation.ts @@ -0,0 +1,261 @@ +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { z } from "zod"; +import type { MuxMessage } from "@/common/types/message"; +import type { HistoryService } from "./historyService"; + +export const CompactionCancellationSchema = z.object({ + version: z.literal(1), + nonce: z.string().min(1), + retainUntilReplacement: z.boolean().optional(), + scope: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("unresolved") }), + z.object({ + kind: z.literal("summary"), + id: z.string(), + sequence: z.number().optional(), + pendingFollowUp: z.record(z.string(), z.unknown()), + }), + ]), +}); +export type CompactionCancellationRecord = z.infer; + +/** Process-local retry ownership; durable nonce + epoch prevent adopting foreign mutations. */ +export interface CompactionCancellationPublication { + attempts: number; + predecessor?: { nonce: string | null | undefined; generation: string | undefined }; +} + +/** Only successfully read bytes with invalid JSON/schema may enter automatic repair. */ +export class MalformedCompactionCancellationError extends Error { + constructor(readonly contents: Uint8Array) { + super("Malformed compaction cancellation record"); + } +} + +/** Cancellation publication survives failed preparation, independently of turn admission epochs. */ +export class CompactionCancellation { + private current: CompactionCancellationRecord | null | undefined; + private generation = 0; + private repairedHistoryRevision = 0; + private replacementNonce?: string; + private pending: Promise = Promise.resolve(); + private unsettled = false; + private mutation?: { + record: CompactionCancellationRecord | null; + retiredNonce?: string; + publication?: CompactionCancellationPublication; + }; + + constructor( + private readonly history: HistoryService, + private readonly workspaceId: string + ) {} + + get needsPersistence(): boolean { + return this.unsettled; + } + + get blocksRecovery(): boolean { + return this.unsettled && !this.isWitnessedRetirement(); + } + + private isWitnessedRetirement(): boolean { + return this.mutation?.record === null && this.mutation.retiredNonce === this.replacementNonce; + } + + private effectiveRecord(): CompactionCancellationRecord | null { + return this.current?.nonce === this.replacementNonce ? null : (this.current ?? null); + } + + get repairRevision(): number { + return this.repairedHistoryRevision; + } + + async read(): Promise { + // Other backends can publish Stop after a previous read (including absence). + // Local in-flight/failed mutations still own their conservative exclusion. + if (this.unsettled && !this.isWitnessedRetirement()) return this.effectiveRecord(); + const generation = this.generation; + const mutation = this.mutation; + const isCurrent = () => generation === this.generation && mutation === this.mutation; + try { + const record = await this.history + .readCompactionCancellation(this.workspaceId) + .catch((error: unknown) => { + if (!(error instanceof MalformedCompactionCancellationError) || !isCurrent()) throw error; + return this.history.repairCompactionCancellation(this.workspaceId, isCurrent, () => { + this.repairedHistoryRevision++; + }); + }); + if (isCurrent()) this.current = record; + } catch (error) { + // An obsolete read must not trigger explicit repair over a newer local Stop. + if (isCurrent()) throw error; + } + return this.effectiveRecord(); + } + + cancel(options?: { retainUntilReplacement?: boolean }): Promise { + // Each explicit Stop is new intent, even during a previous retirement's + // post-commit await. Only retry() may reuse publication identity. + this.current = { + version: 1, + nonce: randomUUID(), + scope: { kind: "unresolved" }, + ...(options?.retainUntilReplacement || this.current?.retainUntilReplacement + ? { retainUntilReplacement: true } + : {}), + }; + this.generation++; + return this.persist(this.current, undefined, { attempts: 0 }); + } + + async readForReplacement(): Promise { + for (;;) { + // An unpublished nonce cannot pass the shared append CAS. Repair only the + // latest blocking publication; witnessed unlink debt remains ancillary. + if (this.blocksRecovery) { + const pending = this.pending; + try { + await pending; + } catch { + // Join first, then retry only the exact failed publication. Concurrent + // readers share the retry instead of continually superseding one another. + if (pending !== this.pending) continue; + const retried = this.retry(); + try { + await retried; + } catch (error) { + if (retried === this.pending) throw error; + } + } + continue; + } + let record: CompactionCancellationRecord | null; + try { + record = await this.read(); + } catch { + // Explicit intervention can replace corrupt/unreadable state with a + // conservative fence, but failed publication must remain visible. + await this.cancel(); + continue; + } + if (!this.blocksRecovery) return record; + } + } + + async narrow(nonce: string, summary: MuxMessage): Promise { + // Exact CAS cannot turn a failed initial publication into apparent success. + await this.pending; + if ( + this.current?.nonce !== nonce || + this.current.scope.kind !== "unresolved" || + this.current.retainUntilReplacement + ) + return; + const metadata = summary.metadata?.muxMetadata; + if (!metadata || !("pendingFollowUp" in metadata) || !metadata.pendingFollowUp) return; + this.current = { + ...this.current, + scope: { + kind: "summary", + id: summary.id, + sequence: summary.metadata?.historySequence, + pendingFollowUp: { ...structuredClone(metadata.pendingFollowUp) }, + }, + }; + await this.persist(this.current); + } + + matches(record: CompactionCancellationRecord, summary: MuxMessage): boolean { + return matchesCompactionCancellation(record, summary); + } + + async flush(): Promise { + // An obsolete publication may have become a no-op behind a newer Stop. + // Success acknowledges the latest mutation, never an absent superseded write. + for (;;) { + const pending = this.pending; + try { + await pending; + } catch (error) { + if (pending !== this.pending) continue; + if (!this.isWitnessedRetirement()) throw error; + } + if (pending === this.pending) return; + } + } + + retireReplacement(nonce: string): Promise { + if (this.current?.nonce !== nonce) return Promise.resolve(); + // Only callers holding an actual durable row witness may retire intent before + // unlink succeeds. The physical deletion mutation remains observable/retryable. + this.replacementNonce = nonce; + return this.retire(nonce); + } + + retry(): Promise { + return this.unsettled && this.mutation + ? this.persist(this.mutation.record, this.mutation.retiredNonce, this.mutation.publication) + : this.pending; + } + + retire(nonce: string): Promise { + if ( + this.current?.nonce !== nonce || + (this.current.retainUntilReplacement && this.replacementNonce !== nonce) + ) + return Promise.resolve(); + this.generation++; + // Keep conservative exclusion in memory until deletion really commits. The + // retry payload remains a deletion, not a republication of that read state. + return this.persist(null, nonce); + } + + private persist( + snapshot: CompactionCancellationRecord | null, + retiredNonce?: string, + publication?: CompactionCancellationPublication + ): Promise { + const generation = this.generation; + const mutation = { record: snapshot, retiredNonce, publication }; + this.mutation = mutation; + this.unsettled = true; + const result = this.pending + .catch(() => undefined) + .then(async () => { + if (publication) publication.attempts++; + await this.history.writeCompactionCancellation( + this.workspaceId, + snapshot, + () => this.generation === generation, + retiredNonce, + publication + ); + if (this.generation === generation && this.mutation === mutation) { + this.unsettled = false; + if (snapshot === null && this.current?.nonce === retiredNonce) this.current = null; + } + }); + this.pending = result; + return result; + } +} + +/** Shared by unlocked policy checks and the locked follow-up append admission. */ +export function matchesCompactionCancellation( + record: CompactionCancellationRecord, + summary: MuxMessage +): boolean { + if (record.scope.kind === "unresolved") return true; + const metadata = summary.metadata?.muxMetadata; + return ( + record.scope.id === summary.id && + record.scope.sequence === summary.metadata?.historySequence && + isDeepStrictEqual( + record.scope.pendingFollowUp, + metadata && "pendingFollowUp" in metadata ? metadata.pendingFollowUp : undefined + ) + ); +} diff --git a/src/node/services/compactionHandler.continuous.test.ts b/src/node/services/compactionHandler.continuous.test.ts index f658def5128..27d984f00f3 100644 --- a/src/node/services/compactionHandler.continuous.test.ts +++ b/src/node/services/compactionHandler.continuous.test.ts @@ -1,13 +1,18 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, spyOn, mock } from "bun:test"; +import * as fsPromises from "fs/promises"; import { mkdir, writeFile } from "node:fs/promises"; import { EventEmitter } from "node:events"; import * as path from "node:path"; import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import assert from "@/common/utils/assert"; import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; +import { HistoryService } from "./historyService"; import { CompactionHandler } from "./compactionHandler"; import { prepareMessagesForProvider } from "./messagePipeline"; import { createTestHistoryService } from "./testHistoryService"; +import { TurnCoordinator } from "./turnCoordinator"; +import { CHAT_FILE_NAME, CHAT_ARCHIVE_FILE_NAME } from "@/common/constants/paths"; +import type { WorkspaceChatMessage } from "@/common/orpc/types"; describe("continuous compaction provider replay", () => { let store: Awaited>; @@ -17,9 +22,361 @@ describe("continuous compaction provider replay", () => { store = await createTestHistoryService(); }); afterEach(async () => { + mock.restore(); await store.cleanup(); }); + it.each(["none", "cleanup", "observer"] as const)( + "publishes committed heartbeat rollback before replacement admission (failure=%s)", + async (failurePoint) => { + const sessionDir = path.join(store.tempDir, "pending"); + await mkdir(sessionDir, { recursive: true }); + const priorDiff = { path: "/tmp/prior.ts", diff: "prior change", truncated: false }; + await writeFile( + path.join(sessionDir, "post-compaction.json"), + JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [priorDiff], + loadedSkills: [], + readFiles: [], + }) + ); + const recent = createMuxMessage("recent", "assistant", ""); + recent.parts = [ + { + type: "dynamic-tool", + toolCallId: "edit", + toolName: "file_edit_replace_string", + state: "output-available", + input: { path: "/tmp/recent.ts" }, + output: { success: true, diff: "recent change" }, + }, + ]; + await store.historyService.appendToHistory(workspaceId, recent); + const emitter = new EventEmitter(); + const emitted: WorkspaceChatMessage[] = []; + emitter.on("chat-event", (event: { message: WorkspaceChatMessage }) => { + emitted.push(event.message); + if (failurePoint === "observer" && event.message.type === "delete") + throw new Error("observer failed after commit"); + }); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter, + }); + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Heartbeat", + pendingFollowUp: { text: "wake", model: "openai:gpt-4o", agentId: "exec" }, + }) + ).success + ).toBe(true); + const boundary = await store.historyService.getLastMessages(workspaceId, 1); + assert(boundary.success, "Expected heartbeat boundary"); + const boundarySequence = boundary.data[0].metadata?.historySequence; + assert(boundarySequence != null, "Expected persisted boundary sequence"); + // The external send has already persisted its row; PREPARING can race + // rollback without needing another history write or its lock. + await store.historyService.appendToHistory( + workspaceId, + createMuxMessage("manual", "user", "manual replacement") + ); + const coordinator = new TurnCoordinator({ + phaseChanged: () => undefined, + drainQueue: () => undefined, + policy: () => Promise.resolve(), + policyError: () => undefined, + }); + const token = coordinator.claimCompactionFollowUp(); + assert(token, "Expected cleanup owner"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const rm = fsPromises.rm; + const historyPath = path.join(store.config.sessionsDir, workspaceId, CHAT_FILE_NAME); + spyOn(fsPromises, "rm").mockImplementation(async (file, options) => { + if (String(file).startsWith(`${historyPath}.continuous-`)) { + entered.resolve(); + await release.promise; + if (failurePoint === "cleanup") throw new Error("post-commit cleanup failed"); + } + return rm(file, options); + }); + const published = mock(() => undefined); + const pending = handler.rollbackHeartbeatContextResetBoundary( + boundary.data[0], + () => coordinator.canClearCompactionFollowUp(token), + published + ); + try { + await entered.promise; + const committedRows = (await fsPromises.readFile(historyPath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as MuxMessage); + expect(committedRows.map((message) => message.id)).toEqual(["manual"]); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + }).status + ).toBe("admitted"); + expect(handler.peekCachedFilePaths()).toEqual([priorDiff.path]); + expect(emitted.filter((message) => message.type === "delete")).toEqual([ + { + type: "delete", + historySequences: [boundarySequence], + }, + ]); + expect(published).toHaveBeenCalledTimes(1); + release.resolve(); + const result = await pending; + expect(result.success).toBe(failurePoint !== "cleanup"); + if (!result.success) expect(result.error).toContain("was deleted"); + expect((await handler.peekPendingState())?.diffs).toEqual([priorDiff]); + const reloaded = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + expect((await reloaded.peekPendingState())?.diffs).toEqual([priorDiff]); + } finally { + release.resolve(); + await pending; + } + } + ); + + it.each([ + "missing", + "replacement sequence", + "foreign clear", + "foreign replacement", + "empty predecessor", + ] as const)( + "settles local heartbeat rollback after a shared-history change (%s)", + async (state) => { + const sessionDir = path.join(store.tempDir, "pending"); + await mkdir(sessionDir, { recursive: true }); + const priorDiff = { path: "/tmp/prior.ts", diff: "prior change", truncated: false }; + await writeFile( + path.join(sessionDir, "post-compaction.json"), + JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [priorDiff], + loadedSkills: [], + readFiles: [], + }) + ); + const recent = createMuxMessage("recent-edit", "assistant", ""); + recent.parts = [ + { + type: "dynamic-tool", + toolCallId: "edit", + toolName: "file_edit_replace_string", + state: "output-available", + input: { path: "/tmp/recent.ts" }, + output: { success: true, diff: "recent change" }, + }, + ]; + if (state !== "empty predecessor") + await store.historyService.appendToHistory(workspaceId, recent); + const emitter = new EventEmitter(); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter, + }); + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Heartbeat", + pendingFollowUp: { text: "wake", model: "openai:gpt-4o", agentId: "exec" }, + }); + const rows = await store.historyService.getLastMessages(workspaceId, 1); + assert(rows.success && rows.data[0], "Expected boundary"); + const boundary = rows.data[0]; + // A second backend can commit the shared history deletion without touching + // this handler's captured rollback or its post-reset memory. + const foreign = new HistoryService(store.config); + if (state === "foreign clear" || state === "foreign replacement") { + expect((await foreign.clearHistory(workspaceId)).success).toBe(true); + if (state === "foreign replacement") + await foreign.appendToHistory( + workspaceId, + createMuxMessage("new-context", "user", "New context") + ); + } else expect((await foreign.deleteMessage(workspaceId, boundary.id)).success).toBe(true); + if (state === "replacement sequence") { + await store.historyService.appendToHistory(workspaceId, { + ...boundary, + metadata: { ...boundary.metadata, historySequence: undefined }, + }); + } + const before = await handler.peekPendingState(); + if (state !== "empty predecessor") + expect(before?.diffs.map((diff) => diff.path)).toContain("/tmp/recent.ts"); + const emit = spyOn(emitter, "emit"); + const published = mock(() => { + expect(handler.peekCachedFilePaths()).toEqual([priorDiff.path]); + }); + expect( + (await handler.rollbackHeartbeatContextResetBoundary(boundary, () => true, published)) + .success + ).toBe(true); + expect(emit).not.toHaveBeenCalled(); + if (state === "missing") { + expect(published).toHaveBeenCalledTimes(1); + expect((await handler.peekPendingState())?.diffs).toEqual([priorDiff]); + } else { + expect(published).not.toHaveBeenCalled(); + expect(await handler.peekPendingState()).toEqual(before); + } + } + ); + + it.each(["veto", "archived"] as const)( + "does not restore or publish a skipped heartbeat rollback (%s)", + async (skipReason) => { + const sessionDir = path.join(store.tempDir, "pending"); + const emitter = new EventEmitter(); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter, + }); + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Heartbeat", + pendingFollowUp: { text: "wake", model: "openai:gpt-4o", agentId: "exec" }, + }) + ).success + ).toBe(true); + const boundary = await store.historyService.getLastMessages(workspaceId, 1); + assert(boundary.success, "Expected heartbeat boundary"); + if (skipReason === "archived") { + await store.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "New boundary", { + compacted: true, + compactionBoundary: true, + compactionEpoch: 2, + }) + ); + } + const emit = spyOn(emitter, "emit"); + const published = mock(() => undefined); + const state = await handler.peekPendingState(); + expect(state).not.toBeNull(); + expect( + ( + await handler.rollbackHeartbeatContextResetBoundary( + boundary.data[0], + () => skipReason !== "veto", + published + ) + ).success + ).toBe(true); + expect(await handler.peekPendingState()).toEqual(state); + expect(emit).not.toHaveBeenCalled(); + expect(published).not.toHaveBeenCalled(); + if (skipReason === "archived") { + const archive = await fsPromises.readFile( + path.join(store.config.sessionsDir, workspaceId, CHAT_ARCHIVE_FILE_NAME), + "utf8" + ); + expect( + archive + .split("\n") + .filter(Boolean) + .map((line) => (JSON.parse(line) as MuxMessage).id) + ).toContain(boundary.data[0].id); + } + } + ); + + it("a held heartbeat rollback unlink cannot consume the replacement rollback snapshot", async () => { + const sessionDir = path.join(store.tempDir, "pending"); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + const followUp = { text: "wake", model: "openai:gpt-4o", agentId: "exec" }; + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "A", + pendingFollowUp: followUp, + }) + ).success + ).toBe(true); + const first = await store.historyService.getLastMessages(workspaceId, 1); + assert(first.success, "Expected first boundary"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const captured = Promise.withResolvers(); + const unlink = fsPromises.unlink; + let held = false; + spyOn(fsPromises, "unlink").mockImplementation(async (file) => { + if (!held && file === path.join(sessionDir, "post-compaction.json")) { + held = true; + entered.resolve(); + await release.promise; + } + return unlink(file); + }); + const internals = handler as unknown as { + captureHeartbeatResetRollbackState(messages: MuxMessage[]): void; + }; + const capture = internals.captureHeartbeatResetRollbackState.bind(handler); + spyOn(internals, "captureHeartbeatResetRollbackState").mockImplementation((messages) => { + capture(messages); + captured.resolve(); + }); + const rollingBack = handler.rollbackHeartbeatContextResetBoundary(first.data[0]); + let replacement: + | ReturnType + | undefined; + try { + await entered.promise; + replacement = handler.appendHeartbeatContextResetBoundary({ + boundaryText: "B", + pendingFollowUp: followUp, + }); + await captured.promise; + release.resolve(); + expect((await rollingBack).success).toBe(true); + expect((await replacement).success).toBe(true); + const reloaded = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + // A's delayed unlink must finish before B's pending snapshot publishes. + expect(await reloaded.peekPendingState()).not.toBeNull(); + const second = await store.historyService.getLastMessages(workspaceId, 1); + assert(second.success, "Expected replacement boundary"); + expect((await handler.rollbackHeartbeatContextResetBoundary(second.data[0])).success).toBe( + true + ); + expect(await handler.peekPendingState()).toBeNull(); + } finally { + release.resolve(); + await rollingBack; + await replacement; + } + }); + it("preserves previously pending attachments when a newer fold is abandoned or crashes", async () => { const sessionDir = path.join(store.tempDir, "pending"); await mkdir(sessionDir, { recursive: true }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 08676aa875d..01bbae95975 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1,8 +1,11 @@ +import type { ContinuousCompactionPublication } from "./continuousCompactionJournal"; import type { EventEmitter } from "events"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { isNonNegativeInteger, isPositiveInteger } from "@/common/utils/numbers"; import * as path from "path"; +import { randomUUID } from "node:crypto"; +import writeFileAtomic from "write-file-atomic"; import type { HistoryService } from "./historyService"; @@ -91,12 +94,24 @@ interface PersistedPostCompactionStateV1 { * model when RLM mode is on. Absent in files written by older builds. */ readFiles: string[]; - /** Only continuous preparation is provisional until this boundary is durable. */ + /** Provisional preparation is usable only after this exact boundary is durable. */ boundaryMessageId?: string; previousState?: PersistedPostCompactionStateV1; + /** Unique per actual write, so equal attachment contents cannot cause ABA cleanup. */ + writeId?: string; +} + +interface PendingStateWriteReceipt { + serialized: string; + previous?: string; +} + +interface PendingStatePreparation { + write?: PendingStateWriteReceipt; } interface HeartbeatResetRollbackState { + sourceRows: Array<{ id: string; sequence: number | undefined }>; postCompactionAttachmentsPending: boolean; cachedFileDiffs: FileEditDiff[]; cachedLoadedSkills: LoadedSkillSnapshot[]; @@ -382,6 +397,9 @@ interface CompactionHandlerOptions { emitter: EventEmitter; /** Called when compaction completes successfully (e.g., to clear idle compaction pending state) */ onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + /** Capture semantic ownership before I/O; durable completion must not retarget a successor. */ + captureCompletionGuard?: () => () => boolean; + getCompactionPublication?: () => ContinuousCompactionPublication | undefined; /** * Called with the terminal outcome of an idle compaction (source === "idle-compaction"), * after the summary is actually persisted (success) or a post-stream persistence failure @@ -411,6 +429,8 @@ export class CompactionHandler { private readonly processedCompactionRequestIds: Set = new Set(); private readonly onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + private readonly captureCompletionGuard?: () => () => boolean; + private readonly getCompactionPublication?: () => ContinuousCompactionPublication | undefined; private readonly onIdleCompactionOutcome?: (success: boolean) => void; /** Flag indicating post-compaction attachments should be generated on next turn */ @@ -419,6 +439,21 @@ export class CompactionHandler { private cachedFileDiffs: FileEditDiff[] = []; /** Rollback snapshot for synthetic heartbeat reset boundaries that get skipped before dispatch. */ private heartbeatResetRollbackState: HeartbeatResetRollbackState | null = null; + private pendingStateWrites: Promise = Promise.resolve(); + private pendingStatePreparation?: PendingStatePreparation; + + private enqueuePendingStateWrite( + write: () => Promise, + publication?: ContinuousCompactionPublication + ): Promise { + const result = this.pendingStateWrites.then(() => + publication + ? this.historyService.withCompactionPublicationWrite(this.workspaceId, publication, write) + : write() + ); + this.pendingStateWrites = result.catch(() => undefined); + return result; + } /** Cached loaded skill snapshots extracted from history before appending compaction summary */ private cachedLoadedSkills: LoadedSkillSnapshot[] = []; /** Cumulative file paths read in summarized epochs (paths only, newest-first, capped). */ @@ -437,14 +472,19 @@ export class CompactionHandler { this.telemetryService = options.telemetryService; this.emitter = options.emitter; this.onCompactionComplete = options.onCompactionComplete; + this.captureCompletionGuard = options.captureCompletionGuard; + this.getCompactionPublication = options.getCompactionPublication; this.onIdleCompactionOutcome = options.onIdleCompactionOutcome; } - private async loadPersistedPendingStateIfNeeded(): Promise { + private async loadPersistedPendingStateIfNeeded( + publication?: ContinuousCompactionPublication + ): Promise { if (this.persistedPendingStateLoaded || this.postCompactionAttachmentsPending) { return; } + const preparation = this.pendingStatePreparation; this.persistedPendingStateLoaded = true; let raw: string; @@ -459,21 +499,21 @@ export class CompactionHandler { parsed = JSON.parse(raw); } catch { log.warn("Invalid post-compaction state JSON; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } let state = coercePersistedPostCompactionState(parsed); if (!state) { log.warn("Invalid post-compaction state schema; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } if (state.boundaryMessageId) { const history = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); if (!history.success) { - this.persistedPendingStateLoaded = false; + if (this.pendingStatePreparation === preparation) this.persistedPendingStateLoaded = false; return; } const boundaryId = history.data.findLast(isDurableContextBoundaryMarker)?.id; @@ -482,11 +522,12 @@ export class CompactionHandler { // nor lose an older, still-pending attachment snapshot. state = state.previousState ?? null; if (!state || (state.boundaryMessageId && state.boundaryMessageId !== boundaryId)) { - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } } } + if (this.pendingStatePreparation !== preparation) return; this.pendingStateBoundaryMessageId = state.boundaryMessageId; this.cachedFileDiffs = state.diffs; this.cachedLoadedSkills = state.loadedSkills; @@ -590,7 +631,7 @@ export class CompactionHandler { async discardPendingStateDurably(reason: string): Promise { await this.discardPendingState(reason); try { - await fsPromises.unlink(this.postCompactionStatePath); + await this.enqueuePendingStateWrite(() => fsPromises.unlink(this.postCompactionStatePath)); } catch (error) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return; @@ -599,47 +640,81 @@ export class CompactionHandler { } } - private async deletePersistedPendingStateBestEffort(): Promise { + private async deletePersistedPendingStateBestEffort( + publication?: ContinuousCompactionPublication, + expectedBytes?: string + ): Promise { try { - await fsPromises.unlink(this.postCompactionStatePath); + if (expectedBytes !== undefined) { + await this.enqueuePendingStateWrite(() => + this.historyService.withCompactionPublicationLock( + this.workspaceId, + publication, + async () => { + if ( + (await fsPromises.readFile(this.postCompactionStatePath, "utf8")) === expectedBytes + ) { + await fsPromises.unlink(this.postCompactionStatePath); + } + } + ) + ); + return; + } + await this.enqueuePendingStateWrite( + () => fsPromises.unlink(this.postCompactionStatePath), + publication + ); } catch { // ignore } } - private captureHeartbeatResetRollbackState(): void { + private captureHeartbeatResetRollbackState(messages: MuxMessage[]): HeartbeatResetRollbackState { this.heartbeatResetRollbackState = { + sourceRows: messages.map((row) => ({ id: row.id, sequence: row.metadata?.historySequence })), postCompactionAttachmentsPending: this.postCompactionAttachmentsPending, cachedFileDiffs: [...this.cachedFileDiffs], cachedLoadedSkills: [...this.cachedLoadedSkills], cachedReadFilePaths: [...this.cachedReadFilePaths], persistedPendingStateLoaded: this.persistedPendingStateLoaded, }; + return this.heartbeatResetRollbackState; } - private async restoreHeartbeatResetRollbackState(): Promise { - const rollbackState = this.heartbeatResetRollbackState; - if (!rollbackState) { - return; + private restoreHeartbeatResetRollbackState( + publication?: ContinuousCompactionPublication, + rollbackState = this.heartbeatResetRollbackState, + persist = true + ): Promise { + if (!rollbackState || rollbackState !== this.heartbeatResetRollbackState) { + return Promise.resolve(); } + // Consume the captured rollback before I/O. A new heartbeat can install its + // own snapshot while this unlink/write is pending; A must never clear B's. + this.heartbeatResetRollbackState = null; + this.postCompactionAttachmentsPending = rollbackState.postCompactionAttachmentsPending; this.cachedFileDiffs = [...rollbackState.cachedFileDiffs]; this.cachedLoadedSkills = [...rollbackState.cachedLoadedSkills]; this.cachedReadFilePaths = [...rollbackState.cachedReadFilePaths]; this.persistedPendingStateLoaded = rollbackState.persistedPendingStateLoaded; + if (!persist) return Promise.resolve(); + if (rollbackState.postCompactionAttachmentsPending) { - await this.persistPendingStateBestEffort( + return this.persistPendingStateBestEffort( this.cachedFileDiffs, this.cachedLoadedSkills, - this.cachedReadFilePaths - ); + this.cachedReadFilePaths, + undefined, + undefined, + publication + ).then(() => undefined); } else { - await this.deletePersistedPendingStateBestEffort(); + return this.deletePersistedPendingStateBestEffort(publication); } - - this.heartbeatResetRollbackState = null; } private async persistPendingStateBestEffort( @@ -647,11 +722,11 @@ export class CompactionHandler { loadedSkills: LoadedSkillSnapshot[], readFiles: string[], boundaryMessageId?: string, - previousState?: PersistedPostCompactionStateV1 - ): Promise { + previousState?: PersistedPostCompactionStateV1, + publication?: ContinuousCompactionPublication + ): Promise { + let receipt: PendingStateWriteReceipt | undefined; try { - await fsPromises.mkdir(this.sessionDir, { recursive: true }); - for (const snapshot of loadedSkills) { assert(snapshot.name.trim().length > 0, "loaded skill snapshot name must not be empty"); } @@ -665,21 +740,95 @@ export class CompactionHandler { ...(boundaryMessageId && { boundaryMessageId, previousState }), }; - await fsPromises.writeFile(this.postCompactionStatePath, JSON.stringify(persisted)); + // Freeze the admitted snapshot, then order writes AND unlinks. A held rollback + // must not erase a newer compaction's pending state after that compaction completes. + const serialized = JSON.stringify({ ...persisted, writeId: randomUUID() }); + await this.enqueuePendingStateWrite(async () => { + await fsPromises.mkdir(this.sessionDir, { recursive: true }); + const previous = await fsPromises + .readFile(this.postCompactionStatePath, "utf8") + .catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + }); + receipt = { serialized, previous }; + await writeFileAtomic(this.postCompactionStatePath, serialized); + }, publication); } catch (error) { log.warn("Failed to persist post-compaction state", { workspaceId: this.workspaceId, error: getErrorMessage(error), }); } + return receipt; + } + + private async cleanupProvisionalPendingState( + preparation: PendingStatePreparation | undefined, + publication: ContinuousCompactionPublication | undefined, + rollbackState?: HeartbeatResetRollbackState + ): Promise { + if (!preparation) return; + const receipt = preparation.write; + try { + await this.enqueuePendingStateWrite(() => + this.historyService.withCompactionPublicationLock( + this.workspaceId, + publication, + async (current) => { + const exactWrite = + receipt !== undefined && + (await fsPromises.readFile(this.postCompactionStatePath, "utf8").then( + (raw) => raw === receipt.serialized, + () => false + )); + if (this.pendingStatePreparation === preparation) { + this.pendingStatePreparation = undefined; + if ((!receipt || exactWrite) && current && rollbackState) { + // This variant restores memory only; no nested pending/history lock. + await this.restoreHeartbeatResetRollbackState(publication, rollbackState, false); + } else { + this.postCompactionAttachmentsPending = false; + this.cachedFileDiffs = []; + this.cachedLoadedSkills = []; + this.cachedReadFilePaths = []; + this.pendingStateBoundaryMessageId = undefined; + this.persistedPendingStateLoaded = false; + if (this.heartbeatResetRollbackState === rollbackState) + this.heartbeatResetRollbackState = null; + } + } + if (!receipt || !exactWrite) return; + // Cleanup owns the exact write even after Stop. Only a still-current + // epoch may restore prior context; reset/edit must never resurrect it. + if (current && receipt.previous !== undefined) { + await writeFileAtomic(this.postCompactionStatePath, receipt.previous); + } else { + await fsPromises.unlink(this.postCompactionStatePath); + } + } + ) + ); + } catch (error) { + log.warn("Failed to clean provisional post-compaction state", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } } async preparePendingStateFromMessages( messages: MuxMessage[], boundaryMessageId?: string, - previousState?: PersistedPostCompactionStateV1 - ): Promise { - await this.loadPersistedPendingStateIfNeeded(); + previousState?: PersistedPostCompactionStateV1, + publication?: ContinuousCompactionPublication + ): Promise { + // Memory ownership starts before the queued write: B may prepare while A's + // exact-file cleanup still holds the lock that B's write is waiting for. + const preparation: PendingStatePreparation = {}; + this.pendingStatePreparation = preparation; + await this.loadPersistedPendingStateIfNeeded(publication); + if (this.pendingStatePreparation !== preparation) return preparation; this.pendingStateBoundaryMessageId = boundaryMessageId; const latestCompactionEpochMessages = sliceMessagesFromLatestCompactionBoundary(messages); @@ -701,13 +850,15 @@ export class CompactionHandler { // Persist pending state before append so pre-boundary diffs survive crashes/restarts. // Best-effort: boundary creation must not fail just because persistence fails. - await this.persistPendingStateBestEffort( + preparation.write = await this.persistPendingStateBestEffort( this.cachedFileDiffs, this.cachedLoadedSkills, this.cachedReadFilePaths, boundaryMessageId, - previousState + previousState, + publication ); + return preparation; } async withContinuousPendingState( @@ -793,11 +944,20 @@ export class CompactionHandler { "appendHeartbeatContextResetBoundary requires non-empty boundary text" ); - const deletePartialResult = await this.historyService.deletePartial(this.workspaceId); + const publication = { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; + const deletePartialResult = await this.historyService.deletePartial( + this.workspaceId, + publication + ); if (!deletePartialResult.success) { log.warn( `Failed to delete partial before heartbeat reset boundary: ${deletePartialResult.error}` ); + return Err(deletePartialResult.error); } const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); @@ -806,9 +966,15 @@ export class CompactionHandler { } const messages = historyResult.data; - await this.loadPersistedPendingStateIfNeeded(); - this.captureHeartbeatResetRollbackState(); - await this.preparePendingStateFromMessages(messages); + await this.loadPersistedPendingStateIfNeeded(publication); + const rollbackState = this.captureHeartbeatResetRollbackState(messages); + const summaryMessageId = createCompactionSummaryMessageId(); + const pendingStateReceipt = await this.preparePendingStateFromMessages( + messages, + summaryMessageId, + undefined, + publication + ); const nextCompactionEpoch = getNextCompactionEpoch(messages); assert( @@ -816,23 +982,18 @@ export class CompactionHandler { "heartbeat reset boundary must compute a positive compaction epoch" ); - const summaryMessage = createMuxMessage( - createCompactionSummaryMessageId(), - "assistant", - params.boundaryText, - { - timestamp: Date.now(), - synthetic: true, - uiVisible: true, - compacted: "heartbeat", - compactionEpoch: nextCompactionEpoch, - compactionBoundary: true, - muxMetadata: { - type: "compaction-summary", - pendingFollowUp: params.pendingFollowUp, - }, - } - ); + const summaryMessage = createMuxMessage(summaryMessageId, "assistant", params.boundaryText, { + timestamp: Date.now(), + synthetic: true, + uiVisible: true, + compacted: "heartbeat", + compactionEpoch: nextCompactionEpoch, + compactionBoundary: true, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: params.pendingFollowUp, + }, + }); assert( summaryMessage.metadata?.compacted === "heartbeat", @@ -848,12 +1009,22 @@ export class CompactionHandler { ); const maxExistingHistorySequence = this.getMaxExistingHistorySequence(messages); + let skipped = false; const persistenceResult = await this.historyService.appendToHistory( this.workspaceId, - summaryMessage + summaryMessage, + { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + skipped = true; + }, + } ); - if (!persistenceResult.success) { - await this.restoreHeartbeatResetRollbackState(); + if (skipped || !persistenceResult.success) { + await this.cleanupProvisionalPendingState(pendingStateReceipt, publication, rollbackState); + if (skipped || persistenceResult.success) return Err("Heartbeat publication was invalidated"); return Err(`Failed to append heartbeat reset boundary: ${persistenceResult.error}`); } @@ -875,7 +1046,9 @@ export class CompactionHandler { } async rollbackHeartbeatContextResetBoundary( - summaryMessage: MuxMessage + summaryMessage: MuxMessage, + isCurrent: () => boolean = () => true, + onCommitted?: () => void ): Promise> { assert( summaryMessage.role === "assistant", @@ -886,22 +1059,51 @@ export class CompactionHandler { "rollbackHeartbeatContextResetBoundary requires a heartbeat reset boundary" ); + let restoration: Promise | undefined; + const restore = (deleted: boolean) => { + // Restoration is local even when another backend committed the shared delete. + // Detach/enqueue state under the lock before any publication can admit B. + restoration = this.restoreHeartbeatResetRollbackState(); + try { + const historySequence = summaryMessage.metadata?.historySequence; + if (deleted && isNonNegativeInteger(historySequence)) { + this.emitChatEvent({ type: "delete", historySequences: [historySequence] }); + } + } finally { + onCommitted?.(); + } + }; + const sourceRows = this.heartbeatResetRollbackState?.sourceRows ?? []; const deleteResult = await this.historyService.deleteMessage( this.workspaceId, - summaryMessage.id + summaryMessage.id, + (messages) => + isCurrent() && + messages.every( + (message) => + message.id !== summaryMessage.id || + message.metadata?.historySequence === summaryMessage.metadata?.historySequence + ), + () => restore(true), + (remaining) => { + // Absence alone also describes a foreign clear/replacement. Restore only + // with positive evidence that this heartbeat's predecessor still survives. + const sequences = new Map(remaining.map((row) => [row.id, row.metadata?.historySequence])); + if ( + sourceRows.length > 0 && + sourceRows.every((row) => row.sequence != null && sequences.get(row.id) === row.sequence) + ) + restore(false); + } ); + // Physical persistence outlives ownership. A later admission may not veto a + // committed rollback; the existing write queue orders it before successor state. + await restoration; if (!deleteResult.success) { - return Err(`Failed to delete heartbeat reset boundary: ${deleteResult.error}`); - } - - await this.restoreHeartbeatResetRollbackState(); - - const historySequence = summaryMessage.metadata?.historySequence; - if (isNonNegativeInteger(historySequence)) { - this.emitChatEvent({ - type: "delete", - historySequences: [historySequence], - }); + const failure = restoration + ? "Heartbeat reset boundary was deleted, but cleanup failed" + : "Failed to delete heartbeat reset boundary"; + return Err(`${failure}: ${deleteResult.error}`); } return Ok(undefined); @@ -929,6 +1131,14 @@ export class CompactionHandler { event: StreamEndEvent, compactionRequestMessageId?: string ): Promise { + // A streamed summary retains its original stream epoch across foreign Stop. + // Standalone callers capture before reading the context used for this completion. + const publication = this.getCompactionPublication?.() ?? { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; + const canComplete = this.captureCompletionGuard?.(); // The current stream identifies its request when available. Synthetic prompt snapshots can // follow that request in history, so the last user row is not always the compaction request. const historyResult = compactionRequestMessageId @@ -1025,7 +1235,8 @@ export class CompactionHandler { event.messageId, compactionRequestMessage.id, isIdleCompaction, - pendingFollowUp + pendingFollowUp, + publication ); if (!result.success) { log.error("Compaction failed:", result.error); @@ -1052,7 +1263,7 @@ export class CompactionHandler { }); // Notify that compaction completed (clears idle compaction pending state) - this.onCompactionComplete?.(result.data); + if (canComplete?.() !== false) this.onCompactionComplete?.(result.data); // Report the idle-compaction success only after the summary is actually persisted, // so the idle loop's failure streak is reset on real success (not just stream end). @@ -1237,8 +1448,10 @@ export class CompactionHandler { params: Parameters[0] & { prepared?: { boundary: MuxMessage; copies: MuxMessage[] }; shouldPersist: (messages: MuxMessage[]) => boolean; + publication?: ContinuousCompactionPublication; } ): Promise { + const canComplete = this.captureCompletionGuard?.(); const { boundary, copies } = params.prepared ?? this.buildContinuousCompactionRows(params); const inputTokens = params.systemMessageTokens + @@ -1258,7 +1471,8 @@ export class CompactionHandler { boundary, copies, false, - params.shouldPersist + params.shouldPersist, + params.publication ); if (!result.success) { log.warn("[continuous-compaction] persist failed", result.error); @@ -1271,15 +1485,16 @@ export class CompactionHandler { const epoch = boundary.metadata.compactionEpoch; assert(isNonNegativeInteger(sequence), "Continuous boundary requires a persisted sequence"); assert(isPositiveInteger(epoch), "Continuous boundary requires an epoch"); - this.onCompactionComplete?.({ - workspaceId: this.workspaceId, - summaryMessageId: boundary.id, - summaryHistorySequence: sequence, - compactionEpoch: epoch, - previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), - compactionRequestMessageId: boundary.id, - preservedTailMessageCount: copies.length, - }); + if (canComplete?.() !== false) + this.onCompactionComplete?.({ + workspaceId: this.workspaceId, + summaryMessageId: boundary.id, + summaryHistorySequence: sequence, + compactionEpoch: epoch, + previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), + compactionRequestMessageId: boundary.id, + preservedTailMessageCount: copies.length, + }); return true; } @@ -1309,7 +1524,8 @@ export class CompactionHandler { streamedSummaryMessageId: string, compactionRequestMessageId: string, isIdleCompaction = false, - pendingFollowUp?: CompactionFollowUpRequest + pendingFollowUp?: CompactionFollowUpRequest, + publication?: ContinuousCompactionPublication ): Promise> { assert(summary.trim().length > 0, "performCompaction requires a non-empty summary"); assert(metadata.model.trim().length > 0, "Compaction summary requires a model"); @@ -1324,18 +1540,19 @@ export class CompactionHandler { // 2. sendQueuedMessages triggers commitPartial // 3. commitPartial finds stale partial.json and appends it to history // By deleting partial first, commitPartial becomes a no-op - const deletePartialResult = await this.historyService.deletePartial(this.workspaceId); + const deletePartialResult = await this.historyService.deletePartial( + this.workspaceId, + publication + ); if (!deletePartialResult.success) { log.warn(`Failed to delete partial before compaction: ${deletePartialResult.error}`); - // Continue anyway - the partial may not exist, which is fine + if (publication) return Err(deletePartialResult.error); } // Extract diffs from the latest compaction epoch only, so append-only history // does not re-inject stale pre-boundary edits after subsequent compactions. // If boundary markers are malformed, slicing self-heals by falling back to // full history instead of crashing or dropping all diffs. - await this.preparePendingStateFromMessages(messages); - const nextCompactionEpoch = getNextCompactionEpoch(messages); assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); @@ -1411,6 +1628,12 @@ export class CompactionHandler { muxMetadata: summaryMuxMetadata, } ); + const pendingStateReceipt = await this.preparePendingStateFromMessages( + messages, + summaryMessage.id, + undefined, + publication + ); if (persistedSummaryHistorySequence !== undefined) { summaryMessage.metadata = { ...(summaryMessage.metadata ?? {}), @@ -1449,21 +1672,39 @@ export class CompactionHandler { summaryMessage.id ); + let skipped = false; const persistenceResult = preservedTailCopies.length > 0 ? await this.historyService.persistBoundaryWithTailCopies( this.workspaceId, summaryMessage, preservedTailCopies, - persistedStreamSummary !== null + persistedStreamSummary !== null, + undefined, + publication ) : persistedStreamSummary - ? await this.historyService.updateHistory(this.workspaceId, summaryMessage) - : await this.historyService.appendToHistory(this.workspaceId, summaryMessage); - if (!persistenceResult.success) { - this.cachedFileDiffs = []; - this.cachedLoadedSkills = []; - await this.deletePersistedPendingStateBestEffort(); + ? await this.historyService.updateHistory( + this.workspaceId, + summaryMessage, + undefined, + undefined, + undefined, + undefined, + publication + ) + : await this.historyService.appendToHistory(this.workspaceId, summaryMessage, { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + skipped = true; + }, + }); + if (skipped || !persistenceResult.success) { + await this.cleanupProvisionalPendingState(pendingStateReceipt, publication); + if (skipped || persistenceResult.success) + return Err("Compaction publication was invalidated"); const operation = preservedTailCopies.length > 0 ? "commit boundary with preserved tail" diff --git a/src/node/services/continuousCompactionJournal.ts b/src/node/services/continuousCompactionJournal.ts index 237af1f1a3f..90b4abb2f10 100644 --- a/src/node/services/continuousCompactionJournal.ts +++ b/src/node/services/continuousCompactionJournal.ts @@ -2,6 +2,9 @@ import { prepareProviderRequestMessages } from "./turnContextAssembler"; import { addInterruptedSentinel } from "@/browser/utils/messages/modelMessageTransform"; import { applyCacheControl } from "@/common/utils/ai/cacheStrategy"; import { promises as fs } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import * as path from "node:path"; +import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; import { isDeepStrictEqual } from "node:util"; import { modelMessageSchema, type ModelMessage } from "ai"; import writeFileAtomic from "write-file-atomic"; @@ -79,24 +82,96 @@ export async function rebuildContinuousPrefix( ]; } -/** Serialized journal I/O plus a synchronous invalidation fence shared by reset and prepareStep. */ +/** Presence distinguishes a captured legacy generation from an unguarded history mutation. */ +export interface ContinuousCompactionPublication { + generation: string | undefined; +} + +interface JournalReadOwnership { + isCurrent: () => boolean; + shouldDiscard: () => boolean; + onRead: (journal: ContinuousCompactionJournal) => void; +} + +/** Journal ownership and publication share the history lock across backend processes. */ export class ContinuousCompactionJournalStore { - private generation = 0; private pending: Promise = Promise.resolve(); constructor( readonly path: string, - private readonly workspaceId: string + private readonly workspaceId: string, + private readonly withHistoryLock: (operation: () => Promise) => Promise, + private readonly canPublishUnderHistoryLock: () => Promise ) {} private enqueue(operation: () => Promise): Promise { - const result = this.pending.then(operation); + const result = this.pending.then(() => this.withHistoryLock(operation)); this.pending = result.catch(() => undefined); return result; } - clear(): Promise { - this.generation++; - return this.enqueue(() => fs.rm(this.path, { force: true })); + async captureGenerationUnderHistoryLock(): Promise { + try { + // The bytes are an opaque version, not semantic configuration. Damaged + // bytes fence older work while fresh capture can still make progress. + const bytes = await fs.readFile( + path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE) + ); + return createHash("sha256").update(bytes).digest("hex"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + captureGeneration(): Promise { + return this.enqueue(() => this.captureGenerationUnderHistoryLock()); + } + + /** Caller already owns the history lock; never join the queue of writers waiting for it. */ + async isPublicationCurrentUnderHistoryLock( + publication: ContinuousCompactionPublication + ): Promise { + return ( + publication.generation === (await this.captureGenerationUnderHistoryLock()) && + (await this.canPublishUnderHistoryLock()) + ); + } + + /** Authoritative repair, unlike an old compactor's identity-scoped cleanup. */ + async invalidateUnderHistoryLock(onAdvanced?: (generation: string) => void): Promise { + const generation = randomUUID(); + await writeFileAtomic( + path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE), + generation, + { mode: 0o600 } + ); + onAdvanced?.(createHash("sha256").update(generation).digest("hex")); + await fs.rm(this.path, { force: true }); + } + + private async clearOwnedUnderHistoryLock(expected: ContinuousCompactionJournal): Promise { + let current: ContinuousCompactionJournal; + try { + current = ContinuousCompactionJournalSchema.parse( + JSON.parse(await fs.readFile(this.path, "utf8")) + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + current.boundary.id === expected.boundary.id && + current.publicationGeneration === expected.publicationGeneration + ) { + await fs.rm(this.path, { force: true }); + } + } + + clear(expected: ContinuousCompactionJournal | undefined): Promise { + // No captured owner means no authority to adopt/delete a foreign journal. + // Still join this store's existing I/O without acquiring a lock or recreating a directory. + if (!expected) return this.pending.then(() => undefined); + return this.enqueue(() => this.clearOwnedUnderHistoryLock(expected)); } exists(): Promise { @@ -112,23 +187,48 @@ export class ContinuousCompactionJournalStore { }); } - read(): Promise { + read(ownership?: JournalReadOwnership): Promise { return this.enqueue(async () => { + // A queued stale read never owned the journal now on disk (possibly B). + if (ownership && !ownership.isCurrent()) return null; + let contents: string; try { - return ContinuousCompactionJournalSchema.parse( - JSON.parse(await fs.readFile(this.path, "utf8")) - ); + contents = await fs.readFile(this.path, "utf8"); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - log.warn("[continuous-compaction] discarded invalid journal", error); - await fs - .rm(this.path, { force: true }) - .catch((cleanupError: unknown) => - log.warn("[continuous-compaction] invalid journal cleanup failed", cleanupError) - ); - } + if ((error as NodeJS.ErrnoException).code !== "ENOENT") + log.warn("[continuous-compaction] journal unavailable", error); + return null; + } + // Settings changes and shutdown preserve recovery even if the read began. + if (ownership && !ownership.isCurrent() && !ownership.shouldDiscard()) return null; + let journal: ContinuousCompactionJournal; + try { + journal = ContinuousCompactionJournalSchema.parse(JSON.parse(contents)); + } catch (error) { + // The shared lock keeps a malformed A cleanup from unlinking a new B. + log.warn("[continuous-compaction] discarded invalid journal", error); + await fs.rm(this.path, { force: true }); + return null; + } + if (ownership && !ownership.isCurrent()) { + if (ownership.shouldDiscard()) await this.clearOwnedUnderHistoryLock(journal); return null; } + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) { + await this.clearOwnedUnderHistoryLock(journal); + return null; + } + if (ownership && !ownership.isCurrent()) { + if (ownership.shouldDiscard()) await this.clearOwnedUnderHistoryLock(journal); + return null; + } + // Transfer exact ownership before releasing the lock, through later fold awaits. + ownership?.onRead(journal); + return journal; }); } @@ -140,16 +240,21 @@ export class ContinuousCompactionJournalStore { providerOptions?: Record; system?: string | ModelMessage; }, - isCurrent: () => boolean + isCurrent: () => boolean, + onCommitted?: (journal: ContinuousCompactionJournal) => void ): Promise { - const generation = this.generation; return this.enqueue(async () => { try { + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) + return null; const current = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) ); - if (generation !== this.generation || !isCurrent() || !isDeepStrictEqual(current, journal)) - return null; + if (!isCurrent() || !isDeepStrictEqual(current, journal)) return null; const prefix = request.prefix.map(exactJson); const parsedPrefix = prefix.map((message) => modelMessageSchema.parse(message)); assert( @@ -165,7 +270,7 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(updated), payload), "Fallback journal dropped request fields" ); - if (generation !== this.generation || !isCurrent()) return null; + if (!isCurrent()) return null; await writeFileAtomic(this.path, JSON.stringify(payload), { mode: 0o600 }); const reread = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) @@ -174,7 +279,9 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(reread), payload), "Fallback journal round-trip mismatch" ); - return generation === this.generation && isCurrent() ? reread : null; + if (!isCurrent()) return null; + onCommitted?.(reread); + return reread; } catch (error) { // Unlike the initial write, this record already describes a consumed request. // Failure must keep it available for P1's durable fold or startup recovery. @@ -190,11 +297,17 @@ export class ContinuousCompactionJournalStore { write( journal: ContinuousCompactionJournal, prefix: ModelMessage[], - isCurrent: () => boolean + isCurrent: () => boolean, + onCommitted?: (journal: ContinuousCompactionJournal) => void ): Promise { - const generation = this.generation; return this.enqueue(async () => { try { + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) + return null; let wire: ContinuousCompactionJournal["prefix"]; try { wire = z.array(z.json()).parse(exactJson(prefix)); @@ -218,20 +331,21 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(parsed), payload), "Journal schema dropped request fields" ); - if (generation !== this.generation || !isCurrent()) return null; + if (!isCurrent()) return null; await writeFileAtomic(this.path, JSON.stringify(payload), { mode: 0o600 }); const reread = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) ); assert(isDeepStrictEqual(exactJson(reread), payload), "Journal round-trip mismatch"); - if (generation !== this.generation || !isCurrent()) { - await fs.rm(this.path, { force: true }); + if (!isCurrent()) { + await this.clearOwnedUnderHistoryLock(journal); return null; } + onCommitted?.(reread); return reread; } catch (error) { log.warn("[continuous-compaction] prefix not swapped: journal failed", error); - await fs.rm(this.path, { force: true }).catch(() => undefined); + await this.clearOwnedUnderHistoryLock(journal).catch(() => undefined); return null; } }); diff --git a/src/node/services/continuousCompactor.test.ts b/src/node/services/continuousCompactor.test.ts index 999aa4088f0..7885552607a 100644 --- a/src/node/services/continuousCompactor.test.ts +++ b/src/node/services/continuousCompactor.test.ts @@ -1,3 +1,6 @@ +import { CompactionCancellation } from "./compactionCancellation"; +import { HistoryService } from "./historyService"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import type { ContinuousPrefixSwap } from "./continuousCompactionJournal"; import { writeFile } from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; @@ -421,9 +424,10 @@ describe("ContinuousCompactor", () => { const release = deferred(); const original = handler.preparePendingStateFromMessages.bind(handler); spyOn(handler, "preparePendingStateFromMessages").mockImplementation(async (...args) => { - await original(...args); + const preparation = await original(...args); entered.resolve(); await release.promise; + return preparation; }); const applying = compactor.observe(context.thresholdPercent, context); await entered.promise; @@ -684,6 +688,98 @@ describe("ContinuousCompactor", () => { return { answer, journal, journalStore, dependencies, swap }; } + async function repairForeignCancellation() { + await writeFile( + `${store.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(store.config); + expect(await new CompactionCancellation(foreign, workspaceId).read()).toBeNull(); + return foreign; + } + + it("foreign repair fences a durable fold that already read its journal", async () => { + const { answer, journal } = await activateJournaledSwap(); + assert(live, "Expected live source"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + streaming = false; + live = undefined; + const entered = deferred(); + const release = deferred(); + const persist = store.historyService.persistBoundaryWithTailCopies.bind(store.historyService); + spyOn(store.historyService, "persistBoundaryWithTailCopies").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return persist(...args); + } + ); + const pending = compactor.observe(context.thresholdPercent, context); + try { + await entered.promise; + await repairForeignCancellation(); + release.resolve(); + expect(await pending).not.toBe("applied"); + expect((await rows()).some((row) => row.id === journal.boundary.id)).toBe(false); + expect(completed).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await pending; + } + }); + + it("foreign repair fences non-journal work already awaiting preparation", async () => { + await seedConversation(); + const entered = deferred(); + const release = deferred(); + prepare.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + }); + const { job } = await start(); + try { + await entered.promise; + await repairForeignCancellation(); + release.resolve(); + await job; + expect(await compactor.observe(context.thresholdPercent, context)).not.toBe("applied"); + expect((await rows())[0].id).toBe("old-user"); + expect(completed).not.toHaveBeenCalled(); + // A newly captured generation may compact the still-valid source normally. + const freshJob = eagerJob(compactor); + jobs.push(freshJob); + await freshJob; + expect(await compactor.observe(context.thresholdPercent, context)).toBe("applied"); + expect(completed).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await job; + } + }); + + it("a failed repair generation write keeps staged durable publication fenced", async () => { + await seedConversation(); + await stage(); + await writeFile( + `${store.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const original = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementationOnce( + Object.assign(() => Promise.reject(new Error("epoch storage unavailable")), { + sync: original.sync, + }) + ); + const failed = await new CompactionCancellation(new HistoryService(store.config), workspaceId) + .read() + .catch((error: unknown) => error); + expect(failed).toBeInstanceOf(Error); + expect(await compactor.observe(context.thresholdPercent, context)).not.toBe("applied"); + expect((await rows())[0].id).toBe("old-user"); + expect(completed).not.toHaveBeenCalled(); + }); + for (const reason of ["disabled", "threshold-changed", "context-changed"]) { it(`preserves consumed journal through ${reason} and finalizes once even after tracker retirement`, async () => { const { answer, journal, journalStore } = await activateJournaledSwap(); @@ -841,6 +937,97 @@ describe("ContinuousCompactor", () => { expect(fastApply).not.toHaveBeenCalled(); }); + it.each( + ["read lock", "read return", "partial commit"].flatMap((stage) => + ["user-interrupt", "shutdown", "disabled"].map((reason) => [stage, reason] as const) + ) + )("recovery ownership during %s honors %s", async (stage, reason) => { + const { answer, journalStore, dependencies } = await activateJournaledSwap(); + assert(live, "Expected live source"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + compactor.reset("shutdown"); + streaming = false; + live = undefined; + compactor = new ContinuousCompactor(dependencies); + const entered = deferred(); + const release = deferred(); + if (stage === "read lock") { + const validate = journalStore.isPublicationCurrentUnderHistoryLock.bind(journalStore); + spyOn(journalStore, "isPublicationCurrentUnderHistoryLock").mockImplementationOnce( + async (...args) => { + const result = await validate(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const read = journalStore.read.bind(journalStore); + spyOn(journalStore, "read").mockImplementationOnce(async (...args) => { + const result = await read(...args); + if (stage === "read return") { + entered.resolve(); + await release.promise; + } else { + const commit = store.historyService.commitPartial.bind(store.historyService); + spyOn(store.historyService, "commitPartial").mockImplementationOnce( + async (...commitArgs) => { + const committed = await commit(...commitArgs); + entered.resolve(); + await release.promise; + return committed; + } + ); + } + return result; + }); + } + const pending = compactor.recover(); + try { + await entered.promise; + compactor.reset(reason); + release.resolve(); + expect(await pending).toBe(false); + expect(await journalStore.exists()).toBe(reason !== "user-interrupt"); + if (reason !== "user-interrupt") { + compactor = new ContinuousCompactor(dependencies); + expect(await compactor.recover()).toBe(true); + } + } finally { + release.resolve(); + await pending; + } + }); + + it("a reset before a queued recovery read starts cannot adopt a foreign journal", async () => { + const { journalStore, dependencies, swap } = await activateJournaledSwap(); + compactor.reset("shutdown"); + streaming = false; + live = undefined; + compactor = new ContinuousCompactor(dependencies); + const release = deferred(); + // Hold only the local queue; B must remain able to publish under the real shared lock. + (journalStore as unknown as { pending: Promise }).pending = release.promise; + const pending = ( + compactor as unknown as { finalizeJournal(): Promise } + ).finalizeJournal(); + try { + compactor.reset("user-interrupt"); + const foreign = new HistoryService(store.config).getContinuousCompactionJournal(workspaceId); + const b = structuredClone(swap.journal); + b.boundary.id = "foreign-after-reset"; + const published = await foreign.write(b, swap.prefix, () => true); + expect(published).not.toBeNull(); + release.resolve(); + expect(await pending).toBe(false); + expect(await foreign.read()).toEqual(published); + } finally { + release.resolve(); + await pending; + } + }); + it("recovers growing crash partials verbatim and retries recovery idempotently after boundary persistence", async () => { const { answer, journal, journalStore, dependencies } = await activateJournaledSwap(); assert(live, "Live fixture missing"); @@ -933,6 +1120,31 @@ describe("ContinuousCompactor", () => { expect(history.filter((row) => row.metadata?.compactionBoundary)).toHaveLength(1); }); + it("late journal finalization cannot clear a replacement journal after reset", async () => { + const { answer, journal, journalStore } = await activateJournaledSwap(); + assert(live, "Live fixture missing"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + streaming = false; + live = undefined; + const entered = deferred(); + const release = deferred(); + const clear = journalStore.clear.bind(journalStore); + spyOn(journalStore, "clear").mockImplementationOnce(async (expected) => { + await clear(expected); + entered.resolve(); + await release.promise; + }); + const recovering = compactor.recover(); + await entered.promise; + compactor.reset("edit"); + const replacement = { ...journal, boundary: { ...journal.boundary, id: "replacement" } }; + expect(await journalStore.write(replacement, [], () => true)).not.toBeNull(); + release.resolve(); + await recovering; + expect((await journalStore.read())?.boundary.id).toBe("replacement"); + }); + it("reset during journal apply cannot append a boundary", async () => { const { answer, dependencies } = await activateJournaledSwap(); assert(live, "Live fixture missing"); diff --git a/src/node/services/continuousCompactor.ts b/src/node/services/continuousCompactor.ts index 14f7a17b284..df6e2c655d0 100644 --- a/src/node/services/continuousCompactor.ts +++ b/src/node/services/continuousCompactor.ts @@ -46,6 +46,8 @@ interface StreamSnapshot { } interface Dependencies { workspaceId: string; + /** Physical session ownership; speculative work must not reserve turn admission. */ + enterExecution?(): Disposable; historyService: HistoryService; compactionHandler: CompactionHandler; streamManager: { @@ -74,7 +76,13 @@ interface Dependencies { apply: (pendingFollowUp?: CompactionFollowUpRequest) => Promise ): Promise; } +interface JournalReadAttempt { + discard: boolean; + journal?: ContinuousCompactionJournal; +} + interface StagedSummary { + publicationGeneration: string | undefined; generation: number; epoch: number; boundarySequence?: number; @@ -133,6 +141,7 @@ export class ContinuousCompactor { private applying: Promise | null = null; private swapAttempted: StagedSummary | null = null; private activeSwap: ContinuousPrefixSwap | null = null; + private journalRead?: JournalReadAttempt; constructor(private readonly deps: Dependencies) { assert(deps.workspaceId.length > 0, "ContinuousCompactor requires a workspace"); @@ -147,23 +156,37 @@ export class ContinuousCompactor { // Hydrating settings must not erase a previous process's journal before recovery. // It also keeps the disabled hot path free of journal I/O when no swap ever activated. const discardJournal = !settingsOnly || this.activeSwap !== null || this.swapAttempted !== null; + const ownedJournal = this.activeSwap?.journal ?? this.journalRead?.journal; this.generation++; - this.job?.abort.abort(); + const job = this.job; this.job = null; this.staged = null; this.swapAttempted = null; this.activeSwap = null; - this.deps.streamManager.clearPrefixSwap?.(this.deps.workspaceId); // Graceful shutdown retains the write-ahead record for ordinary startup recovery. if (discardJournal && reason !== "shutdown") { - this.deps.historyService - .getContinuousCompactionJournal(this.deps.workspaceId) - .clear() - .catch((error: unknown) => log.warn("[continuous-compaction] journal clear failed", error)); + if (this.journalRead) this.journalRead.discard = true; + this.clearJournal(ownedJournal).catch((error: unknown) => + log.warn("[continuous-compaction] journal clear failed", error) + ); + } + // Detach semantic ownership and enqueue the old clear before abort observers can + // synchronously replace work. The original job retains its physical lease until done. + try { + this.deps.streamManager.clearPrefixSwap?.(this.deps.workspaceId); + } finally { + job?.abort.abort(); } log.debug("[continuous-compaction] reset", { workspaceId: this.deps.workspaceId, reason }); } + private async clearJournal(journal: ContinuousCompactionJournal | undefined): Promise { + using _execution = this.deps.enterExecution?.(); + await this.deps.historyService + .getContinuousCompactionJournal(this.deps.workspaceId) + .clear(journal); + } + hasConsumedSwap(): boolean { return this.activeSwap?.consumed === true; } @@ -220,6 +243,7 @@ export class ContinuousCompactor { } if (context.phase !== "mid-stream") { if (await this.finalizeJournal()) return "applied"; + if (generation !== this.generation) return "none"; // A transient history failure must remain recoverable even when future // compaction is disabled; only a discarded/invalid journal releases this latch. if (this.hasConsumedSwap()) return "none"; @@ -263,6 +287,7 @@ export class ContinuousCompactor { done: Promise.resolve(), }; this.job = job; + const execution = this.deps.enterExecution?.(); job.done = this.startEagerJob(job, context) .catch((error: unknown) => { if (!job.abort.signal.aborted) @@ -270,6 +295,7 @@ export class ContinuousCompactor { }) .finally(() => { if (this.job === job) this.job = null; + execution?.[Symbol.dispose](); }); } return usagePercent >= context.thresholdPercent + FORCE_COMPACTION_BUFFER_PERCENT @@ -303,6 +329,12 @@ export class ContinuousCompactor { job: NonNullable, context: ContinuousCompactionContext ): Promise { + // Capture before any source preparation: a later repair must fence this job + // even when the transcript fingerprint itself remains unchanged. + const publicationGeneration = await this.deps.historyService + .getContinuousCompactionJournal(this.deps.workspaceId) + .captureGeneration(); + if (job.generation !== this.generation) return; await this.deps.prepare(); if (job.generation !== this.generation) return; const rows = await this.readSnapshot(); @@ -336,6 +368,7 @@ export class ContinuousCompactor { "Rolling head must have a durable sequence" ); const stagedBase = { + publicationGeneration, generation: job.generation, ...boundaryIdentity(rows), cut, @@ -492,6 +525,7 @@ export class ContinuousCompactor { try { const journal: ContinuousCompactionJournal = { version: 1, + publicationGeneration: staged.publicationGeneration, boundary, staticCopies, liveTailCopySpec: { @@ -566,7 +600,20 @@ export class ContinuousCompactor { private async finalizeJournal(pendingFollowUp?: CompactionFollowUpRequest): Promise { const generation = this.generation; const store = this.deps.historyService.getContinuousCompactionJournal(this.deps.workspaceId); - const journal = await store.read(); + const readAttempt: JournalReadAttempt = { discard: false }; + this.journalRead = readAttempt; + using _readOwnership = { + [Symbol.dispose]: () => { + if (this.journalRead === readAttempt) this.journalRead = undefined; + }, + }; + const journal = await store.read({ + isCurrent: () => generation === this.generation, + shouldDiscard: () => readAttempt.discard, + onRead: (journal) => { + readAttempt.journal = journal; + }, + }); if (generation !== this.generation) return false; if (!journal) { this.activeSwap = null; @@ -581,7 +628,8 @@ export class ContinuousCompactor { journal.staticCopies.every((copy) => rows.some((row) => row.id === copy.id)) && rows.some((row) => row.id === journal.liveTailCopySpec.copyId); if (rows.some((row) => row.id === journal.boundary.id) && copiesPresent) { - await store.clear(); + await store.clear(journal); + if (generation !== this.generation) return false; this.activeSwap = null; return true; } @@ -617,7 +665,8 @@ export class ContinuousCompactor { log.warn("[continuous-compaction] discarded mismatched journal", { workspaceId: this.deps.workspaceId, }); - await store.clear(); + await store.clear(journal); + if (generation !== this.generation) return false; this.activeSwap = null; return false; } @@ -660,6 +709,7 @@ export class ContinuousCompactor { journal.postCompactionAttachments ).reduce((sum, row) => sum + estimateMuxMessageTokens(row), 0), prepared: { boundary, copies: [...journal.staticCopies, liveCopy] }, + publication: { generation: journal.publicationGeneration }, shouldPersist: (current) => generation === this.generation && !this.deps.streamManager.isStreaming(this.deps.workspaceId) && @@ -668,9 +718,14 @@ export class ContinuousCompactor { }, boundary.id ); - if (applied) { - await store.clear(); - this.reset("applied"); + if (applied && generation === this.generation) { + await store.clear(journal); + // This owner has completed cleanup; reset must not queue another unlink + // after recovery returns (or race a subsequent recovery of the same journal). + readAttempt.journal = undefined; + // Persistence may finish after an edit/reset. Its old clear is already ordered + // by the store; a new reset here would enqueue another clear behind replacement work. + if (generation === this.generation) this.reset("applied"); } return applied; } @@ -717,6 +772,7 @@ export class ContinuousCompactor { fingerprint(current) === snapshotFingerprint ); }, + publication: { generation: staged.publicationGeneration }, messages: rows, text: staged.text, model: staged.model, @@ -725,7 +781,7 @@ export class ContinuousCompactor { attachmentTokens: context.attachmentTokens ?? 0, pendingFollowUp, }); - if (applied) this.reset("applied"); + if (applied && staged.generation === this.generation) this.reset("applied"); return applied; } ); diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 3b2d4f7f976..16077ce83a6 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -991,6 +991,191 @@ describe("HistoryService", () => { }); }); + describe("context publication fencing", () => { + it.each( + ["active", "archived"].flatMap((target) => + [false, true].map((afterAdvance) => ({ target, afterAdvance })) + ) + )( + "edit truncation refuses unfenced writes ($target, after advance=$afterAdvance)", + async ({ target, afterAdvance }) => { + const workspaceId = "edit-generation-failure"; + await service.appendToHistory( + workspaceId, + createMuxMessage("target", "user", "Original context") + ); + if (target === "archived") { + await service.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "Summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + } + await service.appendToHistory( + workspaceId, + createMuxMessage("tail", "assistant", "Original answer") + ); + const before = await collectFullHistory(service, workspaceId); + const journal = service.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + const failure = spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce( + async (...args) => { + if (afterAdvance) await invalidate(...args); + throw new Error("edit generation unavailable"); + } + ); + try { + const result = await service.truncateAfterMessage(workspaceId, "target"); + expect(!result.success && result.error).toContain("edit generation unavailable"); + expect(await collectFullHistory(new HistoryService(config), workspaceId)).toEqual(before); + expect((await journal.captureGeneration()) === generation).toBe(!afterAdvance); + expect((await service.truncateAfterMessage(workspaceId, "target")).success).toBe(true); + expect(await collectFullHistory(service, workspaceId)).toEqual([]); + } finally { + failure.mockRestore(); + } + } + ); + + it.each(["missing", "keep tail", "display tail"])( + "edit truncation preserves publication for %s", + async (cut) => { + const workspaceId = "edit-no-context-cut"; + await service.appendToHistory( + workspaceId, + createMuxMessage("target", "user", "Current context") + ); + if (cut === "display tail") { + await service.appendToHistory( + workspaceId, + createMuxMessage("display", "user", "Display only", { + muxMetadata: { type: "workflow-trigger-display", rawCommand: "/wf", runId: "run" }, + }) + ); + } + const journal = service.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "current journal"); + const result = await service.truncateAfterMessage( + workspaceId, + cut === "missing" ? "missing" : "target", + { keepTargetMessage: true } + ); + expect(result.success).toBe(cut !== "missing"); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("current journal"); + } + ); + + it.each([false, true])( + "reset batches validate their original publication before advancing it (admitted=%s)", + async (admitted) => { + const workspaceId = "reset-batch-publication"; + await service.appendToHistory(workspaceId, createMuxMessage("old", "user", "Old context")); + const journal = service.getContinuousCompactionJournal(workspaceId); + const publication = { generation: await journal.captureGeneration() }; + await fs.writeFile(journal.path, "existing journal"); + let skipped = false; + const result = await service.appendManyToHistory( + workspaceId, + [ + createMuxMessage("reset", "assistant", "", { + contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + }), + createMuxMessage("replacement", "user", "Fresh request"), + ], + { + publication, + allowedTailMessageIds: [], + isCurrent: () => admitted, + onSkipped: () => { + skipped = true; + }, + } + ); + expect(result.success).toBe(true); + expect(skipped).toBe(!admitted); + expect((await journal.captureGeneration()) === publication.generation).toBe(!admitted); + expect((await collectFullHistory(service, workspaceId)).map((row) => row.id)).toEqual( + admitted ? ["old", "reset", "replacement"] : ["old"] + ); + if (admitted) { + // The admitted batch succeeds with its old receipt; subsequent foreign + // handoffs carrying that same receipt must now be refused. + let foreignSkipped = false; + await new HistoryService(config).appendToHistory( + workspaceId, + createMuxMessage("stale", "user", "Discarded follow-up"), + { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + foreignSkipped = true; + }, + } + ); + expect(foreignSkipped).toBe(true); + } else { + expect(await fs.readFile(journal.path, "utf8")).toBe("existing journal"); + } + } + ); + + it("refuses a reset batch when its publication generation cannot be written", async () => { + const workspaceId = "reset-batch-generation-failure"; + await service.appendToHistory(workspaceId, createMuxMessage("old", "user", "Old context")); + const failure = spyOn( + service.getContinuousCompactionJournal(workspaceId), + "invalidateUnderHistoryLock" + ).mockRejectedValueOnce(new Error("generation unavailable")); + try { + const result = await service.appendManyToHistory(workspaceId, [ + createMuxMessage("reset", "assistant", "", { + contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + }), + createMuxMessage("replacement", "user", "Fresh request"), + ]); + expect(!result.success && result.error).toContain("generation unavailable"); + expect((await collectFullHistory(service, workspaceId)).map((row) => row.id)).toEqual([ + "old", + ]); + } finally { + failure.mockRestore(); + } + }); + + it.each(["full", "rounded full", "refused full", "no rows", "zero"])( + "only actual full deletions retire the publication generation (%s)", + async (operation) => { + const workspaceId = "full-delete-publication"; + await fs.mkdir(path.join(config.sessionsDir, workspaceId), { recursive: true }); + if (operation !== "no rows") { + await service.appendToHistory( + workspaceId, + createMuxMessage("old", "user", "Old context") + ); + } + const journal = service.getContinuousCompactionJournal(workspaceId); + await fs.writeFile(journal.path, "existing journal"); + const generation = await journal.captureGeneration(); + const result = await service.truncateHistory( + workspaceId, + operation === "zero" ? 0 : operation === "full" || operation === "no rows" ? 1 : 0.5, + { refuseFullDelete: operation === "refused full" } + ); + const removed = operation === "full" || operation === "rounded full"; + expect(result.success).toBe(operation !== "refused full"); + expect((await journal.captureGeneration()) === generation).toBe(!removed); + if (!removed) expect(await fs.readFile(journal.path, "utf8")).toBe("existing journal"); + } + ); + }); + describe("clearHistory", () => { it("should delete chat.jsonl file", async () => { const workspaceId = "workspace1"; @@ -2577,7 +2762,12 @@ describe("HistoryService", () => { }) ); + const journal = service.getContinuousCompactionJournal(wsId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "active publication journal"); expect((await service.truncateHistory(wsId, 0.5)).success).toBe(true); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("active publication journal"); const active = await service.getHistoryFromLatestBoundary(wsId); expect(active.success).toBe(true); @@ -2606,7 +2796,12 @@ describe("HistoryService", () => { }) ); + const journal = service.getContinuousCompactionJournal(wsId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "active publication journal"); expect((await service.truncateHistory(wsId, 0.2)).success).toBe(true); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("active publication journal"); const active = await service.getHistoryFromLatestBoundary(wsId); expect(active.success).toBe(true); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index c96d0f30bd1..ae41b9fbdfd 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -18,9 +18,22 @@ import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrel import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; import * as path from "path"; import { createHash, randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import { renameSync } from "node:fs"; import * as fs from "fs/promises"; -import { ContinuousCompactionJournalStore } from "./continuousCompactionJournal"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import { + CompactionCancellationSchema, + MalformedCompactionCancellationError, + matchesCompactionCancellation, + type CompactionCancellationRecord, + type CompactionCancellationPublication, +} from "./compactionCancellation"; +import { + ContinuousCompactionJournalStore, + type ContinuousCompactionPublication, +} from "./continuousCompactionJournal"; +import { CONTINUOUS_COMPACTION_JOURNAL_FILE } from "@/constants/continuousCompaction"; import writeFileAtomic from "write-file-atomic"; import assert from "node:assert"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; @@ -76,6 +89,16 @@ import { */ const HISTORY_WRITE_LOCK_TIMEOUT_MS = 10_000; +interface CompactionFollowUpAppendCondition { + publication?: ContinuousCompactionPublication; + /** Presence selects manual replacement admission; null captures semantic absence. */ + replacementNonce?: string | null; + summary?: MuxMessage; + allowedTailMessageIds: readonly string[]; + isCurrent: () => boolean; + onSkipped: () => void; +} + interface HistoryTruncateHashes { finalArchiveHash: string | null; finalChatHash: string | null; @@ -324,14 +347,216 @@ export class HistoryService { let journal = this.continuousJournals.get(workspaceId); if (!journal) { journal = new ContinuousCompactionJournalStore( - path.join(this.getSessionDir(workspaceId), "continuous-compaction.json"), - workspaceId + path.join(this.getSessionDir(workspaceId), CONTINUOUS_COMPACTION_JOURNAL_FILE), + workspaceId, + (operation) => + this.withHistoryWriteFileLock(workspaceId, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) + throw new Error(`workspace ${workspaceId} was removed; refusing journal mutation`); + await ensurePrivateDir(this.getSessionDir(workspaceId)); + return operation(); + }), + async () => { + // Raw reads only under the held lock. Even a failed epoch publication + // leaves malformed/unresolved Stop authoritative until exact replacement. + const cancellation = await this.readCompactionCancellation(workspaceId); + return ( + !cancellation || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, cancellation.nonce)) + ); + } ); this.continuousJournals.set(workspaceId, journal); } return journal; } + async readCompactionCancellation( + workspaceId: string + ): Promise { + let contents: Buffer; + try { + contents = await fs.readFile( + path.join(this.getSessionDir(workspaceId), COMPACTION_CANCELLATION_FILE) + ); + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) return null; + // Access/I/O errors do not prove corruption and must remain fail-closed. + throw error; + } + try { + return CompactionCancellationSchema.parse(JSON.parse(contents.toString("utf8"))); + } catch { + throw new MalformedCompactionCancellationError(contents); + } + } + + async hasCompactionReplacementWitness(workspaceId: string, nonce: string): Promise { + return this.withRecoveredHistoryLock(workspaceId, () => + this.hasCompactionReplacementWitnessUnlocked(workspaceId, nonce) + ); + } + + private async hasCompactionReplacementWitnessUnlocked( + workspaceId: string, + nonce: string, + active?: MuxMessage[] + ): Promise { + const matches = (rows: MuxMessage[]) => + rows.some((row) => row.metadata?.compactionCancellationNonce === nonce); + // A later boundary may archive the receipt while ancillary unlink is still + // unavailable. Its exact nonce continues to distinguish a newer explicit Stop. + return ( + matches(active ?? (await this.readChatHistory(workspaceId))) || + matches(await this.readArchivedHistory(workspaceId)) + ); + } + + async repairCompactionCancellation( + workspaceId: string, + isCurrent: () => boolean, + onRepaired: () => void + ): Promise { + return this.fileLocks.withLock(workspaceId, () => + this.withCrossProcessWriteLock(workspaceId, async () => { + if (!isCurrent()) return null; + let malformed: MalformedCompactionCancellationError; + try { + // A newer valid Stop may have replaced the corrupt bytes while we waited. + return await this.readCompactionCancellation(workspaceId); + } catch (error) { + if (!(error instanceof MalformedCompactionCancellationError)) throw error; + malformed = error; + } + if (!isCurrent()) return null; + const cancellationPath = path.join( + this.getSessionDir(workspaceId), + COMPACTION_CANCELLATION_FILE + ); + // Keep the canonical corrupt file as a fail-closed fence through every + // repair await/crash. Quarantine is a copy, never a rename-away gap. + await fs.writeFile(`${cancellationPath}.corrupt.${randomUUID()}`, malformed.contents, { + mode: 0o600, + flag: "wx", + }); + if (!isCurrent()) return null; + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + const historyPath = this.getChatHistoryPath(workspaceId); + const { rows } = await this.readHistoryForRewrite(historyPath); + let changed = false; + const sanitized = this.serializeHistoryRewrite(rows, workspaceId, (row) => { + const metadata = row.metadata?.muxMetadata; + if (!isCompactionSummaryMetadata(metadata) || !metadata.pendingFollowUp) return row; + changed = true; + const { pendingFollowUp: _pendingFollowUp, ...muxMetadata } = metadata; + return { ...row, metadata: { ...row.metadata, muxMetadata } }; + }); + if (!isCurrent()) return null; + if (changed) { + // Repair is a rewrite, so existing history cursors must not remain valid. + // Preserve raw privacy-boundary bytes exactly, just like other rewrites. + invalidateHistoryAppendProvenance(); + if (!(await this.writeGuardedHistory(historyPath, sanitized, isCurrent, onRepaired))) + return null; + } + // Both legacy pending markers and the journal must be durably neutralized + // before absence may authorize normal recovery, including on the next launch. + if (isCurrent()) { + if (!changed) onRepaired(); + await fs.rm(cancellationPath, { force: true }); + } + return null; + }) + ); + } + + async writeCompactionCancellation( + workspaceId: string, + record: CompactionCancellationRecord | null, + isCurrent: () => boolean, + retiredNonce?: string, + publication?: CompactionCancellationPublication + ): Promise { + // This file supplements legacy pendingFollowUp removal when transcript I/O + // fails. Older builds only honor successful cleanup of that legacy marker. + // It must remain writable when transcript reads/recovery are failing. + await this.withHistoryWriteFileLock(workspaceId, async () => { + if (!isCurrent()) return; + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) + throw new Error(`workspace ${workspaceId} was removed; refusing cancellation mutation`); + const cancellationPath = path.join( + this.getSessionDir(workspaceId), + COMPACTION_CANCELLATION_FILE + ); + if (record === null) { + const current = await this.readCompactionCancellation(workspaceId); + if (!isCurrent() || current?.nonce !== retiredNonce) return; + if ( + current?.retainUntilReplacement && + !(await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce)) + ) + return; + if (isCurrent() && current?.nonce === retiredNonce) + await fs.rm(cancellationPath, { force: true }); + return; + } + const current = await this.readCompactionCancellation(workspaceId).catch(() => undefined); + if (record.scope.kind === "summary") { + if (current?.nonce !== record.nonce || current.retainUntilReplacement) return; + } else if (current?.retainUntilReplacement || current === undefined) { + // Carry clear's conservative fence through a foreign Stop. An unreadable + // old sidecar cannot safely prove that this obligation was absent. + record = { ...record, retainUntilReplacement: true }; + } + await ensurePrivateDir(this.getSessionDir(workspaceId)); + if (record.scope.kind === "unresolved") { + // A lost acknowledgement must not erase fresh work after this exact Stop + // already published or was replaced. Transcript failure still permits Stop. + if ( + current?.nonce === record.nonce || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, record.nonce).catch( + () => false + )) + ) + return; + const journal = this.getContinuousCompactionJournal(workspaceId); + const predecessor = { + nonce: current === undefined ? undefined : (current?.nonce ?? null), + generation: await journal.captureGenerationUnderHistoryLock(), + }; + // An unobserved failed attempt cannot adopt any established frontier, + // including a generation whose Stop was already witnessed and unlinked. + if ( + publication && + publication.attempts > 1 && + !publication.predecessor && + (current || predecessor.generation !== undefined) + ) + return; + // Failed A may retry only its own publication frontier. A foreign Stop + // or repair supersedes it; refresh shared state instead of overwriting B. + if (publication?.predecessor && !isDeepStrictEqual(publication.predecessor, predecessor)) + return; + if (publication) publication.predecessor = predecessor; + // Publish a never-reused epoch BEFORE the sidecar. Record advancement + // before journal cleanup so partial failure still has an exact retry CAS. + await journal.invalidateUnderHistoryLock((generation) => { + if (publication) publication.predecessor = { ...predecessor, generation }; + }); + } + const stagedPath = `${cancellationPath}.${randomUUID()}`; + try { + await writeFileAtomic(stagedPath, JSON.stringify(record), { mode: 0o600 }); + if (isCurrent()) { + renameSync(stagedPath, cancellationPath); + if (publication?.predecessor) publication.predecessor.nonce = record.nonce; + } + } finally { + await fs.rm(stagedPath, { force: true }); + } + }); + } + private getSessionDir(workspaceId: string): string { return "getSessionDir" in this.config ? this.config.getSessionDir(workspaceId) @@ -2167,9 +2392,59 @@ export class HistoryService { } /** - * Delete the partial message file for a workspace. + * Pending-state writes must retain their original publication epoch across Stop. + * The callback performs filesystem I/O only; it must not reacquire history or journal queues. */ - async deletePartial(workspaceId: string): Promise> { + async withCompactionPublicationWrite( + workspaceId: string, + publication: ContinuousCompactionPublication, + write: () => Promise + ): Promise { + return this.withCompactionPublicationLock(workspaceId, publication, async (current) => { + if (current) await write(); + }); + } + + /** Exact pending-write cleanup may run after invalidation, but may restore only current state. */ + async withCompactionPublicationLock( + workspaceId: string, + publication: ContinuousCompactionPublication | undefined, + operation: (current: boolean) => Promise + ): Promise { + await this.fileLocks.withLock(workspaceId, () => + this.withHistoryWriteFileLock(workspaceId, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return; + await operation( + !publication || + (await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ); + }) + ); + } + + /** Delete the partial message file for a workspace. */ + async deletePartial( + workspaceId: string, + publication?: ContinuousCompactionPublication + ): Promise> { + if (publication) + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to delete partial", + async () => { + // Compaction cleanup must share the publication lock: an old producer + // cannot unlink the replacement stream's partial while awaiting its epoch. + if ( + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); + return this.deletePartialUnlocked(workspaceId); + } + ); return this.fileLocks.withLock(workspaceId, () => this.deletePartialUnlocked(workspaceId)); } @@ -2447,6 +2722,7 @@ export class HistoryService { `[HISTORY APPEND] Assigned historySequence=${message.metadata.historySequence ?? "unknown"} role=${message.role}` ); + await this.fenceContextResetUnderHistoryLock(workspaceId, [message]); await this.getAppendProvenance(workspaceId).appendChat( Buffer.from(JSON.stringify(historyEntry) + "\n") ); @@ -2457,6 +2733,20 @@ export class HistoryService { } } + private async fenceContextResetUnderHistoryLock( + workspaceId: string, + messages: readonly MuxMessage[] + ): Promise { + if ( + messages.some((message) => getContextBoundaryKind(message) === CONTEXT_BOUNDARY_KINDS.RESET) + ) { + // A reset discards foreign producers' captured context even without a Stop. + // Fence after admission checks and before the write, under this same lock; + // a failed epoch write must never leave a committed reset unfenced. + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } + } + /** Serialize messages as JSONL rows tagged with workspace context. */ private serializeHistoryEntries(messages: readonly MuxMessage[], workspaceId: string): string { return messages.map((msg) => JSON.stringify({ ...msg, workspaceId }) + "\n").join(""); @@ -2679,12 +2969,110 @@ export class HistoryService { } } - async appendToHistory(workspaceId: string, message: MuxMessage): Promise> { - return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to append history", () => - this.appendToHistoryUnderWriteLock(workspaceId, message) + async appendToHistory( + workspaceId: string, + message: MuxMessage, + compactionCondition?: CompactionFollowUpAppendCondition + ): Promise> { + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to append history", + async () => { + if ( + compactionCondition && + !(await this.canAppendCompactionFollowUpUnlocked(workspaceId, compactionCondition)) + ) { + compactionCondition.onSkipped(); + return Ok(undefined); + } + return this.appendToHistoryUnderWriteLock(workspaceId, message); + } ); } + private async canAppendCompactionFollowUpUnlocked( + workspaceId: string, + condition: CompactionFollowUpAppendCondition + ): Promise { + if ( + condition.publication && + !(await this.getContinuousCompactionJournal(workspaceId).isPublicationCurrentUnderHistoryLock( + condition.publication + )) + ) + return false; + if (condition.replacementNonce !== undefined) { + return ( + (await this.matchesReplacementCancellationUnlocked( + workspaceId, + condition.replacementNonce + )) && condition.isCurrent() + ); + } + // Another backend can consume/repair this summary while send preparation + // awaits. Validate its exact intent under the same lock as the new row. + const rows = await this.readChatHistory(workspaceId); + const expected = condition.summary; + if (expected) { + // Parsed UI rows omit malformed reset fences. Admission must also respect + // the provider's raw privacy floor without reacquiring this write lock. + const providerRows = await readProviderHistoryFromLatestBoundary( + { + chat: this.getChatHistoryPath(workspaceId), + archive: this.getChatArchivePath(workspaceId), + }, + 0 + ); + const expectedMeta = expected.metadata?.muxMetadata; + const index = rows.findIndex( + (row) => + row.id === expected.id && + row.metadata?.historySequence === expected.metadata?.historySequence + ); + const current = rows[index]; + const currentMeta = current?.metadata?.muxMetadata; + const matches = + isNonNegativeInteger(expected.metadata?.historySequence) && + providerRows.some( + (row) => + row.id === expected.id && + row.metadata?.historySequence === expected.metadata?.historySequence + ) && + current?.role === "assistant" && + isCompactionSummaryMetadata(expectedMeta) && + expectedMeta.pendingFollowUp != null && + isCompactionSummaryMetadata(currentMeta) && + isDeepStrictEqual(currentMeta.pendingFollowUp, expectedMeta.pendingFollowUp) && + rows + .slice(index + 1) + .every( + (row) => + row.metadata?.rlmPreservedTailCopy === true || + condition.allowedTailMessageIds.includes(row.id) + ); + if (!matches || !condition.isCurrent()) { + return false; + } + } + // Raw reads only: malformed/access failures must not authorize an append, + // and repair would reacquire this lock. Without a source identity, even + // a narrowed Stop must veto the handoff until a replacement is witnessed. + const cancellation = await this.readCompactionCancellation(workspaceId); + if ( + !condition.isCurrent() || + (cancellation && + !(await this.hasCompactionReplacementWitnessUnlocked( + workspaceId, + cancellation.nonce, + rows + )) && + (!expected || matchesCompactionCancellation(cancellation, expected))) + ) { + return false; + } + return true; + } + private async appendToHistoryUnderWriteLock( workspaceId: string, message: MuxMessage @@ -2708,13 +3096,24 @@ export class HistoryService { * the same per-workspace lock every other history mutation takes. Messages * must not carry pre-assigned historySequence values. */ - async appendManyToHistory(workspaceId: string, messages: MuxMessage[]): Promise> { + async appendManyToHistory( + workspaceId: string, + messages: MuxMessage[], + compactionCondition?: CompactionFollowUpAppendCondition + ): Promise> { assert(messages.length > 0, "appendManyToHistory requires at least one message"); return this.withRecoveredHistoryWriteResultLock( workspaceId, "Failed to append history", async () => { try { + if ( + compactionCondition && + !(await this.canAppendCompactionFollowUpUnlocked(workspaceId, compactionCondition)) + ) { + compactionCondition.onSkipped(); + return Ok(undefined); + } await this.refreshSequenceCounterUnderWriteLock(workspaceId); const workspaceDir = this.getSessionDir(workspaceId); await ensurePrivateDir(workspaceDir); @@ -2740,6 +3139,7 @@ export class HistoryService { // temp-and-rename helper the other history mutations use, under the // cross-process append lock (r50) so a foreign backend's row cannot // land between this read and the replace and be silently deleted. + await this.fenceContextResetUnderHistoryLock(workspaceId, messages); await this.getAppendProvenance(workspaceId).appendChat( Buffer.from(this.serializeHistoryEntries(messages, workspaceId)), true @@ -2804,9 +3204,95 @@ export class HistoryService { * always in the active epoch (stream placeholders, compaction summaries), * never in the sealed archive. */ - async updateHistory(workspaceId: string, message: MuxMessage): Promise> { - return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => - this.updateHistoryUnderWriteLock(workspaceId, message) + // Optional ownership predicates are synchronous/pure and may run twice (under + // the lock and immediately before rename). Losing ownership or the target is a successful no-op. + // An optional transform derives field edits from the locked row instead of a stale snapshot. + async updateHistory( + workspaceId: string, + message: MuxMessage, + shouldUpdate?: (current: MuxMessage) => boolean, + updateFromCurrent?: (current: MuxMessage) => MuxMessage, + onCommitted?: () => void, + cancellationNonce?: string, + publication?: ContinuousCompactionPublication + ): Promise> { + assert(!onCommitted || shouldUpdate, "Update commit observers require a conditional mutation"); + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to update history", + async () => { + if ( + publication && + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); + // A live cancellation read can age across another backend's accepted replacement. + // Only the exact still-unwitnessed Stop may erase its captured pending summary. + if (cancellationNonce !== undefined) { + const cancellation = await this.readCompactionCancellation(workspaceId); + if ( + cancellation?.nonce !== cancellationNonce || + !matchesCompactionCancellation(cancellation, message) || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, cancellationNonce)) + ) + return Ok(undefined); + } + return this.updateHistoryUnderWriteLock( + workspaceId, + message, + shouldUpdate, + updateFromCurrent, + onCommitted + ); + } + ); + } + + private async matchesReplacementCancellationUnlocked( + workspaceId: string, + expectedNonce: string | null + ): Promise { + const current = await this.readCompactionCancellation(workspaceId); + // Semantic absence can retain a physical file when witnessed unlink failed. + return ( + (current?.nonce ?? null) === expectedNonce || + (expectedNonce === null && + current != null && + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce))) + ); + } + + async acceptResumeCancellation( + workspaceId: string, + message: MuxMessage, + expectedNonce: string | null, + shouldUpdate: (current: MuxMessage) => boolean, + onCommitted: () => void + ): Promise> { + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to accept resume", + async () => { + // Retry must not stamp an obsolete receipt over a foreign Stop that arrived + // during preparation. Compare shared identity and commit under one lock. + if (!(await this.matchesReplacementCancellationUnlocked(workspaceId, expectedNonce))) + return Ok(undefined); + return this.updateHistoryUnderWriteLock( + workspaceId, + message, + shouldUpdate, + (row) => + expectedNonce === null + ? row + : { + ...row, + metadata: { ...row.metadata, compactionCancellationNonce: expectedNonce }, + }, + onCommitted + ); + } ); } @@ -2867,7 +3353,10 @@ export class HistoryService { private async updateHistoryUnderWriteLock( workspaceId: string, - message: MuxMessage + message: MuxMessage, + shouldUpdate?: (current: MuxMessage) => boolean, + updateFromCurrent?: (current: MuxMessage) => MuxMessage, + onCommitted?: () => void ): Promise> { invalidateHistoryAppendProvenance(); try { @@ -2890,10 +3379,16 @@ export class HistoryService { // Find and replace the message with matching historySequence let found = false; let persistedMessage: MuxMessage | undefined; + let sourceMessage: MuxMessage | undefined; for (let i = 0; i < messages.length; i++) { if (messages[i].metadata?.historySequence === targetSequence) { const existingMessage = messages[i]; assert(existingMessage, "updateHistory matched message must exist"); + if (shouldUpdate && !shouldUpdate(existingMessage)) return Ok(undefined); + sourceMessage = existingMessage; + // Conditional field edits must preserve unrelated late writes. Derive the + // replacement from the row read under this same cross-process write lock. + const updatedMessage = updateFromCurrent?.(existingMessage) ?? message; // Preserve compaction boundary metadata during late in-place rewrites. // Compaction may update an assistant row first, then a late stream rewrite can @@ -2901,14 +3396,14 @@ export class HistoryService { const preservedCompactionMetadata = getCompactionMetadataToPreserve( workspaceId, existingMessage, - message + updatedMessage ); // Preserve the historySequence, update everything else. messages[i] = { - ...message, + ...updatedMessage, metadata: { - ...message.metadata, + ...updatedMessage.metadata, ...(preservedCompactionMetadata ?? {}), historySequence: targetSequence, }, @@ -2921,7 +3416,11 @@ export class HistoryService { } if (!found || !persistedMessage) { - return Err(`No message found with historySequence ${targetSequence}`); + // Conditional cleanup is already settled when its exact target was removed. + // It must not chase a replacement row with the same ID in another sequence. + return shouldUpdate + ? Ok(undefined) + : Err(`No message found with historySequence ${targetSequence}`); } // Rewrite entire file @@ -2932,7 +3431,18 @@ export class HistoryService { ); // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + if (shouldUpdate && sourceMessage) { + const source = sourceMessage; + if ( + !(await this.writeGuardedHistory( + historyPath, + historyEntries, + () => shouldUpdate(source), + onCommitted + )) + ) + return Ok(undefined); + } else await writeFileAtomic(historyPath, historyEntries); // Compaction updates the streamed summary row in-place with boundary // metadata — seal the previous epoch once that lands. Check the persisted @@ -2946,6 +3456,33 @@ export class HistoryService { } } + /** The ownership check, rename and commit publication never yield to new admission. */ + private async writeGuardedHistory( + historyPath: string, + serialized: string | Buffer, + isCurrent: () => boolean, + onCommitted?: () => void + ): Promise { + const stagedPath = `${historyPath}.continuous-${randomUUID()}`; + try { + await writeFileAtomic(stagedPath, serialized, { mode: 0o600 }); + if (!isCurrent()) return false; + renameSync(stagedPath, historyPath); + try { + onCommitted?.(); + } catch (error) { + // Observer failure cannot turn an already committed rewrite into a + // reported write failure. Filesystem cleanup errors still propagate. + log.error("Committed history rewrite publication failed", { + error: getErrorMessage(error), + }); + } + return true; + } finally { + await fs.rm(stagedPath, { force: true }); + } + } + /** * Atomically persist a compaction boundary together with its preserved * keep-recent tail copies (RLM keep-recent floor) in ONE file commit. @@ -2969,13 +3506,23 @@ export class HistoryService { summaryMessage: MuxMessage, tailCopies: readonly MuxMessage[], updateExisting: boolean, - shouldPersist?: (messages: MuxMessage[]) => boolean + shouldPersist?: (messages: MuxMessage[]) => boolean, + publication?: ContinuousCompactionPublication ): Promise> { // Continuous compaction may intentionally keep no tail when no complete turn fits. return this.withRecoveredHistoryWriteResultLock( workspaceId, "Failed to persist compaction boundary with tail copies", async () => { + // A prepared summary or already-read journal can outlive foreign repair. + // Validate its captured generation under this lock before touching sequences. + if ( + publication && + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); invalidateHistoryAppendProvenance(); try { // r52: this path assigns fresh sequences (appended summary + every @@ -3064,16 +3611,12 @@ export class HistoryService { messages.slice(sourceMessages.length) ); if (shouldPersist) { - const stagedPath = `${historyPath}.continuous-${randomUUID()}`; - try { - await writeFileAtomic(stagedPath, serialized, { mode: 0o600 }); - // Bulk I/O remains asynchronous, but the final generation check and - // publication must not yield to reset(), abandonment, or a new stream. - if (!shouldPersist(sourceMessages)) return Err("Compaction snapshot changed"); - renameSync(stagedPath, historyPath); - } finally { - await fs.rm(stagedPath, { force: true }); - } + if ( + !(await this.writeGuardedHistory(historyPath, serialized, () => + shouldPersist(sourceMessages) + )) + ) + return Err("Compaction snapshot changed"); } else { await writeFileAtomic(historyPath, serialized); } @@ -3160,16 +3703,39 @@ export class HistoryService { * * This is safer than truncateAfterMessage for cleanup paths where subsequent * messages may already have been appended. + * A conditional cleanup uses a synchronous/pure predicate under the lock and + * before rename; false returns Ok without changing history. + * Its optional commit observer runs synchronously after rename, before any + * cleanup await. It must not await or reenter the history write lock. */ - async deleteMessage(workspaceId: string, messageId: string): Promise> { + async deleteMessage( + workspaceId: string, + messageId: string, + shouldDelete?: (messages: MuxMessage[]) => boolean, + onCommitted?: () => void, + onAlreadyAbsent?: (remaining: MuxMessage[]) => void + ): Promise> { + assert( + !(onCommitted ?? onAlreadyAbsent) || shouldDelete, + "Delete observers require a conditional cleanup" + ); return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to delete message", () => - this.deleteMessageUnderWriteLock(workspaceId, messageId) + this.deleteMessageUnderWriteLock( + workspaceId, + messageId, + shouldDelete, + onCommitted, + onAlreadyAbsent + ) ); } private async deleteMessageUnderWriteLock( workspaceId: string, - messageId: string + messageId: string, + shouldDelete?: (messages: MuxMessage[]) => boolean, + onCommitted?: () => void, + onAlreadyAbsent?: (remaining: MuxMessage[]) => void ): Promise> { invalidateHistoryAppendProvenance(); try { @@ -3177,9 +3743,30 @@ export class HistoryService { const { rows, messages } = await this.readHistoryForRewrite( this.getChatHistoryPath(workspaceId) ); + // Conditional cleanup is scoped to the observed active context, never its archive. + if (shouldDelete && !shouldDelete(messages)) return Ok(undefined); const filteredMessages = messages.filter((msg) => msg.id !== messageId); if (filteredMessages.length === messages.length) { + if (shouldDelete) { + // Another backend may already have deleted this exact active target. + // An archived row is a replaced context, never an invitation to restore it. + const archived = onAlreadyAbsent ? await this.readArchivedHistory(workspaceId) : []; + if ( + onAlreadyAbsent && + !archived.some((row) => row.id === messageId) && + shouldDelete(messages) + ) { + try { + onAlreadyAbsent([...archived, ...messages]); + } catch (error) { + log.error("Absent history cleanup publication failed", { + error: getErrorMessage(error), + }); + } + } + return Ok(undefined); + } // Not in the active epoch — the row may live in the sealed archive // (rare: cleanup paths almost always target recent rows). const { rows: archiveRows, messages: archiveMessages } = await this.readHistoryForRewrite( @@ -3207,7 +3794,17 @@ export class HistoryService { ); // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + if (shouldDelete) { + if ( + !(await this.writeGuardedHistory( + historyPath, + historyEntries, + () => shouldDelete(messages), + onCommitted + )) + ) + return Ok(undefined); + } else await writeFileAtomic(historyPath, historyEntries); // Keep the in-memory sequence counter monotonic. It's okay to reuse deleted sequence // numbers on restart, but we must not regress within a running process. @@ -3312,6 +3909,11 @@ export class HistoryService { const archiveMaxSeq = await this.getArchiveTailMaxSequence(workspaceId); + // Edit/fork cuts discard captured provider context across backend instances. + // A declined target or keep-target-at-tail no-op must retain its publication. + if (hasProviderEligibleMessages(filterWorkflowDisplayOnlyMessages(removedMessages))) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } // Atomic write prevents corruption if app crashes mid-write await writeFileAtomic(historyPath, historyEntries); @@ -3392,6 +3994,9 @@ export class HistoryService { if (lastArchiveRow && lastArchiveRow.raw.at(-1) !== 10 && activeEpochRows.length > 0) { archiveRows.push({ raw: Buffer.from("\n"), message: undefined }); } + if (hasProviderEligibleMessages(filterWorkflowDisplayOnlyMessages(removedMessages))) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked( workspaceId, null, @@ -3529,6 +4134,11 @@ export class HistoryService { .filter((s): s is number => isNonNegativeInteger(s)); if (percentage >= 1.0) { + // Full deletion also retires captured publications when no Stop exists. + // Keep the epoch advance and destructive rewrite in one locked operation. + if (archiveRows.length > 0 || chatRows.length > 0) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); @@ -3585,6 +4195,7 @@ export class HistoryService { "Truncation would remove every remaining message; retry to run it as a full clear." ); } + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); @@ -3612,6 +4223,11 @@ export class HistoryService { retainedMessages, sanitize ); + // A prefix cut retires captured publications only when it removes active + // provider context; trimming sealed or display-only rows preserves them. + if (activeContextChanged) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked( workspaceId, remainingArchive.length > 0 ? remainingArchive : null, diff --git a/src/node/services/streamManager.continuousCompaction.test.ts b/src/node/services/streamManager.continuousCompaction.test.ts index 691c246e086..1a2c1132b10 100644 --- a/src/node/services/streamManager.continuousCompaction.test.ts +++ b/src/node/services/streamManager.continuousCompaction.test.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { CompactionCancellation } from "./compactionCancellation"; +import { HistoryService } from "./historyService"; +import { removeSessionDirUnderMemoryLocks } from "./workspaceRemoval"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import { assemblePromptPayload } from "./turnContextAssembler"; import type { ActiveTurnThinkingOverride } from "./thinkingOverride"; import { prepareMessagesForProvider } from "./messagePipeline"; @@ -7,6 +11,8 @@ import * as ai from "ai"; import * as atomicWrite from "write-file-atomic"; import { createAnthropic } from "@ai-sdk/anthropic"; import { readFile, writeFile } from "node:fs/promises"; +import { promises as journalFs } from "node:fs"; +import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; import assert from "@/common/utils/assert"; import { createMuxMessage } from "@/common/types/message"; import { @@ -172,6 +178,201 @@ describe("continuous prefix prepareStep and journal", () => { return { ...harness, store, tracker, swap, manager }; } + it("a foreign repair rejects a prefix prepared before its journal publication", async () => { + const { run, tracker, store } = await setup(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = store.write.bind(store); + spyOn(store, "write").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const pending = run(); + try { + await entered.promise; + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(history.config); + expect(await new CompactionCancellation(foreign, workspaceId).read()).toBeNull(); + release.resolve(); + const result = await pending; + expect(tracker.consumedPrefixSwap).toBeUndefined(); + expect(result?.messages).toBeUndefined(); + expect(await foreign.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + } finally { + release.resolve(); + await pending; + } + }); + + it("old identity cleanup cannot invalidate a queued newer journal in the same store", async () => { + const { store, swap } = await setup(); + const a = await store.write(swap.journal, swap.prefix, () => true); + assert(a, "Expected A"); + await store.clear(a); + const b = structuredClone(swap.journal); + b.boundary.id = "newer-boundary"; + const writingB = store.write(b, swap.prefix, () => true); + await store.clear(a); + expect(await writingB).not.toBeNull(); + expect((await store.read())?.boundary.id).toBe(b.boundary.id); + }); + + it.each(["generation write", "journal clear"])( + "failed repair at %s still rejects old prefix publication", + async (failure) => { + const { run, tracker, store } = await setup(); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(history.config); + if (failure === "generation write") { + const original = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementationOnce( + Object.assign(() => Promise.reject(new Error("generation unavailable")), { + sync: original.sync, + }) + ); + } else { + const remove = journalFs.rm; + let failed = false; + spyOn(journalFs, "rm").mockImplementation(async (...args) => { + if (!failed && args[0] === store.path) { + failed = true; + throw new Error("clear unavailable"); + } + return remove(...args); + }); + } + const repair = await new CompactionCancellation(foreign, workspaceId) + .read() + .catch((error: unknown) => error); + expect(repair).toBeInstanceOf(Error); + expect( + await readFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "utf8" + ) + ).toBe("{"); + await run(); + expect(tracker.consumedPrefixSwap).toBeUndefined(); + expect(await store.exists()).toBe(false); + } + ); + + it("opaque corrupt generation bytes fence old work and allow a fresh publication", async () => { + const { store, swap } = await setup(); + const generationPath = `${history.config.sessionsDir}/${workspaceId}/${CONTINUOUS_COMPACTION_GENERATION_FILE}`; + await writeFile(generationPath, Buffer.from([0, 255, 123])); + expect(await store.write(swap.journal, swap.prefix, () => true)).toBeNull(); + const fresh = journalFixture(); + fresh.publicationGeneration = await store.captureGeneration(); + const prefix = await rebuildContinuousPrefix(fresh, workspaceId); + expect(await store.write(fresh, prefix, () => true)).not.toBeNull(); + const foreign = new HistoryService(history.config).getContinuousCompactionJournal(workspaceId); + expect((await foreign.read())?.publicationGeneration).toBe(fresh.publicationGeneration); + }); + + it("old cleanup and fallback cannot replace a fresh foreign journal after repair", async () => { + const { store, swap } = await setup(); + const original = await store.write(swap.journal, swap.prefix, () => true); + assert(original, "Expected original journal"); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreignHistory = new HistoryService(history.config); + expect(await new CompactionCancellation(foreignHistory, workspaceId).read()).toBeNull(); + const foreign = foreignHistory.getContinuousCompactionJournal(workspaceId); + const fresh = journalFixture(); + fresh.publicationGeneration = await foreign.captureGeneration(); + fresh.boundary.id = "foreign-boundary"; + const prefix = await rebuildContinuousPrefix(fresh, workspaceId); + const published = await foreign.write(fresh, prefix, () => true); + assert(published, "Expected fresh publication"); + await store.clear(original); + expect( + await store.recordFallbackPrefix( + original, + { modelString: original.parentModel, prefix: swap.prefix }, + () => true + ) + ).toBeNull(); + expect(await store.write(original, swap.prefix, () => true)).toBeNull(); + // A failed stale write/cleanup must not adopt the newer journal it sees. + await store.clear(undefined); + expect(await foreign.read()).toEqual(published); + expect( + await foreign.recordFallbackPrefix( + published, + { modelString: published.parentModel, prefix }, + () => true + ) + ).not.toBeNull(); + }); + + it("repair does not join a journal writer queued behind its held history lock", async () => { + const { swap } = await setup(); + const foreignHistory = new HistoryService(history.config); + const foreign = foreignHistory.getContinuousCompactionJournal(workspaceId); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const invalidate = foreign.invalidateUnderHistoryLock.bind(foreign); + spyOn(foreign, "invalidateUnderHistoryLock").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return invalidate(); + }); + const repair = new CompactionCancellation(foreignHistory, workspaceId).read(); + let queued: ReturnType | undefined; + try { + await entered.promise; + queued = foreign.write(swap.journal, swap.prefix, () => true); + release.resolve(); + expect(await repair).toBeNull(); + expect(await queued).toBeNull(); + expect(await foreign.exists()).toBe(false); + } finally { + release.resolve(); + await repair; + await queued; + } + }); + + it("journal capture, publication and old cleanup cannot recreate a removed workspace", async () => { + const { store, swap } = await setup(); + const original = await store.write(swap.journal, swap.prefix, () => true); + assert(original, "Expected journal"); + const sessionDir = `${history.config.sessionsDir}/${workspaceId}`; + await removeSessionDirUnderMemoryLocks({ + rootDir: history.config.rootDir, + sessionDir, + workspaceId, + attemptId: "remove-journal-test", + }); + for (const operation of [ + () => store.captureGeneration(), + () => store.write(original, swap.prefix, () => true), + () => store.clear(original), + ]) { + const result = await operation().catch((error: unknown) => error); + expect(result).toHaveProperty("message", expect.stringContaining("removed")); + } + await store.clear(undefined); + expect(await journalFs.stat(sessionDir).catch((error: unknown) => error)).toHaveProperty( + "code", + "ENOENT" + ); + }); + it("journals before returning, swaps once by content identity, and strips retained cache markers", async () => { const { run, tracker, store, swap } = await setup(); const result = await run(); @@ -461,7 +662,7 @@ describe("continuous prefix prepareStep and journal", () => { it.each(["reset-before-write", "abort-after-write"] as const)( "%s fences the swap return and deletes the stale journal", async (mode) => { - const { run, tracker, store, controller } = await setup(); + const { run, tracker, store, controller, swap } = await setup(); let release!: () => void; const gate = new Promise((resolve) => { release = resolve; @@ -487,7 +688,7 @@ describe("continuous prefix prepareStep and journal", () => { let cleared = Promise.resolve(); if (mode === "reset-before-write") { tracker.pendingPrefixSwap = undefined; - cleared = store.clear(); + cleared = store.clear(swap.journal); } else { controller.abort(); } diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index edc09d6ee46..eb476dfe4cc 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -2632,19 +2632,28 @@ export class StreamManager { !abortController.signal.aborted && request.thinkingOverrideState?.pending == null && request.thinkingOverrideState?.applied === thinkingLevel; + const unswappedMessages = effectiveMessages; const journal = await store.write( { ...swap.journal, stepNumber }, swap.prefix, - isCurrent + isCurrent, + (journal) => { + // Consume in the publication lock, so repair orders before or after + // this receipt instead of racing the return from journal.write(). + swap.journal = journal; + swap.consumed = true; + stepTracker.consumedPrefixSwap = swap; + effectiveMessages = swapped; + } ); - if (journal && isCurrent()) { - swap.journal = journal; - swap.consumed = true; - stepTracker.consumedPrefixSwap = swap; - effectiveMessages = swapped; - } else if (journal && stepTracker.pendingPrefixSwap === swap) { - // A thinking change can also arrive after write()'s last fence. - await store.clear(); + if (journal && !isCurrent()) { + // Local thinking/abort changes still win before provider entry. + // Cleanup cannot delete a foreign journal published after our receipt. + effectiveMessages = unswappedMessages; + swap.consumed = false; + if (stepTracker.consumedPrefixSwap === swap) + stepTracker.consumedPrefixSwap = undefined; + await store.clear(journal); } } if (stepTracker.pendingPrefixSwap === swap) stepTracker.pendingPrefixSwap = undefined; @@ -3630,7 +3639,7 @@ export class StreamManager { !streamInfo.abortController.signal.aborted && nextRequest.thinkingOverrideState?.pending == null && nextRequest.thinkingOverrideState?.applied === appliedThinking; - const journal = await this.historyService + await this.historyService .getContinuousCompactionJournal(workspaceId) .recordFallbackPrefix( consumedSwap.journal, @@ -3640,12 +3649,13 @@ export class StreamManager { providerOptions: nextRequest.providerOptions, system: nextRequest.system, }, - isCurrent + isCurrent, + (journal) => { + consumedSwap.journal = journal; + messages = swapped; + } ); - if (journal && isCurrent()) { - consumedSwap.journal = journal; - messages = swapped; - } + if (!isCurrent()) messages = null; } } if (messages) nextRequest.messages = messages; diff --git a/src/node/services/turnCoordinator.test.ts b/src/node/services/turnCoordinator.test.ts index f2f0c922bf7..6780574f5f1 100644 --- a/src/node/services/turnCoordinator.test.ts +++ b/src/node/services/turnCoordinator.test.ts @@ -63,6 +63,271 @@ function reduce(events: CoordinatorEvent[]) { } describe("TurnCoordinator", () => { + test("Stop permits exact follow-up cleanup but refuses dispatch until a new manual turn", () => { + const { coordinator } = setup(); + const retained = coordinator.claimCompactionFollowUp(); + if (!retained) throw new Error("Expected follow-up owner"); + coordinator.invalidateCompaction(true); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + expect(coordinator.isCurrentCompaction(retained)).toBe(false); + expect(coordinator.canClearCompactionFollowUp(retained)).toBe(true); + coordinator.finishCompactionFollowUp(retained); + + const cleanup = coordinator.claimCompactionFollowUpCleanup(); + if (!cleanup) throw new Error("Expected abandoned cleanup owner"); + expect(coordinator.isCurrentCompaction(cleanup)).toBe(false); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: cleanup, + }).status + ).toBe("rejected"); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + }).status + ).toBe("admitted"); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUp()).toBeDefined(); + }); + + test.each([false, true])( + "shutdown retains only exact abandoned cleanup (claimed=%s)", + (claimed) => { + const { coordinator } = setup(); + const ready = claimed ? coordinator.claimCompactionFollowUp() : undefined; + coordinator.invalidateCompaction(true); + coordinator.beginShutdown(); + const cleanup = ready ?? coordinator.claimCompactionFollowUpCleanup(); + if (!cleanup) throw new Error("Expected cleanup through shutdown"); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + expect(coordinator.isCurrentCompaction(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: cleanup, + }).status + ).toBe("rejected"); + coordinator.dispose(); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + coordinator.finishCompactionFollowUp(cleanup); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + } + ); + + test("shutdown preserves ready recovery intent without granting cleanup", () => { + const { coordinator } = setup(); + const ready = coordinator.claimCompactionFollowUp(); + if (!ready) throw new Error("Expected follow-up"); + coordinator.beginShutdown(); + expect(coordinator.canClearCompactionFollowUp(ready)).toBe(false); + expect(coordinator.isCurrentCompaction(ready)).toBe(false); + coordinator.finishCompactionFollowUp(ready); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + }); + + test("work invalidation preserves Stop until an explicit replacement epoch commits", () => { + const { coordinator } = setup(); + const cleanup = coordinator.claimCompactionFollowUp(); + if (!cleanup) throw new Error("Expected cleanup owner"); + coordinator.invalidateCompaction(true); + coordinator.invalidateCompaction(); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + coordinator.invalidateCompaction(false); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUp()).toBeDefined(); + }); + + test("a compaction handoff survives its own admission, but stale completion cannot release its replacement", () => { + const { coordinator } = setup(); + const token = coordinator.beginCompactionObservation("continuous"); + if (!token) throw new Error("Expected observation"); + coordinator.setCompactionStage(token, "stopped"); + const followUp = coordinator.claimCompactionFollowUp(); + if (!followUp) throw new Error("Expected follow-up"); + const own = coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: followUp, + }); + expect(own.status).toBe("admitted"); + expect(coordinator.isCurrentCompaction(token)).toBe(true); + expect(coordinator.isCurrentCompaction(followUp)).toBe(true); + coordinator.finishTurn(coordinator.turnId); + expect( + coordinator.prepare({ kind: "fresh", intent: "direct", expectedTurnId: coordinator.turnId }) + .status + ).toBe("admitted"); + const replacement = coordinator.beginCompactionObservation("continuous"); + if (!replacement) throw new Error("Expected replacement observation"); + coordinator.setCompactionStage(replacement, "stopping"); + expect(coordinator.isCurrentCompaction(token)).toBe(false); + expect(coordinator.finishCompactionObservation(token)).toBe(false); + coordinator.finishCompactionFollowUp(followUp); + expect(coordinator.compactionIntent.observation?.id).toBe(replacement.id); + expect(coordinator.compactionIntent.observation?.stage).toBe("stopping"); + }); + + test("pending compaction wait retires independently of held physical work", async () => { + const { coordinator } = setup(); + const execution = coordinator.enterExecution(); + const a = coordinator.beginCompactionObservation("continuous"); + if (!a) throw new Error("Expected A"); + coordinator.setCompactionStage(a, "stopped"); + let settled = false; + const waiting = coordinator.waitForMidStreamCompactionSettled().then(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + coordinator.invalidateCompaction(false); + await waiting; + expect(settled).toBe(true); + let drained = false; + const draining = coordinator.drain().then(() => { + drained = true; + }); + await Promise.resolve(); + expect(drained).toBe(false); + execution[Symbol.dispose](); + await draining; + }); + + test("old completion cannot release a replacement's reentrantly registered waiter", async () => { + let replacement: ReturnType; + let bWaiting: Promise | undefined; + let bSettled = false; + const { coordinator } = setup({ + phaseChanged: (phase) => { + if (phase !== "preparing") return; + replacement = coordinator.beginCompactionObservation("continuous"); + if (!replacement) throw new Error("Expected B"); + coordinator.setCompactionStage(replacement, "stopping"); + bWaiting = coordinator.waitForMidStreamCompactionSettled().then(() => { + bSettled = true; + }); + }, + }); + const a = coordinator.beginCompactionObservation("continuous"); + if (!a) throw new Error("Expected A"); + coordinator.setCompactionStage(a, "stopped"); + const aWaiting = coordinator.waitForMidStreamCompactionSettled(); + prepare(coordinator); + await aWaiting; + expect(bSettled).toBe(false); + coordinator.finishCompactionObservation(a); + await Promise.resolve(); + expect(bSettled).toBe(false); + if (!replacement) throw new Error("Expected B"); + coordinator.finishCompactionObservation(replacement); + await bWaiting; + expect(bSettled).toBe(true); + }); + + test.each(["install", "phase"] as const)( + "throwing %s callback settles retired waiters without releasing reentrant replacement", + async (source) => { + let b: ReturnType; + let bWaiting: Promise | undefined; + let bSettled = false; + const fail = () => { + b = coordinator.beginCompactionObservation("continuous"); + if (!b) throw new Error("Expected B"); + coordinator.setCompactionStage(b, "stopping"); + bWaiting = coordinator.waitForMidStreamCompactionSettled().then(() => { + bSettled = true; + }); + throw new Error("admission callback failed"); + }; + const { coordinator } = setup({ + phaseChanged: (phase) => { + if (source === "phase" && phase === "preparing") fail(); + }, + }); + const a = coordinator.beginCompactionObservation("continuous"); + if (!a) throw new Error("Expected A"); + coordinator.setCompactionStage(a, "stopped"); + let aSettled = false; + const aWaiting = coordinator.waitForMidStreamCompactionSettled().then(() => { + aSettled = true; + }); + try { + expect(() => + coordinator.prepare( + { kind: "fresh", intent: "direct", expectedTurnId: coordinator.turnId }, + undefined, + source === "install" ? fail : () => undefined + ) + ).toThrow("admission callback failed"); + await Promise.resolve(); + expect(aSettled).toBe(true); + expect(bSettled).toBe(false); + coordinator.finishCompactionObservation(a); + await Promise.resolve(); + expect(bSettled).toBe(false); + if (!b) throw new Error("Expected B"); + coordinator.finishCompactionObservation(b); + await Promise.all([aWaiting, bWaiting]); + } finally { + if (b) coordinator.finishCompactionObservation(b); + } + } + ); + + test("compaction wait preserves dispatch until its own handoff has claimed preparation", async () => { + const { coordinator } = setup(); + const token = coordinator.beginCompactionObservation("continuous"); + if (!token) throw new Error("Expected observation"); + coordinator.setCompactionStage(token, "stopping"); + let settled = false; + const waiting = coordinator.waitForMidStreamCompactionSettled().then(() => { + settled = true; + }); + coordinator.setCompactionStage(token, "stopped"); + coordinator.setCompactionStage(token, "dispatching"); + await Promise.resolve(); + expect(settled).toBe(false); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: token, + }).status + ).toBe("admitted"); + expect(coordinator.phase).toBe("preparing"); + coordinator.finishCompactionObservation(token); + await waiting; + expect(settled).toBe(true); + }); + + test("shutdown releases pending compaction wait without manufacturing idle", async () => { + const { coordinator } = setup(); + prepare(coordinator); + const token = coordinator.beginCompactionObservation("continuous"); + if (!token) throw new Error("Expected observation"); + coordinator.setCompactionStage(token, "stopped"); + const waiting = coordinator.waitForMidStreamCompactionSettled(); + coordinator.beginShutdown(); + await waiting; + expect(coordinator.phase).toBe("preparing"); + expect(coordinator.compactionIntent.observation?.id).toBe(token.id); + }); + test("observed terminal publishes before idle/drain and cannot retire a reentrant replacement", () => { const order: string[] = []; const { coordinator, callbacks } = setup({ diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index 626f9c7d0bf..963cd72e999 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -16,6 +16,7 @@ export type PreparationRequest = intent: "direct" | "resume" | "handoff" | QueueDrainTrigger; expectedTurnId: TurnId; editReservation?: symbol; + compactionHandoff?: CompactionToken; } | { kind: "adopt"; turnId: TurnId }; export type PreparationAdmission = @@ -27,6 +28,22 @@ type ReservationKind = "admission" | "edit" | "manual"; type DecisionKind = "error" | "compaction"; type DecisionOutcome = StreamErrorRecoveryOutcome | boolean; +export interface CompactionToken { + readonly id: symbol; + readonly epoch: number; +} +type CompactionObservation = CompactionToken & { + readonly kind: "continuous" | "legacy"; + readonly stage: "observing" | "stopping" | "stopped" | "dispatching"; +}; +interface CompactionIntent { + readonly epoch: number; + readonly status: "ready" | "abandoned"; + readonly observation?: CompactionObservation; + readonly followUp?: CompactionToken; + readonly summaryId: string | null; +} + type Operation = { readonly id: OperationId; readonly startupMessageId?: string; @@ -64,6 +81,7 @@ export interface CoordinatorState { readonly reservations: ReadonlyArray<{ id: symbol; kind: ReservationKind }>; readonly retry?: symbol; readonly decisions: readonly Decision[]; + readonly compaction: CompactionIntent; } export type CoordinatorEvent = @@ -84,6 +102,15 @@ export type CoordinatorEvent = | { type: "reserve" | "release"; id: symbol; kind: ReservationKind } | { type: "retry-start" | "retry-finish"; id: symbol } | { type: "decision"; kind: DecisionKind; messageId: string; outcome?: DecisionOutcome } + | { type: "compaction-observe"; token: CompactionToken; kind: CompactionObservation["kind"] } + | { type: "compaction-stage"; token: CompactionToken; stage: CompactionObservation["stage"] } + | { type: "compaction-finish"; token: CompactionToken } + | { type: "compaction-invalidate"; abandon: boolean } + | { + type: "compaction-follow-up" | "compaction-follow-up-cleanup" | "compaction-follow-up-finish"; + token: CompactionToken; + } + | { type: "compaction-summary"; summaryId: string | null } | { type: "shutdown" | "dispose" }; type CoordinatorCommand = @@ -103,7 +130,13 @@ type CoordinatorCommand = | { type: "dispose" }; export function initialCoordinatorState(id: TurnId): CoordinatorState { - return { lifetime: "open", turn: { phase: "idle", id }, reservations: [], decisions: [] }; + return { + lifetime: "open", + turn: { phase: "idle", id }, + reservations: [], + decisions: [], + compaction: { epoch: 0, status: "ready", summaryId: null }, + }; } function hasConflictingEdit(state: CoordinatorState, owner?: symbol): boolean { @@ -159,7 +192,9 @@ export function transition( commands.push({ type: "decision", decision: value }); }; const current = state.turn.operation; - if (state.lifetime === "disposed") + // A retained Stop cleanup may finish while destructive disposal drains its lease. + // Only its exact release remains legal; disposal never admits new work. + if (state.lifetime === "disposed" && event.type !== "compaction-follow-up-finish") return { state, commands, @@ -213,6 +248,29 @@ export function transition( admission = { status: "deferred", reason: "busy" }; break; } + const handoff = request.compactionHandoff; + if (handoff != null) { + if ( + state.compaction.status !== "ready" || + handoff.epoch !== state.compaction.epoch || + (state.compaction.observation?.id !== handoff.id && + state.compaction.followUp?.id !== handoff.id) + ) { + admission = { status: "rejected", reason: "retired" }; + break; + } + // This admission belongs to the captured handoff. It changes TurnId, but + // must keep the source intent alive until actual send acceptance is known. + } else { + next = { + ...next, + compaction: { + epoch: state.compaction.epoch + 1, + status: "ready", + summaryId: null, + }, + }; + } if (current) commands.push({ type: "retire", id: current.id }); phase({ phase: "preparing", id: event.id }); } @@ -279,6 +337,15 @@ export function transition( if (current?.delivery === "waiting") { operation({ ...current, stage: "started", messageId: event.payload.messageId }); } + if (next.compaction.status === "abandoned") + next = { + ...next, + compaction: { + epoch: next.compaction.epoch + 1, + status: "ready", + summaryId: null, + }, + }; commands.push({ type: "record-start", turnId: state.turn.id, payload: event.payload }); // Existing raw streams also restore sessions constructed around an already-running engine. phase({ ...next.turn, phase: "streaming" }); @@ -353,6 +420,73 @@ export function transition( case "decision": decision(event.kind, event.messageId, event.outcome); break; + case "compaction-observe": + if ( + state.lifetime === "open" && + !state.compaction.observation && + state.compaction.status === "ready" && + event.token.epoch === state.compaction.epoch + ) + next = { + ...state, + compaction: { + ...state.compaction, + observation: { ...event.token, kind: event.kind, stage: "observing" }, + }, + }; + break; + case "compaction-stage": + if ( + state.compaction.observation?.id === event.token.id && + state.compaction.epoch === event.token.epoch + ) + next = { + ...state, + compaction: { + ...state.compaction, + observation: { ...state.compaction.observation, stage: event.stage }, + }, + }; + break; + case "compaction-finish": + if (state.compaction.observation?.id === event.token.id) + next = { ...state, compaction: { ...state.compaction, observation: undefined } }; + break; + case "compaction-invalidate": + next = { + ...state, + compaction: { + ...state.compaction, + epoch: state.compaction.epoch + 1, + // Abandonment belongs to this canceled epoch, not a replacement context. + status: event.abandon ? "abandoned" : "ready", + // User Stop retains only cleanup ownership for its just-committed boundary. + // Context mutation retires semantic observation immediately; physical leases remain. + observation: event.abandon ? state.compaction.observation : undefined, + followUp: event.abandon ? state.compaction.followUp : undefined, + summaryId: null, + }, + }; + break; + case "compaction-follow-up": + case "compaction-follow-up-cleanup": + if ( + (state.lifetime === "open" || + (state.lifetime === "shutting-down" && event.type === "compaction-follow-up-cleanup")) && + state.compaction.status === + (event.type === "compaction-follow-up" ? "ready" : "abandoned") && + !state.compaction.followUp && + event.token.epoch === state.compaction.epoch + ) + next = { ...state, compaction: { ...state.compaction, followUp: event.token } }; + break; + case "compaction-follow-up-finish": + if (state.compaction.followUp?.id === event.token.id) + next = { ...state, compaction: { ...state.compaction, followUp: undefined } }; + break; + case "compaction-summary": + next = { ...state, compaction: { ...state.compaction, summaryId: event.summaryId } }; + break; case "shutdown": next = { ...state, lifetime: "shutting-down" }; break; @@ -517,6 +651,8 @@ export class TurnCoordinator { >(); private idleWaiters = new Set<() => void>(); private unbusyWaiters = new Set<() => void>(); + private admissionWaiters = new Set<() => void>(); + private compactionWaiters = new Set<() => void>(); private prepared?: { id: TurnId; controller: AbortController }; private thinking: { holder: ActiveTurnThinkingOverride; resource?: Disposable } | null = null; private readonly execution = new TurnExecution(defaultEffectRunner); @@ -534,6 +670,86 @@ export class TurnCoordinator { return this.execution.lease(); } + get compactionIntent(): CompactionIntent { + return this.state.compaction; + } + + get midStreamCompactionPending(): boolean { + const stage = this.state.compaction.observation?.stage; + return stage === "stopping" || stage === "stopped" || stage === "dispatching"; + } + + /** Semantic turn work can retire while an old physical observation still drains. */ + waitForMidStreamCompactionSettled(): Promise { + if (!this.midStreamCompactionPending || this.closing) return Promise.resolve(); + return new Promise((resolve) => this.compactionWaiters.add(resolve)); + } + + beginCompactionObservation(kind: CompactionObservation["kind"]): CompactionToken | undefined { + if (this.admissionBlocked || this.editReserved) return undefined; + const token = { id: Symbol("compaction observation"), epoch: this.state.compaction.epoch }; + this.dispatch({ type: "compaction-observe", token, kind }); + return this.state.compaction.observation?.id === token.id ? token : undefined; + } + + isCurrentCompaction(token: CompactionToken): boolean { + return ( + !this.closing && + this.state.compaction.status === "ready" && + token.epoch === this.state.compaction.epoch && + (this.state.compaction.observation?.id === token.id || + this.state.compaction.followUp?.id === token.id) + ); + } + + setCompactionStage(token: CompactionToken, stage: CompactionObservation["stage"]): void { + this.dispatch({ type: "compaction-stage", token, stage }); + } + + finishCompactionObservation(token: CompactionToken): boolean { + const owns = this.state.compaction.observation?.id === token.id; + this.dispatch({ type: "compaction-finish", token }); + return owns; + } + + invalidateCompaction(abandon = this.state.compaction.status === "abandoned"): void { + this.dispatch({ type: "compaction-invalidate", abandon }); + } + + claimCompactionFollowUp(): CompactionToken | undefined { + const token = { id: Symbol("compaction follow-up"), epoch: this.state.compaction.epoch }; + this.dispatch({ type: "compaction-follow-up", token }); + return this.state.compaction.followUp?.id === token.id ? token : undefined; + } + + claimCompactionFollowUpCleanup(): CompactionToken | undefined { + // Stop may precede the first dispatch claim. This owner can only clear the + // abandoned durable intent; it is never current or admissible for sending. + const token = { + id: Symbol("compaction follow-up cleanup"), + epoch: this.state.compaction.epoch, + }; + this.dispatch({ type: "compaction-follow-up-cleanup", token }); + return this.state.compaction.followUp?.id === token.id ? token : undefined; + } + + finishCompactionFollowUp(token: CompactionToken): void { + this.dispatch({ type: "compaction-follow-up-finish", token }); + } + + canClearCompactionFollowUp(token: CompactionToken): boolean { + // Closing forbids dispatch, but cannot revoke Stop cleanup already joined + // by a physical lease. Replacement still retires the exact token. + return ( + this.state.compaction.followUp?.id === token.id && + (!this.closing || this.state.compaction.status === "abandoned") + ); + } + + recordCompactionSummary(summaryId: string | null): void { + this.dispatch({ type: "compaction-summary", summaryId }); + } + /** Physical completion, distinct from semantic idle and operation policy settlement. */ drain(): Promise { return this.execution.close(); @@ -604,14 +820,36 @@ export class TurnCoordinator { ): Promise | PreparationAdmission | undefined { let launchedPolicy: Promise | undefined; let publicationError: { error: unknown } | undefined; + const previousCompactionEpoch = this.state.compaction.epoch; const result = transition(this.state, event); this.state = result.state; - if (result.admission?.status === "admitted") install?.(); + // Detach before admission/phase callbacks can install a replacement observation + // and waiter. Finishing A must never release B or join A's physical I/O. + const compactionWaiters = + !this.midStreamCompactionPending || this.closing ? this.compactionWaiters : undefined; + if (compactionWaiters) this.compactionWaiters = new Set(); + if (result.admission?.status === "admitted") { + try { + install?.(); + } catch (error) { + // Admission installation precedes phase publication. Its failure still + // releases the detached old batch, never a reentrant replacement's waiters. + for (const resolve of compactionWaiters ?? []) resolve(); + throw error; + } + } // Detach before *any* callback. An idle observer may synchronously admit a new turn and waiter. const idle = result.commands.some( (command) => command.type === "phase" && command.next.phase === "idle" ); const waiters = idle ? this.idleWaiters : undefined; + const admissionWaiters = + !this.admissionBlocked || + this.closing || + previousCompactionEpoch !== this.state.compaction.epoch + ? this.admissionWaiters + : undefined; + if (admissionWaiters) this.admissionWaiters = new Set(); const unbusyWaiters = !this.isBusy() ? this.unbusyWaiters : undefined; if (unbusyWaiters) this.unbusyWaiters = new Set(); const retiredThinking = idle ? this.thinking : undefined; @@ -689,6 +927,8 @@ export class TurnCoordinator { } } } + for (const resolve of compactionWaiters ?? []) resolve(); + for (const resolve of admissionWaiters ?? []) resolve(); for (const resolve of waiters ?? []) resolve(); for (const resolve of unbusyWaiters ?? []) resolve(); retiredThinking?.resource?.[Symbol.dispose](); @@ -808,6 +1048,12 @@ export class TurnCoordinator { return release; } + /** Admission holds are policy waits, not physical work joined by shutdown. */ + waitForAdmissionRelease(): Promise { + if (!this.admissionBlocked || this.closing) return Promise.resolve(); + return new Promise((resolve) => this.admissionWaiters.add(resolve)); + } + waitForIdle(signal?: AbortSignal): Promise { return this.waitForIdleState(false, signal); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e49d9bf1c7e..799b3e6ce06 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,3 +1,7 @@ +import type { CompactionHandler } from "./compactionHandler"; +import { CompactionCancellation } from "./compactionCancellation"; +import { makeTestEffectRunner } from "./di/testEffectRunner"; +import { calculateBackoffDelay } from "@/common/utils/messages/retryState"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -27,7 +31,7 @@ import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import type { SendMessageError } from "@/common/types/errors"; import type { ProjectsConfig } from "@/common/types/project"; import type { Config, SecretsStore } from "@/node/config"; -import type { HistoryService } from "./historyService"; +import { HistoryService } from "./historyService"; import { createTestHistoryService } from "./testHistoryService"; import type { SessionTimingService } from "./sessionTimingService"; import { SessionUsageService } from "./sessionUsageService"; @@ -1877,10 +1881,13 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { }): Promise<"in-flight" | "deferred">; }; try { - // Between the stopped stream and its compaction request the coordinator is idle and no - // stream is running; only the session's pending flag marks the turn work. + // Between the stopped stream and its compaction request the turn is idle, + // but the coordinator still owns the pending observation. const session = service.getOrCreateSession(workspaceId); - Reflect.set(session, "midStreamCompactionPending", true); + const { coordinator } = session as unknown as { coordinator: TurnCoordinator }; + const observation = coordinator.beginCompactionObservation("continuous"); + if (!observation) throw new Error("Expected compaction observation"); + coordinator.setCompactionStage(observation, "stopped"); internal.scheduleBashMonitorWakeReconcileAfterIdle = afterIdle; internal.getDelegatedTurnContinuationSendOptions = () => Promise.resolve({}); internal.sendMessage = sendMessage; @@ -7376,6 +7383,294 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { return { aiService, config, historyService, workspaceService, goalService, cleanup }; } + test.each( + ["legacy", "heartbeat"].flatMap((producer) => + ["absent", "unresolved", "narrowed", "manual before publication", "newer Stop during clear"] + .map((initial) => ({ producer, initial, operation: "full clear" })) + .concat( + [ + "reset", + "destructive replacement", + "active prefix trim", + "sealed prefix trim", + "active edit", + "archived edit", + ].map((operation) => ({ + producer, + initial: "absent", + operation, + })) + ) + ) + )( + "$operation checks captured foreign $producer publication ($initial Stop)", + async ({ producer, initial, operation }) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "clear-foreign-compaction"; + const options = { model: "openai:gpt-4o", agentId: "exec" }; + await config.addWorkspace("/tmp/clear-foreign-project", { + id: workspaceId, + name: workspaceId, + projectName: "clear-foreign-project", + projectPath: "/tmp/clear-foreign-project", + runtimeConfig: { type: "local" }, + }); + const followUp = { text: "Continue discarded work", ...options }; + if (operation === "sealed prefix trim" || operation === "archived edit") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("sealed", "user", "Sealed context") + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("sealed-boundary", "assistant", "Preserved summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + } + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + const cancellation = new CompactionCancellation(historyService, workspaceId); + if (operation === "active prefix trim") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("survivor", "assistant", "Surviving context") + ); + expect(await historyService.classifyTruncationRemoval(workspaceId, 0.25)).toBe("partial"); + } + const foreignHistory = new HistoryService(config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config, + historyService: foreignHistory, + }); + const handler = (foreign.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = foreignHistory.appendToHistory.bind(foreignHistory); + spyOn(foreignHistory, "appendToHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return append(...args); + }); + const publication = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Captured old context", + pendingFollowUp: followUp, + }) + : handler.handleCompletion({ + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Captured old context" }], + metadata: { model: options.model, duration: 1 }, + }); + const clock = makeTestEffectRunner(); + let fresh: Awaited> | undefined; + try { + await entered.promise; + if (initial === "unresolved" || initial === "narrowed") { + await cancellation.cancel(); + if (initial === "narrowed") { + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + await cancellation.narrow( + record.nonce, + createMuxMessage("older-summary", "assistant", "Older work", { + historySequence: 10, + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp }, + }) + ); + } + } + const mutation = + operation === "active edit" || operation === "archived edit" + ? await (async () => { + const editor = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + }); + try { + return await editor.session.sendMessage("Edited context", { + ...options, + editMessageId: operation === "active edit" ? "old-request" : "sealed", + }); + } finally { + await editor.session.dispose(); + } + })() + : operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "destructive replacement" + ? await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "Fresh context") + ) + : await workspaceService.truncateHistory( + workspaceId, + operation === "active prefix trim" + ? 0.25 + : operation === "sealed prefix trim" + ? 0.01 + : undefined + ); + expect(mutation.success).toBe(true); + if (operation === "active prefix trim") { + const trimmed = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(trimmed.success && trimmed.data.map((row) => row.id)).toEqual(["survivor"]); + } + if (operation === "sealed prefix trim") { + const retainedIds: string[] = []; + await historyService.iterateFullHistory(workspaceId, "forward", (rows) => { + retainedIds.push(...rows.map((row) => row.id)); + }); + expect(retainedIds).toEqual(["sealed-boundary", "old-request"]); + } + if (initial === "newer Stop during clear") + await new CompactionCancellation(new HistoryService(config), workspaceId).cancel(); + if (initial === "manual before publication") { + const replacement = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + }); + try { + expect( + (await replacement.session.sendMessage("Fresh explicit work", options)).success + ).toBe(true); + } finally { + await replacement.session.dispose(); + } + } + release.resolve(); + await publication.catch(() => undefined); + const published = await foreignHistory.getLastMessages(workspaceId, 10); + expect( + published.success && + published.data.some((row) => row.metadata?.muxMetadata?.type === "compaction-summary") + ).toBe(operation === "sealed prefix trim"); + if ( + operation === "sealed prefix trim" || + operation === "active edit" || + operation === "archived edit" + ) + return; + fresh = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, + }); + const stream = spyOn(fresh.aiService, "streamMessage"); + await fresh.session.runStartupRecovery(); + await clock.adjust(calculateBackoffDelay(6) * 2); + if (initial === "manual before publication") { + const rows = await fresh.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { text: "Fresh explicit work" }, + ]); + } else { + expect(stream).not.toHaveBeenCalled(); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + expect((await fresh.session.sendMessage("Fresh explicit work", options)).success).toBe( + true + ); + expect(stream).toHaveBeenCalledTimes(1); + } + } finally { + release.resolve(); + await publication.catch(() => undefined); + await fresh?.session.dispose(); + await foreign.session.dispose(); + await workspaceService.disposeSession(workspaceId); + await clock.dispose(); + await cleanup(); + } + } + ); + + test.each( + ["reset", "replace", "active prefix trim"].flatMap((operation) => + [false, true].map((afterAdvance) => ({ operation, afterAdvance })) + ) + )( + "$operation refuses history mutation when publication fencing fails (after advance=$afterAdvance)", + async ({ operation, afterAdvance }) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "context-publication-failure"; + await config.addWorkspace("/tmp/context-publication-project", { + id: workspaceId, + name: workspaceId, + projectName: "context-publication-project", + projectPath: "/tmp/context-publication-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "Original context") + ); + if (operation === "active prefix trim") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("survivor", "assistant", "Surviving context") + ); + } + const journal = historyService.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + const failure = spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce( + async (...args) => { + if (afterAdvance) await invalidate(...args); + throw new Error("context publication unavailable"); + } + ); + const mutate = () => + operation === "reset" + ? workspaceService.resetContext(workspaceId) + : operation === "active prefix trim" + ? workspaceService.truncateHistory(workspaceId, 0.25) + : workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "Fresh context") + ); + try { + const result = await mutate(); + expect(!result.success && result.error).toContain("context publication unavailable"); + const freshHistory = new HistoryService(config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual( + operation === "active prefix trim" ? ["old-user", "survivor"] : ["old-user"] + ); + const failedGeneration = await freshHistory + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(); + expect(failedGeneration === generation).toBe(!afterAdvance); + // A partial epoch advance stays retired. Retry publishes a fresh epoch rather + // than restoring an old producer's authority after storage becomes writable. + expect((await mutate()).success).toBe(true); + expect(await journal.captureGeneration()).not.toBe(failedGeneration); + } finally { + failure.mockRestore(); + await workspaceService.disposeSession(workspaceId); + await cleanup(); + } + } + ); + test("requireIdle sends carry a live idle-admission probe re-evaluated at session gates", async () => { // Codex P1 (PRRT_kwDOPxxmWM6cJ6NI): the preflight count check at // sendMessage entry is a one-shot snapshot — a manual send can enter @@ -7503,9 +7798,11 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { const { workspaceService, cleanup } = await createServices(); const workspaceId = "idle-wait-pending-compaction"; const session = workspaceService.getOrCreateSession(workspaceId); - const settle = Reflect.get(session, "settleMidStreamCompaction") as () => void; + const { coordinator } = session as unknown as { coordinator: TurnCoordinator }; + const observation = coordinator.beginCompactionObservation("legacy"); + if (!observation) throw new Error("Expected compaction observation"); try { - Reflect.set(session, "midStreamCompactionPending", true); + coordinator.setCompactionStage(observation, "stopped"); let resolved = false; const waitPromise = workspaceService.waitForIdleAndNoQueuedMessages(workspaceId).then(() => { resolved = true; @@ -7514,7 +7811,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { expect(resolved).toBe(false); // The compaction request never became a turn: no stream event fires, only the window closes. - settle.call(session); + coordinator.finishCompactionObservation(observation); await waitPromise; expect(resolved).toBe(true); } finally { @@ -8155,6 +8452,527 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } }); + test.each(["lazy access", "shutdown", "external registration"] as const)( + "failed disposal retains cleanup through %s and storage recovery", + async (retry) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "retained-disposal-cleanup"; + let external: Awaited> | undefined; + try { + await config.addWorkspace("/tmp/retained-cleanup-project", { + id: workspaceId, + name: workspaceId, + projectName: "retained-cleanup-project", + projectPath: "/tmp/retained-cleanup-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Prior work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + const session = workspaceService.getOrCreateSession(workspaceId); + await session.interruptStream({ abandonPartial: true }); + const internal = session as unknown as { dispatchPendingFollowUp(): Promise }; + const read = spyOn(historyService, "getLastMessages").mockRejectedValue( + new Error("storage unavailable") + ); + await internal.dispatchPendingFollowUp().catch(() => undefined); + const failure = await workspaceService + .disposeSession(workspaceId) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + const state = workspaceService as unknown as { + sessions: Map; + failedCompactionCleanups: Map }>; + }; + expect(state.sessions.has(workspaceId)).toBe(false); + expect(state.failedCompactionCleanups.has(workspaceId)).toBe(true); + read.mockRestore(); + const recoveredRead = spyOn(historyService, "getLastMessages"); + if (retry === "external registration") { + external = await createAgentSessionHarness({ workspaceId, config, historyService }); + const supplied = external.session; + expect(() => workspaceService.registerSession(workspaceId, supplied)).toThrow("cleanup"); + expect(state.sessions.has(workspaceId)).toBe(false); + await state.failedCompactionCleanups.get(workspaceId)?.retry; + workspaceService.registerSession(workspaceId, supplied); + } else if (retry === "lazy access") { + expect(() => workspaceService.getOrCreateSession(workspaceId)).toThrow("cleanup"); + const first = state.failedCompactionCleanups.get(workspaceId)?.retry; + expect(first).toBeDefined(); + expect(() => workspaceService.getOrCreateSession(workspaceId)).toThrow("cleanup"); + expect(state.failedCompactionCleanups.get(workspaceId)?.retry).toBe(first); + await first; + } else { + workspaceService.beginShutdown(); + await state.failedCompactionCleanups.get(workspaceId)?.retry; + } + expect(recoveredRead).toHaveBeenCalledTimes(1); + expect(state.failedCompactionCleanups.has(workspaceId)).toBe(false); + const rows = await historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + if (retry !== "shutdown") { + const reopened = workspaceService.getOrCreateSession(workspaceId); + expect(reopened).not.toBe(session); + const send = spyOn(reopened, "sendMessage"); + await reopened.runStartupRecovery(); + expect(send).not.toHaveBeenCalled(); + } else { + const restarted = await createAgentSessionHarness({ + workspaceId, + config, + historyService, + }); + try { + const send = spyOn(restarted.session, "sendMessage"); + await restarted.session.runStartupRecovery(); + expect(send).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } + } + } finally { + await external?.session.dispose(); + await cleanup(); + } + } + ); + + test("disposeSession reports failed durable cleanup after removing old registry entries", async () => { + const { config, workspaceService, cleanup } = await createServices(); + const workspaceId = "failed-cleanup-disposal"; + try { + await config.addWorkspace("/tmp/failed-cleanup-project", { + id: workspaceId, + name: workspaceId, + projectName: "failed-cleanup-project", + projectPath: "/tmp/failed-cleanup-project", + runtimeConfig: { type: "local" }, + }); + const session = workspaceService.getOrCreateSession(workspaceId); + const dispose = session.dispose.bind(session); + spyOn(session, "dispose").mockImplementationOnce(async () => { + await dispose(); + throw new Error("unresolved stopped cleanup"); + }); + const failure = await workspaceService + .disposeSession(workspaceId) + .catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "unresolved stopped cleanup"); + const state = workspaceService as unknown as { + sessions: Map; + sessionSubscriptions: Map; + }; + expect(state.sessions.has(workspaceId)).toBe(false); + expect(state.sessionSubscriptions.has(workspaceId)).toBe(false); + expect(workspaceService.getOrCreateSession(workspaceId)).not.toBe(session); + } finally { + await cleanup(); + } + }); + + test.each([false, true])( + "no-op reset preserves a foreign Stop during cleanup (prior Stop=%s)", + async (priorStop) => { + const { config, workspaceService, cleanup } = await createServices(); + const workspaceId = "noop-reset-cancellation"; + await config.addWorkspace("/tmp/noop-reset-project", { + id: workspaceId, + name: workspaceId, + projectName: "noop-reset-project", + projectPath: "/tmp/noop-reset-project", + runtimeConfig: { type: "local" }, + }); + const session = workspaceService.getOrCreateSession(workspaceId); + if (priorStop) { + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + } + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const clear = session.clearPostCompactionState.bind(session); + spyOn(session, "clearPostCompactionState").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + await clear(); + }); + const reset = workspaceService.resetContext(workspaceId); + const foreignHistory = new HistoryService(config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config, + historyService: foreignHistory, + }); + try { + await entered.promise; + await foreignHistory.appendToHistory( + workspaceId, + createMuxMessage("foreign-summary", "assistant", "foreign context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue foreign", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + await foreign.session.interruptStream({ abandonPartial: true }); + await foreign.session.retryPendingCompactionCleanup(); + const stopped = await foreignHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await reset).toEqual(Ok("noop")); + expect(await new HistoryService(config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await reset; + await foreign.session.dispose(); + await cleanup(); + } + } + ); + + test("destructive replacement publishes its durable witness before reporting failed retirement", async () => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "replacement-retirement-witness"; + await config.addWorkspace("/tmp/replacement-witness-project", { + id: workspaceId, + name: workspaceId, + projectName: "replacement-witness-project", + projectPath: "/tmp/replacement-witness-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + const nonce = await session.getCompactionCancellationNonce(); + const events: WorkspaceChatMessage[] = []; + const detach = session.onChatEvent(({ message }) => events.push(message)); + const write = historyService.writeCompactionCancellation.bind(historyService); + const failedRetire = spyOn(historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("replacement retirement unavailable"); + return write(...args); + } + ); + try { + const result = await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue replacement", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + expect(!result.success && result.error).toContain("replacement retirement unavailable"); + const freshHistory = new HistoryService(config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.map((row) => [row.id, row.metadata?.compactionCancellationNonce]) + ).toEqual([["replacement", nonce]]); + expect( + events + .filter((event) => event.type === "delete" || event.type === "message") + .map((event) => event.type) + ).toEqual(["delete", "message"]); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).toHaveBeenCalledTimes(1); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + const tail = await freshHistory.getLastMessages(workspaceId, 1); + expect(tail.success && tail.data[0].parts).toMatchObject([ + { type: "text", text: "Continue replacement" }, + ]); + } finally { + await restarted.session.dispose(); + } + } finally { + detach(); + failedRetire.mockRestore(); + await cleanup(); + } + }); + + test.each([ + ["reset", false], + ["reset", true], + ["clear", false], + ["clear", true], + ["replace", false], + ["replace", true], + ] as const)( + "%s preserves a newer Stop after its durable write (prior Stop=%s)", + async (operation, priorStop) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "context-captured-cancellation"; + await config.addWorkspace("/tmp/context-cancellation-project", { + id: workspaceId, + name: workspaceId, + projectName: "context-cancellation-project", + projectPath: "/tmp/context-cancellation-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + if (priorStop) { + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + } + const oldNonce = await session.getCompactionCancellationNonce(); + let newNonce: string | undefined; + const stop = async () => { + await session.interruptStream({ abandonPartial: true }); + newNonce = await session.getCompactionCancellationNonce(); + }; + if (operation === "reset") { + const append = historyService.appendToHistory.bind(historyService); + spyOn(historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await stop(); + return result; + }); + } else if (operation === "clear") { + const truncate = historyService.truncateHistory.bind(historyService); + spyOn(historyService, "truncateHistory").mockImplementationOnce(async (...args) => { + const result = await truncate(...args); + await stop(); + return result; + }); + } else { + const clear = historyService.clearHistory.bind(historyService); + spyOn(historyService, "clearHistory").mockImplementationOnce(async (...args) => { + const result = await clear(...args); + await stop(); + return result; + }); + } + try { + const result = + operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "clear" + ? await workspaceService.truncateHistory(workspaceId) + : await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue replacement", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + expect(result.success).toBe(true); + await session.retryPendingCompactionCleanup(); + expect(newNonce).toBeDefined(); + expect(newNonce).not.toBe(oldNonce); + const freshHistory = new HistoryService(config); + expect((await freshHistory.readCompactionCancellation(workspaceId))?.nonce).toBe(newNonce); + if (operation === "replace") { + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data[0].metadata?.compactionCancellationNonce).toBe(oldNonce); + const restarted = await createAgentSessionHarness({ + workspaceId, + config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } + } + } finally { + await cleanup(); + } + } + ); + + test.each(["reset", "clear", "replace"] as const)( + "committed %s still publishes invalidation and hygiene when cancellation retirement fails", + async (operation) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "cancel-retirement-hygiene"; + await config.addWorkspace("/tmp/cancel-retirement-project", { + id: workspaceId, + name: workspaceId, + projectName: "cancel-retirement-project", + projectPath: "/tmp/cancel-retirement-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + const clearState = spyOn(session, "clearFileState"); + const clearCarryover = spyOn(session, "clearPostCompactionState"); + const discard = spyOn(sandboxHostService, "discardScope"); + const write = historyService.writeCompactionCancellation.bind(historyService); + const failedRetire = spyOn(historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("cancel unlink unavailable"); + return write(...args); + } + ); + const epochs = (workspaceService as unknown as { contextMutationEpochs: Map }) + .contextMutationEpochs; + const before = epochs.get(workspaceId) ?? 0; + try { + const result = + operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "clear" + ? await workspaceService.truncateHistory(workspaceId) + : await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context") + ); + expect(result.success).toBe(operation === "clear"); + if (operation === "clear") + expect(await historyService.readCompactionCancellation(workspaceId)).toMatchObject({ + retainUntilReplacement: true, + scope: { kind: "unresolved" }, + }); + else expect(!result.success && result.error).toContain("cancel unlink unavailable"); + expect(epochs.get(workspaceId)).toBe(before + 1); + if (operation === "replace") { + const persisted = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(persisted.success && persisted.data.map((row) => row.id)).toEqual(["replacement"]); + } + if (operation !== "replace") expect(clearState).toHaveBeenCalled(); + expect(clearCarryover).toHaveBeenCalled(); + expect(discard).toHaveBeenCalled(); + failedRetire.mockRestore(); + if (operation === "reset") { + expect(await workspaceService.resetContext(workspaceId)).toEqual(Ok("noop")); + expect(await historyService.readCompactionCancellation(workspaceId)).toBeNull(); + } + } finally { + failedRetire.mockRestore(); + await cleanup(); + } + } + ); + + test.each([ + "temporary", + "busy", + "preflight", + "failed partial discard", + "failed reset read", + ] as const)( + "temporary or failed context admission preserves Stop cleanup (%s)", + async (operation) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "stop-context-admission"; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let pending: Promise | undefined; + try { + await config.addWorkspace("/tmp/stop-context-project", { + id: workspaceId, + name: workspaceId, + projectName: "stop-context-project", + projectPath: "/tmp/stop-context-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Prior work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + const session = workspaceService.getOrCreateSession(workspaceId); + const internal = session as unknown as { + coordinator: TurnCoordinator; + dispatchPendingFollowUp(): Promise; + }; + await session.interruptStream({ abandonPartial: true }); + const update = historyService.updateHistory.bind(historyService); + spyOn(historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + pending = internal.dispatchPendingFollowUp(); + await entered.promise; + const token = internal.coordinator.compactionIntent.followUp; + if (!token) throw new Error("Expected stopped cleanup"); + if (operation === "busy") + spyOn(session, "hasActiveOrPendingTurnWork").mockReturnValueOnce(true); + const serviceState = workspaceService as unknown as { + preflightSendCounts: Map; + }; + if (operation === "preflight") serviceState.preflightSendCounts.set(workspaceId, 1); + if (operation === "failed partial discard") + spyOn(historyService, "deletePartial").mockResolvedValueOnce(Err("disk unavailable")); + if (operation === "failed reset read") + spyOn(historyService, "getHistoryFromLatestBoundary").mockResolvedValueOnce( + Err("disk unavailable") + ); + if (operation.startsWith("failed")) + expect((await workspaceService.resetContext(workspaceId)).success).toBe(false); + else { + const held = workspaceService.acquireIdleTurnExclusion(workspaceId); + expect(held.success).toBe(operation === "temporary"); + if (held.success) held.data[Symbol.dispose](); + } + serviceState.preflightSendCounts.delete(workspaceId); + expect(internal.coordinator.canClearCompactionFollowUp(token)).toBe(true); + release.resolve(); + expect(await pending).toBe(false); + const rows = await historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + } finally { + release.resolve(); + await pending?.catch(() => undefined); + await cleanup(); + } + } + ); + test("acquireIdleTurnExclusion refuses busy workspaces and blocks turn admission while held (r40)", async () => { // /refine publication rides this exclusion: it must fail closed when a // turn is active and, while held, refuse new turn admission so the diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index b21fdd796f2..1eda21d5d80 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -1817,6 +1817,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { private shuttingDown = false; private readonly shutdownSessions = new Set(); private readonly pendingWorkspaceCleanup = new Set>(); + // Retired sessions release all live resources, but their exact canceled-history + // obligations survive until I/O recovers. Never recreate a ready recovery owner first. + private readonly failedCompactionCleanups = new Map< + string, + { + owners: Set; + retry?: Promise; + } + >(); private readonly providerConfigChangedListener = (): void => { const liveSessions = new Map([ ...this.sessions.entries(), @@ -3123,14 +3132,31 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ - private advanceContextMutationEpoch(workspaceId: string): void { + private bumpContextMutationEpoch(workspaceId: string): void { this.contextMutationEpochs.set( workspaceId, (this.contextMutationEpochs.get(workspaceId) ?? 0) + 1 ); } + /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ + private async advanceContextMutationEpoch( + workspaceId: string, + cancellationNonce?: string | null + ): Promise> { + this.bumpContextMutationEpoch(workspaceId); + try { + await this.sessions.get(workspaceId)?.contextMutationCommitted(cancellationNonce); + return Ok(undefined); + } catch (error) { + // History already changed. Callers must finish its other cleanup before + // reporting a cancellation-retirement failure or releasing admission. + return Err( + `History changed, but canceled compaction state could not be retired: ${getErrorMessage(error)}` + ); + } + } + /** * Admission guard for context-discarding history mutations (r40): reject * new sends at the door (contextMutationWorkspaces), block turn admission @@ -4114,6 +4140,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { beginShutdown(): void { if (this.shuttingDown) return; this.shuttingDown = true; + for (const workspaceId of this.failedCompactionCleanups.keys()) { + this.retryFailedCompactionCleanup(workspaceId).catch((error: unknown) => + log.warn("Stopped compaction cleanup retry failed during shutdown", { workspaceId, error }) + ); + } // Capture before disposal can remove transient instances from either registry. for (const session of [ ...this.sessions.values(), @@ -4157,6 +4188,17 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return; } + if (this.hasBlockingCompactionCleanup(trimmed)) { + this.retryFailedCompactionCleanup(trimmed).then( + () => { + if (!this.shuttingDown) this.startStartupRecovery(trimmed); + }, + (error: unknown) => + log.warn("Stopped compaction cleanup retry failed", { workspaceId: trimmed, error }) + ); + return; + } + const existingSession = this.sessions.get(trimmed) ?? this.transientStartupRecoverySessions.get(trimmed); if (existingSession) { @@ -4193,8 +4235,27 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); } + private hasBlockingCompactionCleanup(workspaceId: string): boolean { + return [...(this.failedCompactionCleanups.get(workspaceId)?.owners ?? [])].some( + (owner) => owner.hasBlockingCompactionCleanup + ); + } + + private assertCompactionCleanupSettled(workspaceId: string): void { + if (this.failedCompactionCleanups.has(workspaceId)) { + this.retryFailedCompactionCleanup(workspaceId).catch((error: unknown) => + log.warn("Stopped compaction cleanup retry failed", { workspaceId, error }) + ); + if (this.hasBlockingCompactionCleanup(workspaceId)) + throw new Error( + "Stopped compaction cleanup is still pending. Please retry opening this workspace." + ); + } + } + private createSession(workspaceId: string): AgentSession { if (this.shuttingDown) throw new Error("Server is shutting down"); + this.assertCompactionCleanupSettled(workspaceId); return new AgentSession({ effectRunner: this.effectRunner, appFiberScope: this.appFiberScope, @@ -4216,7 +4277,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { args.workspacePath, args.runtimeConfig ), - onContextWindowRollover: () => this.advanceContextMutationEpoch(workspaceId), + onContextWindowRollover: () => this.bumpContextMutationEpoch(workspaceId), onCompactionComplete: (metadata) => { this.schedulePostCompactionMetadataRefresh(workspaceId); // Compaction marks a long session with accumulated learnings: harvest @@ -4337,7 +4398,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { assert(!this.sessions.has(workspaceId), `session already registered for ${workspaceId}`); if (this.transientStartupRecoverySessions.get(workspaceId) === session) { this.transientStartupRecoverySessions.delete(workspaceId); - } + } else this.assertCompactionCleanupSettled(workspaceId); this.sessions.set(workspaceId, session); this.attachSessionSubscriptions(workspaceId, session); @@ -4356,6 +4417,39 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return this.sessions.get(trimmed)?.countQueuedAgentPeerMessages() ?? 0; } + private retryFailedCompactionCleanup(workspaceId: string): Promise { + const obligation = this.failedCompactionCleanups.get(workspaceId); + if (!obligation) return Promise.resolve(); + if (obligation.retry) return obligation.retry; + const settled = Promise.withResolvers(); + obligation.retry = settled.promise; + let failure: PromiseRejectedResult | undefined; + this.trackWorkspaceCleanup(async () => { + const owners = [...obligation.owners]; + const results = await Promise.allSettled( + owners.map((owner) => owner.retryPendingCompactionCleanup()) + ); + for (const owner of owners) + if (!owner.hasPendingCompactionCleanup) obligation.owners.delete(owner); + if ( + obligation.owners.size === 0 && + this.failedCompactionCleanups.get(workspaceId) === obligation + ) + this.failedCompactionCleanups.delete(workspaceId); + failure = results.find((result) => result.status === "rejected"); + if (!failure && obligation.owners.size > 0) + failure = { + status: "rejected", + reason: new Error("Stopped compaction cleanup remains pending"), + }; + }).then(() => { + obligation.retry = undefined; + if (failure) settled.reject(failure.reason); + else settled.resolve(); + }, settled.reject); + return settled.promise; + } + public disposeSession(workspaceId: string): Promise { const trimmed = workspaceId.trim(); const transientSession = this.transientStartupRecoverySessions.get(trimmed); @@ -4366,10 +4460,24 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { clearTimeout(refreshTimer); this.postCompactionRefreshTimers.delete(trimmed); } + let failure: PromiseRejectedResult | undefined; return this.trackWorkspaceCleanup(async () => { // Start both synchronous admission latches before the first await. Registry identity // remains visible through terminal delivery and prevents a late old disposer removing B. - await Promise.all([transientSession?.dispose(), session?.dispose()]); + const settled = await Promise.allSettled([ + this.retryFailedCompactionCleanup(trimmed), + transientSession?.dispose(), + session?.dispose(), + ]); + for (const owner of [transientSession, session]) { + if (!owner?.hasPendingCompactionCleanup) continue; + let obligation = this.failedCompactionCleanups.get(trimmed); + if (!obligation) { + obligation = { owners: new Set() }; + this.failedCompactionCleanups.set(trimmed, obligation); + } + obligation.owners.add(owner); + } if (this.transientStartupRecoverySessions.get(trimmed) === transientSession) { this.transientStartupRecoverySessions.delete(trimmed); } @@ -4378,6 +4486,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (this.sessionSubscriptions.get(trimmed) === subscriptions) this.sessionSubscriptions.delete(trimmed); if (this.sessions.get(trimmed) === session) this.sessions.delete(trimmed); + // Failed durable cleanup must be reported after both old sessions release + // resources and identity-guarded registry teardown has completed. + failure = settled.find((result) => result.status === "rejected"); + }).then(() => { + // The guardian's physical cleanup promise remains fulfilled; the public + // caller still receives unresolved durable work after registries are clean. + if (failure) throw failure.reason; }); } @@ -11325,7 +11440,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const result = await session.sendMessage(message, continuationSendState.options, { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), onContextWindowRollover: () => { - this.advanceContextMutationEpoch(workspaceId); + this.bumpContextMutationEpoch(workspaceId); admissionEpoch = this.contextMutationEpochs.get(workspaceId) ?? 0; }, synthetic: internal?.synthetic, @@ -12522,6 +12637,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { refuseRowRemoval: truncationScope === "none", requireFullDelete: truncationScope === "all", }); + if (isFullClear) { + try { + await this.getOrCreateSession(workspaceId).fenceCompactionForContextClear(); + } catch (error) { + return Err( + `Cannot clear history: compaction cancellation could not be persisted (${getErrorMessage(error)})` + ); + } + } const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { @@ -12534,9 +12658,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). - if (isFullClear) { - this.advanceContextMutationEpoch(workspaceId); - } + const cancellationRetirement = isFullClear + ? await this.advanceContextMutationEpoch(workspaceId, null) + : Ok(undefined); // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the // registration too or the next send would re-emit the discarded summary @@ -12618,7 +12742,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - return Ok(undefined); + return cancellationRetirement; } async resetContext(workspaceId: string): Promise> { @@ -12683,6 +12807,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + const cancellationNonce = + await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce(); const historyResult = await this.historyService.getHistoryFromLatestBoundary(workspaceId); if (!historyResult.success) { return Err(`Failed to read active context before reset: ${historyResult.error}`); @@ -12723,6 +12849,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { `may reappear after a restart; retry once the session storage is writable.` ); } + try { + await this.getOrCreateSession(workspaceId).retireCompactionCancellation( + cancellationNonce ?? null + ); + } catch (error) { + return Err( + `Nothing to reset, but canceled compaction state could not be retired: ${getErrorMessage(error)}` + ); + } return Ok("noop"); } @@ -12733,6 +12868,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { { timestamp: Date.now(), contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + compactionCancellationNonce: cancellationNonce, } ); @@ -12743,7 +12879,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the boundary is durable — sends that entered before it must not // be admitted afterwards (their content references the discarded // context). - this.advanceContextMutationEpoch(workspaceId); + const cancellationRetirement = await this.advanceContextMutationEpoch( + workspaceId, + boundaryMessage.metadata?.compactionCancellationNonce ?? null + ); // r43: drop any settled-but-unconsumed branch-summary registration — // its row now sits behind the new boundary, and the next send would // otherwise re-emit that pre-reset summary into the live transcript. @@ -12787,6 +12926,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + if (!cancellationRetirement.success) return cancellationRetirement; return Ok("reset"); } finally { admissionGuard[Symbol.dispose](); @@ -12823,6 +12963,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { try { let messageToAppend = summaryMessage; let deletedSequences: number[] = []; + let cancellationRetirement: Result = Ok(undefined); if (replaceMode === "append-compaction-boundary") { assert( @@ -12938,6 +13079,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + const cancellationNonce = !isCompaction + ? ((await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce()) ?? null) + : null; + if (cancellationNonce) { + // The replacement row itself proves supersession if sidecar retirement + // fails. Capture before destructive I/O so a newer Stop keeps its nonce. + messageToAppend = { + ...messageToAppend, + metadata: { + ...messageToAppend.metadata, + compactionCancellationNonce: cancellationNonce, + }, + }; + } this.sessions.get(workspaceId)?.clearUsageState(); const clearResult = await this.clearHistoryWithRetiredBashMonitorWakes( workspaceId, @@ -12950,7 +13105,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (!isCompaction) { // r41: the destructive replacement is durable — refuse sends that // entered before it (see contextMutationEpochs). - this.advanceContextMutationEpoch(workspaceId); + cancellationRetirement = await this.advanceContextMutationEpoch( + workspaceId, + cancellationNonce + ); // r43: same branch-summary hygiene as full clear, and same r44 // ordering — drop the registration only after the clear commits // (see truncateHistory). @@ -13042,7 +13200,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.sessions.get(workspaceId)?.clearFileState(); } - return Ok(undefined); + return cancellationRetirement; } catch (error) { const message = getErrorMessage(error); return Err(`Failed to replace history: ${message}`); diff --git a/tests/ui/compaction/compaction.test.ts b/tests/ui/compaction/compaction.test.ts index f2aa78c4efa..e6bd0becc47 100644 --- a/tests/ui/compaction/compaction.test.ts +++ b/tests/ui/compaction/compaction.test.ts @@ -9,7 +9,7 @@ import "../dom"; import { waitFor } from "@testing-library/react"; -import { preloadTestModules, type TestEnvironment } from "../../ipc/setup"; +import { preloadTestModules, setupProviders, type TestEnvironment } from "../../ipc/setup"; import { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; @@ -19,6 +19,7 @@ import { WORKSPACE_DEFAULTS } from "@/constants/workspaceDefaults"; import { updatePersistedState } from "@/browser/hooks/usePersistedState"; import { getAutoCompactionThresholdKey } from "@/common/constants/storage"; import { workspaceStore } from "@/browser/stores/WorkspaceStore"; +import { KNOWN_MODELS } from "@/common/constants/knownModels"; interface ServiceContainerPrivates { backgroundProcessManager: BackgroundProcessManager; @@ -123,7 +124,17 @@ describe("Compaction UI (mock AI router)", () => { }, 60_000); test("auto-compacts after context_exceeded and resumes", async () => { - const app = await createAppHarness({ branchPrefix: "compaction-ui" }); + const app = await createAppHarness({ + branchPrefix: "compaction-ui", + beforeRenderEnvironment: async (env) => { + // Auto-recovery needs an available compaction model even with the mock router; + // make that independent of local credentials and the catalog's context-window ordering. + await setupProviders(env, { anthropic: { apiKey: "dummy" } }); + await env.orpc.config.updateAgentAiDefaults({ + agentAiDefaults: { compact: { modelString: KNOWN_MODELS.HAIKU.id } }, + }); + }, + }); try { const triggerMessage = "Trigger context error";