From 44d48d3b8d507d8acf36521421188afe37cd60bd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 13:39:00 +0200 Subject: [PATCH 01/17] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20coordinate=20c?= =?UTF-8?q?ompaction=20and=20durable=20continuations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction work can finish after its originating turn has been canceled or replaced. Give compaction observations and durable continuation handoffs explicit coordinator ownership so late work cannot clear a replacement's state, replay a continuation after Stop, or report a handoff that never reached durable acceptance. - Track semantic compaction intent, observation stages, and handoff tokens in the turn coordinator. A continuation preserves its ownership across its own admission; unrelated manual or edited turns retire it. - Keep original eager summary jobs and journal cleanup physically leased through reset and shutdown. Reuse the existing journal queue, and guard completion callbacks against retired intent. - Guard summary updates and heartbeat-boundary cleanup under the existing history lock and at final publication. Serialize pending-state writes and unlinks, and detach rollback snapshots before awaiting I/O. - Count a durable continuation as accepted only when its acceptance callback runs. Preserve ordinary continuation queue priority and the existing manual-input vetoes for optional and goal continuations. - 2,039 backend tests passed across session/compaction, history, workspace/task/container, and CLI suites. - 15 UI tests passed across compaction, interruption, and send modes; one existing compaction test remains skipped. - Deterministic regressions cover held preparation, summary generation, journal clearing, history reads/writes, Stop, replacement, rollback, and successful sends that never reach acceptance. - The automatic-compaction UI fixture now explicitly configures its mock provider and compaction model. The original fixture timed out on unchanged main without those prerequisites; the corrected fixture passes on both main and this branch. - The separate credential-backed context-limit UI suite could not run locally because `OPENAI_API_KEY` is unavailable. The mock-provider compaction suite covers context-limit recovery. This changes ownership across continuous, legacy, and heartbeat compaction and startup replay. Cancellation ordering, manual-turn priority, and durable cleanup are the primary regression risks; the held-I/O and UI tests exercise those boundaries. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I8ff308b1367d223a86fe3914da56c8e928f334df --- .../agentSession.compactionAcceptance.test.ts | 89 ++++++ ...gentSession.continueMessageAgentId.test.ts | 265 +++++++++++++++-- .../agentSession.continuousCompaction.test.ts | 76 ++++- ...tSession.drainQueuedMessagesIfIdle.test.ts | 4 +- .../agentSession.goalAutoPause.test.ts | 118 ++++---- .../agentSession.scopedLifetimes.test.ts | 113 +++++++ src/node/services/agentSession.ts | 277 ++++++++++++------ .../compactionHandler.continuous.test.ts | 69 ++++- src/node/services/compactionHandler.ts | 70 +++-- src/node/services/continuousCompactor.test.ts | 25 ++ src/node/services/continuousCompactor.ts | 37 ++- src/node/services/historyService.ts | 81 +++-- src/node/services/turnCoordinator.test.ts | 31 ++ src/node/services/turnCoordinator.ts | 177 ++++++++++- tests/ui/compaction/compaction.test.ts | 15 +- 15 files changed, 1218 insertions(+), 229 deletions(-) create mode 100644 src/node/services/agentSession.compactionAcceptance.test.ts diff --git a/src/node/services/agentSession.compactionAcceptance.test.ts b/src/node/services/agentSession.compactionAcceptance.test.ts new file mode 100644 index 00000000000..92d6d324b59 --- /dev/null +++ b/src/node/services/agentSession.compactionAcceptance.test.ts @@ -0,0 +1,89 @@ +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { createMuxMessage } from "@/common/types/message"; +import * as branchSummary from "./branchSummary"; +import { createAgentSessionHarness } from "./agentSession.testHarness"; + +afterEach(() => mock.restore()); + +test("disposal during real pre-acceptance preparation does not report a continued handoff", async () => { + const workspaceId = "unaccepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + spyOn(branchSummary, "awaitPendingBranchSummary").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return null; + }); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + const pending = dispatch.dispatchPendingFollowUp(); + let disposal: Promise | undefined; + try { + await entered.promise; + disposal = h.session.dispose(); + release.resolve(); + expect(await pending).toBe(false); + await disposal; + expect(stream).not.toHaveBeenCalled(); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((message) => message.role === "user")).toBe(false); + } finally { + release.resolve(); + await pending; + await disposal; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("an accepted real handoff stays continued after a subsequent manual replacement", async () => { + const workspaceId = "accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }) + ); + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementationOnce(async (...args) => { + const result = await send(...args); + expect(result.success).toBe(true); + await h.session.interruptStream(); + expect((await send("manual replacement", options)).success).toBe(true); + return result; + }); + try { + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + expect(await dispatch.dispatchPendingFollowUp()).toBe(true); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && + history.data.filter((message) => message.role === "user").map((message) => message.parts) + ).toMatchObject([ + [{ type: "text", text: "Continue" }], + [{ type: "text", text: "manual replacement" }], + ]); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index 6390fc0178a..d7e54d9a4a3 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -1,5 +1,5 @@ import type { TurnCoordinator } from "./turnCoordinator"; -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; @@ -26,16 +26,31 @@ interface AutoRetryResumeRequest { } interface SessionInternals { - dispatchPendingFollowUp: () => Promise; + coordinator: TurnCoordinator; + dispatchPendingFollowUp: (summaryId?: string) => Promise; sendMessage: ( message: string, options?: SendOptions, - internal?: { synthetic?: boolean; agentInitiated?: boolean } + internal?: { + synthetic?: boolean; + agentInitiated?: boolean; + onAccepted?: () => void | Promise; + } ) => Promise; runStartupRecovery: () => Promise; lastAutoRetryResumeRequest?: AutoRetryResumeRequest; } +function mockAcceptedSend( + send: SessionInternals["sendMessage"] = () => Promise.resolve({ success: true }) +) { + return mock(async (...args: Parameters) => { + const result = await send(...args); + if (result.success) await args[2]?.onAccepted?.(); + return result; + }); +} + const idleFollowUp = (): CompactionFollowUpRequest => ({ text: "heartbeat follow-up", model: "openai:gpt-4o", @@ -187,7 +202,7 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-1", legacyFollowUp), ]); - internals.sendMessage = mock( + internals.sendMessage = mockAcceptedSend( ( message: string, options?: SendOptions, @@ -207,6 +222,216 @@ describe("AgentSession continue-message agentId fallback", () => { expect(dispatchedInternal?.synthetic).toBe(true); }); + test.each([false, true])( + "a completed manual replacement retires a held follow-up read (targeted=%s)", + async (targeted) => { + const { internals, historyService } = await createSession([ + compactionSummaryMessage("held-summary", { + text: "old request", + agentId: "exec", + model: "openai:gpt-4o", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = targeted ? "getHistoryFromLatestBoundary" : "getLastMessages"; + const original = historyService[read].bind(historyService); + spyOn(historyService, read).mockImplementationOnce(async (workspaceId: string) => { + const result = await original(workspaceId, 1); + entered.resolve(); + await release.promise; + return result; + }); + const send = mockAcceptedSend(() => Promise.resolve({ success: true as const })); + internals.sendMessage = send; + const pending = internals.dispatchPendingFollowUp(targeted ? "held-summary" : undefined); + try { + await entered.promise; + const admitted = internals.coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: internals.coordinator.turnId, + }); + expect(admitted.status).toBe("admitted"); + if (admitted.status !== "admitted") throw new Error("Expected replacement"); + await historyService.appendToHistory("ws", createMuxMessage("manual", "user", "new work")); + internals.coordinator.finishTurn(admitted.turnId); + release.resolve(); + expect(await pending).toBe(false); + expect(send).not.toHaveBeenCalled(); + expect(internals.lastAutoRetryResumeRequest).toBeUndefined(); + } finally { + release.resolve(); + await pending; + } + } + ); + + test("concurrent durable dispatch has one owner and preserves ordinary request priority", async () => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("held-summary", { + text: "interrupted request", + agentId: "exec", + model: "openai:gpt-4o", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const send = mockAcceptedSend(async () => { + entered.resolve(); + await release.promise; + await historyService.appendToHistory( + "ws", + createMuxMessage("accepted", "user", "interrupted request") + ); + return { success: true as const }; + }); + internals.sendMessage = send; + session.queueMessage("later manual work", { agentId: "exec", model: "openai:gpt-4o" }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + expect(await internals.dispatchPendingFollowUp()).toBe(false); + release.resolve(); + expect(await pending).toBe(true); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(send).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await pending; + } + }); + + test("a delayed follow-up cleanup cannot overwrite a replacement summary", async () => { + const summary = compactionSummaryMessage("summary", { + text: "obsolete goal", + goalKind: "goal_continuation", + agentId: "exec", + model: "openai:gpt-4o", + }); + const { session, internals, historyService } = await createSession([summary]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = historyService.updateHistory.bind(historyService); + spyOn(historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + using _mutation = session.holdTurnAdmission(); + expect( + ( + await update("ws", { + ...summary, + parts: [{ type: "text", text: "replacement summary" }], + metadata: { + ...summary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "replacement request", + agentId: "exec", + model: "openai:gpt-4o", + }, + }, + }, + }) + ).success + ).toBe(true); + release.resolve(); + expect(await pending).toBe(false); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].parts).toEqual([ + { type: "text", text: "replacement summary" }, + ]); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement request" + ); + } finally { + release.resolve(); + await pending; + } + }); + + test("a delayed heartbeat rollback cannot delete a replacement context", async () => { + const summary = heartbeatBoundaryMessage(); + const { session, internals, historyService } = await createSession([summary]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const remove = historyService.deleteMessage.bind(historyService); + spyOn(historyService, "deleteMessage").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return remove(...args); + }); + session.queueMessage("manual replacement", { agentId: "exec", model: "openai:gpt-4o" }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + using _mutation = session.holdTurnAdmission(); + expect( + ( + await historyService.updateHistory("ws", { + ...summary, + parts: [{ type: "text", text: "new context" }], + }) + ).success + ).toBe(true); + release.resolve(); + expect(await pending).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary("ws"); + expect(history.success && history.data[0].parts).toEqual([ + { type: "text", text: "new context" }, + ]); + } finally { + release.resolve(); + await pending; + } + }); + + test("Stop during a held summary read clears the canceled durable handoff", async () => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("summary", { + text: "resume", + model: "openai:gpt-4o", + agentId: "exec", + }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = historyService.getHistoryFromLatestBoundary.bind(historyService); + spyOn(historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const send = mockAcceptedSend(() => Promise.resolve({ success: true as const })); + internals.sendMessage = send; + const pending = internals.dispatchPendingFollowUp("summary"); + try { + await entered.promise; + await session.interruptStream({ abandonPartial: true }); + release.resolve(); + expect(await pending).toBe(false); + expect(send).not.toHaveBeenCalled(); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + } finally { + release.resolve(); + await pending; + } + }); + test("dispatchPendingFollowUp aliases legacy exclusive-PTC experiments", async () => { // An older build can persist {programmaticToolCalling: false, // programmaticToolCallingExclusive: true}; dispatch copies raw persisted @@ -227,7 +452,7 @@ describe("AgentSession continue-message agentId fallback", () => { }, }), ]); - internals.sendMessage = mock((_message: string, options?: SendOptions) => { + internals.sendMessage = mockAcceptedSend((_message: string, options?: SendOptions) => { dispatchedOptions = options; return Promise.resolve({ success: true as const }); }); @@ -248,7 +473,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentInitiated: true, }), ]); - internals.sendMessage = mock( + internals.sendMessage = mockAcceptedSend( ( _message: string, _options?: SendOptions, @@ -275,7 +500,7 @@ describe("AgentSession continue-message agentId fallback", () => { strictAgentResolution: true, }), ]); - internals.sendMessage = mock((_message: string, options?: SendOptions) => { + internals.sendMessage = mockAcceptedSend((_message: string, options?: SendOptions) => { dispatchedOptions = options; return Promise.resolve({ success: true as const }); }); @@ -312,7 +537,7 @@ describe("AgentSession continue-message agentId fallback", () => { ], archivedConfig ); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); const dispatched = await internals.dispatchPendingFollowUp(); @@ -330,7 +555,7 @@ describe("AgentSession continue-message agentId fallback", () => { const { session, historyService, internals } = await createSession([ compactionSummaryMessage("summary-idle-only", idleFollowUp()), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); session.queueMessage( "user returned", { model: "openai:gpt-4o", agentId: "exec" }, @@ -356,7 +581,7 @@ describe("AgentSession continue-message agentId fallback", () => { earlierMessage, heartbeatBoundaryMessage(), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); session.queueMessage( "user returned", { model: "openai:gpt-4o", agentId: "exec" }, @@ -386,7 +611,7 @@ describe("AgentSession continue-message agentId fallback", () => { earlierMessage, heartbeatBoundaryMessage(), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); (session as unknown as { hasExternalSendPreflight?: () => boolean }).hasExternalSendPreflight = () => true; @@ -408,7 +633,7 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-active-turn", idleFollowUp()), ]); const busyInternals = internals as SessionInternals & { isBusy: () => boolean }; - busyInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + busyInternals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); busyInternals.isBusy = () => true; const dispatched = await busyInternals.dispatchPendingFollowUp(); @@ -427,7 +652,7 @@ describe("AgentSession continue-message agentId fallback", () => { test("dispatchPendingFollowUp keeps heartbeat reset boundaries once a non-idle turn has started", async () => { const { historyService, internals } = await createSession([heartbeatBoundaryMessage()]); const busyInternals = internals as SessionInternals & { isBusy: () => boolean }; - busyInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + busyInternals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); busyInternals.isBusy = () => true; const dispatched = await busyInternals.dispatchPendingFollowUp(); @@ -449,7 +674,9 @@ describe("AgentSession continue-message agentId fallback", () => { compactionSummaryMessage("summary-completing-turn", idleFollowUp()), ]); const completingInternals = internals as SessionInternals & { coordinator: TurnCoordinator }; - completingInternals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + completingInternals.sendMessage = mockAcceptedSend(() => + Promise.resolve({ success: true as const }) + ); completingInternals.coordinator.beginPolicy(completingInternals.coordinator.turnId); const dispatched = await completingInternals.dispatchPendingFollowUp(); @@ -478,7 +705,7 @@ describe("AgentSession continue-message agentId fallback", () => { }, agentInitiated: true, }; - internals.sendMessage = mock(() => + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: false as const, error: { type: "runtime_start_failed", message: "startup failed" }, @@ -544,7 +771,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ]); - internals.sendMessage = mock(() => { + internals.sendMessage = mockAcceptedSend(() => { sendCount += 1; return Promise.resolve({ success: true as const }); }); @@ -563,7 +790,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ]); - internals.sendMessage = mock(() => { + internals.sendMessage = mockAcceptedSend(() => { sendCount += 1; if (sendCount === 1) { return Promise.resolve({ @@ -596,7 +823,7 @@ describe("AgentSession continue-message agentId fallback", () => { preservedTailCopy("tail-copy-1", "user", "original user message"), preservedTailCopy("tail-copy-2", "assistant", "original assistant reply"), ]); - internals.sendMessage = mock((message: string) => { + internals.sendMessage = mockAcceptedSend((message: string) => { dispatchedMessage = message; return Promise.resolve({ success: true as const }); }); @@ -621,7 +848,7 @@ describe("AgentSession continue-message agentId fallback", () => { createMuxMessage("post-compaction-turn", "assistant", "new turn after compaction"), preservedTailCopy("tail-copy-2", "assistant", "trailing copy"), ]); - internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = mockAcceptedSend(() => Promise.resolve({ success: true as const })); const dispatched = await internals.dispatchPendingFollowUp(); diff --git a/src/node/services/agentSession.continuousCompaction.test.ts b/src/node/services/agentSession.continuousCompaction.test.ts index 6d442b229c6..2cdc81344a0 100644 --- a/src/node/services/agentSession.continuousCompaction.test.ts +++ b/src/node/services/agentSession.continuousCompaction.test.ts @@ -24,6 +24,7 @@ import { type AgentSessionHarness, } from "./agentSession.testHarness"; import type { ContinuousCompactor } from "./continuousCompactor"; +import type { CompactionToken, TurnCoordinator } from "./turnCoordinator"; const workspaceId = "continuous-session"; const model = "openai:gpt-4o"; @@ -34,6 +35,10 @@ const sendOptions: SendMessageOptions = { }; interface SessionInternals { + coordinator: TurnCoordinator; + runContinuousCompactionObservation( + observe: (token: CompactionToken) => Promise + ): Promise; continuousCompactor: ContinuousCompactor; activeStreamContext?: { modelString: string; @@ -42,7 +47,8 @@ interface SessionInternals { }; finishContinuousCompaction: ( applied: boolean, - context: NonNullable + context: NonNullable, + token: CompactionToken ) => Promise; interruptForContinuousCompaction: ( apply: (followUp?: CompactionFollowUpRequest) => Promise @@ -60,9 +66,13 @@ async function applyThenFinish( const state = internals(session); const context = state.activeStreamContext; if (!context) throw new Error("Expected active stream context"); - const applied = await state.interruptForContinuousCompaction(apply); - await state.finishContinuousCompaction(applied, context); - return applied; + return ( + (await state.runContinuousCompactionObservation(async (token) => { + const applied = await state.interruptForContinuousCompaction(apply); + await state.finishContinuousCompaction(applied, context, token); + return applied; + })) ?? false + ); } function deferred() { @@ -244,16 +254,22 @@ describe("AgentSession continuous compaction wiring", () => { await h.historyService.writePartial(workspaceId, source); streaming = false; if (mode === "failed-consumed-apply") { - Reflect.set(h.session, "continuousCompactionStopped", true); + const token = internals(h.session).coordinator.beginCompactionObservation("continuous"); + assert(token, "Expected observation"); + internals(h.session).coordinator.setCompactionStage(token, "stopped"); const reset = spyOn(compactor, "reset"); - await internals(h.session).finishContinuousCompaction(false, { - modelString: model, - options: sendOptions, - providersConfig: null, - }); + await internals(h.session).finishContinuousCompaction( + false, + { + modelString: model, + options: sendOptions, + providersConfig: null, + }, + token + ); expect(reset).not.toHaveBeenCalled(); expect(await store.read()).not.toBeNull(); - Reflect.set(h.session, "continuousCompactionStopped", false); + internals(h.session).coordinator.finishCompactionObservation(token); } if (mode !== "startup") { const terminal = Reflect.get(h.session, "observeContinuousCompactionAtStreamEnd") as ( @@ -499,6 +515,36 @@ describe("AgentSession continuous compaction wiring", () => { }); }); + test("a late durable completion cannot replace a successor's summary target", async () => { + const h = await setup(); + const handler = (h.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const tail = createMuxMessage("tail", "user", "Retain this request"); + await h.historyService.appendToHistory(workspaceId, tail); + const history = await rows(h); + const persist = h.historyService.persistBoundaryWithTailCopies.bind(h.historyService); + spyOn(h.historyService, "persistBoundaryWithTailCopies").mockImplementationOnce( + async (...args) => { + const result = await persist(...args); + internals(h.session).coordinator.invalidateCompaction(); + internals(h.session).coordinator.recordCompactionSummary("successor-summary"); + return result; + } + ); + expect( + await handler.persistContinuousCompaction({ + messages: history, + text: "Old summary", + model, + tail: [tail], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + }) + ).toBe(true); + expect(internals(h.session).coordinator.compactionIntent.summaryId).toBe("successor-summary"); + }); + test("invalidation contains an initial wait rejection and safely stops the blocked stream", async () => { const h = await setup(); const compactor = internals(h.session).continuousCompactor; @@ -574,10 +620,10 @@ describe("AgentSession continuous compaction wiring", () => { const invoked = deferred(); const drain = spyOn(h.session, "drainQueuedMessagesIfIdle").mockImplementation(() => undefined); const run = Reflect.get(h.session, "runContinuousCompactionObservation") as ( - observe: () => Promise + observe: (token: CompactionToken) => Promise ) => Promise; - const first = run.call(h.session, async () => { - Reflect.set(h.session, "midStreamCompactionPending", true); + const first = run.call(h.session, async (token) => { + internals(h.session).coordinator.setCompactionStage(token, "stopping"); await release.promise; }); const observe = spyOn(internals(h.session).continuousCompactor, "observe").mockImplementation( @@ -805,7 +851,6 @@ describe("AgentSession continuous compaction wiring", () => { spyOn(internals(h.session).continuousCompactor, "waitForIdle").mockReturnValueOnce( observationFinished.promise ); - Reflect.set(h.session, "continuousCompactionObserving", true); } h.aiEmitter.emit(eventType, { type: eventType, @@ -815,7 +860,6 @@ describe("AgentSession continuous compaction wiring", () => { }); if (eventType === "prefix-swap-invalidated") { expect(order).toEqual([]); - Reflect.set(h.session, "continuousCompactionObserving", false); observationFinished.resolve(); } await resumed.promise; diff --git a/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts b/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts index 1a1e7306458..5532e1b8e67 100644 --- a/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts +++ b/src/node/services/agentSession.drainQueuedMessagesIfIdle.test.ts @@ -92,7 +92,9 @@ describe("AgentSession.drainQueuedMessagesIfIdle", () => { [ "a pending mid-stream compaction owns the next dispatch", (s) => { - s.midStreamCompactionPending = true; + const token = s.coordinator.beginCompactionObservation("legacy"); + if (!token) throw new Error("Expected compaction owner"); + s.coordinator.setCompactionStage(token, "stopping"); }, 0, ], diff --git a/src/node/services/agentSession.goalAutoPause.test.ts b/src/node/services/agentSession.goalAutoPause.test.ts index 36b3711c98e..5b20be47a02 100644 --- a/src/node/services/agentSession.goalAutoPause.test.ts +++ b/src/node/services/agentSession.goalAutoPause.test.ts @@ -485,65 +485,71 @@ describe("AgentSession goal safety hooks", () => { await session.dispose(); }); - test("a recovered budget wrap-up follow-up installs its missing reservation", async () => { - // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): a crash between wrap-up send - // acceptance and tryMarkBudgetLimitInjected leaves the goal unmarked. - // The redispatched follow-up must install the reservation or the - // recovered stream's end arms a second wrap-up. - const workspaceId = "compaction-followup-wrapup-reservation"; - const { session, goalService, historyService, cleanup } = - await createSessionHarness(workspaceId); - cleanups.push(cleanup); - const created = await setGoalOk(goalService, { - workspaceId, - objective: "Recover the owed wrap-up", - budgetCents: 100, - }); - await goalService.recordStreamAccounting({ - workspaceId, - costUsd: 1.25, - streamStartedAtMs: created.createdAtMs + 1, - streamOriginKind: "goal_continuation", - }); - expect(await goalService.getGoal(workspaceId)).toMatchObject({ - status: "budget_limited", - budgetLimitInjectedForGoalId: null, - }); - const summary = createMuxMessage( - `summary-${crypto.randomUUID()}`, - "assistant", - "Compacted conversation.", - { - muxMetadata: { - type: "compaction-summary", - pendingFollowUp: { - text: "Wrap up the budget-limited goal.", - agentId: "exec", - model: "openai:gpt-4o", - goalKind: GOAL_BUDGET_LIMIT_KIND, - goalId: created.goalId, + test.each([true, false])( + "a recovered wrap-up reserves only an accepted handoff (accepted=%s)", + async (accepted) => { + // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): a crash between wrap-up send + // acceptance and tryMarkBudgetLimitInjected leaves the goal unmarked. + // The redispatched follow-up must install the reservation or the + // recovered stream's end arms a second wrap-up. + const workspaceId = "compaction-followup-wrapup-reservation"; + const { session, goalService, historyService, cleanup } = + await createSessionHarness(workspaceId); + cleanups.push(cleanup); + const created = await setGoalOk(goalService, { + workspaceId, + objective: "Recover the owed wrap-up", + budgetCents: 100, + }); + await goalService.recordStreamAccounting({ + workspaceId, + costUsd: 1.25, + streamStartedAtMs: created.createdAtMs + 1, + streamOriginKind: "goal_continuation", + }); + expect(await goalService.getGoal(workspaceId)).toMatchObject({ + status: "budget_limited", + budgetLimitInjectedForGoalId: null, + }); + const summary = createMuxMessage( + `summary-${crypto.randomUUID()}`, + "assistant", + "Compacted conversation.", + { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Wrap up the budget-limited goal.", + agentId: "exec", + model: "openai:gpt-4o", + goalKind: GOAL_BUDGET_LIMIT_KIND, + goalId: created.goalId, + }, }, - }, - } - ); - expect((await historyService.appendToHistory(workspaceId, summary)).success).toBe(true); - const sendSpy = spyOn(session, "sendMessage").mockImplementation(() => - Promise.resolve(Ok(undefined)) - ); + } + ); + expect((await historyService.appendToHistory(workspaceId, summary)).success).toBe(true); + const sendSpy = spyOn(session, "sendMessage").mockImplementation( + async (_message, _options, internal) => { + if (accepted) await internal?.onAccepted?.(); + return Ok(undefined); + } + ); - const dispatched = await ( - session as unknown as { dispatchPendingFollowUp: (id?: string) => Promise } - ).dispatchPendingFollowUp(); + const dispatched = await ( + session as unknown as { dispatchPendingFollowUp: (id?: string) => Promise } + ).dispatchPendingFollowUp(); - expect(dispatched).toBe(true); - expect(sendSpy).toHaveBeenCalledTimes(1); - expect(await goalService.getGoal(workspaceId)).toMatchObject({ - status: "budget_limited", - budgetLimitInjectedForGoalId: created.goalId, - }); - sendSpy.mockRestore(); - await session.dispose(); - }); + expect(dispatched).toBe(accepted); + expect(sendSpy).toHaveBeenCalledTimes(1); + expect(await goalService.getGoal(workspaceId)).toMatchObject({ + status: "budget_limited", + budgetLimitInjectedForGoalId: accepted ? created.goalId : null, + }); + sendSpy.mockRestore(); + await session.dispose(); + } + ); test("malformed persisted follow-up goal IDs are discarded during recovery", async () => { const workspaceId = "compaction-followup-malformed-goal-id"; diff --git a/src/node/services/agentSession.scopedLifetimes.test.ts b/src/node/services/agentSession.scopedLifetimes.test.ts index 6425c41520a..c56ea401220 100644 --- a/src/node/services/agentSession.scopedLifetimes.test.ts +++ b/src/node/services/agentSession.scopedLifetimes.test.ts @@ -3,11 +3,124 @@ import { Effect, Exit, Scope } from "effect"; import { Err } from "@/common/types/result"; import { defaultEffectRunner as runner } from "./di/effectRunner"; import { createAgentSessionHarness } from "./agentSession.testHarness"; +import type { ContinuousCompactor } from "./continuousCompactor"; +import { createMuxMessage } from "@/common/types/message"; const workspaceId = "scoped-turn"; const options = { model: "openai:gpt-4o", agentId: "exec" }; describe("AgentSession scoped turn lifetimes", () => { + test("a canceled non-cooperative summary stays joined while its replacement applies", async () => { + const h = await createAgentSessionHarness({ workspaceId }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const { continuousCompactor: compactor } = h.session as unknown as { + continuousCompactor: ContinuousCompactor; + }; + const { deps } = compactor as unknown as { + deps: ConstructorParameters[0]; + }; + const summarize = spyOn(deps, "summarize") + .mockImplementationOnce(async (_head, signal) => { + entered.resolve(); + await release.promise; + expect(signal.aborted).toBe(true); + return { text: "obsolete summary", model: options.model }; + }) + .mockResolvedValue({ text: "replacement summary", model: options.model }); + const context = { + enabled: true, + model: options.model, + contextWindowTokens: 100_000, + thresholdPercent: 80, + phase: "on-send" as const, + }; + let closing: Promise | undefined; + try { + for (const row of [ + createMuxMessage("old-user", "user", "Investigate"), + createMuxMessage("old-answer", "assistant", "earlier investigation ".repeat(4_000)), + createMuxMessage("recent-user", "user", "Implement"), + createMuxMessage("recent-answer", "assistant", "Done"), + ]) + expect((await h.historyService.appendToHistory(workspaceId, row)).success).toBe(true); + await compactor.observe(75, context); + await entered.promise; + { + using _mutation = h.session.holdTurnAdmission(); + } + expect(h.session.isBusy()).toBe(false); + await compactor.observe(75, context); + const replacement = (compactor as unknown as { job: { done: Promise } }).job; + await replacement.done; + expect(await compactor.observe(80, context)).toBe("applied"); + expect(summarize).toHaveBeenCalledTimes(2); + let closed = false; + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await runner.runPromise(Effect.yieldNow); + expect(closed).toBe(false); + release.resolve(); + await closing; + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data[0].parts).toMatchObject([ + { type: "text", text: "replacement summary" }, + ]); + } finally { + release.resolve(); + await closing; + await h.session.dispose(); + await h.cleanup(); + mock.restore(); + } + }); + + test("shutdown joins an eager job whose prepare synchronously closes the session", async () => { + const h = await createAgentSessionHarness({ workspaceId }); + const stream = spyOn(h.aiService, "streamMessage"); + const release = Promise.withResolvers(); + const { continuousCompactor } = h.session as unknown as { + continuousCompactor: ContinuousCompactor; + }; + const { deps } = continuousCompactor as unknown as { + deps: { prepare(): Promise }; + }; + spyOn(deps, "prepare").mockImplementation(() => { + h.session.beginShutdown(); + return release.promise; + }); + let closed = false; + let closing: Promise | undefined; + try { + await continuousCompactor.observe(75, { + enabled: true, + model: options.model, + contextWindowTokens: 100_000, + thresholdPercent: 80, + phase: "on-send", + }); + // Reset detaches the job immediately; its original preparation still owns I/O. + expect(h.session.closingSignal.aborted).toBe(true); + expect(h.session.isBusy()).toBe(false); + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await runner.runPromise(Effect.yieldNow); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(closed).toBe(true); + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await closing; + await h.session.dispose(); + await h.cleanup(); + mock.restore(); + } + }); + test("queue clear publication cannot outrun its cancellation refund", async () => { const appFiberScope = Scope.makeUnsafe("parallel"); const entered = Promise.withResolvers(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 988e9cc8fcf..f07d5b51d35 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -10,6 +10,7 @@ import type { Dirent } from "fs"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import { PlatformPaths } from "@/common/utils/paths"; import { log } from "@/node/services/log"; +import { isDeepStrictEqual } from "node:util"; import { eventSpine } from "@/node/services/events/eventSpine"; import { ProvidersConfigStore, type Config } from "@/node/config"; import { @@ -18,6 +19,7 @@ import { type TurnId, type QueueDrainTrigger, type OperationId, + type CompactionToken, type StreamErrorRecoveryOutcome, } from "./turnCoordinator"; export type { StreamErrorRecoveryOutcome } from "./turnCoordinator"; @@ -676,6 +678,7 @@ interface CachedMemoryContext { } interface SendMessageInternalOptions { + compactionHandoff?: CompactionToken; preparation?: PreparationAttempt; /** A dequeued send keeps its admission owner through acceptance and startup failure. */ turnReservation?: TurnId; @@ -787,7 +790,6 @@ export class AgentSession { []; private readonly coordinator = new TurnCoordinator({ streamStarted: (payload) => { - this.continuousCompactionAbandoned = false; this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; @@ -875,11 +877,17 @@ export class AgentSession { private lastSystemMessageTokens?: number; /** Prevent duplicate mid-stream compaction interrupts while we are already transitioning. */ - private midStreamCompactionPending = false; - private continuousCompactionAbandoned = false; - private continuousCompactionStopped = false; - private continuousCompactionObserving = false; - private continuousCompactionObservation: Promise | null = null; + private readonly compactionObservations = new Map>(); + private get midStreamCompactionPending(): boolean { + const stage = this.coordinator.compactionIntent.observation?.stage; + return stage === "stopping" || stage === "stopped" || stage === "dispatching"; + } + private get continuousCompactionAbandoned(): boolean { + return this.coordinator.compactionIntent.status === "abandoned"; + } + private get continuousCompactionObserving(): boolean { + return this.coordinator.compactionIntent.observation?.kind === "continuous"; + } /** Tracks file state for detecting external edits. */ private readonly fileChangeTracker = new FileChangeTracker(); @@ -1016,7 +1024,9 @@ export class AgentSession { * dispatch must target it by ID; null for default (RLM-off) compactions so * their "last message is the summary" staleness guard stays byte-identical. */ - private pendingCompactionFollowUpSummaryId: string | null = null; + private get pendingCompactionFollowUpSummaryId(): string | null { + return this.coordinator.compactionIntent.summaryId; + } constructor(options: AgentSessionOptions) { assert(options, "AgentSession requires options"); @@ -1069,6 +1079,10 @@ export class AgentSession { this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, + captureCompletionGuard: () => { + const epoch = this.coordinator.compactionIntent.epoch; + return () => !this.coordinator.closing && this.coordinator.compactionIntent.epoch === epoch; + }, historyService: this.historyService, sessionDir: path.join(this.config.sessionsDir, this.workspaceId), telemetryService, @@ -1079,8 +1093,9 @@ export class AgentSession { // follow-up dispatch can target it directly. // Reset on every completion: a resumeless continuous fold may precede a // legacy compaction whose current follow-up is on its final summary row. - this.pendingCompactionFollowUpSummaryId = - (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null; + this.coordinator.recordCompactionSummary( + (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null + ); onCompactionComplete?.(metadata); }, onIdleCompactionOutcome, @@ -1093,6 +1108,7 @@ export class AgentSession { this.continuousCompactor = new ContinuousCompactor({ workspaceId: this.workspaceId, + enterExecution: () => this.coordinator.enterExecution(), historyService: this.historyService, compactionHandler: this.compactionHandler, streamManager: { @@ -3276,6 +3292,7 @@ export class AgentSession { */ const rollbackPersistedTurnRows = async (): Promise => { if (persistedCancelableMessageIds.length === 0) return true; + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("delete-messages"); const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, @@ -3568,6 +3585,7 @@ export class AgentSession { return refuseBeforeAcceptance( createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) ); + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("edit"); // Ensure no in-flight completion code can append after we truncate. if (this.isBusy()) { @@ -3798,7 +3816,13 @@ export class AgentSession { const providersConfigForCompaction = this.getProvidersConfigSafe(); // Recover before measuring pressure so the old pre-swap usage cannot force another fold. - if (await this.continuousCompactor.recover()) this.clearUsageState(); + if (await this.continuousCompactor.recover()) { + if (isAdmissionStale() || this.coordinator.closing) + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); + this.clearUsageState(); + } const compactionResult = this.compactionMonitor.checkBeforeSend({ model: modelForStream, usage: this.getUsageState(), @@ -3822,6 +3846,10 @@ export class AgentSession { phase: "on-send", }) : "none"; + if (isAdmissionStale() || this.coordinator.closing) + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); if (continuousResult === "applied") this.clearUsageState(); if (await cancelBeforeAcceptance()) return Ok(undefined); @@ -4273,6 +4301,7 @@ export class AgentSession { intent: "direct", expectedTurnId: attempt.expectedTurn, editReservation: attempt.editReservation?.id, + compactionHandoff: internal?.compactionHandoff, }, preparedTurnAbortController, (turnId) => { @@ -5043,6 +5072,7 @@ export class AgentSession { this.deferQueuedFlushUntilAfterEdit ) return; + const epoch = this.coordinator.compactionIntent.epoch; try { const context = this.getContinuousCompactionContext(model, options); if (!context.enabled && !this.continuousCompactor.hasConsumedSwap()) { @@ -5063,7 +5093,12 @@ export class AgentSession { ...context, phase: "stream-end", }); - if (result === "applied") this.clearUsageState(); + if ( + result === "applied" && + epoch === this.coordinator.compactionIntent.epoch && + !this.coordinator.closing + ) + this.clearUsageState(); } catch (error) { await this.recoverContinuousCompactionFailure(error); } @@ -5096,46 +5131,40 @@ export class AgentSession { } private async waitForContinuousCompactionObservation(): Promise { - while (this.continuousCompactionObservation) await this.continuousCompactionObservation; + const observation = this.coordinator.compactionIntent.observation; + if (observation) await this.compactionObservations.get(observation.id); } private async runContinuousCompactionObservation( - observe: () => Promise + observe: (token: CompactionToken) => Promise ): Promise { if (this.coordinator.closing) return undefined; - // Own the actual apply and its continuation; the compactor's detached eager summary job - // retains its existing cancellation contract until the later compaction migration. using _execution = this.coordinator.enterExecution(); - if (this.continuousCompactionObservation) { - await this.continuousCompactionObservation; + const token = this.coordinator.beginCompactionObservation("continuous"); + if (!token) { + await this.waitForContinuousCompactionObservation(); return undefined; } - let finish!: () => void; - const observation = new Promise((resolve) => { - finish = resolve; - }); - this.continuousCompactionObservation = observation; - this.continuousCompactionObserving = true; + const observation = Promise.withResolvers(); + this.compactionObservations.set(token.id, observation.promise); try { - return await observe(); + return await observe(token); } catch (error) { - await this.recoverContinuousCompactionFailure(error, true); + if (this.coordinator.isCurrentCompaction(token)) + await this.recoverContinuousCompactionFailure(error, true); return undefined; } finally { - // Reserve through dispatch and cleanup, not just the compactor's apply latch. - // Waiters/duplicate invalidations never own or clear these flags. - if (this.continuousCompactionObservation === observation) { - this.midStreamCompactionPending = false; - this.continuousCompactionStopped = false; - this.continuousCompactionObserving = false; - this.continuousCompactionObservation = null; + // Semantic ownership can move to B while A's Promise is still finishing. + // Always release A's waiter; only its current owner may unblock queue dispatch. + this.compactionObservations.delete(token.id); + if (this.coordinator.finishCompactionObservation(token)) { try { this.drainQueuedMessagesIfIdle(); } catch (error) { log.warn("[continuous-compaction] queued drain failed", error); } } - finish(); + observation.resolve(); } } @@ -5143,7 +5172,10 @@ export class AgentSession { apply: (pendingFollowUp?: CompactionFollowUpRequest) => Promise ): Promise { const context = this.activeStreamContext; + const token = this.coordinator.compactionIntent.observation; if ( + !token || + !this.coordinator.isCurrentCompaction(token) || this.midStreamCompactionPending || !context?.options || this.coordinator.disposed || @@ -5151,17 +5183,17 @@ export class AgentSession { ) { return false; } - this.midStreamCompactionPending = true; + this.coordinator.setCompactionStage(token, "stopping"); const stopped = await this.streamManager.stopStream(this.workspaceId, { abortReason: "system", }); - if (!stopped.success) return false; - this.continuousCompactionStopped = true; + if (!stopped.success || !this.coordinator.isCurrentCompaction(token)) return false; + this.coordinator.setCompactionStage(token, "stopped"); await this.waitForIdle(); if ( this.coordinator.disposed || this.coordinator.closing || - this.continuousCompactionAbandoned || + !this.coordinator.isCurrentCompaction(token) || this.isWorkspaceArchivedOnDisk() ) return false; @@ -5190,13 +5222,19 @@ export class AgentSession { private async finishContinuousCompaction( applied: boolean, - context: NonNullable + context: NonNullable, + token: CompactionToken ): Promise { assert( !this.continuousCompactor.isApplying(), "Continue must dispatch after the apply latch clears" ); - if (!this.continuousCompactionStopped || !context.options) return; + if ( + this.coordinator.compactionIntent.observation?.id !== token.id || + this.coordinator.compactionIntent.observation.stage !== "stopped" || + !context.options + ) + return; // A consumed journal is an outstanding durable obligation, not a failed // speculative summary. Leave it retryable instead of resetting into legacy compaction. if (!applied && this.continuousCompactor.hasConsumedSwap()) return; @@ -5206,7 +5244,7 @@ export class AgentSession { // We already interrupted the turn, so recover using its captured context // rather than relying on activeStreamContext (cleared by stream-abort). if ( - this.continuousCompactionAbandoned || + !this.coordinator.isCurrentCompaction(token) || this.coordinator.disposed || this.coordinator.closing || this.isWorkspaceArchivedOnDisk() || @@ -5229,6 +5267,7 @@ export class AgentSession { reason: "mid-stream", }); } + if (!this.coordinator.isCurrentCompaction(token)) return; const fallback = pressure.shouldForceCompact ? this.buildAutoCompactionRequest({ baseOptions: context.options, @@ -5237,30 +5276,35 @@ export class AgentSession { }) : undefined; this.continuousCompactor.reset("failed-fast-apply"); + this.coordinator.setCompactionStage(token, "dispatching"); const sent = await this.sendMessage( fallback?.messageText ?? followUp.text, fallback ? { ...fallback.sendOptions, muxMetadata: fallback.metadata } : context.options, { synthetic: true, + compactionHandoff: token, agentInitiated: fallback?.agentInitiated ?? context.agentInitiated, goalKind: fallback ? undefined : context.goalKind, goalId: fallback ? undefined : context.goalId, - admissionStale: () => this.continuousCompactionAbandoned, + admissionStale: () => !this.coordinator.isCurrentCompaction(token), } ); - if (!sent.success && !sent.failureHandled && !this.continuousCompactionAbandoned) { + if (!sent.success && !sent.failureHandled && this.coordinator.isCurrentCompaction(token)) { this.emitChatEvent(createStreamErrorMessage(buildStreamErrorEventData(sent.error))); } return; } - this.lastUsageState = undefined; + if (this.coordinator.isCurrentCompaction(token)) this.lastUsageState = undefined; const summaryId = this.pendingCompactionFollowUpSummaryId; await this.dispatchPendingFollowUp( summaryId ?? undefined, - () => this.continuousCompactionAbandoned + () => !this.coordinator.isCurrentCompaction(token) ); - if (this.pendingCompactionFollowUpSummaryId === summaryId) - this.pendingCompactionFollowUpSummaryId = null; + if ( + this.coordinator.isCurrentCompaction(token) && + this.pendingCompactionFollowUpSummaryId === summaryId + ) + this.coordinator.recordCompactionSummary(null); } private async interruptForCompaction(): Promise { @@ -5274,10 +5318,12 @@ export class AgentSession { return; } + const token = this.coordinator.beginCompactionObservation("legacy"); + if (!token) return; const interruptedUserMessageId = this.activeStreamUserMessageId; this.continuousCompactor.reset("legacy-fallback"); - this.midStreamCompactionPending = true; + this.coordinator.setCompactionStage(token, "stopping"); try { const stopResult = await this.streamManager.stopStream(this.workspaceId, { abortReason: "system", @@ -5291,7 +5337,7 @@ export class AgentSession { } await this.waitForIdle(); - if (this.coordinator.disposed) { + if (!this.coordinator.isCurrentCompaction(token)) { return; } @@ -5312,6 +5358,8 @@ export class AgentSession { reason: "mid-stream", }); + if (!this.coordinator.isCurrentCompaction(token)) return; + this.coordinator.setCompactionStage(token, "dispatching"); const autoCompactionRequest = this.buildAutoCompactionRequest({ followUpContent, baseOptions: streamContext.options, @@ -5324,9 +5372,14 @@ export class AgentSession { ...autoCompactionRequest.sendOptions, muxMetadata: autoCompactionRequest.metadata, }, - { synthetic: true, agentInitiated: autoCompactionRequest.agentInitiated } + { + synthetic: true, + agentInitiated: autoCompactionRequest.agentInitiated, + compactionHandoff: token, + admissionStale: () => !this.coordinator.isCurrentCompaction(token), + } ); - if (!sendResult.success) { + if (!sendResult.success && this.coordinator.isCurrentCompaction(token)) { log.warn("Failed to dispatch mid-stream compaction request", { workspaceId: this.workspaceId, error: sendResult.error, @@ -5358,10 +5411,9 @@ export class AgentSession { } } } finally { - this.midStreamCompactionPending = false; // Preflight drains deferred to this pending compaction have no other retry: if the // compaction request never became a turn, release the queue now (no-op when it did). - this.drainQueuedMessagesIfIdle(); + if (this.coordinator.finishCompactionObservation(token)) this.drainQueuedMessagesIfIdle(); } } @@ -5392,7 +5444,7 @@ export class AgentSession { // a future boundary, so neither joins policy here. const interruptedPolicy = this.coordinator.captureInterruptSettlement(options?.soft); if (options?.abandonPartial || this.midStreamCompactionPending) { - this.continuousCompactionAbandoned = true; + this.coordinator.invalidateCompaction(true); this.continuousCompactor.reset("user-interrupt"); } @@ -5839,6 +5891,7 @@ export class AgentSession { } private async clearFailedAssistantMessage(messageId: string, reason: string): Promise { + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("delete-message"); const [partialResult, deleteMessageResult] = await Promise.all([ this.historyService.deletePartial(this.workspaceId), @@ -6683,7 +6736,7 @@ export class AgentSession { // RLM keep-recent floor: when tail copies were appended the summary is // not the last row, so target it by ID (stashed in onCompactionComplete). const rlmSummaryId = this.pendingCompactionFollowUpSummaryId; - this.pendingCompactionFollowUpSummaryId = null; + this.coordinator.recordCompactionSummary(null); continuedAfterCompaction = await this.dispatchPendingFollowUp(rlmSummaryId ?? undefined); if ( !this.coordinator.isCurrentTurn(turn) || @@ -6901,13 +6954,12 @@ export class AgentSession { !this.streamManager.isStreaming(this.workspaceId) ) return; - await this.runContinuousCompactionObservation(async () => { + await this.runContinuousCompactionObservation(async (token) => { const result = await this.continuousCompactor.observe(0, { ...this.getContinuousCompactionContext(context.modelString, context.options), phase: "mid-stream", }); - this.midStreamCompactionPending = false; - await this.finishContinuousCompaction(result === "applied", context); + await this.finishContinuousCompaction(result === "applied", context, token); }); } catch (error) { await this.recoverContinuousCompactionFailure(error); @@ -6971,16 +7023,17 @@ export class AgentSession { if (continuousContext.enabled || consumedSwapPending) { // One usage handler owns the eventual resume; observe itself shares its // latch result, which must not dispatch the continuation twice. - const observed = await this.runContinuousCompactionObservation(async () => { + const observed = await this.runContinuousCompactionObservation(async (token) => { const result = await this.continuousCompactor.observe(usagePercent, { ...continuousContext, phase: "mid-stream", }); if (this.midStreamCompactionPending) { - await this.finishContinuousCompaction(result === "applied", streamContext); + await this.finishContinuousCompaction(result === "applied", streamContext, token); return undefined; } - if (result === "applied") this.clearUsageState(); + if (result === "applied" && this.coordinator.isCurrentCompaction(token)) + this.clearUsageState(); return result; }); if (observed === undefined) return; @@ -7157,6 +7210,7 @@ export class AgentSession { * deleting the partial removes the discarded transcript's tail durably. */ async discardAutoRetryForContextMutation(): Promise> { + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("context-mutation"); this.retryManager.cancel(); this.setAutoRetryResumeState(undefined); @@ -7184,6 +7238,7 @@ export class AgentSession { * check. */ holdTurnAdmission(): Disposable { + this.coordinator.invalidateCompaction(); this.continuousCompactor.reset("context-mutation"); return this.coordinator.reserve("admission"); } @@ -7839,7 +7894,22 @@ export class AgentSession { return false; } using _execution = this.coordinator.enterExecution(); + // Claim before history I/O: a send admitted and completed during that read + // must not make an obsolete summary look like a fresh idle continuation. + const token = this.coordinator.claimCompactionFollowUp(); + if (!token) return false; + try { + return await this.dispatchOwnedCompactionFollowUp(token, summaryMessageId, cancelResume); + } finally { + this.coordinator.finishCompactionFollowUp(token); + } + } + private async dispatchOwnedCompactionFollowUp( + token: CompactionToken, + summaryMessageId?: string, + cancelResume?: () => boolean + ): Promise { let summaryMessage: MuxMessage | undefined; if (summaryMessageId) { const historyResult = await this.historyService.getHistoryFromLatestBoundary( @@ -7918,10 +7988,18 @@ export class AgentSession { return false; } + if (!this.coordinator.isCurrentCompaction(token)) { + // Stop cancels dispatch while retaining ownership of its durable cleanup. + // A replacement/edit drops that ownership, so it cannot erase the new intent. + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + // A user can abandon after the boundary commits but before its continuation // dispatches. Keep the fold, but remove the crash-recoverable resume intent. if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -7945,7 +8023,7 @@ export class AgentSession { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -7962,7 +8040,7 @@ export class AgentSession { summaryMessageId: lastMessage.id, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } @@ -7991,7 +8069,8 @@ export class AgentSession { await this.skipIdleRuleFollowUp( lastMessage, hasQueuedMessages || hasExternalPreflightSend, - hasActiveNonCompletingTurn + hasActiveNonCompletingTurn, + token ); return false; } @@ -8026,7 +8105,7 @@ export class AgentSession { workspaceId: this.workspaceId, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } goalAdmissionStale = admission.admissionStale; @@ -8044,13 +8123,13 @@ export class AgentSession { this.hasExternalSendPreflight?.() === true || (this.isBusy() && this.coordinator.phase !== "completing") : undefined; - const followUpAdmissionStale = - idleRuleStale != null || goalAdmissionStale != null || cancelResume != null - ? () => - idleRuleStale?.() === true || - goalAdmissionStale?.() === true || - cancelResume?.() === true - : undefined; + // Ordinary durable follow-ups reconstruct the interrupted request and precede + // queued input. Optional/goal/requireIdle continuations yield to manual work. + const followUpAdmissionStale = () => + !this.coordinator.isCurrentCompaction(token) || + idleRuleStale?.() === true || + goalAdmissionStale?.() === true || + cancelResume?.() === true; log.debug("Dispatching pending follow-up from compaction summary", { workspaceId: this.workspaceId, @@ -8131,13 +8210,14 @@ export class AgentSession { } if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // The compaction summary is now the source of truth for the next live resume // request. Pre-arm retry state from the reconstructed follow-up so failures // before stream startup do not fall back to the already-completed compact turn. + if (!this.coordinator.isCurrentCompaction(token)) return false; this.setAutoRetryResumeState( options, followUp.agentInitiated, @@ -8150,8 +8230,13 @@ export class AgentSession { // before sendQueuedMessages() runs, preventing race conditions. // Mark as synthetic so recovery/background dispatches do not implicitly // re-enable auto-retry after a user explicitly opted out. + let accepted = false; const sendResult = await this.sendMessage(finalText, options, { synthetic: true, + compactionHandoff: token, + onAccepted: () => { + accepted = true; + }, agentInitiated: followUp.agentInitiated, goalKind: persistedGoalKind, // Keep the re-dispatched continuation row goal-scoped so a replaced @@ -8165,7 +8250,7 @@ export class AgentSession { }); if (!sendResult.success) { if (cancelResume?.()) { - await this.clearPendingFollowUpFromSummary(lastMessage); + await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // A stale-admission refusal is the idle rule (or a goal transition) @@ -8179,7 +8264,8 @@ export class AgentSession { await this.skipIdleRuleFollowUp( lastMessage, this.hasPendingManualFollowUp() || this.hasExternalSendPreflight?.() === true, - this.isBusy() && this.coordinator.phase !== "completing" + this.isBusy() && this.coordinator.phase !== "completing", + token ); return false; } @@ -8187,6 +8273,11 @@ export class AgentSession { throw new Error(`Failed to dispatch pending follow-up: ${message}`); } + // A successful send can be a pre-acceptance no-op (for example disposal + // during branch-summary preparation). Only the durable acceptance callback + // acknowledges this handoff. Later manual admission cannot undo that fact. + if (!accepted) return false; + // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): if the original wrap-up dispatcher // crashed between send acceptance and its tryMarkBudgetLimitInjected // commit, this redispatched follow-up owns the wrap-up now — install the @@ -8224,25 +8315,31 @@ export class AgentSession { private async skipIdleRuleFollowUp( summaryMessage: MuxMessage, hasUserContention: boolean, - hasActiveNonCompletingTurn: boolean + hasActiveNonCompletingTurn: boolean, + token: CompactionToken ): Promise { if ( summaryMessage.metadata?.compacted === "heartbeat" && hasUserContention && !hasActiveNonCompletingTurn ) { - const rollbackResult = - await this.compactionHandler.rollbackHeartbeatContextResetBoundary(summaryMessage); + const rollbackResult = await this.compactionHandler.rollbackHeartbeatContextResetBoundary( + summaryMessage, + () => this.coordinator.canClearCompactionFollowUp(token) + ); if (!rollbackResult.success) { throw new Error(`Failed to rollback heartbeat reset boundary: ${rollbackResult.error}`); } - this.onPostCompactionStateChange?.(); + if (this.coordinator.canClearCompactionFollowUp(token)) this.onPostCompactionStateChange?.(); } else { - await this.clearPendingFollowUpFromSummary(summaryMessage); + await this.clearPendingFollowUpFromSummary(summaryMessage, token); } } - private async clearPendingFollowUpFromSummary(summaryMessage: MuxMessage): Promise { + private async clearPendingFollowUpFromSummary( + summaryMessage: MuxMessage, + token: CompactionToken + ): Promise { assert( summaryMessage.role === "assistant", "clearPendingFollowUpFromSummary requires an assistant summary message" @@ -8259,13 +8356,19 @@ export class AgentSession { } const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; - const updateResult = await this.historyService.updateHistory(this.workspaceId, { - ...summaryMessage, - metadata: { - ...(summaryMessage.metadata ?? {}), - muxMetadata: muxMetadataWithoutFollowUp, + const updateResult = await this.historyService.updateHistory( + this.workspaceId, + { + ...summaryMessage, + metadata: { + ...(summaryMessage.metadata ?? {}), + muxMetadata: muxMetadataWithoutFollowUp, + }, }, - }); + (current) => + this.coordinator.canClearCompactionFollowUp(token) && + isDeepStrictEqual(current, summaryMessage) + ); if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } diff --git a/src/node/services/compactionHandler.continuous.test.ts b/src/node/services/compactionHandler.continuous.test.ts index f658def5128..da4461474f5 100644 --- a/src/node/services/compactionHandler.continuous.test.ts +++ b/src/node/services/compactionHandler.continuous.test.ts @@ -1,4 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, spyOn, mock } from "bun:test"; +import * as fsPromises from "fs/promises"; import { mkdir, writeFile } from "node:fs/promises"; import { EventEmitter } from "node:events"; import * as path from "node:path"; @@ -17,9 +18,75 @@ describe("continuous compaction provider replay", () => { store = await createTestHistoryService(); }); afterEach(async () => { + mock.restore(); await store.cleanup(); }); + it("a held heartbeat rollback unlink cannot consume the replacement rollback snapshot", async () => { + const sessionDir = path.join(store.tempDir, "pending"); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + const followUp = { text: "wake", model: "openai:gpt-4o", agentId: "exec" }; + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "A", + pendingFollowUp: followUp, + }) + ).success + ).toBe(true); + const first = await store.historyService.getLastMessages(workspaceId, 1); + assert(first.success, "Expected first boundary"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const captured = Promise.withResolvers(); + const unlink = fsPromises.unlink; + let held = false; + spyOn(fsPromises, "unlink").mockImplementation(async (file) => { + if (!held && file === path.join(sessionDir, "post-compaction.json")) { + held = true; + entered.resolve(); + await release.promise; + } + return unlink(file); + }); + const internals = handler as unknown as { captureHeartbeatResetRollbackState(): void }; + const capture = internals.captureHeartbeatResetRollbackState.bind(handler); + spyOn(internals, "captureHeartbeatResetRollbackState").mockImplementation(() => { + capture(); + captured.resolve(); + }); + const rollingBack = handler.rollbackHeartbeatContextResetBoundary(first.data[0]); + let replacement: + | ReturnType + | undefined; + try { + await entered.promise; + replacement = handler.appendHeartbeatContextResetBoundary({ + boundaryText: "B", + pendingFollowUp: followUp, + }); + await captured.promise; + release.resolve(); + expect((await rollingBack).success).toBe(true); + expect((await replacement).success).toBe(true); + const second = await store.historyService.getLastMessages(workspaceId, 1); + assert(second.success, "Expected replacement boundary"); + expect((await handler.rollbackHeartbeatContextResetBoundary(second.data[0])).success).toBe( + true + ); + expect(await handler.peekPendingState()).toBeNull(); + } finally { + release.resolve(); + await rollingBack; + await replacement; + } + }); + it("preserves previously pending attachments when a newer fold is abandoned or crashes", async () => { const sessionDir = path.join(store.tempDir, "pending"); await mkdir(sessionDir, { recursive: true }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 08676aa875d..719c209fde6 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -382,6 +382,8 @@ interface CompactionHandlerOptions { emitter: EventEmitter; /** Called when compaction completes successfully (e.g., to clear idle compaction pending state) */ onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + /** Capture semantic ownership before I/O; durable completion must not retarget a successor. */ + captureCompletionGuard?: () => () => boolean; /** * Called with the terminal outcome of an idle compaction (source === "idle-compaction"), * after the summary is actually persisted (success) or a post-stream persistence failure @@ -411,6 +413,7 @@ export class CompactionHandler { private readonly processedCompactionRequestIds: Set = new Set(); private readonly onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + private readonly captureCompletionGuard?: () => () => boolean; private readonly onIdleCompactionOutcome?: (success: boolean) => void; /** Flag indicating post-compaction attachments should be generated on next turn */ @@ -419,6 +422,13 @@ export class CompactionHandler { private cachedFileDiffs: FileEditDiff[] = []; /** Rollback snapshot for synthetic heartbeat reset boundaries that get skipped before dispatch. */ private heartbeatResetRollbackState: HeartbeatResetRollbackState | null = null; + private pendingStateWrites: Promise = Promise.resolve(); + + private enqueuePendingStateWrite(write: () => Promise): Promise { + const result = this.pendingStateWrites.then(write); + this.pendingStateWrites = result.catch(() => undefined); + return result; + } /** Cached loaded skill snapshots extracted from history before appending compaction summary */ private cachedLoadedSkills: LoadedSkillSnapshot[] = []; /** Cumulative file paths read in summarized epochs (paths only, newest-first, capped). */ @@ -437,6 +447,7 @@ export class CompactionHandler { this.telemetryService = options.telemetryService; this.emitter = options.emitter; this.onCompactionComplete = options.onCompactionComplete; + this.captureCompletionGuard = options.captureCompletionGuard; this.onIdleCompactionOutcome = options.onIdleCompactionOutcome; } @@ -590,7 +601,7 @@ export class CompactionHandler { async discardPendingStateDurably(reason: string): Promise { await this.discardPendingState(reason); try { - await fsPromises.unlink(this.postCompactionStatePath); + await this.enqueuePendingStateWrite(() => fsPromises.unlink(this.postCompactionStatePath)); } catch (error) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return; @@ -601,7 +612,7 @@ export class CompactionHandler { private async deletePersistedPendingStateBestEffort(): Promise { try { - await fsPromises.unlink(this.postCompactionStatePath); + await this.enqueuePendingStateWrite(() => fsPromises.unlink(this.postCompactionStatePath)); } catch { // ignore } @@ -623,6 +634,10 @@ export class CompactionHandler { return; } + // Consume the captured rollback before I/O. A new heartbeat can install its + // own snapshot while this unlink/write is pending; A must never clear B's. + this.heartbeatResetRollbackState = null; + this.postCompactionAttachmentsPending = rollbackState.postCompactionAttachmentsPending; this.cachedFileDiffs = [...rollbackState.cachedFileDiffs]; this.cachedLoadedSkills = [...rollbackState.cachedLoadedSkills]; @@ -638,8 +653,6 @@ export class CompactionHandler { } else { await this.deletePersistedPendingStateBestEffort(); } - - this.heartbeatResetRollbackState = null; } private async persistPendingStateBestEffort( @@ -650,8 +663,6 @@ export class CompactionHandler { previousState?: PersistedPostCompactionStateV1 ): Promise { try { - await fsPromises.mkdir(this.sessionDir, { recursive: true }); - for (const snapshot of loadedSkills) { assert(snapshot.name.trim().length > 0, "loaded skill snapshot name must not be empty"); } @@ -665,7 +676,13 @@ export class CompactionHandler { ...(boundaryMessageId && { boundaryMessageId, previousState }), }; - await fsPromises.writeFile(this.postCompactionStatePath, JSON.stringify(persisted)); + // Freeze the admitted snapshot, then order writes AND unlinks. A held rollback + // must not erase a newer compaction's pending state after that compaction completes. + const serialized = JSON.stringify(persisted); + await this.enqueuePendingStateWrite(async () => { + await fsPromises.mkdir(this.sessionDir, { recursive: true }); + await fsPromises.writeFile(this.postCompactionStatePath, serialized); + }); } catch (error) { log.warn("Failed to persist post-compaction state", { workspaceId: this.workspaceId, @@ -875,7 +892,8 @@ export class CompactionHandler { } async rollbackHeartbeatContextResetBoundary( - summaryMessage: MuxMessage + summaryMessage: MuxMessage, + isCurrent: () => boolean = () => true ): Promise> { assert( summaryMessage.role === "assistant", @@ -888,14 +906,25 @@ export class CompactionHandler { const deleteResult = await this.historyService.deleteMessage( this.workspaceId, - summaryMessage.id + summaryMessage.id, + (messages) => + isCurrent() && + messages.some( + (message) => + message.id === summaryMessage.id && + message.metadata?.historySequence === summaryMessage.metadata?.historySequence + ) ); if (!deleteResult.success) { return Err(`Failed to delete heartbeat reset boundary: ${deleteResult.error}`); } + if (!isCurrent()) return Ok(undefined); + await this.restoreHeartbeatResetRollbackState(); + if (!isCurrent()) return Ok(undefined); + const historySequence = summaryMessage.metadata?.historySequence; if (isNonNegativeInteger(historySequence)) { this.emitChatEvent({ @@ -929,6 +958,7 @@ export class CompactionHandler { event: StreamEndEvent, compactionRequestMessageId?: string ): Promise { + const canComplete = this.captureCompletionGuard?.(); // The current stream identifies its request when available. Synthetic prompt snapshots can // follow that request in history, so the last user row is not always the compaction request. const historyResult = compactionRequestMessageId @@ -1052,7 +1082,7 @@ export class CompactionHandler { }); // Notify that compaction completed (clears idle compaction pending state) - this.onCompactionComplete?.(result.data); + if (canComplete?.() !== false) this.onCompactionComplete?.(result.data); // Report the idle-compaction success only after the summary is actually persisted, // so the idle loop's failure streak is reset on real success (not just stream end). @@ -1239,6 +1269,7 @@ export class CompactionHandler { shouldPersist: (messages: MuxMessage[]) => boolean; } ): Promise { + const canComplete = this.captureCompletionGuard?.(); const { boundary, copies } = params.prepared ?? this.buildContinuousCompactionRows(params); const inputTokens = params.systemMessageTokens + @@ -1271,15 +1302,16 @@ export class CompactionHandler { const epoch = boundary.metadata.compactionEpoch; assert(isNonNegativeInteger(sequence), "Continuous boundary requires a persisted sequence"); assert(isPositiveInteger(epoch), "Continuous boundary requires an epoch"); - this.onCompactionComplete?.({ - workspaceId: this.workspaceId, - summaryMessageId: boundary.id, - summaryHistorySequence: sequence, - compactionEpoch: epoch, - previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), - compactionRequestMessageId: boundary.id, - preservedTailMessageCount: copies.length, - }); + if (canComplete?.() !== false) + this.onCompactionComplete?.({ + workspaceId: this.workspaceId, + summaryMessageId: boundary.id, + summaryHistorySequence: sequence, + compactionEpoch: epoch, + previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), + compactionRequestMessageId: boundary.id, + preservedTailMessageCount: copies.length, + }); return true; } diff --git a/src/node/services/continuousCompactor.test.ts b/src/node/services/continuousCompactor.test.ts index 999aa4088f0..2ca295a0f35 100644 --- a/src/node/services/continuousCompactor.test.ts +++ b/src/node/services/continuousCompactor.test.ts @@ -933,6 +933,31 @@ describe("ContinuousCompactor", () => { expect(history.filter((row) => row.metadata?.compactionBoundary)).toHaveLength(1); }); + it("late journal finalization cannot clear a replacement journal after reset", async () => { + const { answer, journal, journalStore } = await activateJournaledSwap(); + assert(live, "Live fixture missing"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + streaming = false; + live = undefined; + const entered = deferred(); + const release = deferred(); + const clear = journalStore.clear.bind(journalStore); + spyOn(journalStore, "clear").mockImplementationOnce(async () => { + await clear(); + entered.resolve(); + await release.promise; + }); + const recovering = compactor.recover(); + await entered.promise; + compactor.reset("edit"); + const replacement = { ...journal, boundary: { ...journal.boundary, id: "replacement" } }; + expect(await journalStore.write(replacement, [], () => true)).not.toBeNull(); + release.resolve(); + await recovering; + expect((await journalStore.read())?.boundary.id).toBe("replacement"); + }); + it("reset during journal apply cannot append a boundary", async () => { const { answer, dependencies } = await activateJournaledSwap(); assert(live, "Live fixture missing"); diff --git a/src/node/services/continuousCompactor.ts b/src/node/services/continuousCompactor.ts index 14f7a17b284..a21339f674f 100644 --- a/src/node/services/continuousCompactor.ts +++ b/src/node/services/continuousCompactor.ts @@ -46,6 +46,8 @@ interface StreamSnapshot { } interface Dependencies { workspaceId: string; + /** Physical session ownership; speculative work must not reserve turn admission. */ + enterExecution?(): Disposable; historyService: HistoryService; compactionHandler: CompactionHandler; streamManager: { @@ -148,22 +150,32 @@ export class ContinuousCompactor { // It also keeps the disabled hot path free of journal I/O when no swap ever activated. const discardJournal = !settingsOnly || this.activeSwap !== null || this.swapAttempted !== null; this.generation++; - this.job?.abort.abort(); + const job = this.job; this.job = null; this.staged = null; this.swapAttempted = null; this.activeSwap = null; - this.deps.streamManager.clearPrefixSwap?.(this.deps.workspaceId); // Graceful shutdown retains the write-ahead record for ordinary startup recovery. if (discardJournal && reason !== "shutdown") { - this.deps.historyService - .getContinuousCompactionJournal(this.deps.workspaceId) - .clear() - .catch((error: unknown) => log.warn("[continuous-compaction] journal clear failed", error)); + this.clearJournal().catch((error: unknown) => + log.warn("[continuous-compaction] journal clear failed", error) + ); + } + // Detach semantic ownership and enqueue the old clear before abort observers can + // synchronously replace work. The original job retains its physical lease until done. + try { + this.deps.streamManager.clearPrefixSwap?.(this.deps.workspaceId); + } finally { + job?.abort.abort(); } log.debug("[continuous-compaction] reset", { workspaceId: this.deps.workspaceId, reason }); } + private async clearJournal(): Promise { + using _execution = this.deps.enterExecution?.(); + await this.deps.historyService.getContinuousCompactionJournal(this.deps.workspaceId).clear(); + } + hasConsumedSwap(): boolean { return this.activeSwap?.consumed === true; } @@ -220,6 +232,7 @@ export class ContinuousCompactor { } if (context.phase !== "mid-stream") { if (await this.finalizeJournal()) return "applied"; + if (generation !== this.generation) return "none"; // A transient history failure must remain recoverable even when future // compaction is disabled; only a discarded/invalid journal releases this latch. if (this.hasConsumedSwap()) return "none"; @@ -263,6 +276,7 @@ export class ContinuousCompactor { done: Promise.resolve(), }; this.job = job; + const execution = this.deps.enterExecution?.(); job.done = this.startEagerJob(job, context) .catch((error: unknown) => { if (!job.abort.signal.aborted) @@ -270,6 +284,7 @@ export class ContinuousCompactor { }) .finally(() => { if (this.job === job) this.job = null; + execution?.[Symbol.dispose](); }); } return usagePercent >= context.thresholdPercent + FORCE_COMPACTION_BUFFER_PERCENT @@ -582,6 +597,7 @@ export class ContinuousCompactor { rows.some((row) => row.id === journal.liveTailCopySpec.copyId); if (rows.some((row) => row.id === journal.boundary.id) && copiesPresent) { await store.clear(); + if (generation !== this.generation) return false; this.activeSwap = null; return true; } @@ -618,6 +634,7 @@ export class ContinuousCompactor { workspaceId: this.deps.workspaceId, }); await store.clear(); + if (generation !== this.generation) return false; this.activeSwap = null; return false; } @@ -668,9 +685,11 @@ export class ContinuousCompactor { }, boundary.id ); - if (applied) { + if (applied && generation === this.generation) { await store.clear(); - this.reset("applied"); + // Persistence may finish after an edit/reset. Its old clear is already ordered + // by the store; a new reset here would enqueue another clear behind replacement work. + if (generation === this.generation) this.reset("applied"); } return applied; } @@ -725,7 +744,7 @@ export class ContinuousCompactor { attachmentTokens: context.attachmentTokens ?? 0, pendingFollowUp, }); - if (applied) this.reset("applied"); + if (applied && staged.generation === this.generation) this.reset("applied"); return applied; } ); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index b28affd6350..55188638d0f 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2533,15 +2533,22 @@ export class HistoryService { * always in the active epoch (stream placeholders, compaction summaries), * never in the sealed archive. */ - async updateHistory(workspaceId: string, message: MuxMessage): Promise> { + // Optional ownership predicates are synchronous/pure and may run twice (under + // the lock and immediately before rename). Losing ownership is a successful no-op. + async updateHistory( + workspaceId: string, + message: MuxMessage, + shouldUpdate?: (current: MuxMessage) => boolean + ): Promise> { return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => - this.updateHistoryUnderWriteLock(workspaceId, message) + this.updateHistoryUnderWriteLock(workspaceId, message, shouldUpdate) ); } private async updateHistoryUnderWriteLock( workspaceId: string, - message: MuxMessage + message: MuxMessage, + shouldUpdate?: (current: MuxMessage) => boolean ): Promise> { try { const historyPath = this.getChatHistoryPath(workspaceId); @@ -2562,10 +2569,13 @@ export class HistoryService { // Find and replace the message with matching historySequence let found = false; let persistedMessage: MuxMessage | undefined; + let sourceMessage: MuxMessage | undefined; for (let i = 0; i < messages.length; i++) { if (messages[i].metadata?.historySequence === targetSequence) { const existingMessage = messages[i]; assert(existingMessage, "updateHistory matched message must exist"); + if (shouldUpdate && !shouldUpdate(existingMessage)) return Ok(undefined); + sourceMessage = existingMessage; // Preserve compaction boundary metadata during late in-place rewrites. // Compaction may update an assistant row first, then a late stream rewrite can @@ -2599,7 +2609,13 @@ export class HistoryService { const historyEntries = this.serializeHistoryEntries(messages, workspaceId); // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + if (shouldUpdate && sourceMessage) { + const source = sourceMessage; + if ( + !(await this.writeGuardedHistory(historyPath, historyEntries, () => shouldUpdate(source))) + ) + return Ok(undefined); + } else await writeFileAtomic(historyPath, historyEntries); // Compaction updates the streamed summary row in-place with boundary // metadata — seal the previous epoch once that lands. Check the persisted @@ -2613,6 +2629,23 @@ export class HistoryService { } } + /** The final ownership check and rename do not yield to a context reset/new admission. */ + private async writeGuardedHistory( + historyPath: string, + serialized: string, + isCurrent: () => boolean + ): Promise { + const stagedPath = `${historyPath}.continuous-${randomUUID()}`; + try { + await writeFileAtomic(stagedPath, serialized, { mode: 0o600 }); + if (!isCurrent()) return false; + renameSync(stagedPath, historyPath); + return true; + } finally { + await fs.rm(stagedPath, { force: true }); + } + } + /** * Atomically persist a compaction boundary together with its preserved * keep-recent tail copies (RLM keep-recent floor) in ONE file commit. @@ -2723,16 +2756,12 @@ export class HistoryService { const serialized = this.serializeHistoryEntries(messages, workspaceId); if (shouldPersist) { - const stagedPath = `${historyPath}.continuous-${randomUUID()}`; - try { - await writeFileAtomic(stagedPath, serialized, { mode: 0o600 }); - // Bulk I/O remains asynchronous, but the final generation check and - // publication must not yield to reset(), abandonment, or a new stream. - if (!shouldPersist(sourceMessages)) return Err("Compaction snapshot changed"); - renameSync(stagedPath, historyPath); - } finally { - await fs.rm(stagedPath, { force: true }); - } + if ( + !(await this.writeGuardedHistory(historyPath, serialized, () => + shouldPersist(sourceMessages) + )) + ) + return Err("Compaction snapshot changed"); } else { await writeFileAtomic(historyPath, serialized); } @@ -2816,20 +2845,29 @@ export class HistoryService { * * This is safer than truncateAfterMessage for cleanup paths where subsequent * messages may already have been appended. + * A conditional cleanup uses a synchronous/pure predicate under the lock and + * before rename; false returns Ok without changing history. */ - async deleteMessage(workspaceId: string, messageId: string): Promise> { + async deleteMessage( + workspaceId: string, + messageId: string, + shouldDelete?: (messages: MuxMessage[]) => boolean + ): Promise> { return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to delete message", () => - this.deleteMessageUnderWriteLock(workspaceId, messageId) + this.deleteMessageUnderWriteLock(workspaceId, messageId, shouldDelete) ); } private async deleteMessageUnderWriteLock( workspaceId: string, - messageId: string + messageId: string, + shouldDelete?: (messages: MuxMessage[]) => boolean ): Promise> { try { // Structural rewrite requires full file content const messages = await this.readChatHistory(workspaceId); + // Conditional cleanup is scoped to the observed active context, never its archive. + if (shouldDelete && !shouldDelete(messages)) return Ok(undefined); const filteredMessages = messages.filter((msg) => msg.id !== messageId); if (filteredMessages.length === messages.length) { @@ -2854,7 +2892,14 @@ export class HistoryService { const historyEntries = this.serializeHistoryEntries(filteredMessages, workspaceId); // Atomic write prevents corruption if app crashes mid-write - await writeFileAtomic(historyPath, historyEntries); + if (shouldDelete) { + if ( + !(await this.writeGuardedHistory(historyPath, historyEntries, () => + shouldDelete(messages) + )) + ) + return Ok(undefined); + } else await writeFileAtomic(historyPath, historyEntries); // Keep the in-memory sequence counter monotonic. It's okay to reuse deleted sequence // numbers on restart, but we must not regress within a running process. diff --git a/src/node/services/turnCoordinator.test.ts b/src/node/services/turnCoordinator.test.ts index 9d660a21b75..deb466e9de0 100644 --- a/src/node/services/turnCoordinator.test.ts +++ b/src/node/services/turnCoordinator.test.ts @@ -63,6 +63,37 @@ function reduce(events: CoordinatorEvent[]) { } describe("TurnCoordinator", () => { + test("a compaction handoff survives its own admission, but stale completion cannot release its replacement", () => { + const { coordinator } = setup(); + const token = coordinator.beginCompactionObservation("continuous"); + if (!token) throw new Error("Expected observation"); + coordinator.setCompactionStage(token, "stopped"); + const followUp = coordinator.claimCompactionFollowUp(); + if (!followUp) throw new Error("Expected follow-up"); + const own = coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: followUp, + }); + expect(own.status).toBe("admitted"); + expect(coordinator.isCurrentCompaction(token)).toBe(true); + expect(coordinator.isCurrentCompaction(followUp)).toBe(true); + coordinator.finishTurn(coordinator.turnId); + expect( + coordinator.prepare({ kind: "fresh", intent: "direct", expectedTurnId: coordinator.turnId }) + .status + ).toBe("admitted"); + const replacement = coordinator.beginCompactionObservation("continuous"); + if (!replacement) throw new Error("Expected replacement observation"); + coordinator.setCompactionStage(replacement, "stopping"); + expect(coordinator.isCurrentCompaction(token)).toBe(false); + expect(coordinator.finishCompactionObservation(token)).toBe(false); + coordinator.finishCompactionFollowUp(followUp); + expect(coordinator.compactionIntent.observation?.id).toBe(replacement.id); + expect(coordinator.compactionIntent.observation?.stage).toBe("stopping"); + }); + test("session scope releases thinking and idle listeners even without another idle event", async () => { const scope = Scope.makeUnsafe("parallel"); const { coordinator } = setup(); diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index 3c030320024..440b37ea3c1 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -16,6 +16,7 @@ export type PreparationRequest = intent: "direct" | "resume" | "handoff" | QueueDrainTrigger; expectedTurnId: TurnId; editReservation?: symbol; + compactionHandoff?: CompactionToken; } | { kind: "adopt"; turnId: TurnId }; export type PreparationAdmission = @@ -27,6 +28,22 @@ type ReservationKind = "admission" | "edit" | "manual"; type DecisionKind = "error" | "compaction"; type DecisionOutcome = StreamErrorRecoveryOutcome | boolean; +export interface CompactionToken { + readonly id: symbol; + readonly epoch: number; +} +type CompactionObservation = CompactionToken & { + readonly kind: "continuous" | "legacy"; + readonly stage: "observing" | "stopping" | "stopped" | "dispatching"; +}; +interface CompactionIntent { + readonly epoch: number; + readonly status: "ready" | "abandoned"; + readonly observation?: CompactionObservation; + readonly followUp?: CompactionToken; + readonly summaryId: string | null; +} + type Operation = { readonly id: OperationId; readonly startupMessageId?: string; @@ -58,6 +75,7 @@ export interface CoordinatorState { readonly reservations: ReadonlyArray<{ id: symbol; kind: ReservationKind }>; readonly retry?: symbol; readonly decisions: readonly Decision[]; + readonly compaction: CompactionIntent; } export type CoordinatorEvent = @@ -76,6 +94,12 @@ export type CoordinatorEvent = | { type: "reserve" | "release"; id: symbol; kind: ReservationKind } | { type: "retry-start" | "retry-finish"; id: symbol } | { type: "decision"; kind: DecisionKind; messageId: string; outcome?: DecisionOutcome } + | { type: "compaction-observe"; token: CompactionToken; kind: CompactionObservation["kind"] } + | { type: "compaction-stage"; token: CompactionToken; stage: CompactionObservation["stage"] } + | { type: "compaction-finish"; token: CompactionToken } + | { type: "compaction-invalidate"; abandon: boolean } + | { type: "compaction-follow-up" | "compaction-follow-up-finish"; token: CompactionToken } + | { type: "compaction-summary"; summaryId: string | null } | { type: "shutdown" | "dispose" }; type CoordinatorCommand = @@ -95,7 +119,13 @@ type CoordinatorCommand = | { type: "dispose" }; export function initialCoordinatorState(id: TurnId): CoordinatorState { - return { lifetime: "open", turn: { phase: "idle", id }, reservations: [], decisions: [] }; + return { + lifetime: "open", + turn: { phase: "idle", id }, + reservations: [], + decisions: [], + compaction: { epoch: 0, status: "ready", summaryId: null }, + }; } function hasConflictingEdit(state: CoordinatorState, owner?: symbol): boolean { @@ -205,6 +235,28 @@ export function transition( admission = { status: "deferred", reason: "busy" }; break; } + const handoff = request.compactionHandoff; + if (handoff != null) { + if ( + handoff.epoch !== state.compaction.epoch || + (state.compaction.observation?.id !== handoff.id && + state.compaction.followUp?.id !== handoff.id) + ) { + admission = { status: "rejected", reason: "retired" }; + break; + } + // This admission belongs to the captured handoff. It changes TurnId, but + // must keep the source intent alive until actual send acceptance is known. + } else { + next = { + ...next, + compaction: { + epoch: state.compaction.epoch + 1, + status: "ready", + summaryId: null, + }, + }; + } if (current) commands.push({ type: "retire", id: current.id }); phase({ phase: "preparing", id: event.id }); } @@ -252,6 +304,15 @@ export function transition( if (current?.delivery === "waiting") { operation({ ...current, stage: "started", messageId: event.payload.messageId }); } + if (next.compaction.status === "abandoned") + next = { + ...next, + compaction: { + epoch: next.compaction.epoch + 1, + status: "ready", + summaryId: null, + }, + }; commands.push({ type: "record-start", turnId: state.turn.id, payload: event.payload }); // Existing raw streams also restore sessions constructed around an already-running engine. phase({ ...next.turn, phase: "streaming" }); @@ -326,6 +387,68 @@ export function transition( case "decision": decision(event.kind, event.messageId, event.outcome); break; + case "compaction-observe": + if ( + state.lifetime === "open" && + !state.compaction.observation && + state.compaction.status === "ready" && + event.token.epoch === state.compaction.epoch + ) + next = { + ...state, + compaction: { + ...state.compaction, + observation: { ...event.token, kind: event.kind, stage: "observing" }, + }, + }; + break; + case "compaction-stage": + if ( + state.compaction.observation?.id === event.token.id && + state.compaction.epoch === event.token.epoch + ) + next = { + ...state, + compaction: { + ...state.compaction, + observation: { ...state.compaction.observation, stage: event.stage }, + }, + }; + break; + case "compaction-finish": + if (state.compaction.observation?.id === event.token.id) + next = { ...state, compaction: { ...state.compaction, observation: undefined } }; + break; + case "compaction-invalidate": + next = { + ...state, + compaction: { + ...state.compaction, + epoch: state.compaction.epoch + 1, + status: event.abandon ? "abandoned" : state.compaction.status, + // User Stop retains only cleanup ownership for its just-committed boundary. + // Context mutation retires semantic observation immediately; physical leases remain. + observation: event.abandon ? state.compaction.observation : undefined, + followUp: event.abandon ? state.compaction.followUp : undefined, + summaryId: null, + }, + }; + break; + case "compaction-follow-up": + if ( + state.lifetime === "open" && + !state.compaction.followUp && + event.token.epoch === state.compaction.epoch + ) + next = { ...state, compaction: { ...state.compaction, followUp: event.token } }; + break; + case "compaction-follow-up-finish": + if (state.compaction.followUp?.id === event.token.id) + next = { ...state, compaction: { ...state.compaction, followUp: undefined } }; + break; + case "compaction-summary": + next = { ...state, compaction: { ...state.compaction, summaryId: event.summaryId } }; + break; case "shutdown": next = { ...state, lifetime: "shutting-down" }; break; @@ -507,6 +630,58 @@ export class TurnCoordinator { return this.execution.lease(); } + get compactionIntent(): CompactionIntent { + return this.state.compaction; + } + + beginCompactionObservation(kind: CompactionObservation["kind"]): CompactionToken | undefined { + if (this.admissionBlocked || this.editReserved) return undefined; + const token = { id: Symbol("compaction observation"), epoch: this.state.compaction.epoch }; + this.dispatch({ type: "compaction-observe", token, kind }); + return this.state.compaction.observation?.id === token.id ? token : undefined; + } + + isCurrentCompaction(token: CompactionToken): boolean { + return ( + !this.closing && + token.epoch === this.state.compaction.epoch && + (this.state.compaction.observation?.id === token.id || + this.state.compaction.followUp?.id === token.id) + ); + } + + setCompactionStage(token: CompactionToken, stage: CompactionObservation["stage"]): void { + this.dispatch({ type: "compaction-stage", token, stage }); + } + + finishCompactionObservation(token: CompactionToken): boolean { + const owns = this.state.compaction.observation?.id === token.id; + this.dispatch({ type: "compaction-finish", token }); + return owns; + } + + invalidateCompaction(abandon = false): void { + this.dispatch({ type: "compaction-invalidate", abandon }); + } + + claimCompactionFollowUp(): CompactionToken | undefined { + const token = { id: Symbol("compaction follow-up"), epoch: this.state.compaction.epoch }; + this.dispatch({ type: "compaction-follow-up", token }); + return this.state.compaction.followUp?.id === token.id ? token : undefined; + } + + finishCompactionFollowUp(token: CompactionToken): void { + this.dispatch({ type: "compaction-follow-up-finish", token }); + } + + canClearCompactionFollowUp(token: CompactionToken): boolean { + return !this.closing && this.state.compaction.followUp?.id === token.id; + } + + recordCompactionSummary(summaryId: string | null): void { + this.dispatch({ type: "compaction-summary", summaryId }); + } + /** Physical completion, distinct from semantic idle and operation policy settlement. */ drain(): Promise { return this.execution.close(); diff --git a/tests/ui/compaction/compaction.test.ts b/tests/ui/compaction/compaction.test.ts index f2aa78c4efa..e6bd0becc47 100644 --- a/tests/ui/compaction/compaction.test.ts +++ b/tests/ui/compaction/compaction.test.ts @@ -9,7 +9,7 @@ import "../dom"; import { waitFor } from "@testing-library/react"; -import { preloadTestModules, type TestEnvironment } from "../../ipc/setup"; +import { preloadTestModules, setupProviders, type TestEnvironment } from "../../ipc/setup"; import { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; @@ -19,6 +19,7 @@ import { WORKSPACE_DEFAULTS } from "@/constants/workspaceDefaults"; import { updatePersistedState } from "@/browser/hooks/usePersistedState"; import { getAutoCompactionThresholdKey } from "@/common/constants/storage"; import { workspaceStore } from "@/browser/stores/WorkspaceStore"; +import { KNOWN_MODELS } from "@/common/constants/knownModels"; interface ServiceContainerPrivates { backgroundProcessManager: BackgroundProcessManager; @@ -123,7 +124,17 @@ describe("Compaction UI (mock AI router)", () => { }, 60_000); test("auto-compacts after context_exceeded and resumes", async () => { - const app = await createAppHarness({ branchPrefix: "compaction-ui" }); + const app = await createAppHarness({ + branchPrefix: "compaction-ui", + beforeRenderEnvironment: async (env) => { + // Auto-recovery needs an available compaction model even with the mock router; + // make that independent of local credentials and the catalog's context-window ordering. + await setupProviders(env, { anthropic: { apiKey: "dummy" } }); + await env.orpc.config.updateAgentAiDefaults({ + agentAiDefaults: { compact: { modelString: KNOWN_MODELS.HAIKU.id } }, + }); + }, + }); try { const triggerMessage = "Trigger context error"; From 4af8d68d3b63a9a01a1c326662bfb4166034f9d3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 14:05:39 +0200 Subject: [PATCH 02/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20accepted?= =?UTF-8?q?=20compaction=20handoffs=20and=20Stop=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record supersession at the session failure source so replacement after durable acceptance cannot restart the predecessor's recovery path. Genuine startup failures retain their error handling even when a replacement arrives later. Reject new dispatch claims after Stop and allow a separate guarded cleanup claim to remove the abandoned durable follow-up without sending it or changing a replacement's history. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: Ia08d923a50745a76ccc2bab5235820ebf5b1e90e --- .../agentSession.admissionGates.test.ts | 1 + .../agentSession.compactionAcceptance.test.ts | 115 ++++++++++++++++ ...gentSession.continueMessageAgentId.test.ts | 129 +++++++++++------- src/node/services/agentSession.ts | 48 +++++-- src/node/services/turnCoordinator.test.ts | 35 +++++ src/node/services/turnCoordinator.ts | 21 ++- 6 files changed, 282 insertions(+), 67 deletions(-) diff --git a/src/node/services/agentSession.admissionGates.test.ts b/src/node/services/agentSession.admissionGates.test.ts index 523f113e27d..3c905923c04 100644 --- a/src/node/services/agentSession.admissionGates.test.ts +++ b/src/node/services/agentSession.admissionGates.test.ts @@ -211,6 +211,7 @@ describe("AgentSession.sendMessage (admission gates)", () => { expect(result).toEqual({ success: false, error: { type: "unknown", raw: CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE }, + superseded: true, }); // Accepted, then notified so delivered-state bookkeeping can revert // (terminal-attention outbox contract, r41) — and the stale snapshot diff --git a/src/node/services/agentSession.compactionAcceptance.test.ts b/src/node/services/agentSession.compactionAcceptance.test.ts index 92d6d324b59..23d4fee5d24 100644 --- a/src/node/services/agentSession.compactionAcceptance.test.ts +++ b/src/node/services/agentSession.compactionAcceptance.test.ts @@ -1,10 +1,125 @@ import { afterEach, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; +import { Err } from "@/common/types/result"; +import type { TurnCoordinator } from "./turnCoordinator"; import * as branchSummary from "./branchSummary"; import { createAgentSessionHarness } from "./agentSession.testHarness"; afterEach(() => mock.restore()); +test.each(["before preparation", "during provider startup"] as const)( + "an accepted handoff stays continued when replaced %s before send returns an error", + async (replacementPoint) => { + const workspaceId = "replaced-accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + dispatchPendingFollowUp(): Promise; + }; + const send = h.session.sendMessage.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + if (replacementPoint === "before preparation") { + spyOn(h.session, "sendMessage").mockImplementationOnce((message, sendOptions, internal) => + send(message, sendOptions, { + ...internal, + onAccepted: async () => { + await internal?.onAccepted?.(); + entered.resolve(); + await release.promise; + }, + }) + ); + } else { + stream.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return Err({ type: "runtime_start_failed", message: "retired startup failed" }); + }); + } + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }) + ); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + // A blocked startup can be retired before its engine call returns; its + // durable continuation still belongs to the predecessor's completed handoff. + if (replacementPoint === "during provider startup") + internals.coordinator.preemptPreparation(); + expect((await send("manual replacement", options)).success).toBe(true); + release.resolve(); + expect(await pending).toBe(true); + expect(stream).toHaveBeenCalledTimes(replacementPoint === "before preparation" ? 1 : 2); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && + history.data.filter((message) => message.role === "user").map((message) => message.parts) + ).toMatchObject([ + [{ type: "text", text: "Continue" }], + [{ type: "text", text: "manual replacement" }], + ]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "an accepted handoff still reports a current provider startup failure (later replacement=%s)", + async (replaceAfterFailure) => { + const workspaceId = "failed-accepted-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Err({ type: "runtime_start_failed", message: "provider startup failed" }) + ); + if (replaceAfterFailure) { + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementationOnce(async (...args) => { + const result = await send(...args); + expect(result.success).toBe(false); + expect( + (await send("manual replacement", { model: "openai:gpt-4o", agentId: "exec" })).success + ).toBe(true); + return result; + }); + } + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + try { + const dispatch = h.session as unknown as { dispatchPendingFollowUp(): Promise }; + const failure = await dispatch.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).toHaveProperty("message", expect.stringContaining("provider startup failed")); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && history.data.find((message) => message.role === "user")?.parts + ).toMatchObject([{ type: "text", text: "Continue" }]); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + test("disposal during real pre-acceptance preparation does not report a continued handoff", async () => { const workspaceId = "unaccepted-compaction"; const h = await createAgentSessionHarness({ workspaceId }); diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index d7e54d9a4a3..821bdebe832 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -302,60 +302,64 @@ describe("AgentSession continue-message agentId fallback", () => { } }); - test("a delayed follow-up cleanup cannot overwrite a replacement summary", async () => { - const summary = compactionSummaryMessage("summary", { - text: "obsolete goal", - goalKind: "goal_continuation", - agentId: "exec", - model: "openai:gpt-4o", - }); - const { session, internals, historyService } = await createSession([summary]); - const entered = Promise.withResolvers(); - const release = Promise.withResolvers(); - const update = historyService.updateHistory.bind(historyService); - spyOn(historyService, "updateHistory").mockImplementationOnce(async (...args) => { - entered.resolve(); - await release.promise; - return update(...args); - }); - const pending = internals.dispatchPendingFollowUp(); - try { - await entered.promise; - using _mutation = session.holdTurnAdmission(); - expect( - ( - await update("ws", { - ...summary, - parts: [{ type: "text", text: "replacement summary" }], - metadata: { - ...summary.metadata, - muxMetadata: { - type: "compaction-summary", - pendingFollowUp: { - text: "replacement request", - agentId: "exec", - model: "openai:gpt-4o", + test.each([false, true])( + "a delayed follow-up cleanup cannot overwrite a replacement summary (abandoned=%s)", + async (abandoned) => { + const summary = compactionSummaryMessage("summary", { + text: "obsolete goal", + goalKind: "goal_continuation", + agentId: "exec", + model: "openai:gpt-4o", + }); + const { session, internals, historyService } = await createSession([summary]); + if (abandoned) await session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = historyService.updateHistory.bind(historyService); + spyOn(historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const pending = internals.dispatchPendingFollowUp(); + try { + await entered.promise; + using _mutation = session.holdTurnAdmission(); + expect( + ( + await update("ws", { + ...summary, + parts: [{ type: "text", text: "replacement summary" }], + metadata: { + ...summary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "replacement request", + agentId: "exec", + model: "openai:gpt-4o", + }, }, }, - }, - }) - ).success - ).toBe(true); - release.resolve(); - expect(await pending).toBe(false); - const history = await historyService.getLastMessages("ws", 1); - expect(history.success && history.data[0].parts).toEqual([ - { type: "text", text: "replacement summary" }, - ]); - expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( - "pendingFollowUp.text", - "replacement request" - ); - } finally { - release.resolve(); - await pending; + }) + ).success + ).toBe(true); + release.resolve(); + expect(await pending).toBe(false); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].parts).toEqual([ + { type: "text", text: "replacement summary" }, + ]); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement request" + ); + } finally { + release.resolve(); + await pending; + } } - }); + ); test("a delayed heartbeat rollback cannot delete a replacement context", async () => { const summary = heartbeatBoundaryMessage(); @@ -393,6 +397,29 @@ describe("AgentSession continue-message agentId fallback", () => { } }); + test.each([false, true])( + "Stop before a follow-up claim clears its durable handoff (targeted=%s)", + async (targeted) => { + const { session, internals, historyService } = await createSession([ + compactionSummaryMessage("summary", { + text: "resume", + model: "openai:gpt-4o", + agentId: "exec", + }), + ]); + const send = mockAcceptedSend(); + internals.sendMessage = send; + await session.interruptStream({ abandonPartial: true }); + expect(await internals.dispatchPendingFollowUp(targeted ? "summary" : undefined)).toBe(false); + expect(send).not.toHaveBeenCalled(); + const history = await historyService.getLastMessages("ws", 1); + expect(history.success && history.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await internals.dispatchPendingFollowUp()).toBe(false); + } + ); + test("Stop during a held summary read clears the canceled durable handoff", async () => { const { session, internals, historyService } = await createSession([ compactionSummaryMessage("summary", { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index f07d5b51d35..90fb5c9dcb7 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -227,10 +227,12 @@ type SessionCompactionContext = ContinuousCompactionContext & { * Result shape for turn-starting session methods. failureHandled marks errors * whose retry/abandon bookkeeping already ran inside streamWithHistory, so * callers must not re-handle them (would double-increment backoff attempts). + * superseded records ownership loss at the failure source, before cleanup awaits; + * an accepted durable handoff remains acknowledged even though its startup retired. */ type AgentSessionResult = | { success: true; data: T } - | { success: false; error: SendMessageError; failureHandled?: true }; + | { success: false; error: SendMessageError; failureHandled?: true; superseded?: true }; /** * Tracked file state for detecting external edits. @@ -4289,7 +4291,7 @@ export class AgentSession { // callback to revert it — returning without notifying would strand // that bookkeeping (r41). await this.settlePreparationFailure(attempt, error); - return Err(error); + return { success: false, error, superseded: true }; } const preparedTurnAbortController = new AbortController(); @@ -4312,13 +4314,15 @@ export class AgentSession { } ); if (admission.status !== "admitted") { - return Err( - createUnknownSendMessageError( + return { + success: false, + error: createUnknownSendMessageError( admission.status === "rejected" && admission.reason === "closing" ? SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE : CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE - ) - ); + ), + superseded: true, + }; } const preparedTurn = admission.turnId; @@ -5476,6 +5480,8 @@ export class AgentSession { preStartErrors?: StreamErrorPayload[] | null, preparation?: PreparationAttempt ): Promise> { + const superseded = + !this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation); if (preparation) { await this.settlePreparationFailure(preparation, error); // Recovery may synchronously claim a follow-up. Its predecessor's callback has @@ -5490,7 +5496,12 @@ export class AgentSession { for (const payload of preStartErrors ?? []) { this.coordinator.resolveErrorDecision(payload.messageId, "terminal"); } - return { success: false, error, failureHandled: true }; + return { + success: false, + error, + failureHandled: true, + superseded: superseded ? true : undefined, + }; } // Collected pre-start error events own this failure; the branches below @@ -5824,7 +5835,12 @@ export class AgentSession { for (const payload of preStartErrors) { this.coordinator.resolveErrorDecision(payload.messageId, "terminal"); } - return { success: false, error: streamResult.error, failureHandled: true }; + return { + success: false, + error: streamResult.error, + failureHandled: true, + superseded: true, + }; } return await fail(streamResult.error, acpPromptId, preStartErrors); } @@ -7896,7 +7912,9 @@ export class AgentSession { using _execution = this.coordinator.enterExecution(); // Claim before history I/O: a send admitted and completed during that read // must not make an obsolete summary look like a fresh idle continuation. - const token = this.coordinator.claimCompactionFollowUp(); + const token = + this.coordinator.claimCompactionFollowUp() ?? + this.coordinator.claimCompactionFollowUpCleanup(); if (!token) return false; try { return await this.dispatchOwnedCompactionFollowUp(token, summaryMessageId, cancelResume); @@ -8248,15 +8266,15 @@ export class AgentSession { // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, }); - if (!sendResult.success) { - if (cancelResume?.()) { + if (!sendResult.success && !(accepted && sendResult.superseded)) { + if (!accepted && cancelResume?.()) { await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. - if (followUpAdmissionStale?.() === true) { + if (!accepted && followUpAdmissionStale?.() === true) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, @@ -8273,9 +8291,9 @@ export class AgentSession { throw new Error(`Failed to dispatch pending follow-up: ${message}`); } - // A successful send can be a pre-acceptance no-op (for example disposal - // during branch-summary preparation). Only the durable acceptance callback - // acknowledges this handoff. Later manual admission cannot undo that fact. + // Success can be a pre-acceptance no-op. Conversely, an accepted send may + // return a captured supersession error before startup; that cannot undo its + // durable handoff. Genuine startup errors still follow the failure path above. if (!accepted) return false; // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): if the original wrap-up dispatcher diff --git a/src/node/services/turnCoordinator.test.ts b/src/node/services/turnCoordinator.test.ts index deb466e9de0..4f535df530a 100644 --- a/src/node/services/turnCoordinator.test.ts +++ b/src/node/services/turnCoordinator.test.ts @@ -63,6 +63,41 @@ function reduce(events: CoordinatorEvent[]) { } describe("TurnCoordinator", () => { + test("Stop permits exact follow-up cleanup but refuses dispatch until a new manual turn", () => { + const { coordinator } = setup(); + const retained = coordinator.claimCompactionFollowUp(); + if (!retained) throw new Error("Expected follow-up owner"); + coordinator.invalidateCompaction(true); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + expect(coordinator.isCurrentCompaction(retained)).toBe(false); + expect(coordinator.canClearCompactionFollowUp(retained)).toBe(true); + coordinator.finishCompactionFollowUp(retained); + + const cleanup = coordinator.claimCompactionFollowUpCleanup(); + if (!cleanup) throw new Error("Expected abandoned cleanup owner"); + expect(coordinator.isCurrentCompaction(cleanup)).toBe(false); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: cleanup, + }).status + ).toBe("rejected"); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + }).status + ).toBe("admitted"); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUp()).toBeDefined(); + }); + test("a compaction handoff survives its own admission, but stale completion cannot release its replacement", () => { const { coordinator } = setup(); const token = coordinator.beginCompactionObservation("continuous"); diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index 440b37ea3c1..a6086a25c74 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -98,7 +98,10 @@ export type CoordinatorEvent = | { type: "compaction-stage"; token: CompactionToken; stage: CompactionObservation["stage"] } | { type: "compaction-finish"; token: CompactionToken } | { type: "compaction-invalidate"; abandon: boolean } - | { type: "compaction-follow-up" | "compaction-follow-up-finish"; token: CompactionToken } + | { + type: "compaction-follow-up" | "compaction-follow-up-cleanup" | "compaction-follow-up-finish"; + token: CompactionToken; + } | { type: "compaction-summary"; summaryId: string | null } | { type: "shutdown" | "dispose" }; @@ -238,6 +241,7 @@ export function transition( const handoff = request.compactionHandoff; if (handoff != null) { if ( + state.compaction.status !== "ready" || handoff.epoch !== state.compaction.epoch || (state.compaction.observation?.id !== handoff.id && state.compaction.followUp?.id !== handoff.id) @@ -435,8 +439,11 @@ export function transition( }; break; case "compaction-follow-up": + case "compaction-follow-up-cleanup": if ( state.lifetime === "open" && + state.compaction.status === + (event.type === "compaction-follow-up" ? "ready" : "abandoned") && !state.compaction.followUp && event.token.epoch === state.compaction.epoch ) @@ -644,6 +651,7 @@ export class TurnCoordinator { isCurrentCompaction(token: CompactionToken): boolean { return ( !this.closing && + this.state.compaction.status === "ready" && token.epoch === this.state.compaction.epoch && (this.state.compaction.observation?.id === token.id || this.state.compaction.followUp?.id === token.id) @@ -670,6 +678,17 @@ export class TurnCoordinator { return this.state.compaction.followUp?.id === token.id ? token : undefined; } + claimCompactionFollowUpCleanup(): CompactionToken | undefined { + // Stop may precede the first dispatch claim. This owner can only clear the + // abandoned durable intent; it is never current or admissible for sending. + const token = { + id: Symbol("compaction follow-up cleanup"), + epoch: this.state.compaction.epoch, + }; + this.dispatch({ type: "compaction-follow-up-cleanup", token }); + return this.state.compaction.followUp?.id === token.id ? token : undefined; + } + finishCompactionFollowUp(token: CompactionToken): void { this.dispatch({ type: "compaction-follow-up-finish", token }); } From 4e5bed7367294fed1deb5fdc02b250766a424668 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 14:32:18 +0200 Subject: [PATCH 03/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20publish=20heartbeat?= =?UTF-8?q?=20rollback=20at=20the=20history=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the captured pending state and publish boundary deletion synchronously with the guarded history commit. A replacement admitted during later cleanup can no longer observe a deleted boundary with stale attachments. Join the ordered persistence afterward and preserve committed-write facts through cleanup errors. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: If698c58a4512d3a41536e56f413a003e5f0dc82f --- src/node/services/agentSession.ts | 4 +- .../compactionHandler.continuous.test.ts | 199 ++++++++++++++++++ src/node/services/compactionHandler.ts | 49 +++-- src/node/services/historyService.ts | 33 ++- 4 files changed, 255 insertions(+), 30 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 90fb5c9dcb7..0e732cb5ea8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -8343,12 +8343,12 @@ export class AgentSession { ) { const rollbackResult = await this.compactionHandler.rollbackHeartbeatContextResetBoundary( summaryMessage, - () => this.coordinator.canClearCompactionFollowUp(token) + () => this.coordinator.canClearCompactionFollowUp(token), + this.onPostCompactionStateChange ); if (!rollbackResult.success) { throw new Error(`Failed to rollback heartbeat reset boundary: ${rollbackResult.error}`); } - if (this.coordinator.canClearCompactionFollowUp(token)) this.onPostCompactionStateChange?.(); } else { await this.clearPendingFollowUpFromSummary(summaryMessage, token); } diff --git a/src/node/services/compactionHandler.continuous.test.ts b/src/node/services/compactionHandler.continuous.test.ts index da4461474f5..2f9a25eddf6 100644 --- a/src/node/services/compactionHandler.continuous.test.ts +++ b/src/node/services/compactionHandler.continuous.test.ts @@ -9,6 +9,9 @@ import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/util import { CompactionHandler } from "./compactionHandler"; import { prepareMessagesForProvider } from "./messagePipeline"; import { createTestHistoryService } from "./testHistoryService"; +import { TurnCoordinator } from "./turnCoordinator"; +import { CHAT_FILE_NAME, CHAT_ARCHIVE_FILE_NAME } from "@/common/constants/paths"; +import type { WorkspaceChatMessage } from "@/common/orpc/types"; describe("continuous compaction provider replay", () => { let store: Awaited>; @@ -22,6 +25,194 @@ describe("continuous compaction provider replay", () => { await store.cleanup(); }); + it.each(["none", "cleanup", "observer"] as const)( + "publishes committed heartbeat rollback before replacement admission (failure=%s)", + async (failurePoint) => { + const sessionDir = path.join(store.tempDir, "pending"); + await mkdir(sessionDir, { recursive: true }); + const priorDiff = { path: "/tmp/prior.ts", diff: "prior change", truncated: false }; + await writeFile( + path.join(sessionDir, "post-compaction.json"), + JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [priorDiff], + loadedSkills: [], + readFiles: [], + }) + ); + const recent = createMuxMessage("recent", "assistant", ""); + recent.parts = [ + { + type: "dynamic-tool", + toolCallId: "edit", + toolName: "file_edit_replace_string", + state: "output-available", + input: { path: "/tmp/recent.ts" }, + output: { success: true, diff: "recent change" }, + }, + ]; + await store.historyService.appendToHistory(workspaceId, recent); + const emitter = new EventEmitter(); + const emitted: WorkspaceChatMessage[] = []; + emitter.on("chat-event", (event: { message: WorkspaceChatMessage }) => { + emitted.push(event.message); + if (failurePoint === "observer" && event.message.type === "delete") + throw new Error("observer failed after commit"); + }); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter, + }); + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Heartbeat", + pendingFollowUp: { text: "wake", model: "openai:gpt-4o", agentId: "exec" }, + }) + ).success + ).toBe(true); + const boundary = await store.historyService.getLastMessages(workspaceId, 1); + assert(boundary.success, "Expected heartbeat boundary"); + const boundarySequence = boundary.data[0].metadata?.historySequence; + assert(boundarySequence != null, "Expected persisted boundary sequence"); + // The external send has already persisted its row; PREPARING can race + // rollback without needing another history write or its lock. + await store.historyService.appendToHistory( + workspaceId, + createMuxMessage("manual", "user", "manual replacement") + ); + const coordinator = new TurnCoordinator({ + phaseChanged: () => undefined, + drainQueue: () => undefined, + policy: () => Promise.resolve(), + policyError: () => undefined, + }); + const token = coordinator.claimCompactionFollowUp(); + assert(token, "Expected cleanup owner"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const rm = fsPromises.rm; + const historyPath = path.join(store.config.sessionsDir, workspaceId, CHAT_FILE_NAME); + spyOn(fsPromises, "rm").mockImplementation(async (file, options) => { + if (String(file).startsWith(`${historyPath}.continuous-`)) { + entered.resolve(); + await release.promise; + if (failurePoint === "cleanup") throw new Error("post-commit cleanup failed"); + } + return rm(file, options); + }); + const published = mock(() => undefined); + const pending = handler.rollbackHeartbeatContextResetBoundary( + boundary.data[0], + () => coordinator.canClearCompactionFollowUp(token), + published + ); + try { + await entered.promise; + const committedRows = (await fsPromises.readFile(historyPath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as MuxMessage); + expect(committedRows.map((message) => message.id)).toEqual(["manual"]); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + }).status + ).toBe("admitted"); + expect(handler.peekCachedFilePaths()).toEqual([priorDiff.path]); + expect(emitted.filter((message) => message.type === "delete")).toEqual([ + { + type: "delete", + historySequences: [boundarySequence], + }, + ]); + expect(published).toHaveBeenCalledTimes(1); + release.resolve(); + const result = await pending; + expect(result.success).toBe(failurePoint !== "cleanup"); + if (!result.success) expect(result.error).toContain("was deleted"); + expect((await handler.peekPendingState())?.diffs).toEqual([priorDiff]); + const reloaded = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + expect((await reloaded.peekPendingState())?.diffs).toEqual([priorDiff]); + } finally { + release.resolve(); + await pending; + } + } + ); + + it.each(["veto", "archived"] as const)( + "does not restore or publish a skipped heartbeat rollback (%s)", + async (skipReason) => { + const sessionDir = path.join(store.tempDir, "pending"); + const emitter = new EventEmitter(); + const handler = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter, + }); + expect( + ( + await handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Heartbeat", + pendingFollowUp: { text: "wake", model: "openai:gpt-4o", agentId: "exec" }, + }) + ).success + ).toBe(true); + const boundary = await store.historyService.getLastMessages(workspaceId, 1); + assert(boundary.success, "Expected heartbeat boundary"); + if (skipReason === "archived") { + await store.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "New boundary", { + compacted: true, + compactionBoundary: true, + compactionEpoch: 2, + }) + ); + } + const emit = spyOn(emitter, "emit"); + const published = mock(() => undefined); + const state = await handler.peekPendingState(); + expect(state).not.toBeNull(); + expect( + ( + await handler.rollbackHeartbeatContextResetBoundary( + boundary.data[0], + () => skipReason !== "veto", + published + ) + ).success + ).toBe(true); + expect(await handler.peekPendingState()).toEqual(state); + expect(emit).not.toHaveBeenCalled(); + expect(published).not.toHaveBeenCalled(); + if (skipReason === "archived") { + const archive = await fsPromises.readFile( + path.join(store.config.sessionsDir, workspaceId, CHAT_ARCHIVE_FILE_NAME), + "utf8" + ); + expect( + archive + .split("\n") + .filter(Boolean) + .map((line) => (JSON.parse(line) as MuxMessage).id) + ).toContain(boundary.data[0].id); + } + } + ); + it("a held heartbeat rollback unlink cannot consume the replacement rollback snapshot", async () => { const sessionDir = path.join(store.tempDir, "pending"); const handler = new CompactionHandler({ @@ -74,6 +265,14 @@ describe("continuous compaction provider replay", () => { release.resolve(); expect((await rollingBack).success).toBe(true); expect((await replacement).success).toBe(true); + const reloaded = new CompactionHandler({ + workspaceId, + historyService: store.historyService, + sessionDir, + emitter: new EventEmitter(), + }); + // A's delayed unlink must finish before B's pending snapshot publishes. + expect(await reloaded.peekPendingState()).not.toBeNull(); const second = await store.historyService.getLastMessages(workspaceId, 1); assert(second.success, "Expected replacement boundary"); expect((await handler.rollbackHeartbeatContextResetBoundary(second.data[0])).success).toBe( diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 719c209fde6..e9919cd0e2c 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -628,10 +628,10 @@ export class CompactionHandler { }; } - private async restoreHeartbeatResetRollbackState(): Promise { + private restoreHeartbeatResetRollbackState(): Promise { const rollbackState = this.heartbeatResetRollbackState; if (!rollbackState) { - return; + return Promise.resolve(); } // Consume the captured rollback before I/O. A new heartbeat can install its @@ -645,13 +645,13 @@ export class CompactionHandler { this.persistedPendingStateLoaded = rollbackState.persistedPendingStateLoaded; if (rollbackState.postCompactionAttachmentsPending) { - await this.persistPendingStateBestEffort( + return this.persistPendingStateBestEffort( this.cachedFileDiffs, this.cachedLoadedSkills, this.cachedReadFilePaths ); } else { - await this.deletePersistedPendingStateBestEffort(); + return this.deletePersistedPendingStateBestEffort(); } } @@ -893,7 +893,8 @@ export class CompactionHandler { async rollbackHeartbeatContextResetBoundary( summaryMessage: MuxMessage, - isCurrent: () => boolean = () => true + isCurrent: () => boolean = () => true, + onCommitted?: () => void ): Promise> { assert( summaryMessage.role === "assistant", @@ -904,6 +905,7 @@ export class CompactionHandler { "rollbackHeartbeatContextResetBoundary requires a heartbeat reset boundary" ); + let restoration: Promise | undefined; const deleteResult = await this.historyService.deleteMessage( this.workspaceId, summaryMessage.id, @@ -913,24 +915,29 @@ export class CompactionHandler { (message) => message.id === summaryMessage.id && message.metadata?.historySequence === summaryMessage.metadata?.historySequence - ) + ), + () => { + // The boundary is gone. Restore memory and enqueue its immutable disk + // snapshot before publishing events that may admit a replacement turn. + restoration = this.restoreHeartbeatResetRollbackState(); + try { + const historySequence = summaryMessage.metadata?.historySequence; + if (isNonNegativeInteger(historySequence)) { + this.emitChatEvent({ type: "delete", historySequences: [historySequence] }); + } + } finally { + onCommitted?.(); + } + } ); + // Physical persistence outlives ownership. A later admission may not veto a + // committed rollback; the existing write queue orders it before successor state. + await restoration; if (!deleteResult.success) { - return Err(`Failed to delete heartbeat reset boundary: ${deleteResult.error}`); - } - - if (!isCurrent()) return Ok(undefined); - - await this.restoreHeartbeatResetRollbackState(); - - if (!isCurrent()) return Ok(undefined); - - const historySequence = summaryMessage.metadata?.historySequence; - if (isNonNegativeInteger(historySequence)) { - this.emitChatEvent({ - type: "delete", - historySequences: [historySequence], - }); + const failure = restoration + ? "Heartbeat reset boundary was deleted, but cleanup failed" + : "Failed to delete heartbeat reset boundary"; + return Err(`${failure}: ${deleteResult.error}`); } return Ok(undefined); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 55188638d0f..ad790754a59 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2629,17 +2629,27 @@ export class HistoryService { } } - /** The final ownership check and rename do not yield to a context reset/new admission. */ + /** The ownership check, rename and commit publication never yield to new admission. */ private async writeGuardedHistory( historyPath: string, serialized: string, - isCurrent: () => boolean + isCurrent: () => boolean, + onCommitted?: () => void ): Promise { const stagedPath = `${historyPath}.continuous-${randomUUID()}`; try { await writeFileAtomic(stagedPath, serialized, { mode: 0o600 }); if (!isCurrent()) return false; renameSync(stagedPath, historyPath); + try { + onCommitted?.(); + } catch (error) { + // Observer failure cannot turn an already committed rewrite into a + // reported write failure. Filesystem cleanup errors still propagate. + log.error("Committed history rewrite publication failed", { + error: getErrorMessage(error), + }); + } return true; } finally { await fs.rm(stagedPath, { force: true }); @@ -2847,21 +2857,26 @@ export class HistoryService { * messages may already have been appended. * A conditional cleanup uses a synchronous/pure predicate under the lock and * before rename; false returns Ok without changing history. + * Its optional commit observer runs synchronously after rename, before any + * cleanup await. It must not await or reenter the history write lock. */ async deleteMessage( workspaceId: string, messageId: string, - shouldDelete?: (messages: MuxMessage[]) => boolean + shouldDelete?: (messages: MuxMessage[]) => boolean, + onCommitted?: () => void ): Promise> { + assert(!onCommitted || shouldDelete, "Delete commit observers require a conditional cleanup"); return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to delete message", () => - this.deleteMessageUnderWriteLock(workspaceId, messageId, shouldDelete) + this.deleteMessageUnderWriteLock(workspaceId, messageId, shouldDelete, onCommitted) ); } private async deleteMessageUnderWriteLock( workspaceId: string, messageId: string, - shouldDelete?: (messages: MuxMessage[]) => boolean + shouldDelete?: (messages: MuxMessage[]) => boolean, + onCommitted?: () => void ): Promise> { try { // Structural rewrite requires full file content @@ -2871,6 +2886,7 @@ export class HistoryService { const filteredMessages = messages.filter((msg) => msg.id !== messageId); if (filteredMessages.length === messages.length) { + if (shouldDelete) return Ok(undefined); // Not in the active epoch — the row may live in the sealed archive // (rare: cleanup paths almost always target recent rows). const archiveMessages = await this.readArchivedHistory(workspaceId); @@ -2894,8 +2910,11 @@ export class HistoryService { // Atomic write prevents corruption if app crashes mid-write if (shouldDelete) { if ( - !(await this.writeGuardedHistory(historyPath, historyEntries, () => - shouldDelete(messages) + !(await this.writeGuardedHistory( + historyPath, + historyEntries, + () => shouldDelete(messages), + onCommitted )) ) return Ok(undefined); From 75808743a0da523ed67e006abc8b8116a82299e6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 14:55:19 +0200 Subject: [PATCH 04/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retain=20stopped=20?= =?UTF-8?q?handoff=20cleanup=20through=20row=20rollback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep rollback scoped to the compaction handoff that persisted the rows. Stop retains abandoned cleanup ownership; a superseded handoff cannot invalidate a successor. Cover held append before and after commit, successor preservation, and restart recovery. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I0b812c3898c4c0c9e54de4f41dde21e8d35eb4de --- .../agentSession.compactionAcceptance.test.ts | 112 ++++++++++++++++++ src/node/services/agentSession.ts | 12 +- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/src/node/services/agentSession.compactionAcceptance.test.ts b/src/node/services/agentSession.compactionAcceptance.test.ts index 23d4fee5d24..0632fdb4d46 100644 --- a/src/node/services/agentSession.compactionAcceptance.test.ts +++ b/src/node/services/agentSession.compactionAcceptance.test.ts @@ -7,6 +7,118 @@ import { createAgentSessionHarness } from "./agentSession.testHarness"; afterEach(() => mock.restore()); +test.each([ + ["before", false], + ["after", false], + ["before", true], + ["after", true], +] as const)( + "Stop %s handoff append commits clears only its own pending continuation (successor=%s)", + async (commitPoint, hasSuccessor) => { + const workspaceId = "stopped-appending-compaction"; + const h = await createAgentSessionHarness({ workspaceId }); + const options = { model: "openai:gpt-4o", agentId: "exec" }; + const summary = createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }); + await h.historyService.appendToHistory(workspaceId, summary); + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + dispatchPendingFollowUp(): Promise; + }; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementation(async (...args) => { + const isHandoff = + args[1].role === "user" && + args[1].parts.some((part) => part.type === "text" && part.text === "Continue"); + if (!isHandoff) return append(...args); + if (commitPoint === "before") { + entered.resolve(); + await release.promise; + return append(...args); + } + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = internals.dispatchPendingFollowUp(); + let restarted: Awaited> | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + let successor: ReturnType; + if (hasSuccessor) { + expect((await h.session.sendMessage("manual replacement", options)).success).toBe(true); + successor = internals.coordinator.beginCompactionObservation("continuous"); + expect(successor).toBeDefined(); + // A's late rollback must not retire B's compaction work or erase a + // replacement summary, even when its history row reuses the source ID. + await h.historyService.updateHistory( + workspaceId, + createMuxMessage("summary", "assistant", "Replacement boundary", { + ...summary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Replacement follow-up", ...options }, + }, + }) + ); + } + release.resolve(); + expect(await pending).toBe(false); + expect(stream).toHaveBeenCalledTimes(hasSuccessor ? 1 : 0); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + if (!history.success) throw new Error(history.error); + expect( + history.data + .filter((message) => message.role === "user") + .map((message) => + message.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("") + ) + ).toEqual(hasSuccessor ? ["manual replacement"] : []); + const source = history.data.find((message) => message.id === summary.id); + if (hasSuccessor) { + if (!successor) throw new Error("Expected successor compaction"); + expect(internals.coordinator.isCurrentCompaction(successor)).toBe(true); + expect(source?.metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "Replacement follow-up" + ); + } else { + expect(source?.metadata?.muxMetadata).not.toHaveProperty("pendingFollowUp"); + await h.session.dispose(); + restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: h.historyService, + }); + const recovered = restarted.session as unknown as { + dispatchPendingFollowUp(): Promise; + }; + const resumed = spyOn(restarted.aiService, "streamMessage"); + expect(await recovered.dispatchPendingFollowUp()).toBe(false); + expect(resumed).not.toHaveBeenCalled(); + } + } finally { + release.resolve(); + await pending.catch(() => undefined); + await restarted?.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + test.each(["before preparation", "during provider startup"] as const)( "an accepted handoff stays continued when replaced %s before send returns an error", async (replacementPoint) => { diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0e732cb5ea8..e1e503748a2 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -3294,8 +3294,16 @@ export class AgentSession { */ const rollbackPersistedTurnRows = async (): Promise => { if (persistedCancelableMessageIds.length === 0) return true; - this.coordinator.invalidateCompaction(); - this.continuousCompactor.reset("delete-messages"); + const handoff = internal?.compactionHandoff; + const ownsCleanup = handoff != null && this.coordinator.canClearCompactionFollowUp(handoff); + // Stop retains this handoff's summary cleanup even while its user-row append + // unwinds. A replaced handoff may delete its own rows, but cannot retire its successor. + if (handoff == null || ownsCleanup || this.coordinator.isCurrentCompaction(handoff)) { + this.coordinator.invalidateCompaction( + ownsCleanup && this.coordinator.compactionIntent.status === "abandoned" + ); + this.continuousCompactor.reset("delete-messages"); + } const rollbackResult = await this.historyService.deleteMessages( this.workspaceId, persistedCancelableMessageIds From 83ed03e3fb5979d0477c7440d6724c3d8fb78e5c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 15:36:43 +0200 Subject: [PATCH 05/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20settle=20durable=20?= =?UTF-8?q?handoffs=20and=20retain=20shutdown=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acknowledge the compaction handoff at the irrevocable row frontier, preserve actual startup errors, and keep abandoned cleanup joined through shutdown. Handle rejected goal work with captured cleanup ownership without removing an accepted continuation's recovery marker. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I44b6643d5f3c64d8e9bf1f1b8235dce240ffe0b5 --- .../agentSession.compactionShutdown.test.ts | 382 ++++++++++++++++++ src/node/services/agentSession.ts | 115 +++++- src/node/services/turnCoordinator.test.ts | 40 ++ src/node/services/turnCoordinator.ts | 14 +- 4 files changed, 534 insertions(+), 17 deletions(-) create mode 100644 src/node/services/agentSession.compactionShutdown.test.ts diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts new file mode 100644 index 00000000000..2a99529928b --- /dev/null +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -0,0 +1,382 @@ +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { createMuxMessage } from "@/common/types/message"; +import { Err, Ok } from "@/common/types/result"; +import type { TurnCompletion } from "./streamManager"; +import type { TurnCoordinator } from "./turnCoordinator"; +import type { CompactionHandler } from "./compactionHandler"; +import type { HistoryService } from "./historyService"; +import { log } from "./log"; +import { ExtensionMetadataService } from "./ExtensionMetadataService"; +import { WorkspaceGoalService } from "./workspaceGoalService"; +import { createTestHistoryService } from "./testHistoryService"; +import { createAgentSessionHarness } from "./agentSession.testHarness"; + +const workspaceId = "compaction-shutdown"; +const options = { model: "openai:gpt-4o", agentId: "exec" }; +const summary = () => + createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", ...options }, + }, + }); + +async function setup() { + const history = await createTestHistoryService(); + const goalService = new WorkspaceGoalService( + history.config, + history.historyService, + new ExtensionMetadataService(`${history.config.rootDir}/extension.json`), + { recordGoalLifecycleEvent: mock(() => undefined) } + ); + const h = await createAgentSessionHarness({ + ...history, + workspaceId, + workspaceGoalService: goalService, + }); + const internals = h.session as unknown as { + coordinator: TurnCoordinator; + activeCompactionRequest?: { id: string; modelString: string }; + compactionHandler: CompactionHandler; + dispatchPendingFollowUp(summaryId?: string, cancelResume?: () => boolean): Promise; + }; + return { ...h, cleanup: history.cleanup, goalService, internals }; +} + +afterEach(() => mock.restore()); + +test.each(["Stop", "replacement", "shutdown", "dispose"] as const)( + "a durable handoff survives %s during goal synchronization", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const sync = h.goalService.syncGoalModeWithChatTail.bind(h.goalService); + spyOn(h.goalService, "syncGoalModeWithChatTail").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return sync(...args); + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + let closing: Promise | undefined; + try { + await entered.promise; + if (action === "Stop") await h.session.interruptStream({ abandonPartial: true }); + if (action === "replacement") + expect((await h.session.sendMessage("manual replacement", options)).success).toBe(true); + if (action === "shutdown") closing = h.session.finishShutdown(); + if (action === "dispose") closing = h.session.dispose(); + release.resolve(); + expect(await pending).toBe(true); + await closing; + expect(stream).toHaveBeenCalledTimes(action === "replacement" ? 1 : 0); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + history.success && history.data.find((row) => row.role === "user")?.parts + ).toMatchObject([{ type: "text", text: "Continue" }]); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "a real goal-sync failure retains the durable handoff and reports failure (Stop=%s)", + async (stop) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + spyOn(h.goalService, "syncGoalModeWithChatTail").mockImplementationOnce(async () => { + if (stop) await h.session.interruptStream({ abandonPartial: true }); + throw new Error("goal reconciliation failed"); + }); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).toHaveProperty( + "message", + expect.stringContaining("goal reconciliation failed") + ); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((row) => row.role === "user")).toBe(true); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["Stop", "replacement"] as const)( + "a synchronous row listener preserves the durable handoff on %s", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + let replacement: Promise | undefined; + const detach = h.session.onChatEvent(({ message }) => { + if (message.type !== "message" || message.role !== "user") return; + detach(); + if (action === "Stop") replacement = h.session.interruptStream({ abandonPartial: true }); + else { + // Simulate reentrant manual admission in the same synchronous event publication. + expect( + h.internals.coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: h.internals.coordinator.turnId, + }).status + ).toBe("admitted"); + } + }); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + await replacement; + expect(stream).not.toHaveBeenCalled(); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + } finally { + detach(); + await replacement; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["shutdown", "dispose"] as const)( + "Stop cleanup retains its exact guarded commit while %s joins it", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const history = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const write = history.writeGuardedHistory.bind(history); + spyOn(history, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stream = spyOn(h.aiService, "streamMessage"); + // A ready owner is already clearing a canceled resume when Stop and shutdown arrive. + const pending = h.internals.dispatchPendingFollowUp(undefined, () => true); + let closing: Promise | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + let closed = false; + closing = (action === "shutdown" ? h.session.finishShutdown() : h.session.dispose()).then( + () => { + closed = true; + } + ); + await Promise.resolve(); + expect(closed).toBe(false); + release.resolve(); + expect(await pending).toBe(false); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + await pending; + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +async function expectNoRecovery( + config: Awaited>["config"], + historyService: HistoryService +) { + const restarted = await createAgentSessionHarness({ workspaceId, config, historyService }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + const recovered = restarted.session as unknown as { + dispatchPendingFollowUp(): Promise; + }; + expect(await recovered.dispatchPendingFollowUp()).toBe(false); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } +} + +test.each(["success", "goal rejection", "boundary publication rejection"] as const)( + "Stop then shutdown before terminal cleanup removes its durable intent (%s)", + async (outcome) => { + const h = await setup(); + const completion = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Ok({ messageId: "assistant", completion: completion.promise }) + ); + spyOn(h.internals.compactionHandler, "handleCompletion").mockImplementationOnce(async () => { + await h.historyService.appendToHistory(workspaceId, summary()); + if (outcome === "boundary publication rejection") { + entered.resolve(); + await release.promise; + throw new Error("boundary publication failed"); + } + return true; + }); + spyOn(h.goalService, "applyPendingAfterStreamEnd").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (outcome === "goal rejection") throw new Error("pending goal application failed"); + return null; + }); + const errors = spyOn(log, "error"); + const consumer = spyOn(h.internals.coordinator, "consumeCompletion"); + let closing: Promise | undefined; + try { + expect((await h.session.sendMessage("original", options)).success).toBe(true); + h.internals.activeCompactionRequest = { id: "compact-request", modelString: options.model }; + completion.resolve({ + status: "completed", + streamEnd: { + type: "stream-end", + workspaceId, + metadata: { model: options.model }, + parts: [], + }, + }); + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + let closed = false; + closing = h.session.finishShutdown().then(() => { + closed = true; + }); + await Promise.resolve(); + expect(closed).toBe(false); + release.resolve(); + const policy = consumer.mock.results.at(-1); + if (policy?.type !== "return") throw new Error("Expected terminal consumer"); + await policy.value; + await closing; + expect(stream).toHaveBeenCalledTimes(1); + if (outcome !== "success") + expect(errors).toHaveBeenCalledWith("stream-end cleanup failed", { + workspaceId, + error: + outcome === "goal rejection" + ? "pending goal application failed" + : "boundary publication failed", + }); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + completion.resolve({ status: "aborted", abortReason: "user" }); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "Stop during goal redispatch admission finishes cleanup through shutdown (rejection=%s)", + async (reject) => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + ...options, + goalKind: "goal_continuation", + goalId: "00000000-0000-4000-8000-000000000001", + }, + }, + }, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage"); + spyOn(h.goalService, "buildGoalRedispatchAdmission").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (reject) throw new Error("goal admission read failed"); + return { admissible: true, admissionStale: () => false }; + }); + const pending = h.internals.dispatchPendingFollowUp(); + let closing: Promise | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + closing = h.session.finishShutdown(); + release.resolve(); + const result = await pending.catch((error: unknown) => error); + if (reject) expect(result).toHaveProperty("message", "goal admission read failed"); + else expect(result).toBe(false); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(stream).not.toHaveBeenCalled(); + await expectNoRecovery(h.config, h.historyService); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("a failed Stop rollback acknowledges the retained durable continuation", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const stream = spyOn(h.aiService, "streamMessage"); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await h.session.interruptStream({ abandonPartial: true }); + return result; + }); + spyOn(h.historyService, "deleteMessages").mockResolvedValueOnce(Err("disk unavailable")); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(true); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e1e503748a2..145774d6601 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -679,6 +679,11 @@ interface CachedMemoryContext { includesHotMemories: boolean; } +interface CompactionFollowUpDispatch { + summary?: MuxMessage; + accepted: boolean; +} + interface SendMessageInternalOptions { compactionHandoff?: CompactionToken; preparation?: PreparationAttempt; @@ -691,6 +696,8 @@ interface SendMessageInternalOptions { /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ goalId?: string; startStreamInBackground?: boolean; + /** Synchronous receipt at the rollback frontier, before goal sync or event observers. */ + onRowsDurable?: () => void; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -3326,18 +3333,26 @@ export class AgentSession { const markRowsDurable = (): void => { if (attempt.durability !== "rollback-eligible") return; attempt.durability = "durable"; + internal?.onRowsDurable?.(); if ((internal?.preTurnMessages?.length ?? 0) > 0) internal?.onPreTurnRowsPersisted?.(); }; const accept = async (): Promise => { await internal?.onAccepted?.(); attempt.durability = "accepted"; }; + // Capture known admission refusal where it occurs; later token changes cannot + // turn a genuine goal/provider failure into a successful historical handoff. + const refuseAdmission = (error: SendMessageError): AgentSessionResult => ({ + success: false, + error, + ...(attempt.durability !== "rollback-eligible" ? { superseded: true as const } : {}), + }); const refuseBeforeAcceptance = async ( error: SendMessageError ): Promise> => { if (attempt.durability === "rollback-eligible" && !(await rollbackPersistedTurnRows())) markRowsDurable(); - return Err(error); + return refuseAdmission(error); }; let cancellationHandled = false; let cancellationDisabled = false; @@ -4154,7 +4169,7 @@ export class AgentSession { // refunds keep the charge — refunding here would leave provider-visible rows uncharged. markRowsDurable(); } - return Err( + return refuseAdmission( createUnknownSendMessageError( "Send refused: the caller's admission became stale before the turn was accepted." ) @@ -4205,7 +4220,7 @@ export class AgentSession { // creation) lands in the holder the stream's prepareStep will read. const turnThinkingOverride: ActiveTurnThinkingOverride = {}; if (isAdmissionStale()) - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAdmission(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); this.coordinator.acceptThinkingOverride( turnThinkingOverride, attempt.owner ?? attempt.expectedTurn @@ -4255,10 +4270,14 @@ export class AgentSession { // A fresh accepted user send supersedes any persisted startup-abandon // classification from previous turns. if (isAdmissionStale()) - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAdmission( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); await this.clearStartupAutoRetryAbandon(); if (isAdmissionStale()) - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAdmission( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); this.retryManager.cancel(); this.retryManager.setEnabled(true); await this.persistAutoRetryEnabledPreference(true); @@ -4267,7 +4286,7 @@ export class AgentSession { // Same-session retry should resume the exact accepted request we just finalized // in history, even if runtime warmup fails before streamWithHistory() starts. if (isAdmissionStale()) - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + return refuseAdmission(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); try { await accept(); @@ -6664,6 +6683,8 @@ export class AgentSession { let emittedStreamEnd = false; const completedCompactionRequest = this.activeCompactionRequest; let continuedAfterCompaction = false; + let handledCompaction = false; + let followUpDispatchStarted = false; try { this.activeCompactionRequest = undefined; @@ -6683,6 +6704,7 @@ export class AgentSession { streamEndPayload, completedCompactionRequest?.id ); + handledCompaction = handled; if (!this.coordinator.isCurrentTurn(turn) || !this.coordinator.isCurrentOperation(operation)) return; @@ -6761,6 +6783,7 @@ export class AgentSession { // not the last row, so target it by ID (stashed in onCompactionComplete). const rlmSummaryId = this.pendingCompactionFollowUpSummaryId; this.coordinator.recordCompactionSummary(null); + followUpDispatchStarted = true; continuedAfterCompaction = await this.dispatchPendingFollowUp(rlmSummaryId ?? undefined); if ( !this.coordinator.isCurrentTurn(turn) || @@ -6837,6 +6860,27 @@ export class AgentSession { // Best-effort; don't mask the original error. } } + + // Goal/accounting or boundary-publication failures can bypass dispatch + // after the boundary commits. History revalidation also makes an uncommitted + // compaction request a no-op here. + // Stop still owns that cleanup; the terminal lease joins it before shutdown ends. + // Once dispatch began, its own receipt protects any irrevocable continuation row. + if ( + (handledCompaction || completedCompactionRequest != null) && + !followUpDispatchStarted && + this.coordinator.isCurrentOperation(operation) && + this.coordinator.compactionIntent.status === "abandoned" + ) { + try { + await this.dispatchPendingFollowUp(this.pendingCompactionFollowUpSummaryId ?? undefined); + } catch (cleanupError) { + log.warn("Abandoned compaction follow-up cleanup failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(cleanupError), + }); + } + } } finally { if (completedCompactionRequest != null) { this.coordinator.resolveCompactionDecision( @@ -7914,9 +7958,14 @@ export class AgentSession { summaryMessageId?: string, cancelResume?: () => boolean ): Promise { - if (this.coordinator.disposed || this.coordinator.closing) { + if ( + this.coordinator.disposed || + (this.coordinator.closing && this.coordinator.compactionIntent.status !== "abandoned") + ) { return false; } + // A leased terminal producer can reach its abandoned boundary only after + // shutdown begins. Join its cleanup while continuing to forbid new sends. using _execution = this.coordinator.enterExecution(); // Claim before history I/O: a send admitted and completed during that read // must not make an obsolete summary look like a fresh idle continuation. @@ -7924,8 +7973,34 @@ export class AgentSession { this.coordinator.claimCompactionFollowUp() ?? this.coordinator.claimCompactionFollowUpCleanup(); if (!token) return false; + const dispatch: CompactionFollowUpDispatch = { accepted: false }; try { - return await this.dispatchOwnedCompactionFollowUp(token, summaryMessageId, cancelResume); + return await this.dispatchOwnedCompactionFollowUp( + token, + dispatch, + summaryMessageId, + cancelResume + ); + } catch (error) { + // Rejections must settle the same abandoned obligation as normal refusals. + // Keep the captured payload guard and durable receipt: a replacement summary + // or already-persisted Continue must never lose its recovery marker here. + if ( + !dispatch.accepted && + dispatch.summary && + this.coordinator.compactionIntent.status === "abandoned" && + this.coordinator.canClearCompactionFollowUp(token) + ) { + try { + await this.clearPendingFollowUpFromSummary(dispatch.summary, token); + } catch (cleanupError) { + log.warn("Abandoned compaction follow-up cleanup failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(cleanupError), + }); + } + } + throw error; } finally { this.coordinator.finishCompactionFollowUp(token); } @@ -7933,6 +8008,7 @@ export class AgentSession { private async dispatchOwnedCompactionFollowUp( token: CompactionToken, + dispatch: CompactionFollowUpDispatch, summaryMessageId?: string, cancelResume?: () => boolean ): Promise { @@ -8013,6 +8089,7 @@ export class AgentSession { if (!isCompactionSummaryMetadata(muxMeta) || !muxMeta.pendingFollowUp) { return false; } + dispatch.summary = lastMessage; if (!this.coordinator.isCurrentCompaction(token)) { // Stop cancels dispatch while retaining ownership of its durable cleanup. @@ -8137,6 +8214,12 @@ export class AgentSession { goalAdmissionStale = admission.admissionStale; } + if (!this.coordinator.isCurrentCompaction(token)) { + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + // Codex P1 (PRRT_kwDOPxxmWM6cQt3j): the queue/busy sample above ages // across the awaited goal read and the send's own preflight. Re-evaluate // the idle rule through the send-admission gates — all of them run before @@ -8256,12 +8339,16 @@ export class AgentSession { // before sendQueuedMessages() runs, preventing race conditions. // Mark as synthetic so recovery/background dispatches do not implicitly // re-enable auto-retry after a user explicitly opted out. - let accepted = false; const sendResult = await this.sendMessage(finalText, options, { synthetic: true, compactionHandoff: token, + // Goal sync and synchronous message observers run after rollback becomes + // forbidden but before onAccepted. Stop cannot erase that durable handoff. + onRowsDurable: () => { + dispatch.accepted = true; + }, onAccepted: () => { - accepted = true; + dispatch.accepted = true; }, agentInitiated: followUp.agentInitiated, goalKind: persistedGoalKind, @@ -8274,15 +8361,15 @@ export class AgentSession { // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, }); - if (!sendResult.success && !(accepted && sendResult.superseded)) { - if (!accepted && cancelResume?.()) { + if (!sendResult.success && !(dispatch.accepted && sendResult.superseded)) { + if (!dispatch.accepted && cancelResume?.()) { await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } // A stale-admission refusal is the idle rule (or a goal transition) // working as intended, not a recovery failure: route it through the // same skip path as the pre-send check instead of throwing. - if (!accepted && followUpAdmissionStale?.() === true) { + if (!dispatch.accepted && followUpAdmissionStale?.() === true) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, @@ -8302,7 +8389,7 @@ export class AgentSession { // Success can be a pre-acceptance no-op. Conversely, an accepted send may // return a captured supersession error before startup; that cannot undo its // durable handoff. Genuine startup errors still follow the failure path above. - if (!accepted) return false; + if (!dispatch.accepted) return false; // Codex P2 (PRRT_kwDOPxxmWM6cRJEE): if the original wrap-up dispatcher // crashed between send acceptance and its tryMarkBudgetLimitInjected diff --git a/src/node/services/turnCoordinator.test.ts b/src/node/services/turnCoordinator.test.ts index 4f535df530a..a4d052fc608 100644 --- a/src/node/services/turnCoordinator.test.ts +++ b/src/node/services/turnCoordinator.test.ts @@ -98,6 +98,46 @@ describe("TurnCoordinator", () => { expect(coordinator.claimCompactionFollowUp()).toBeDefined(); }); + test.each([false, true])( + "shutdown retains only exact abandoned cleanup (claimed=%s)", + (claimed) => { + const { coordinator } = setup(); + const ready = claimed ? coordinator.claimCompactionFollowUp() : undefined; + coordinator.invalidateCompaction(true); + coordinator.beginShutdown(); + const cleanup = ready ?? coordinator.claimCompactionFollowUpCleanup(); + if (!cleanup) throw new Error("Expected cleanup through shutdown"); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + expect(coordinator.isCurrentCompaction(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect( + coordinator.prepare({ + kind: "fresh", + intent: "direct", + expectedTurnId: coordinator.turnId, + compactionHandoff: cleanup, + }).status + ).toBe("rejected"); + coordinator.dispose(); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(true); + coordinator.finishCompactionFollowUp(cleanup); + expect(coordinator.canClearCompactionFollowUp(cleanup)).toBe(false); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + } + ); + + test("shutdown preserves ready recovery intent without granting cleanup", () => { + const { coordinator } = setup(); + const ready = coordinator.claimCompactionFollowUp(); + if (!ready) throw new Error("Expected follow-up"); + coordinator.beginShutdown(); + expect(coordinator.canClearCompactionFollowUp(ready)).toBe(false); + expect(coordinator.isCurrentCompaction(ready)).toBe(false); + coordinator.finishCompactionFollowUp(ready); + expect(coordinator.claimCompactionFollowUp()).toBeUndefined(); + expect(coordinator.claimCompactionFollowUpCleanup()).toBeUndefined(); + }); + test("a compaction handoff survives its own admission, but stale completion cannot release its replacement", () => { const { coordinator } = setup(); const token = coordinator.beginCompactionObservation("continuous"); diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index a6086a25c74..6f99a431437 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -184,7 +184,9 @@ export function transition( commands.push({ type: "decision", decision: value }); }; const current = state.turn.operation; - if (state.lifetime === "disposed") + // A retained Stop cleanup may finish while destructive disposal drains its lease. + // Only its exact release remains legal; disposal never admits new work. + if (state.lifetime === "disposed" && event.type !== "compaction-follow-up-finish") return { state, commands, @@ -441,7 +443,8 @@ export function transition( case "compaction-follow-up": case "compaction-follow-up-cleanup": if ( - state.lifetime === "open" && + (state.lifetime === "open" || + (state.lifetime === "shutting-down" && event.type === "compaction-follow-up-cleanup")) && state.compaction.status === (event.type === "compaction-follow-up" ? "ready" : "abandoned") && !state.compaction.followUp && @@ -694,7 +697,12 @@ export class TurnCoordinator { } canClearCompactionFollowUp(token: CompactionToken): boolean { - return !this.closing && this.state.compaction.followUp?.id === token.id; + // Closing forbids dispatch, but cannot revoke Stop cleanup already joined + // by a physical lease. Replacement still retires the exact token. + return ( + this.state.compaction.followUp?.id === token.id && + (!this.closing || this.state.compaction.status === "abandoned") + ); } recordCompactionSummary(summaryId: string | null): void { From e5a0b0c4b583b3c73496d42b00baa42cd1fd3077 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 16:00:06 +0200 Subject: [PATCH 06/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20scope=20cleanup=20t?= =?UTF-8?q?o=20the=20pending=20handoff=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry an initial failed summary read once under the retained abandoned owner, update only the captured pending handoff from the locked current row, and reset abandonment when a replacement context advances the epoch. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: Ifb99c5fbc1fca3d7c91b98ec29794a96afa00928 --- .../agentSession.compactionShutdown.test.ts | 227 ++++++++++++++++++ src/node/services/agentSession.ts | 40 ++- src/node/services/historyService.ts | 26 +- src/node/services/turnCoordinator.ts | 3 +- 4 files changed, 275 insertions(+), 21 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index 2a99529928b..82e23d69f82 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -380,3 +380,230 @@ test("a failed Stop rollback acknowledges the retained durable continuation", as await h.cleanup(); } }); + +test.each([ + [false, false, false], + [true, false, false], + [false, true, false], + [true, true, false], + [false, false, true], + [true, false, true], +] as const)( + "a failed initial follow-up read retries only abandoned ownership (targeted=%s, replacement=%s, result error=%s)", + async (targeted, replacement, resultError) => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, boundary); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = targeted ? "getHistoryFromLatestBoundary" : "getLastMessages"; + spyOn(h.historyService, read).mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + if (resultError) return Err("initial history read failed"); + throw new Error("initial history read failed"); + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(targeted ? boundary.id : undefined); + let closing: Promise | undefined; + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + if (replacement) { + using _mutation = h.session.holdTurnAdmission(); + await h.historyService.updateHistory(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "replacement", ...options }, + }, + }, + }); + } + closing = h.session.finishShutdown(); + release.resolve(); + const failure = await pending.catch((error: unknown) => error); + expect(failure).toHaveProperty( + "message", + expect.stringContaining("initial history read failed") + ); + await closing; + const rows = await h.historyService.getLastMessages(workspaceId, 1); + if (replacement) + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement" + ); + else { + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } + expect(stream).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["fields", "pending request", "message ID", "sequence"] as const)( + "abandoned cleanup matches handoff identity and preserves newer summary fields (%s)", + async (changed) => { + const h = await setup(); + const boundary = summary(); + boundary.metadata = { + ...boundary.metadata, + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }; + await h.historyService.appendToHistory(workspaceId, boundary); + await h.session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + const rewritten = { + ...boundary, + id: changed === "message ID" ? "replacement-summary" : boundary.id, + parts: [{ type: "text" as const, text: "finalized summary" }], + metadata: { + ...boundary.metadata, + model: "openai:gpt-4o", + duration: 42, + muxMetadata: { + type: "compaction-summary" as const, + pendingFollowUp: { + text: changed === "pending request" ? "replacement" : "Continue", + ...options, + }, + }, + }, + }; + if (changed === "sequence") { + await h.historyService.deleteMessages(workspaceId, [boundary.id]); + await h.historyService.appendToHistory(workspaceId, { + ...rewritten, + metadata: { ...rewritten.metadata, historySequence: undefined }, + }); + } else if (changed === "fields") { + // Real late finalization omits compaction metadata; HistoryService must + // preserve the handoff before cleanup merges only its pending field away. + await update(workspaceId, { + ...rewritten, + metadata: { + historySequence: boundary.metadata?.historySequence, + model: "openai:gpt-4o", + duration: 42, + }, + }); + } else await update(workspaceId, rewritten); + release.resolve(); + expect(await pending).toBe(false); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "finalized summary" }, + ]); + expect(rows.success && rows.data[0].metadata).toHaveProperty("duration", 42); + expect(rows.success && rows.data[0].metadata).toHaveProperty("compactionBoundary", true); + if (changed !== "fields") + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp" + ); + else { + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + await expectNoRecovery(h.config, h.historyService); + } + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("context replacement after Stop dispatches B while A's held cleanup is retired", async () => { + const h = await setup(); + const boundary = summary(); + await h.historyService.appendToHistory(workspaceId, boundary); + await h.session.interruptStream({ abandonPartial: true }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + const stale = h.internals.dispatchPendingFollowUp(); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await entered.promise; + { + using _mutation = h.session.holdTurnAdmission(); + await update(workspaceId, { + ...boundary, + metadata: { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "replacement", ...options }, + }, + }, + }); + } + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + release.resolve(); + expect(await stale).toBe(false); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.find((row) => row.role === "user")?.parts).toMatchObject([ + { type: "text", text: "replacement" }, + ]); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty( + "pendingFollowUp.text", + "replacement" + ); + } finally { + release.resolve(); + await stale; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("a persistently unreadable abandoned follow-up retries once and preserves the initial error", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages") + .mockRejectedValueOnce(new Error("original read failure")) + .mockRejectedValueOnce(new Error("cleanup read failure")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "original read failure"); + expect(read).toHaveBeenCalledTimes(2); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 145774d6601..7f9e6773ab9 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -7987,12 +7987,16 @@ export class AgentSession { // or already-persisted Continue must never lose its recovery marker here. if ( !dispatch.accepted && - dispatch.summary && this.coordinator.compactionIntent.status === "abandoned" && this.coordinator.canClearCompactionFollowUp(token) ) { try { - await this.clearPendingFollowUpFromSummary(dispatch.summary, token); + if (dispatch.summary) await this.clearPendingFollowUpFromSummary(dispatch.summary, token); + else { + // The initial read can fail before capturing a summary. Retry once under + // the same abandoned owner; this path cannot admit a send or reclaim B. + await this.dispatchOwnedCompactionFollowUp(token, dispatch, summaryMessageId); + } } catch (cleanupError) { log.warn("Abandoned compaction follow-up cleanup failed", { workspaceId: this.workspaceId, @@ -8468,19 +8472,31 @@ export class AgentSession { return; } - const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = muxMeta; const updateResult = await this.historyService.updateHistory( this.workspaceId, - { - ...summaryMessage, - metadata: { - ...(summaryMessage.metadata ?? {}), - muxMetadata: muxMetadataWithoutFollowUp, - }, + summaryMessage, + (current) => { + const currentMeta = current.metadata?.muxMetadata; + // Summary finalization may change content/usage without replacing the + // handoff. Only its durable identity and pending request authorize cleanup. + return ( + this.coordinator.canClearCompactionFollowUp(token) && + current.id === summaryMessage.id && + current.metadata?.historySequence === summaryMessage.metadata?.historySequence && + current.role === "assistant" && + isCompactionSummaryMetadata(currentMeta) && + isDeepStrictEqual(currentMeta.pendingFollowUp, muxMeta.pendingFollowUp) + ); }, - (current) => - this.coordinator.canClearCompactionFollowUp(token) && - isDeepStrictEqual(current, summaryMessage) + (current) => { + const currentMeta = current.metadata?.muxMetadata; + assert(isCompactionSummaryMetadata(currentMeta), "Cleanup requires the guarded summary"); + const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = currentMeta; + return { + ...current, + metadata: { ...current.metadata, muxMetadata: muxMetadataWithoutFollowUp }, + }; + } ); if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index ad790754a59..ab14b2a4431 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2534,21 +2534,24 @@ export class HistoryService { * never in the sealed archive. */ // Optional ownership predicates are synchronous/pure and may run twice (under - // the lock and immediately before rename). Losing ownership is a successful no-op. + // the lock and immediately before rename). Losing ownership or the target is a successful no-op. + // An optional transform derives field edits from the locked row instead of a stale snapshot. async updateHistory( workspaceId: string, message: MuxMessage, - shouldUpdate?: (current: MuxMessage) => boolean + shouldUpdate?: (current: MuxMessage) => boolean, + updateFromCurrent?: (current: MuxMessage) => MuxMessage ): Promise> { return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => - this.updateHistoryUnderWriteLock(workspaceId, message, shouldUpdate) + this.updateHistoryUnderWriteLock(workspaceId, message, shouldUpdate, updateFromCurrent) ); } private async updateHistoryUnderWriteLock( workspaceId: string, message: MuxMessage, - shouldUpdate?: (current: MuxMessage) => boolean + shouldUpdate?: (current: MuxMessage) => boolean, + updateFromCurrent?: (current: MuxMessage) => MuxMessage ): Promise> { try { const historyPath = this.getChatHistoryPath(workspaceId); @@ -2576,6 +2579,9 @@ export class HistoryService { assert(existingMessage, "updateHistory matched message must exist"); if (shouldUpdate && !shouldUpdate(existingMessage)) return Ok(undefined); sourceMessage = existingMessage; + // Conditional field edits must preserve unrelated late writes. Derive the + // replacement from the row read under this same cross-process write lock. + const updatedMessage = updateFromCurrent?.(existingMessage) ?? message; // Preserve compaction boundary metadata during late in-place rewrites. // Compaction may update an assistant row first, then a late stream rewrite can @@ -2583,14 +2589,14 @@ export class HistoryService { const preservedCompactionMetadata = getCompactionMetadataToPreserve( workspaceId, existingMessage, - message + updatedMessage ); // Preserve the historySequence, update everything else. messages[i] = { - ...message, + ...updatedMessage, metadata: { - ...message.metadata, + ...updatedMessage.metadata, ...(preservedCompactionMetadata ?? {}), historySequence: targetSequence, }, @@ -2602,7 +2608,11 @@ export class HistoryService { } if (!found || !persistedMessage) { - return Err(`No message found with historySequence ${targetSequence}`); + // Conditional cleanup is already settled when its exact target was removed. + // It must not chase a replacement row with the same ID in another sequence. + return shouldUpdate + ? Ok(undefined) + : Err(`No message found with historySequence ${targetSequence}`); } // Rewrite entire file diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index 6f99a431437..e4155f712ce 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -431,7 +431,8 @@ export function transition( compaction: { ...state.compaction, epoch: state.compaction.epoch + 1, - status: event.abandon ? "abandoned" : state.compaction.status, + // Abandonment belongs to this canceled epoch, not a replacement context. + status: event.abandon ? "abandoned" : "ready", // User Stop retains only cleanup ownership for its just-committed boundary. // Context mutation retires semantic observation immediately; physical leases remain. observation: event.abandon ? state.compaction.observation : undefined, From 7ca91fd5ba0c187b9226331742687a11d1971efe Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 17:44:42 +0200 Subject: [PATCH 07/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20persist=20compactio?= =?UTF-8?q?n=20cancellation=20and=20resume=20held=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep canceled compaction intent durable across failed cleanup and process restarts. Record explicit replacement and Retry acceptance in history, give each Stop a distinct nonce, and retry the original failed storage mutation. Wait for temporary admission holds without consuming startup recovery, including holds encountered during send preparation. Preserve committed context invalidation and cleanup when cancellation retirement fails. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I43462c94a0b5e89d1eb9e87390fc7b22e9e2a2fa --- .../constants/compactionCancellation.ts | 1 + src/common/orpc/schemas/message.ts | 1 + src/common/types/message.ts | 2 + .../agentSession.compactionShutdown.test.ts | 526 +++++++++++++++++- ...gentSession.continueMessageAgentId.test.ts | 4 +- .../agentSession.continuousCompaction.test.ts | 2 +- .../agentSession.scopedLifetimes.test.ts | 2 +- src/node/services/agentSession.ts | 208 ++++++- .../services/compactionCancellation.test.ts | 143 +++++ src/node/services/compactionCancellation.ts | 140 +++++ src/node/services/historyService.ts | 82 ++- src/node/services/turnCoordinator.ts | 16 + src/node/services/workspaceService.test.ts | 60 ++ src/node/services/workspaceService.ts | 37 +- 14 files changed, 1186 insertions(+), 38 deletions(-) create mode 100644 src/common/constants/compactionCancellation.ts create mode 100644 src/node/services/compactionCancellation.test.ts create mode 100644 src/node/services/compactionCancellation.ts diff --git a/src/common/constants/compactionCancellation.ts b/src/common/constants/compactionCancellation.ts new file mode 100644 index 00000000000..9a4c6789b44 --- /dev/null +++ b/src/common/constants/compactionCancellation.ts @@ -0,0 +1 @@ +export const COMPACTION_CANCELLATION_FILE = "compaction-cancellation.json"; diff --git a/src/common/orpc/schemas/message.ts b/src/common/orpc/schemas/message.ts index b27ee8fac29..a46a8110e1b 100644 --- a/src/common/orpc/schemas/message.ts +++ b/src/common/orpc/schemas/message.ts @@ -152,6 +152,7 @@ export const MuxMessageSchema = z.object({ metadata: z .object({ historySequence: z.number().optional(), + compactionCancellationNonce: z.string().optional().catch(undefined), // Step cuts are an optimization; malformed legacy metadata must not block chat replay. stepStartPartIndices: z.array(z.number()).optional().catch(undefined), timestamp: z.number().optional(), diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 15921616ec9..fe80050fe9e 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -889,6 +889,8 @@ export interface ModelFallbackRecord { // Our custom metadata type export interface MuxMetadata { + /** Durable explicit replacement of a stopped compaction intent, safe across sidecar retirement crashes. */ + compactionCancellationNonce?: string; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index d4a0f0871ed..d961e444671 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1,10 +1,13 @@ +import { writeFile } from "node:fs/promises"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import type { ContinuousCompactor } from "./continuousCompactor"; import { afterEach, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import type { CompactionHandler } from "./compactionHandler"; -import type { HistoryService } from "./historyService"; +import { HistoryService } from "./historyService"; import { log } from "./log"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; import { WorkspaceGoalService } from "./workspaceGoalService"; @@ -421,7 +424,7 @@ test.each([ }, }, }); - h.session.contextMutationCommitted(); + await h.session.contextMutationCommitted(); } closing = h.session.finishShutdown(); release.resolve(); @@ -569,7 +572,7 @@ test("context replacement after Stop dispatches B while A's held cleanup is reti }, }, }); - h.session.contextMutationCommitted(); + await h.session.contextMutationCommitted(); } expect(await h.internals.dispatchPendingFollowUp()).toBe(true); release.resolve(); @@ -689,7 +692,7 @@ test("retrying retired cleanup never dispatches the replacement handoff", async }, }, }); - h.session.contextMutationCommitted(); + await h.session.contextMutationCommitted(); } await h.session.retryPendingCompactionCleanup(); expect(stream).not.toHaveBeenCalled(); @@ -700,3 +703,518 @@ test("retrying retired cleanup never dispatches the replacement handoff", async await h.cleanup(); } }); + +test("stopped follow-up stays canceled after failed cleanup and a fresh process session", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const read = spyOn(h.historyService, "getLastMessages").mockRejectedValue( + new Error("history unavailable") + ); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + await h.session.finishShutdown().catch(() => undefined); + read.mockRestore(); + const freshHistory = new HistoryService(h.config); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } + } finally { + read.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("an initial follow-up waits for admission release without consuming recovery", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const hold = h.session.holdTurnAdmission(); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const pending = h.internals.dispatchPendingFollowUp(); + await Promise.resolve(); + expect(h.internals.coordinator.compactionIntent.followUp).toBeUndefined(); + expect(stream).not.toHaveBeenCalled(); + hold[Symbol.dispose](); + expect(await pending).toBe(true); + expect(stream).toHaveBeenCalledTimes(1); + } finally { + hold[Symbol.dispose](); + await h.session.dispose(); + await h.cleanup(); + } +}, 1000); + +test.each([ + "release", + "nested release", + "Stop", + "replacement", + "shutdown", + "hold during read", + "hold during prepare", +] as const)( + "startup follow-up admission wait handles %s without another recovery trigger", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + let hold = + action === "hold during read" || action === "hold during prepare" + ? undefined + : h.session.holdTurnAdmission(); + const second = action === "nested release" ? h.session.holdTurnAdmission() : undefined; + const waiting = Promise.withResolvers(); + const wait = h.internals.coordinator.waitForAdmissionRelease.bind(h.internals.coordinator); + spyOn(h.internals.coordinator, "waitForAdmissionRelease").mockImplementation(() => { + waiting.resolve(); + return wait(); + }); + if (action === "hold during read") { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await read(...args); + hold = h.session.holdTurnAdmission(); + return result; + }); + } + if (action === "hold during prepare") { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + hold = h.session.holdTurnAdmission(); + return result; + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const recovery = h.session.runStartupRecovery(); + try { + await waiting.promise; + // No physical lease is waiting for a policy hold: shutdown can drain it. + await h.internals.coordinator.drain(); + expect(stream).not.toHaveBeenCalled(); + if (action === "Stop") await h.session.interruptStream({ abandonPartial: true }); + if (action === "replacement") { + await h.historyService.clearHistory(workspaceId); + await h.session.contextMutationCommitted(); + } + if (action === "shutdown") await h.session.finishShutdown(); + hold?.[Symbol.dispose](); + if (second) { + await Promise.resolve(); + expect(stream).not.toHaveBeenCalled(); + second[Symbol.dispose](); + } + await recovery; + expect(stream).toHaveBeenCalledTimes( + action === "release" || + action === "nested release" || + action === "hold during read" || + action === "hold during prepare" + ? 1 + : 0 + ); + } finally { + hold?.[Symbol.dispose](); + second?.[Symbol.dispose](); + await recovery; + await h.session.dispose(); + await h.cleanup(); + } + }, + 2000 +); + +test("a delayed cancellation write survives failed manual preparation", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + try { + await h.session.interruptStream({ abandonPartial: true }); + await entered.promise; + spyOn(h.historyService, "appendToHistory").mockResolvedValueOnce( + Err("replacement append failed") + ); + expect((await h.session.sendMessage("replacement", options)).success).toBe(false); + release.resolve(); + await h.session.dispose(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + await expectNoRecovery(h.config, freshHistory); + } finally { + release.resolve(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("a durable replacement witness survives a crash before cancellation retirement", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const nonce = await h.session.getCompactionCancellationNonce(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("cancellation unlink failed"); + return write(...args); + } + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.session + .sendMessage("replacement", options) + .catch((error: unknown) => error); + expect(result).toHaveProperty("message", "cancellation unlink failed"); + expect(stream).not.toHaveBeenCalled(); + const freshHistory = new HistoryService(h.config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && + rows.data.find((row) => row.role === "user")?.metadata?.compactionCancellationNonce + ).toBe(nonce); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + try { + await restarted.session.runStartupRecovery(); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + const after = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + after.success && after.data.filter((row) => row.role === "user").map((row) => row.parts) + ).toMatchObject([[{ type: "text", text: "replacement" }]]); + } finally { + await restarted.session.dispose(); + } + } finally { + writes.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("cancellation persistence failures fail teardown until an explicit durable retry", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValue( + new Error("cancellation disk unavailable") + ); + try { + await h.session.interruptStream({ abandonPartial: true }); + const failure = await h.session.dispose().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "cancellation disk unavailable"); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + expect(h.aiEmitter.listenerCount("stream-start")).toBe(0); + write.mockRestore(); + await h.session.retryPendingCompactionCleanup(); + expect(h.session.hasPendingCompactionCleanup).toBe(false); + await expectNoRecovery(h.config, new HistoryService(h.config)); + } finally { + write.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("an exact cancellation survives failed summary writes across a fresh service", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const update = spyOn(h.historyService, "updateHistory").mockResolvedValue( + Err("history rewrite unavailable") + ); + try { + await h.internals.dispatchPendingFollowUp().catch(() => undefined); + await h.session.finishShutdown().catch(() => undefined); + const freshHistory = new HistoryService(h.config); + expect((await freshHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + await expectNoRecovery(h.config, freshHistory); + } finally { + update.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } +}); + +test("an unreadable cancellation blocks journal and follow-up startup recovery", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const compactor = (h.session as unknown as { continuousCompactor: ContinuousCompactor }) + .continuousCompactor; + const recover = spyOn(compactor, "recover"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recover).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(stream).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("ordinary manual sends do not write cancellation state or attach a witness", async () => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation"); + try { + expect((await h.session.sendMessage("ordinary user", options)).success).toBe(true); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.find((row) => row.role === "user")?.metadata + ).not.toHaveProperty("compactionCancellationNonce"); + expect(write).not.toHaveBeenCalled(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("Stop during cancellation loading retains exact summary cleanup", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = h.historyService.readCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "readCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return read(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + release.resolve(); + expect(await pending).toBe(false); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + release.resolve(); + await pending; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["manual", "context replacement"] as const)( + "explicit %s repairs a corrupt cancellation without permitting old automatic recovery", + async (action) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + if (action === "manual") { + expect((await h.session.sendMessage("explicit repair", options)).success).toBe(true); + } else { + using _hold = h.session.holdTurnAdmission(); + await h.historyService.clearHistory(workspaceId); + await h.session.contextMutationCommitted(); + } + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user").length).toBe( + action === "manual" ? 1 : 0 + ); + expect(stream).toHaveBeenCalledTimes(action === "manual" ? 1 : 0); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("explicit resume durably supersedes an earlier Stop without appending another user row", async () => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "original request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).not.toBeNull(); + try { + expect((await h.session.resumeStream(options)).success).toBe(true); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user").length).toBe(1); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("a second Stop after the resume witness commits cannot be retired by the first nonce", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "request")); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const firstNonce = await h.session.getCompactionCancellationNonce(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const history = h.historyService as unknown as { + writeGuardedHistory( + path: string, + serialized: string, + guard: () => boolean, + committed?: () => void + ): Promise; + }; + const write = history.writeGuardedHistory.bind(history); + spyOn(history, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + const committed = await write(...args); + entered.resolve(); + await release.promise; + return committed; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const resume = h.session.resumeStream(options); + try { + await entered.promise; + await h.session.interruptStream({ abandonPartial: true }); + const secondNonce = await h.session.getCompactionCancellationNonce(); + expect(secondNonce).not.toBe(firstNonce); + release.resolve(); + expect(await resume).toEqual(Ok({ started: false })); + await h.session.retryPendingCompactionCleanup(); + expect(stream).not.toHaveBeenCalled(); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(secondNonce); + } finally { + release.resolve(); + await resume; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["history failure", "witness no-op", "automatic"] as const)( + "resume keeps cancellation before explicit durable acceptance (%s)", + async (outcome) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const nonce = await h.session.getCompactionCancellationNonce(); + if (outcome === "history failure") + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockResolvedValueOnce( + Err("resume history unavailable") + ); + if (outcome === "witness no-op") + spyOn(h.historyService, "updateHistory").mockResolvedValueOnce(Ok(undefined)); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.resumeStream(options, { automatic: outcome === "automatic" }); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(nonce); + expect(stream).toHaveBeenCalledTimes(outcome === "automatic" ? 1 : 0); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["user row", "summary only", "failed unlink"] as const)( + "accepted explicit resume allows B's follow-up across restart (%s)", + async (shape) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + shape === "summary only" ? summary() : createMuxMessage("user", "user", "request") + ); + await h.session.interruptStream({ abandonPartial: true }); + await h.session.retryPendingCompactionCleanup(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (shape === "failed unlink" && args[1] === null) throw new Error("resume unlink failed"); + return write(...args); + } + ); + try { + await h.session.resumeStream(options).catch(() => undefined); + // A new process first reconciles an already-durable Retry receipt if unlink failed. + const freshHistory = new HistoryService(h.config); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + try { + await restarted.session.runStartupRecovery(); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await restarted.session.dispose(); + } + await freshHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + const next = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const stream = spyOn(next.aiService, "streamMessage"); + try { + await next.session.runStartupRecovery(); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "Continue B" }, + ]); + } finally { + await next.session.dispose(); + } + } finally { + writes.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); + } + } +); diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index fdb5fa7ebb9..7e9f9fd3a73 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -344,7 +344,7 @@ describe("AgentSession continue-message agentId fallback", () => { }) ).success ).toBe(true); - session.contextMutationCommitted(); + await session.contextMutationCommitted(); release.resolve(); expect(await pending).toBe(false); const history = await historyService.getLastMessages("ws", 1); @@ -386,7 +386,7 @@ describe("AgentSession continue-message agentId fallback", () => { }) ).success ).toBe(true); - session.contextMutationCommitted(); + await session.contextMutationCommitted(); release.resolve(); expect(await pending).toBe(false); const history = await historyService.getHistoryFromLatestBoundary("ws"); diff --git a/src/node/services/agentSession.continuousCompaction.test.ts b/src/node/services/agentSession.continuousCompaction.test.ts index af75e79989a..5831a7fdaa3 100644 --- a/src/node/services/agentSession.continuousCompaction.test.ts +++ b/src/node/services/agentSession.continuousCompaction.test.ts @@ -1187,7 +1187,7 @@ describe("AgentSession continuous compaction wiring", () => { const reset = spyOn(internals(h.session).continuousCompactor, "reset"); using _admission = h.session.holdTurnAdmission(); expect(reset).not.toHaveBeenCalled(); - h.session.contextMutationCommitted(); + await h.session.contextMutationCommitted(); expect(reset).toHaveBeenCalled(); reset.mockClear(); await h.session.discardAutoRetryForContextMutation(); diff --git a/src/node/services/agentSession.scopedLifetimes.test.ts b/src/node/services/agentSession.scopedLifetimes.test.ts index f555b66ad15..f3f0752ddc8 100644 --- a/src/node/services/agentSession.scopedLifetimes.test.ts +++ b/src/node/services/agentSession.scopedLifetimes.test.ts @@ -48,7 +48,7 @@ describe("AgentSession scoped turn lifetimes", () => { await entered.promise; { using _mutation = h.session.holdTurnAdmission(); - h.session.contextMutationCommitted(); + await h.session.contextMutationCommitted(); } expect(h.session.isBusy()).toBe(false); await compactor.observe(75, context); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index bf554431846..bba82ba3ad0 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -5,6 +5,7 @@ import * as path from "path"; import assert from "@/common/utils/assert"; import { EventEmitter } from "events"; import { Effect, Fiber } from "effect"; +import { CompactionCancellation } from "./compactionCancellation"; import { StartupRecovery, type StartupRecoveryOutcome } from "./startupRecovery"; import { mkdir, readdir, readFile, unlink, writeFile } from "fs/promises"; import type { Dirent } from "fs"; @@ -233,7 +234,13 @@ type SessionCompactionContext = ContinuousCompactionContext & { */ type AgentSessionResult = | { success: true; data: T } - | { success: false; error: SendMessageError; failureHandled?: true; superseded?: true }; + | { + success: false; + error: SendMessageError; + failureHandled?: true; + superseded?: true; + admissionDeferred?: true; + }; /** * Tracked file state for detecting external edits. @@ -776,6 +783,7 @@ interface PreparationAttempt { failureAttempts?: number; failure?: SendMessageError; onFailure?: (error: SendMessageError) => Promise | void; + resumeCancellation?: { nonce: string; epoch: number }; } export class AgentSession { @@ -786,6 +794,7 @@ export class AgentSession { private readonly workspaceId: string; private readonly config: Config; private readonly historyService: HistoryService; + private readonly compactionCancellation: CompactionCancellation; private readonly aiService: AgentSessionAIService; private readonly streamManager: AgentSessionStreamManager; private readonly mcpServerManager?: MCPServerManager; @@ -865,10 +874,16 @@ export class AgentSession { // Persisted compaction follow-ups precede goal continuation recovery. Each successful // step is checkpointed, so retrying a later read cannot replay an earlier side effect. steps: [ + () => this.runStartupRecoveryStep(() => this.reconcileCompactionCancellation()), () => this.runStartupRecoveryStep(() => this.requireGoalAcknowledgmentForCrashRecoveredPartial()), - () => this.runStartupRecoveryStep(() => this.continuousCompactor.recover()), - () => this.runStartupRecoveryStep(() => this.dispatchPendingFollowUp()), + () => + this.runStartupRecoveryStep(async () => { + // An unresolved Stop also excludes a journal that has not published its summary yet. + if ((await this.compactionCancellation.read())?.scope.kind !== "unresolved") + await this.continuousCompactor.recover(); + }), + () => this.dispatchPendingFollowUp(), () => this.runStartupRecoveryStep(() => this.workspaceGoalService?.recoverPendingDispatchAfterRestart(this.workspaceId) @@ -1071,6 +1086,7 @@ export class AgentSession { this.workspaceId = trimmedWorkspaceId; this.config = config; this.historyService = historyService; + this.compactionCancellation = new CompactionCancellation(historyService, trimmedWorkspaceId); this.aiService = aiService; const streamManagerCandidate = streamManager ?? aiService; assert( @@ -1265,6 +1281,7 @@ export class AgentSession { get hasPendingCompactionCleanup(): boolean { return ( + this.compactionCancellation.needsPersistence || this.compactionCleanupRetry != null || (this.deferredCompactionCleanup != null && this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token)) @@ -1273,6 +1290,8 @@ export class AgentSession { retryPendingCompactionCleanup(): Promise { if (this.compactionCleanupRetry) return this.compactionCleanupRetry; + if (this.compactionCancellation.needsPersistence) + return this.compactionCancellation.retry().then(() => this.retryPendingCompactionCleanup()); const deferred = this.deferredCompactionCleanup; if (!deferred) return Promise.resolve(); if (!this.coordinator.canClearCompactionFollowUp(deferred.token)) { @@ -1363,6 +1382,7 @@ export class AgentSession { // A failed producer may defer cleanup while drain is running. Retry only // after its lease settles; retaining its token must never self-join drain. try { + await this.compactionCancellation.retry(); await this.retryPendingCompactionCleanup(); } catch (error) { unresolvedCompactionCleanup = @@ -1646,6 +1666,7 @@ export class AgentSession { goalKind: request.goalKind, goalId: request.goalId, retrySignal: signal, + automatic: true, }); // Interrupting the scheduling fiber cannot cancel resumeStream's original Promise. // Its late settlement must not mutate a replacement retry or accepted manual turn. @@ -2545,6 +2566,24 @@ export class AgentSession { return retryRequest?.model ?? null; } + private async reconcileCompactionCancellation(): Promise { + const cancellation = await this.compactionCancellation.read(); + if (!cancellation) return; + const history = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); + if (!history.success) throw new Error(history.error); + // A replacement witness is part of the row's atomic commit. The sidecar can + // safely retire even if the previous process died before its unlink completed. + if ( + history.data.length === 0 || + history.data.some((row) => row.metadata?.compactionCancellationNonce === cancellation.nonce) + ) + await this.compactionCancellation.retire(cancellation.nonce); + } + + getCompactionCancellationNonce(): Promise { + return this.compactionCancellation.readForReplacement().then((record) => record?.nonce); + } + private async runStartupRecoveryStep(step: () => unknown): Promise { if (this.coordinator.closing) return; using _execution = this.coordinator.enterExecution(); @@ -3429,6 +3468,26 @@ export class AgentSession { markRowsDurable(); return refuseAdmission(error); }; + const refuseContextMutation = (): Promise> => { + // Only an intact, pre-persistence handoff can wait and retry. Capture the + // refusal here; later state or error text cannot reclassify real failures. + if ( + this.coordinator.admissionBlocked && + !isAdmissionStale() && + internal?.compactionHandoff != null && + this.coordinator.isCurrentCompaction(internal.compactionHandoff) && + attempt.durability === "rollback-eligible" && + persistedCancelableMessageIds.length === 0 + ) + return Promise.resolve({ + success: false, + error: createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE), + admissionDeferred: true, + }); + return refuseBeforeAcceptance( + createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) + ); + }; let cancellationHandled = false; let cancellationDisabled = false; const cancelBeforeAcceptance = async (): Promise => { @@ -3740,9 +3799,7 @@ export class AgentSession { // other. The epoch probe (r41) also refuses edits whose target rows a // completed mutation already discarded. if (this.coordinator.admissionBlocked || isAdmissionStale()) { - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + return refuseContextMutation(); } if (this.coordinator.closing) { return refuseBeforeAcceptance( @@ -3852,6 +3909,9 @@ export class AgentSession { ? await this.withKeepRecentTailStamp(typedMuxMetadata, optionsForStream) : typedMuxMetadata; + const compactionCancellationNonce = isManualUserMessage + ? await this.getCompactionCancellationNonce() + : undefined; const userMessage = createMuxMessage( messageId, "user", @@ -3862,6 +3922,7 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, retrySendOptions: pickStartupRetrySendOptions(optionsForStream, agentInitiated, goalKind), muxMetadata: stampedMuxMetadata, // Pass through frontend metadata as black-box + ...(compactionCancellationNonce ? { compactionCancellationNonce } : {}), ...(acpPromptId != null ? { acpPromptId } : {}), ...(goalKind != null ? { kind: goalKind } : {}), // Scope goal-loop rows to their goal so a replaced goal's continuation @@ -4033,15 +4094,13 @@ export class AgentSession { autoCompactionRequest.agentInitiated ), muxMetadata: autoCompactionRequest.metadata, + ...(compactionCancellationNonce ? { compactionCancellationNonce } : {}), synthetic: true, uiVisible: true, } ); - if (this.coordinator.admissionBlocked || isAdmissionStale()) - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + if (this.coordinator.admissionBlocked || isAdmissionStale()) return refuseContextMutation(); if (this.coordinator.closing) return refuseBeforeAcceptance( createUnknownSendMessageError(SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE) @@ -4085,9 +4144,7 @@ export class AgentSession { // after a mutation commits; this check and the PREPARING gate remain // backstops for entry-accounting bypasses. if (this.coordinator.admissionBlocked || isAdmissionStale()) { - return refuseBeforeAcceptance( - createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE) - ); + return refuseContextMutation(); } // Still pre-persist: a row appended now would read as a dispatched turn on the next startup // while streamWithHistory's own latch check keeps its stream from ever running. @@ -4261,6 +4318,9 @@ export class AgentSession { // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. markRowsDurable(); + // No replacement provider/compaction may produce B before A's fence retires. + if (compactionCancellationNonce) + await this.compactionCancellation.retire(compactionCancellationNonce); try { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { @@ -4509,6 +4569,7 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; goalId?: string; retrySignal?: AbortSignal; + automatic?: boolean; } ): Promise> { this.assertNotDisposed("resumeStream"); @@ -4583,6 +4644,17 @@ export class AgentSession { ); if (admission.status !== "admitted") return Ok({ started: false }); const preparedTurn = admission.turnId; + if (!internal?.automatic && internal?.agentInitiated !== true) { + const epoch = this.coordinator.compactionIntent.epoch; + const nonce = await this.getCompactionCancellationNonce(); + if ( + !this.coordinator.isCurrentTurn(preparedTurn) || + this.coordinator.compactionIntent.epoch !== epoch || + this.coordinator.closing + ) + return Ok({ started: false }); + if (nonce) attempt.resumeCancellation = { nonce, epoch }; + } this.setAutoRetryResumeState( optionsForStream, internal?.agentInitiated, @@ -5551,6 +5623,18 @@ export class AgentSession { const interruptedPolicy = this.coordinator.captureInterruptSettlement(options?.soft); if (options?.abandonPartial || this.midStreamCompactionPending) { this.coordinator.invalidateCompaction(true); + // Register durable cancellation before any Stop await. Its independent nonce + // survives a later manual preparation that fails before committing its row. + const execution = this.coordinator.enterExecution(); + this.compactionCancellation + .cancel() + .finally(() => execution[Symbol.dispose]()) + .catch((error: unknown) => + log.warn("Failed to persist compaction cancellation", { + workspaceId: this.workspaceId, + error, + }) + ); this.continuousCompactor.reset("user-interrupt"); } @@ -5888,6 +5972,47 @@ export class AgentSession { // collect them so the Err path resolves each exactly once. const preStartErrors: StreamErrorPayload[] = []; this.coordinator.configureOperation(operation, this.activeCompactionRequest != null); + const resumedCancellation = preparation?.resumeCancellation; + if (resumedCancellation) { + const resumedUser = + lastUserMessage ?? requestMessages.findLast((row) => row.role === "user"); + if (!resumedUser) + return await fail( + createUnknownSendMessageError("Cannot accept resume without a user row") + ); + let committed = false; + // Retry is explicit user intent but reuses its existing row. Persist that + // acceptance before engine entry; failed preflight or a guarded no-op keeps Stop. + const witnessed = await this.historyService.updateHistory( + this.workspaceId, + resumedUser, + (current) => + !isStreamStartAborted() && + this.coordinator.compactionIntent.epoch === resumedCancellation.epoch && + current.id === resumedUser.id && + current.role === "user" && + current.metadata?.historySequence === resumedUser.metadata?.historySequence, + (current) => ({ + ...current, + metadata: { + ...current.metadata, + compactionCancellationNonce: resumedCancellation.nonce, + }, + }), + () => { + committed = true; + } + ); + if (committed) await this.compactionCancellation.retire(resumedCancellation.nonce); + if (!witnessed.success) return await fail(createUnknownSendMessageError(witnessed.error)); + if ( + !committed || + isStreamStartAborted() || + this.coordinator.compactionIntent.epoch !== resumedCancellation.epoch + ) + return Ok(undefined); + } + const streamResult = await this.aiService.streamMessage({ messages: requestMessages, workspaceId: this.workspaceId, @@ -7415,9 +7540,15 @@ export class AgentSession { return this.coordinator.reserve("admission"); } - contextMutationCommitted(): void { + async contextMutationCommitted(): Promise { this.coordinator.invalidateCompaction(false); this.continuousCompactor.reset("context-mutation"); + await this.retireCompactionCancellation(); + } + + async retireCompactionCancellation(): Promise { + const cancellation = await this.compactionCancellation.readForReplacement(); + if (cancellation) await this.compactionCancellation.retire(cancellation.nonce); } /** @@ -8067,6 +8198,23 @@ export class AgentSession { summaryMessageId?: string, cancelResume?: () => boolean ): Promise { + const epoch = this.coordinator.compactionIntent.epoch; + for (;;) { + // Keep the recovery checkpoint pending while a temporary hold excludes sends. + // Wait outside the I/O lease so shutdown can cancel it without joining itself. + while (this.coordinator.admissionBlocked && !this.coordinator.closing) { + await this.coordinator.waitForAdmissionRelease(); + if (this.coordinator.compactionIntent.epoch !== epoch) return false; + } + const result = await this.tryDispatchPendingFollowUp(summaryMessageId, cancelResume); + if (result !== "deferred") return result; + } + } + + private async tryDispatchPendingFollowUp( + summaryMessageId?: string, + cancelResume?: () => boolean + ): Promise { if ( this.deferredCompactionCleanup && !this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token) @@ -8080,9 +8228,6 @@ export class AgentSession { ) { return false; } - // A temporary admission hold is not a replacement. Keep deferred work for - // its next lifecycle opportunity rather than competing with context writes. - if (deferred && this.coordinator.admissionBlocked && !this.coordinator.closing) return false; this.deferredCompactionCleanup = undefined; const targetSummaryId = deferred ? deferred.summaryMessageId : summaryMessageId; // A leased terminal producer can reach its abandoned boundary only after @@ -8097,6 +8242,7 @@ export class AgentSession { if (!token) return false; const dispatch: CompactionFollowUpDispatch = deferred?.dispatch ?? { accepted: false }; try { + await this.compactionCancellation.flush(); if (deferred && dispatch.summary) { await this.clearPendingFollowUpFromSummary(dispatch.summary, token); return false; @@ -8145,7 +8291,7 @@ export class AgentSession { dispatch: CompactionFollowUpDispatch, summaryMessageId?: string, cancelResume?: () => boolean - ): Promise { + ): Promise { let summaryMessage: MuxMessage | undefined; if (summaryMessageId) { const historyResult = await this.historyService.getHistoryFromLatestBoundary( @@ -8233,6 +8379,17 @@ export class AgentSession { return false; } + const cancellation = await this.compactionCancellation.read(); + if (!this.coordinator.isCurrentCompaction(token)) { + if (this.coordinator.canClearCompactionFollowUp(token)) + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + if (cancellation && this.compactionCancellation.matches(cancellation, lastMessage)) { + await this.clearPendingFollowUpFromSummary(lastMessage, token); + return false; + } + // A user can abandon after the boundary commits but before its continuation // dispatches. Keep the fold, but remove the crash-recoverable resume intent. if (cancelResume?.()) { @@ -8468,6 +8625,10 @@ export class AgentSession { persistedGoalId ); + // A hold may arrive during history/goal reads. Release this claim and wait + // at the outer checkpoint rather than turning temporary exclusion into failure. + if (this.coordinator.admissionBlocked) return "deferred"; + // Await sendMessage to ensure the follow-up is persisted before returning. // This guarantees ordering: the follow-up message is written to history // before sendQueuedMessages() runs, preventing race conditions. @@ -8495,6 +8656,7 @@ export class AgentSession { // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, }); + if (!sendResult.success && sendResult.admissionDeferred) return "deferred"; if (!sendResult.success && !(dispatch.accepted && sendResult.superseded)) { if (!dispatch.accepted && cancelResume?.()) { await this.clearPendingFollowUpFromSummary(lastMessage, token); @@ -8602,6 +8764,14 @@ export class AgentSession { return; } + if (!this.coordinator.canClearCompactionFollowUp(token)) return; + const cancellation = await this.compactionCancellation.read(); + if (!this.coordinator.canClearCompactionFollowUp(token)) return; + const canceled = + cancellation != null && this.compactionCancellation.matches(cancellation, summaryMessage); + if (canceled) await this.compactionCancellation.narrow(cancellation.nonce, summaryMessage); + if (!this.coordinator.canClearCompactionFollowUp(token)) return; + const updateResult = await this.historyService.updateHistory( this.workspaceId, summaryMessage, @@ -8631,6 +8801,8 @@ export class AgentSession { if (!updateResult.success) { throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } + if (canceled && this.coordinator.canClearCompactionFollowUp(token)) + await this.compactionCancellation.retire(cancellation.nonce); } /** diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts new file mode 100644 index 00000000000..f48827a347c --- /dev/null +++ b/src/node/services/compactionCancellation.test.ts @@ -0,0 +1,143 @@ +import { afterEach, expect, mock, spyOn, test } from "bun:test"; +import { CompactionCancellation } from "./compactionCancellation"; +import { createTestHistoryService } from "./testHistoryService"; +import { HistoryService } from "./historyService"; +import { createMuxMessage } from "@/common/types/message"; + +afterEach(() => mock.restore()); + +test("retired cancellation writes and clears cannot overwrite the successor nonce", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-order"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stoppedA = state.cancel(); + try { + await entered.promise; + const a = await state.read(); + if (!a) throw new Error("Expected cancellation A"); + const retired = state.retire(a.nonce); + const stoppedB = state.cancel(); + const b = await state.read(); + expect(b?.nonce).not.toBe(a.nonce); + release.resolve(); + await Promise.all([stoppedA, retired, stoppedB]); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(b?.nonce); + } finally { + release.resolve(); + await stoppedA; + await state.flush(); + await h.cleanup(); + } +}); + +test("late exact narrowing cannot replace another service's newer cancellation", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-foreign"; + const a = new CompactionCancellation(h.historyService, workspaceId); + await a.cancel(); + const record = await a.read(); + if (!record) throw new Error("Expected cancellation"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const narrowing = a.narrow( + record.nonce, + createMuxMessage("summary-a", "assistant", "summary", { + historySequence: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + try { + await entered.promise; + const b = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await b.cancel(); + const successor = await b.read(); + release.resolve(); + await narrowing; + await a.retire(record.nonce); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(successor?.nonce); + } finally { + release.resolve(); + await narrowing; + await h.cleanup(); + } +}); + +test("failed retirement retries deletion while retaining conservative exclusion", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-retire"; + const state = new CompactionCancellation(h.historyService, workspaceId); + try { + await state.cancel(); + const record = await state.read(); + if (!record) throw new Error("Expected cancellation"); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink failed") + ); + const failure = await state.retire(record.nonce).catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "unlink failed"); + expect(state.needsPersistence).toBe(true); + expect((await state.read())?.nonce).toBe(record.nonce); + await state.retry(); + expect(state.needsPersistence).toBe(false); + expect(await state.read()).toBeNull(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await h.cleanup(); + } +}); + +test("failed initial publication cannot be acknowledged by exact narrowing", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-first-write"; + const state = new CompactionCancellation(h.historyService, workspaceId); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("first write failed") + ); + try { + const failure = await state.cancel().catch((error: unknown) => error); + expect(failure).toHaveProperty("message", "first write failed"); + const record = await state.read(); + if (!record) throw new Error("Expected retained cancellation"); + const narrowed = await state + .narrow( + record.nonce, + createMuxMessage("summary", "assistant", "summary", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ) + .catch((error: unknown) => error); + expect(narrowed).toHaveProperty("message", "first write failed"); + expect(state.needsPersistence).toBe(true); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + await state.retry(); + expect( + (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce + ).toBe(record.nonce); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts new file mode 100644 index 00000000000..e8236cb4ef1 --- /dev/null +++ b/src/node/services/compactionCancellation.ts @@ -0,0 +1,140 @@ +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { z } from "zod"; +import type { MuxMessage } from "@/common/types/message"; +import type { HistoryService } from "./historyService"; + +export const CompactionCancellationSchema = z.object({ + version: z.literal(1), + nonce: z.string().min(1), + scope: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("unresolved") }), + z.object({ + kind: z.literal("summary"), + id: z.string(), + sequence: z.number().optional(), + pendingFollowUp: z.record(z.string(), z.unknown()), + }), + ]), +}); +export type CompactionCancellationRecord = z.infer; + +/** Cancellation publication survives failed preparation, independently of turn admission epochs. */ +export class CompactionCancellation { + private current: CompactionCancellationRecord | null | undefined; + private generation = 0; + private pending: Promise = Promise.resolve(); + private unsettled = false; + private mutation?: { record: CompactionCancellationRecord | null; retiredNonce?: string }; + + constructor( + private readonly history: HistoryService, + private readonly workspaceId: string + ) {} + + get needsPersistence(): boolean { + return this.unsettled; + } + + async read(): Promise { + if (this.current !== undefined) return this.current; + const generation = this.generation; + const record = await this.history.readCompactionCancellation(this.workspaceId); + if (generation === this.generation) this.current = record; + return this.current ?? null; + } + + cancel(): Promise { + // Each explicit Stop is new intent, even during a previous retirement's + // post-commit await. Only retry() may reuse publication identity. + this.current = { version: 1, nonce: randomUUID(), scope: { kind: "unresolved" } }; + this.generation++; + return this.persist(this.current); + } + + async readForReplacement(): Promise { + try { + return await this.read(); + } catch { + // Explicit user intervention may repair corrupt state. First publish a + // conservative fence; failed writes still refuse the replacement safely. + await this.cancel(); + return this.read(); + } + } + + async narrow(nonce: string, summary: MuxMessage): Promise { + // Exact CAS cannot turn a failed initial publication into apparent success. + await this.pending; + if (this.current?.nonce !== nonce || this.current.scope.kind !== "unresolved") return; + const metadata = summary.metadata?.muxMetadata; + if (!metadata || !("pendingFollowUp" in metadata) || !metadata.pendingFollowUp) return; + this.current = { + ...this.current, + scope: { + kind: "summary", + id: summary.id, + sequence: summary.metadata?.historySequence, + pendingFollowUp: { ...structuredClone(metadata.pendingFollowUp) }, + }, + }; + await this.persist(this.current); + } + + matches(record: CompactionCancellationRecord, summary: MuxMessage): boolean { + if (record.scope.kind === "unresolved") return true; + const metadata = summary.metadata?.muxMetadata; + return ( + record.scope.id === summary.id && + record.scope.sequence === summary.metadata?.historySequence && + isDeepStrictEqual( + record.scope.pendingFollowUp, + metadata && "pendingFollowUp" in metadata ? metadata.pendingFollowUp : undefined + ) + ); + } + + flush(): Promise { + return this.pending; + } + + retry(): Promise { + return this.unsettled && this.mutation + ? this.persist(this.mutation.record, this.mutation.retiredNonce) + : this.pending; + } + + retire(nonce: string): Promise { + if (this.current?.nonce !== nonce) return Promise.resolve(); + this.generation++; + // Keep conservative exclusion in memory until deletion really commits. The + // retry payload remains a deletion, not a republication of that read state. + return this.persist(null, nonce); + } + + private persist( + snapshot: CompactionCancellationRecord | null, + retiredNonce?: string + ): Promise { + const generation = this.generation; + const mutation = { record: snapshot, retiredNonce }; + this.mutation = mutation; + this.unsettled = true; + const result = this.pending + .catch(() => undefined) + .then(async () => { + await this.history.writeCompactionCancellation( + this.workspaceId, + snapshot, + () => this.generation === generation, + retiredNonce + ); + if (this.generation === generation && this.mutation === mutation) { + this.unsettled = false; + if (snapshot === null && this.current?.nonce === retiredNonce) this.current = null; + } + }); + this.pending = result; + return result; + } +} diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index ae51b2f78ff..c6f4e6847bf 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2,6 +2,11 @@ import * as path from "path"; import { createHash, randomUUID } from "node:crypto"; import { renameSync } from "node:fs"; import * as fs from "fs/promises"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import { + CompactionCancellationSchema, + type CompactionCancellationRecord, +} from "./compactionCancellation"; import { ContinuousCompactionJournalStore } from "./continuousCompactionJournal"; import writeFileAtomic from "write-file-atomic"; import assert from "node:assert"; @@ -241,6 +246,61 @@ export class HistoryService { return journal; } + async readCompactionCancellation( + workspaceId: string + ): Promise { + try { + return CompactionCancellationSchema.parse( + JSON.parse( + await fs.readFile( + path.join(this.getSessionDir(workspaceId), COMPACTION_CANCELLATION_FILE), + "utf8" + ) + ) + ); + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) return null; + // An unreadable cancellation record must never authorize automatic recovery. + throw error; + } + } + + async writeCompactionCancellation( + workspaceId: string, + record: CompactionCancellationRecord | null, + isCurrent: () => boolean, + retiredNonce?: string + ): Promise { + // This file must remain writable when transcript reads/recovery are failing. + await this.withHistoryWriteFileLock(workspaceId, async () => { + if (!isCurrent()) return; + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) + throw new Error(`workspace ${workspaceId} was removed; refusing cancellation mutation`); + const cancellationPath = path.join( + this.getSessionDir(workspaceId), + COMPACTION_CANCELLATION_FILE + ); + if (record === null) { + const current = await this.readCompactionCancellation(workspaceId); + if (isCurrent() && current?.nonce === retiredNonce) + await fs.rm(cancellationPath, { force: true }); + return; + } + if (record.scope.kind === "summary") { + const current = await this.readCompactionCancellation(workspaceId); + if (current?.nonce !== record.nonce) return; + } + await ensurePrivateDir(this.getSessionDir(workspaceId)); + const stagedPath = `${cancellationPath}.${randomUUID()}`; + try { + await writeFileAtomic(stagedPath, JSON.stringify(record), { mode: 0o600 }); + if (isCurrent()) renameSync(stagedPath, cancellationPath); + } finally { + await fs.rm(stagedPath, { force: true }); + } + }); + } + private getSessionDir(workspaceId: string): string { return "getSessionDir" in this.config ? this.config.getSessionDir(workspaceId) @@ -2540,10 +2600,18 @@ export class HistoryService { workspaceId: string, message: MuxMessage, shouldUpdate?: (current: MuxMessage) => boolean, - updateFromCurrent?: (current: MuxMessage) => MuxMessage + updateFromCurrent?: (current: MuxMessage) => MuxMessage, + onCommitted?: () => void ): Promise> { + assert(!onCommitted || shouldUpdate, "Update commit observers require a conditional mutation"); return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => - this.updateHistoryUnderWriteLock(workspaceId, message, shouldUpdate, updateFromCurrent) + this.updateHistoryUnderWriteLock( + workspaceId, + message, + shouldUpdate, + updateFromCurrent, + onCommitted + ) ); } @@ -2551,7 +2619,8 @@ export class HistoryService { workspaceId: string, message: MuxMessage, shouldUpdate?: (current: MuxMessage) => boolean, - updateFromCurrent?: (current: MuxMessage) => MuxMessage + updateFromCurrent?: (current: MuxMessage) => MuxMessage, + onCommitted?: () => void ): Promise> { try { const historyPath = this.getChatHistoryPath(workspaceId); @@ -2622,7 +2691,12 @@ export class HistoryService { if (shouldUpdate && sourceMessage) { const source = sourceMessage; if ( - !(await this.writeGuardedHistory(historyPath, historyEntries, () => shouldUpdate(source))) + !(await this.writeGuardedHistory( + historyPath, + historyEntries, + () => shouldUpdate(source), + onCommitted + )) ) return Ok(undefined); } else await writeFileAtomic(historyPath, historyEntries); diff --git a/src/node/services/turnCoordinator.ts b/src/node/services/turnCoordinator.ts index 54a04aa7e12..0fc7df454e7 100644 --- a/src/node/services/turnCoordinator.ts +++ b/src/node/services/turnCoordinator.ts @@ -651,6 +651,7 @@ export class TurnCoordinator { >(); private idleWaiters = new Set<() => void>(); private unbusyWaiters = new Set<() => void>(); + private admissionWaiters = new Set<() => void>(); private prepared?: { id: TurnId; controller: AbortController }; private thinking: { holder: ActiveTurnThinkingOverride; resource?: Disposable } | null = null; private readonly execution = new TurnExecution(defaultEffectRunner); @@ -807,6 +808,7 @@ export class TurnCoordinator { ): Promise | PreparationAdmission | undefined { let launchedPolicy: Promise | undefined; let publicationError: { error: unknown } | undefined; + const previousCompactionEpoch = this.state.compaction.epoch; const result = transition(this.state, event); this.state = result.state; if (result.admission?.status === "admitted") install?.(); @@ -815,6 +817,13 @@ export class TurnCoordinator { (command) => command.type === "phase" && command.next.phase === "idle" ); const waiters = idle ? this.idleWaiters : undefined; + const admissionWaiters = + !this.admissionBlocked || + this.closing || + previousCompactionEpoch !== this.state.compaction.epoch + ? this.admissionWaiters + : undefined; + if (admissionWaiters) this.admissionWaiters = new Set(); const unbusyWaiters = !this.isBusy() ? this.unbusyWaiters : undefined; if (unbusyWaiters) this.unbusyWaiters = new Set(); const retiredThinking = idle ? this.thinking : undefined; @@ -892,6 +901,7 @@ export class TurnCoordinator { } } } + for (const resolve of admissionWaiters ?? []) resolve(); for (const resolve of waiters ?? []) resolve(); for (const resolve of unbusyWaiters ?? []) resolve(); retiredThinking?.resource?.[Symbol.dispose](); @@ -1011,6 +1021,12 @@ export class TurnCoordinator { return release; } + /** Admission holds are policy waits, not physical work joined by shutdown. */ + waitForAdmissionRelease(): Promise { + if (!this.admissionBlocked || this.closing) return Promise.resolve(); + return new Promise((resolve) => this.admissionWaiters.add(resolve)); + } + waitForIdle(signal?: AbortSignal): Promise { return this.waitForIdleState(false, signal); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index bbb1852554f..18916e53ad2 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7259,6 +7259,66 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } }); + test.each(["reset", "clear", "replace"] as const)( + "committed %s still publishes invalidation and hygiene when cancellation retirement fails", + async (operation) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "cancel-retirement-hygiene"; + await config.addWorkspace("/tmp/cancel-retirement-project", { + id: workspaceId, + name: workspaceId, + projectName: "cancel-retirement-project", + projectPath: "/tmp/cancel-retirement-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + const clearState = spyOn(session, "clearFileState"); + const clearCarryover = spyOn(session, "clearPostCompactionState"); + const discard = spyOn(sandboxHostService, "discardScope"); + const write = historyService.writeCompactionCancellation.bind(historyService); + const failedRetire = spyOn(historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("cancel unlink unavailable"); + return write(...args); + } + ); + const epochs = (workspaceService as unknown as { contextMutationEpochs: Map }) + .contextMutationEpochs; + const before = epochs.get(workspaceId) ?? 0; + try { + const result = + operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "clear" + ? await workspaceService.truncateHistory(workspaceId) + : await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context") + ); + expect(result.success).toBe(false); + expect(!result.success && result.error).toContain("cancel unlink unavailable"); + expect(epochs.get(workspaceId)).toBe(before + 1); + if (operation !== "replace") expect(clearState).toHaveBeenCalled(); + expect(clearCarryover).toHaveBeenCalled(); + expect(discard).toHaveBeenCalled(); + failedRetire.mockRestore(); + if (operation === "reset") { + expect(await workspaceService.resetContext(workspaceId)).toEqual(Ok("noop")); + expect(await historyService.readCompactionCancellation(workspaceId)).toBeNull(); + } + } finally { + failedRetire.mockRestore(); + await cleanup(); + } + } + ); + test.each([ "temporary", "busy", diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1fe1737a53c..4f693c4250e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3068,12 +3068,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ - private advanceContextMutationEpoch(workspaceId: string): void { - this.sessions.get(workspaceId)?.contextMutationCommitted(); + private async advanceContextMutationEpoch(workspaceId: string): Promise> { this.contextMutationEpochs.set( workspaceId, (this.contextMutationEpochs.get(workspaceId) ?? 0) + 1 ); + try { + await this.sessions.get(workspaceId)?.contextMutationCommitted(); + return Ok(undefined); + } catch (error) { + // History already changed. Callers must finish its other cleanup before + // reporting a cancellation-retirement failure or releasing admission. + return Err( + `History changed, but canceled compaction state could not be retired: ${getErrorMessage(error)}` + ); + } } /** @@ -12476,9 +12485,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). - if (isFullClear) { - this.advanceContextMutationEpoch(workspaceId); - } + const cancellationRetirement = isFullClear + ? await this.advanceContextMutationEpoch(workspaceId) + : Ok(undefined); // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the // registration too or the next send would re-emit the discarded summary @@ -12560,7 +12569,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - return Ok(undefined); + return cancellationRetirement; } async resetContext(workspaceId: string): Promise> { @@ -12665,6 +12674,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { `may reappear after a restart; retry once the session storage is writable.` ); } + try { + await this.getOrCreateSession(workspaceId).retireCompactionCancellation(); + } catch (error) { + return Err( + `Nothing to reset, but canceled compaction state could not be retired: ${getErrorMessage(error)}` + ); + } return Ok("noop"); } @@ -12675,6 +12691,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { { timestamp: Date.now(), contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + compactionCancellationNonce: + await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce(), } ); @@ -12685,7 +12703,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the boundary is durable — sends that entered before it must not // be admitted afterwards (their content references the discarded // context). - this.advanceContextMutationEpoch(workspaceId); + const cancellationRetirement = await this.advanceContextMutationEpoch(workspaceId); // r43: drop any settled-but-unconsumed branch-summary registration — // its row now sits behind the new boundary, and the next send would // otherwise re-emit that pre-reset summary into the live transcript. @@ -12766,6 +12784,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } + if (!cancellationRetirement.success) return cancellationRetirement; return Ok("reset"); } finally { admissionGuard[Symbol.dispose](); @@ -12802,6 +12821,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { try { let messageToAppend = summaryMessage; let deletedSequences: number[] = []; + let cancellationRetirement: Result = Ok(undefined); if (replaceMode === "append-compaction-boundary") { assert( @@ -12929,7 +12949,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (!isCompaction) { // r41: the destructive replacement is durable — refuse sends that // entered before it (see contextMutationEpochs). - this.advanceContextMutationEpoch(workspaceId); + cancellationRetirement = await this.advanceContextMutationEpoch(workspaceId); // r43: same branch-summary hygiene as full clear, and same r44 // ordering — drop the registration only after the clear commits // (see truncateHistory). @@ -12981,6 +13001,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { deletedSequences = clearResult.data; } + if (!cancellationRetirement.success) return cancellationRetirement; const appendResult = await this.historyService.appendToHistory(workspaceId, messageToAppend); if (!appendResult.success) { return Err(`Failed to append summary message: ${appendResult.error}`); From 507b70ab6ddb0fbffa5f60fb04c989031f9c1dc0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 18:16:18 +0200 Subject: [PATCH 08/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20refresh=20shared=20?= =?UTF-8?q?cancellation=20and=20complete=20history=20replacement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refresh settled cancellation records while preserving local mutation debt and fencing stale reads. Require a locked summary match before narrowing and a durable cleanup receipt before retirement, so obsolete cleanup cannot remove a successor Stop. Complete replacement append and chat publication before returning cancellation-retirement errors. Capture exact cancellation identity, including absence, across reset, clear, replacement, and no-op reset cleanup. Add real-history regressions for shared backends, held I/O, failed writes, and fresh-service recovery. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I6381dea8142e3c2fc01bf67d3320a1d7955cfde8 --- .../agentSession.compactionShutdown.test.ts | 176 +++++++++++- src/node/services/agentSession.ts | 37 ++- .../services/compactionCancellation.test.ts | 89 +++++++ src/node/services/compactionCancellation.ts | 15 +- src/node/services/workspaceService.test.ts | 250 +++++++++++++++++- src/node/services/workspaceService.ts | 48 +++- 6 files changed, 590 insertions(+), 25 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index d961e444671..19eae3380d6 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -941,8 +941,11 @@ test("an exact cancellation survives failed summary writes across a fresh servic const h = await setup(); await h.historyService.appendToHistory(workspaceId, summary()); await h.session.interruptStream({ abandonPartial: true }); - const update = spyOn(h.historyService, "updateHistory").mockResolvedValue( - Err("history rewrite unavailable") + const historyWrites = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const update = spyOn(historyWrites, "writeGuardedHistory").mockRejectedValue( + new Error("history rewrite unavailable") ); try { await h.internals.dispatchPendingFollowUp().catch(() => undefined); @@ -1218,3 +1221,172 @@ test.each(["user row", "summary only", "failed unlink"] as const)( } } ); + +test.each(["cached absence", "changed nonce"] as const)( + "a live backend observes another backend's durable Stop after %s", + async (cached) => { + const a = await setup(); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const bDispatch = ( + b.session as unknown as { dispatchPendingFollowUp(): Promise } + ).dispatchPendingFollowUp.bind(b.session); + const bWrites = bHistory as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + const failedCleanup = spyOn(bWrites, "writeGuardedHistory").mockRejectedValue( + new Error("cleanup unavailable in backend B") + ); + const stream = spyOn(a.aiService, "streamMessage"); + try { + await a.historyService.appendToHistory(workspaceId, summary()); + if (cached === "changed nonce") { + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + await bDispatch().catch(() => undefined); + expect((await bHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + } + const cachedNonce = await a.session.getCompactionCancellationNonce(); + if (cached === "changed nonce") { + await bHistory.clearHistory(workspaceId); + await b.session.contextMutationCommitted(); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + } + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + await bDispatch().catch(() => undefined); + expect( + (await new HistoryService(a.config).readCompactionCancellation(workspaceId))?.nonce + ).not.toBe(cachedNonce); + expect(await a.internals.dispatchPendingFollowUp()).toBe(false); + expect(stream).not.toHaveBeenCalled(); + } finally { + failedCleanup.mockRestore(); + await a.session.dispose().catch(() => undefined); + await b.session.dispose().catch(() => undefined); + await a.cleanup(); + } + } +); + +test("stale cross-backend summary cleanup cannot narrow or retire a successor Stop", async () => { + const a = await setup(); + await a.historyService.appendToHistory(workspaceId, summary()); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = a.historyService.readCompactionCancellation.bind(a.historyService); + spyOn(a.historyService, "readCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return read(...args); + }); + const stale = a.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await bHistory.clearHistory(workspaceId); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + const canceledB = await bHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await stale).toBe(false); + expect(await new HistoryService(a.config).readCompactionCancellation(workspaceId)).toEqual( + canceledB + ); + await expectNoRecovery(a.config, new HistoryService(a.config)); + } finally { + release.resolve(); + await stale; + await a.session.dispose(); + await b.session.dispose(); + await a.cleanup(); + } +}); + +test("failed cleanup cannot narrow a foreign replacement's newer Stop and preserves its original error", async () => { + const a = await setup(); + await a.historyService.appendToHistory(workspaceId, summary()); + await a.session.interruptStream({ abandonPartial: true }); + await a.session.retryPendingCompactionCleanup(); + const bHistory = new HistoryService(a.config); + const b = await createAgentSessionHarness({ + workspaceId, + config: a.config, + historyService: bHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const writes = a.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce( + new Error("original guarded rewrite failed") + ); + const update = a.historyService.updateHistory.bind(a.historyService); + spyOn(a.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + const result = await update(...args); + entered.resolve(); + await release.promise; + return result; + }); + const cleanup = a.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + try { + await entered.promise; + await bHistory.clearHistory(workspaceId); + await bHistory.appendToHistory( + workspaceId, + createMuxMessage("summary-b", "assistant", "B compacted", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", ...options }, + }, + }) + ); + await b.session.interruptStream({ abandonPartial: true }); + await b.session.retryPendingCompactionCleanup(); + const canceledB = await bHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await cleanup).toHaveProperty( + "message", + expect.stringContaining("original guarded rewrite failed") + ); + expect(await new HistoryService(a.config).readCompactionCancellation(workspaceId)).toEqual( + canceledB + ); + await expectNoRecovery(a.config, new HistoryService(a.config)); + } finally { + release.resolve(); + await cleanup; + await a.session.dispose(); + await b.session.dispose(); + await a.cleanup(); + } +}); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index bba82ba3ad0..dc4736e0e7a 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -7540,15 +7540,19 @@ export class AgentSession { return this.coordinator.reserve("admission"); } - async contextMutationCommitted(): Promise { + async contextMutationCommitted(cancellationNonce?: string | null): Promise { this.coordinator.invalidateCompaction(false); this.continuousCompactor.reset("context-mutation"); - await this.retireCompactionCancellation(); + await this.retireCompactionCancellation(cancellationNonce); } - async retireCompactionCancellation(): Promise { - const cancellation = await this.compactionCancellation.readForReplacement(); - if (cancellation) await this.compactionCancellation.retire(cancellation.nonce); + async retireCompactionCancellation(cancellationNonce?: string | null): Promise { + // null captures absence before a mutation; undefined preserves explicit repair callers. + const nonce = + cancellationNonce === undefined + ? (await this.compactionCancellation.readForReplacement())?.nonce + : cancellationNonce; + if (nonce) await this.compactionCancellation.retire(nonce); } /** @@ -8769,9 +8773,8 @@ export class AgentSession { if (!this.coordinator.canClearCompactionFollowUp(token)) return; const canceled = cancellation != null && this.compactionCancellation.matches(cancellation, summaryMessage); - if (canceled) await this.compactionCancellation.narrow(cancellation.nonce, summaryMessage); - if (!this.coordinator.canClearCompactionFollowUp(token)) return; - + let matched = false; + let committed = false; const updateResult = await this.historyService.updateHistory( this.workspaceId, summaryMessage, @@ -8789,6 +8792,7 @@ export class AgentSession { ); }, (current) => { + matched = true; const currentMeta = current.metadata?.muxMetadata; assert(isCompactionSummaryMetadata(currentMeta), "Cleanup requires the guarded summary"); const { pendingFollowUp: _pendingFollowUp, ...muxMetadataWithoutFollowUp } = currentMeta; @@ -8796,12 +8800,27 @@ export class AgentSession { ...current, metadata: { ...current.metadata, muxMetadata: muxMetadataWithoutFollowUp }, }; + }, + () => { + committed = true; } ); if (!updateResult.success) { + // Only a target verified under the write lock may narrow an unresolved Stop. + // A conditional no-op for obsolete A must leave a foreign successor C intact. + if (canceled && matched && this.coordinator.canClearCompactionFollowUp(token)) { + try { + await this.compactionCancellation.narrow(cancellation.nonce, summaryMessage); + } catch (error) { + log.warn("Failed to narrow canceled compaction cleanup", { + workspaceId: this.workspaceId, + error, + }); + } + } throw new Error(`Failed to clear skipped pending follow-up: ${updateResult.error}`); } - if (canceled && this.coordinator.canClearCompactionFollowUp(token)) + if (committed && canceled && this.coordinator.canClearCompactionFollowUp(token)) await this.compactionCancellation.retire(cancellation.nonce); } diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index f48827a347c..caa9e5df3d2 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -141,3 +141,92 @@ test("failed initial publication cannot be acknowledged by exact narrowing", asy await h.cleanup(); } }); + +test.each([ + ["cancel", false], + ["cancel", true], + ["narrow", false], + ["narrow", true], + ["retire", false], + ["retire", true], +] as const)( + "a held shared read cannot overwrite local %s (read error=%s)", + async (action, reject) => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-refresh-race"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const initial = await state.read(); + if (!initial) throw new Error("Expected initial cancellation"); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const read = h.historyService.readCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "readCompactionCancellation").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + if (reject) throw new Error("obsolete shared read failed"); + return result; + } + ); + const reading = state.readForReplacement(); + try { + await entered.promise; + if (action === "cancel") await state.cancel(); + if (action === "retire") await state.retire(initial.nonce); + if (action === "narrow") + await state.narrow( + initial.nonce, + createMuxMessage("summary", "assistant", "summary", { + historySequence: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + const committed = await new HistoryService(h.config).readCompactionCancellation(workspaceId); + release.resolve(); + expect(await reading).toEqual(committed); + expect(await state.read()).toEqual(committed); + expect(state.needsPersistence).toBe(false); + } finally { + release.resolve(); + await reading; + await h.cleanup(); + } + } +); + +test("refreshing a foreign nonce does not reactivate settled mutation debt", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-refresh-debt"; + const a = new CompactionCancellation(h.historyService, workspaceId); + const b = new CompactionCancellation(new HistoryService(h.config), workspaceId); + try { + await a.cancel(); + const old = await a.read(); + if (!old) throw new Error("Expected old cancellation"); + await b.cancel(); + const current = await b.read(); + expect(await a.read()).toEqual(current); + await a.retry(); + await a.retire(old.nonce); + await a.narrow( + old.nonce, + createMuxMessage("old", "assistant", "old", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "A", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + expect(a.needsPersistence).toBe(false); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + current + ); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index e8236cb4ef1..74581029c07 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -37,10 +37,19 @@ export class CompactionCancellation { } async read(): Promise { - if (this.current !== undefined) return this.current; + // Other backends can publish Stop after a previous read (including absence). + // Local in-flight/failed mutations still own their conservative exclusion. + if (this.unsettled) return this.current ?? null; const generation = this.generation; - const record = await this.history.readCompactionCancellation(this.workspaceId); - if (generation === this.generation) this.current = record; + const mutation = this.mutation; + const isCurrent = () => generation === this.generation && mutation === this.mutation; + try { + const record = await this.history.readCompactionCancellation(this.workspaceId); + if (isCurrent()) this.current = record; + } catch (error) { + // An obsolete read must not trigger explicit repair over a newer local Stop. + if (isCurrent()) throw error; + } return this.current ?? null; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 18916e53ad2..a99e14565dc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -26,7 +26,7 @@ import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import type { SendMessageError } from "@/common/types/errors"; import type { ProjectsConfig } from "@/common/types/project"; import type { Config, SecretsStore } from "@/node/config"; -import type { HistoryService } from "./historyService"; +import { HistoryService } from "./historyService"; import { createTestHistoryService } from "./testHistoryService"; import type { SessionTimingService } from "./sessionTimingService"; import { SessionUsageService } from "./sessionUsageService"; @@ -7259,6 +7259,250 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } }); + test.each([false, true])( + "no-op reset preserves a foreign Stop during cleanup (prior Stop=%s)", + async (priorStop) => { + const { config, workspaceService, cleanup } = await createServices(); + const workspaceId = "noop-reset-cancellation"; + await config.addWorkspace("/tmp/noop-reset-project", { + id: workspaceId, + name: workspaceId, + projectName: "noop-reset-project", + projectPath: "/tmp/noop-reset-project", + runtimeConfig: { type: "local" }, + }); + const session = workspaceService.getOrCreateSession(workspaceId); + if (priorStop) { + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + } + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const clear = session.clearPostCompactionState.bind(session); + spyOn(session, "clearPostCompactionState").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + await clear(); + }); + const reset = workspaceService.resetContext(workspaceId); + const foreignHistory = new HistoryService(config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config, + historyService: foreignHistory, + }); + try { + await entered.promise; + await foreignHistory.appendToHistory( + workspaceId, + createMuxMessage("foreign-summary", "assistant", "foreign context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue foreign", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + await foreign.session.interruptStream({ abandonPartial: true }); + await foreign.session.retryPendingCompactionCleanup(); + const stopped = await foreignHistory.readCompactionCancellation(workspaceId); + release.resolve(); + expect(await reset).toEqual(Ok("noop")); + expect(await new HistoryService(config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await reset; + await foreign.session.dispose(); + await cleanup(); + } + } + ); + + test("destructive replacement publishes its durable witness before reporting failed retirement", async () => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "replacement-retirement-witness"; + await config.addWorkspace("/tmp/replacement-witness-project", { + id: workspaceId, + name: workspaceId, + projectName: "replacement-witness-project", + projectPath: "/tmp/replacement-witness-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + const nonce = await session.getCompactionCancellationNonce(); + const events: WorkspaceChatMessage[] = []; + const detach = session.onChatEvent(({ message }) => events.push(message)); + const write = historyService.writeCompactionCancellation.bind(historyService); + const failedRetire = spyOn(historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("replacement retirement unavailable"); + return write(...args); + } + ); + try { + const result = await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue replacement", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + expect(!result.success && result.error).toContain("replacement retirement unavailable"); + const freshHistory = new HistoryService(config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.map((row) => [row.id, row.metadata?.compactionCancellationNonce]) + ).toEqual([["replacement", nonce]]); + expect( + events + .filter((event) => event.type === "delete" || event.type === "message") + .map((event) => event.type) + ).toEqual(["delete", "message"]); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).toHaveBeenCalledTimes(1); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + const tail = await freshHistory.getLastMessages(workspaceId, 1); + expect(tail.success && tail.data[0].parts).toMatchObject([ + { type: "text", text: "Continue replacement" }, + ]); + } finally { + await restarted.session.dispose(); + } + } finally { + detach(); + failedRetire.mockRestore(); + await cleanup(); + } + }); + + test.each([ + ["reset", false], + ["reset", true], + ["clear", false], + ["clear", true], + ["replace", false], + ["replace", true], + ] as const)( + "%s preserves a newer Stop after its durable write (prior Stop=%s)", + async (operation, priorStop) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "context-captured-cancellation"; + await config.addWorkspace("/tmp/context-cancellation-project", { + id: workspaceId, + name: workspaceId, + projectName: "context-cancellation-project", + projectPath: "/tmp/context-cancellation-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "old context") + ); + const session = workspaceService.getOrCreateSession(workspaceId); + if (priorStop) { + await session.interruptStream({ abandonPartial: true }); + await session.retryPendingCompactionCleanup(); + } + const oldNonce = await session.getCompactionCancellationNonce(); + let newNonce: string | undefined; + const stop = async () => { + await session.interruptStream({ abandonPartial: true }); + newNonce = await session.getCompactionCancellationNonce(); + }; + if (operation === "reset") { + const append = historyService.appendToHistory.bind(historyService); + spyOn(historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await stop(); + return result; + }); + } else if (operation === "clear") { + const truncate = historyService.truncateHistory.bind(historyService); + spyOn(historyService, "truncateHistory").mockImplementationOnce(async (...args) => { + const result = await truncate(...args); + await stop(); + return result; + }); + } else { + const clear = historyService.clearHistory.bind(historyService); + spyOn(historyService, "clearHistory").mockImplementationOnce(async (...args) => { + const result = await clear(...args); + await stop(); + return result; + }); + } + try { + const result = + operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "clear" + ? await workspaceService.truncateHistory(workspaceId) + : await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "new context", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue replacement", + model: "openai:gpt-4o", + agentId: "exec", + }, + }, + }) + ); + expect(result.success).toBe(true); + await session.retryPendingCompactionCleanup(); + expect(newNonce).toBeDefined(); + expect(newNonce).not.toBe(oldNonce); + const freshHistory = new HistoryService(config); + expect((await freshHistory.readCompactionCancellation(workspaceId))?.nonce).toBe(newNonce); + if (operation === "replace") { + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data[0].metadata?.compactionCancellationNonce).toBe(oldNonce); + const restarted = await createAgentSessionHarness({ + workspaceId, + config, + historyService: freshHistory, + }); + const stream = spyOn(restarted.aiService, "streamMessage"); + try { + await restarted.session.runStartupRecovery(); + expect(stream).not.toHaveBeenCalled(); + } finally { + await restarted.session.dispose(); + } + } + } finally { + await cleanup(); + } + } + ); + test.each(["reset", "clear", "replace"] as const)( "committed %s still publishes invalidation and hygiene when cancellation retirement fails", async (operation) => { @@ -7304,6 +7548,10 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { expect(result.success).toBe(false); expect(!result.success && result.error).toContain("cancel unlink unavailable"); expect(epochs.get(workspaceId)).toBe(before + 1); + if (operation === "replace") { + const persisted = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(persisted.success && persisted.data.map((row) => row.id)).toEqual(["replacement"]); + } if (operation !== "replace") expect(clearState).toHaveBeenCalled(); expect(clearCarryover).toHaveBeenCalled(); expect(discard).toHaveBeenCalled(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 4f693c4250e..669710888ae 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3068,13 +3068,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ - private async advanceContextMutationEpoch(workspaceId: string): Promise> { + private async advanceContextMutationEpoch( + workspaceId: string, + cancellationNonce?: string | null + ): Promise> { this.contextMutationEpochs.set( workspaceId, (this.contextMutationEpochs.get(workspaceId) ?? 0) + 1 ); try { - await this.sessions.get(workspaceId)?.contextMutationCommitted(); + await this.sessions.get(workspaceId)?.contextMutationCommitted(cancellationNonce); return Ok(undefined); } catch (error) { // History already changed. Callers must finish its other cleanup before @@ -12473,6 +12476,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { refuseRowRemoval: truncationScope === "none", requireFullDelete: truncationScope === "all", }); + const cancellationNonce = isFullClear + ? ((await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce()) ?? null) + : null; const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { @@ -12486,7 +12492,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). const cancellationRetirement = isFullClear - ? await this.advanceContextMutationEpoch(workspaceId) + ? await this.advanceContextMutationEpoch(workspaceId, cancellationNonce) : Ok(undefined); // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the @@ -12634,6 +12640,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + const cancellationNonce = + await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce(); const historyResult = await this.historyService.getHistoryFromLatestBoundary(workspaceId); if (!historyResult.success) { return Err(`Failed to read active context before reset: ${historyResult.error}`); @@ -12675,7 +12683,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } try { - await this.getOrCreateSession(workspaceId).retireCompactionCancellation(); + await this.getOrCreateSession(workspaceId).retireCompactionCancellation( + cancellationNonce ?? null + ); } catch (error) { return Err( `Nothing to reset, but canceled compaction state could not be retired: ${getErrorMessage(error)}` @@ -12691,8 +12701,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { { timestamp: Date.now(), contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, - compactionCancellationNonce: - await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce(), + compactionCancellationNonce: cancellationNonce, } ); @@ -12703,7 +12712,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the boundary is durable — sends that entered before it must not // be admitted afterwards (their content references the discarded // context). - const cancellationRetirement = await this.advanceContextMutationEpoch(workspaceId); + const cancellationRetirement = await this.advanceContextMutationEpoch( + workspaceId, + boundaryMessage.metadata?.compactionCancellationNonce ?? null + ); // r43: drop any settled-but-unconsumed branch-summary registration — // its row now sits behind the new boundary, and the next send would // otherwise re-emit that pre-reset summary into the live transcript. @@ -12937,6 +12949,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + const cancellationNonce = !isCompaction + ? ((await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce()) ?? null) + : null; + if (cancellationNonce) { + // The replacement row itself proves supersession if sidecar retirement + // fails. Capture before destructive I/O so a newer Stop keeps its nonce. + messageToAppend = { + ...messageToAppend, + metadata: { + ...messageToAppend.metadata, + compactionCancellationNonce: cancellationNonce, + }, + }; + } this.sessions.get(workspaceId)?.clearUsageState(); const clearResult = await this.clearHistoryWithRetiredBashMonitorWakes( workspaceId, @@ -12949,7 +12975,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (!isCompaction) { // r41: the destructive replacement is durable — refuse sends that // entered before it (see contextMutationEpochs). - cancellationRetirement = await this.advanceContextMutationEpoch(workspaceId); + cancellationRetirement = await this.advanceContextMutationEpoch( + workspaceId, + cancellationNonce + ); // r43: same branch-summary hygiene as full clear, and same r44 // ordering — drop the registration only after the clear commits // (see truncateHistory). @@ -13001,7 +13030,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { deletedSequences = clearResult.data; } - if (!cancellationRetirement.success) return cancellationRetirement; const appendResult = await this.historyService.appendToHistory(workspaceId, messageToAppend); if (!appendResult.success) { return Err(`Failed to append summary message: ${appendResult.error}`); @@ -13042,7 +13070,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.sessions.get(workspaceId)?.clearFileState(); } - return Ok(undefined); + return cancellationRetirement; } catch (error) { const message = getErrorMessage(error); return Err(`Failed to replace history: ${message}`); From 4b17334007364e2a54e5a9c3caec92a3a8250629 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 18:51:21 +0200 Subject: [PATCH 09/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20self-heal=20malform?= =?UTF-8?q?ed=20compaction=20cancellation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distinguish malformed JSON/schema bytes from storage errors. Recheck corruption under the history write lock, preserve a private quarantine copy, and durably sanitize the continuous journal and pending follow-ups before removing the corrupt cancellation fence. Failed repair remains blocked and retriable; newer cancellation and local persistence debt retain ownership. Publish repair receipts with the history commit and carry them through follow-up admission/rollback checks, preventing stale snapshots from dispatching during concurrent recovery. Cover corruption, each repair-stage failure, fresh-service recovery, and concurrent Stop. Document that the sidecar supplements legacy cleanup and cannot add support to unmodified older readers. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I972fea04b4cd6bfc34f3a516b0c73f5d534b2de1 --- .../agentSession.compactionShutdown.test.ts | 245 +++++++++++++++++- src/node/services/agentSession.ts | 5 + .../services/compactionCancellation.test.ts | 110 ++++++++ src/node/services/compactionCancellation.ts | 21 +- src/node/services/historyService.ts | 82 +++++- 5 files changed, 448 insertions(+), 15 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index 19eae3380d6..be68f255ca5 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1,4 +1,5 @@ -import { writeFile } from "node:fs/promises"; +import * as fs from "node:fs/promises"; +import { readFile, readdir, writeFile } from "node:fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import type { ContinuousCompactor } from "./continuousCompactor"; import { afterEach, expect, mock, spyOn, test } from "bun:test"; @@ -964,27 +965,261 @@ test("an exact cancellation survives failed summary writes across a fresh servic } }); -test("an unreadable cancellation blocks journal and follow-up startup recovery", async () => { +test.each(["{", JSON.stringify({ version: 1, nonce: "broken", scope: {} })])( + "malformed cancellation %s self-heals before automatic startup recovery", + async (malformed) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const sessionDir = `${h.config.sessionsDir}/${workspaceId}`; + await writeFile(`${sessionDir}/${COMPACTION_CANCELLATION_FILE}`, malformed); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + await writeFile(journal.path, "untrusted interrupted journal"); + const compactor = (h.session as unknown as { continuousCompactor: ContinuousCompactor }) + .continuousCompactor; + const recoverJournal = compactor.recover.bind(compactor); + const recover = spyOn(compactor, "recover").mockImplementation(async () => { + // The old journal must be gone before normal recovery can inspect it. + expect(await journal.exists()).toBe(false); + return recoverJournal(); + }); + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recoverGoal).toHaveBeenCalledTimes(1); + expect(recover).toHaveBeenCalledTimes(1); + expect(stream).not.toHaveBeenCalled(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(await freshHistory.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + const files = await readdir(sessionDir); + const preserved = await Promise.all( + files.map((file) => readFile(`${sessionDir}/${file}`, "utf8").catch(() => "")) + ); + expect(preserved).toContain(malformed); + await expectNoRecovery(h.config, freshHistory); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([ + [false, false], + [true, false], + [false, true], + [true, true], +] as const)( + "direct follow-up dispatch discards its pre-repair summary (targeted=%s, during history=%s)", + async (targeted, duringHistory) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + if (duringHistory) { + if (targeted) { + const read = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const captured = await read(...args); + await h.session.getCompactionCancellationNonce(); + return captured; + } + ); + } else { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const captured = await read(...args); + await h.session.getCompactionCancellationNonce(); + return captured; + }); + } + } + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp(targeted ? "summary" : undefined)).toBe( + false + ); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["goal admission", "preparation", "row persistence"] as const)( + "late repair during %s invalidates an unaccepted follow-up", + async (stage) => { + const h = await setup(); + const boundary = summary(); + if (stage === "goal admission") { + boundary.metadata = { + ...boundary.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Continue", + ...options, + goalKind: "goal_continuation", + goalId: "00000000-0000-4000-8000-000000000001", + }, + }, + }; + } + await h.historyService.appendToHistory(workspaceId, boundary); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "goal admission") { + spyOn(h.goalService, "buildGoalRedispatchAdmission").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return { admissible: true, admissionStale: () => false }; + }); + } else if (stage === "preparation") { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + } + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await h.session.runStartupRecovery(); + release.resolve(); + expect(await pending).toBe(false); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data[0].metadata?.muxMetadata).not.toHaveProperty( + "pendingFollowUp" + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("cancellation storage-access failures still block automatic startup recovery", async () => { const h = await setup(); await h.historyService.appendToHistory(workspaceId, summary()); - await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValue( + Object.assign(new Error("cancellation inaccessible"), { code: "EACCES" }) + ); const compactor = (h.session as unknown as { continuousCompactor: ContinuousCompactor }) .continuousCompactor; const recover = spyOn(compactor, "recover"); + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); const stream = spyOn(h.aiService, "streamMessage"); try { await h.session.runStartupRecovery(); expect(recover).not.toHaveBeenCalled(); + expect(recoverGoal).not.toHaveBeenCalled(); expect(stream).not.toHaveBeenCalled(); const failure = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); - expect(failure).toBeInstanceOf(Error); - expect(stream).not.toHaveBeenCalled(); + expect(failure).toHaveProperty("code", "EACCES"); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); } finally { await h.session.dispose(); await h.cleanup(); } }); +test.each(["quarantine", "journal", "history", "unlink"] as const)( + "failed corrupt cancellation repair at %s retains the fence and retries safely", + async (step) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const cancellationPath = `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`; + await writeFile(cancellationPath, "{"); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + await writeFile(journal.path, "interrupted journal"); + const failure = new Error(`${step} unavailable`); + if (step === "quarantine") { + const write = fs.writeFile; + let failed = false; + spyOn(fs, "writeFile").mockImplementation(async (...args) => { + if (!failed && typeof args[0] === "string" && args[0].includes(".corrupt.")) { + failed = true; + throw failure; + } + return write(...args); + }); + } else if (step === "journal") { + spyOn(journal, "clear").mockRejectedValueOnce(failure); + } else if (step === "unlink") { + const remove = fs.rm; + let failed = false; + spyOn(fs, "rm").mockImplementation(async (...args) => { + if (!failed && typeof args[0] === "string" && args[0] === cancellationPath) { + failed = true; + throw failure; + } + return remove(...args); + }); + } else { + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, contents: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce(failure); + } + const recoverGoal = spyOn(h.goalService, "recoverPendingDispatchAfterRestart"); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.session.runStartupRecovery(); + expect(recoverGoal).not.toHaveBeenCalled(); + expect(stream).not.toHaveBeenCalled(); + expect(await readFile(cancellationPath, "utf8")).toBe("{"); + expect(await journal.exists()).toBe(step === "quarantine" || step === "journal"); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + const metadata = rows.success && rows.data[0].metadata?.muxMetadata; + if (step === "unlink") expect(metadata).not.toHaveProperty("pendingFollowUp"); + else expect(metadata).toHaveProperty("pendingFollowUp"); + await h.session.runStartupRecovery(); + expect(recoverGoal).toHaveBeenCalledTimes(1); + expect(stream).not.toHaveBeenCalled(); + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(await freshHistory.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + await expectNoRecovery(h.config, freshHistory); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + test("ordinary manual sends do not write cancellation state or attach a witness", async () => { const h = await setup(); const write = spyOn(h.historyService, "writeCompactionCancellation"); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index dc4736e0e7a..35d6e4ce5da 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -8296,6 +8296,7 @@ export class AgentSession { summaryMessageId?: string, cancelResume?: () => boolean ): Promise { + const repairRevision = this.compactionCancellation.repairRevision; let summaryMessage: MuxMessage | undefined; if (summaryMessageId) { const historyResult = await this.historyService.getHistoryFromLatestBoundary( @@ -8389,6 +8390,9 @@ export class AgentSession { await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; } + // A repairing read may have removed this captured follow-up from history. + // Its now-absent fence cannot authorize the stale request we read before repair. + if (repairRevision !== this.compactionCancellation.repairRevision) return false; if (cancellation && this.compactionCancellation.matches(cancellation, lastMessage)) { await this.clearPendingFollowUpFromSummary(lastMessage, token); return false; @@ -8530,6 +8534,7 @@ export class AgentSession { // Ordinary durable follow-ups reconstruct the interrupted request and precede // queued input. Optional/goal/requireIdle continuations yield to manual work. const followUpAdmissionStale = () => + repairRevision !== this.compactionCancellation.repairRevision || !this.coordinator.isCurrentCompaction(token) || idleRuleStale?.() === true || goalAdmissionStale?.() === true || diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index caa9e5df3d2..6af1dac4744 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -1,3 +1,5 @@ +import { writeFile } from "node:fs/promises"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import { afterEach, expect, mock, spyOn, test } from "bun:test"; import { CompactionCancellation } from "./compactionCancellation"; import { createTestHistoryService } from "./testHistoryService"; @@ -230,3 +232,111 @@ test("refreshing a foreign nonce does not reactivate settled mutation debt", asy await h.cleanup(); } }); + +test("corrupt-read repair preserves a newer valid foreign cancellation", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-race"; + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "work")); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const state = new CompactionCancellation(h.historyService, workspaceId); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const repair = h.historyService.repairCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "repairCompactionCancellation").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return repair(...args); + } + ); + const pending = state.read(); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const newer = await foreign.read(); + release.resolve(); + expect(await pending).toEqual(newer); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + newer + ); + } finally { + release.resolve(); + await pending; + await h.cleanup(); + } +}); + +test.each(["publication", "retirement"] as const)( + "corrupt shared bytes cannot discard local cancellation %s debt", + async (mutation) => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-debt"; + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "work")); + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected cancellation"); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("write unavailable") + ); + await (mutation === "publication" ? state.cancel() : state.retire(original.nonce)).catch( + () => undefined + ); + const owned = await state.read(); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const repair = spyOn(h.historyService, "repairCompactionCancellation"); + try { + expect(await state.read()).toEqual(owned); + expect(state.needsPersistence).toBe(true); + expect(repair).not.toHaveBeenCalled(); + } finally { + await h.cleanup(); + } + } +); + +test("a local Stop during corrupt repair prevents its obsolete history commit", async () => { + const h = await createTestHistoryService(); + const workspaceId = "cancel-corrupt-local"; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("summary", "assistant", "work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }) + ); + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + const state = new CompactionCancellation(h.historyService, workspaceId); + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, contents: string, guard: () => boolean): Promise; + }; + const write = writes.writeGuardedHistory.bind(writes); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(writes, "writeGuardedHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const pending = state.read(); + try { + await entered.promise; + const stop = state.cancel(); + const newer = await state.read(); + release.resolve(); + expect(await pending).toEqual(newer); + await stop; + const freshHistory = new HistoryService(h.config); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toEqual(newer); + const rows = await freshHistory.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + } finally { + release.resolve(); + await pending; + await state.flush(); + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 74581029c07..1e057fe76f4 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -19,10 +19,18 @@ export const CompactionCancellationSchema = z.object({ }); export type CompactionCancellationRecord = z.infer; +/** Only successfully read bytes with invalid JSON/schema may enter automatic repair. */ +export class MalformedCompactionCancellationError extends Error { + constructor(readonly contents: Uint8Array) { + super("Malformed compaction cancellation record"); + } +} + /** Cancellation publication survives failed preparation, independently of turn admission epochs. */ export class CompactionCancellation { private current: CompactionCancellationRecord | null | undefined; private generation = 0; + private repairedHistoryRevision = 0; private pending: Promise = Promise.resolve(); private unsettled = false; private mutation?: { record: CompactionCancellationRecord | null; retiredNonce?: string }; @@ -36,6 +44,10 @@ export class CompactionCancellation { return this.unsettled; } + get repairRevision(): number { + return this.repairedHistoryRevision; + } + async read(): Promise { // Other backends can publish Stop after a previous read (including absence). // Local in-flight/failed mutations still own their conservative exclusion. @@ -44,7 +56,14 @@ export class CompactionCancellation { const mutation = this.mutation; const isCurrent = () => generation === this.generation && mutation === this.mutation; try { - const record = await this.history.readCompactionCancellation(this.workspaceId); + const record = await this.history + .readCompactionCancellation(this.workspaceId) + .catch((error: unknown) => { + if (!(error instanceof MalformedCompactionCancellationError) || !isCurrent()) throw error; + return this.history.repairCompactionCancellation(this.workspaceId, isCurrent, () => { + this.repairedHistoryRevision++; + }); + }); if (isCurrent()) this.current = record; } catch (error) { // An obsolete read must not trigger explicit repair over a newer local Stop. diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index c6f4e6847bf..0c950106df8 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -5,6 +5,7 @@ import * as fs from "fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import { CompactionCancellationSchema, + MalformedCompactionCancellationError, type CompactionCancellationRecord, } from "./compactionCancellation"; import { ContinuousCompactionJournalStore } from "./continuousCompactionJournal"; @@ -249,20 +250,81 @@ export class HistoryService { async readCompactionCancellation( workspaceId: string ): Promise { + let contents: Buffer; try { - return CompactionCancellationSchema.parse( - JSON.parse( - await fs.readFile( - path.join(this.getSessionDir(workspaceId), COMPACTION_CANCELLATION_FILE), - "utf8" - ) - ) + contents = await fs.readFile( + path.join(this.getSessionDir(workspaceId), COMPACTION_CANCELLATION_FILE) ); } catch (error) { if (isErrnoWithCode(error, "ENOENT")) return null; - // An unreadable cancellation record must never authorize automatic recovery. + // Access/I/O errors do not prove corruption and must remain fail-closed. throw error; } + try { + return CompactionCancellationSchema.parse(JSON.parse(contents.toString("utf8"))); + } catch { + throw new MalformedCompactionCancellationError(contents); + } + } + + async repairCompactionCancellation( + workspaceId: string, + isCurrent: () => boolean, + onRepaired: () => void + ): Promise { + return this.fileLocks.withLock(workspaceId, () => + this.withCrossProcessWriteLock(workspaceId, async () => { + if (!isCurrent()) return null; + let malformed: MalformedCompactionCancellationError; + try { + // A newer valid Stop may have replaced the corrupt bytes while we waited. + return await this.readCompactionCancellation(workspaceId); + } catch (error) { + if (!(error instanceof MalformedCompactionCancellationError)) throw error; + malformed = error; + } + if (!isCurrent()) return null; + const cancellationPath = path.join( + this.getSessionDir(workspaceId), + COMPACTION_CANCELLATION_FILE + ); + // Keep the canonical corrupt file as a fail-closed fence through every + // repair await/crash. Quarantine is a copy, never a rename-away gap. + await fs.writeFile(`${cancellationPath}.corrupt.${randomUUID()}`, malformed.contents, { + mode: 0o600, + flag: "wx", + }); + if (!isCurrent()) return null; + await this.getContinuousCompactionJournal(workspaceId).clear(); + const rows = await this.readChatHistory(workspaceId); + let changed = false; + const sanitized = rows.map((row) => { + const metadata = row.metadata?.muxMetadata; + if (!isCompactionSummaryMetadata(metadata) || !metadata.pendingFollowUp) return row; + changed = true; + const { pendingFollowUp: _pendingFollowUp, ...muxMetadata } = metadata; + return { ...row, metadata: { ...row.metadata, muxMetadata } }; + }); + if (!isCurrent()) return null; + if ( + changed && + !(await this.writeGuardedHistory( + this.getChatHistoryPath(workspaceId), + this.serializeHistoryEntries(sanitized, workspaceId), + isCurrent, + onRepaired + )) + ) + return null; + // Both legacy pending markers and the journal must be durably neutralized + // before absence may authorize normal recovery, including on the next launch. + if (isCurrent()) { + if (!changed) onRepaired(); + await fs.rm(cancellationPath, { force: true }); + } + return null; + }) + ); } async writeCompactionCancellation( @@ -271,7 +333,9 @@ export class HistoryService { isCurrent: () => boolean, retiredNonce?: string ): Promise { - // This file must remain writable when transcript reads/recovery are failing. + // This file supplements legacy pendingFollowUp removal when transcript I/O + // fails. Older builds only honor successful cleanup of that legacy marker. + // It must remain writable when transcript reads/recovery are failing. await this.withHistoryWriteFileLock(workspaceId, async () => { if (!isCurrent()) return; if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) From 1e84781767f67f4ac08c6fbde02b02e34c75101a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 19:16:20 +0200 Subject: [PATCH 10/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20guard=20compaction?= =?UTF-8?q?=20continuation=20append=20across=20backends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revalidate captured summary identity, pending follow-up, permitted trailing rows, and cancellation under the same cross-process history lock as the continuation append. Apply the condition to both direct user-row persistence and the on-send compaction request that carries its follow-up. Ordinary appends retain their existing path. Skipped writes refuse through existing admission/rollback handling without creating acceptance receipts. Snapshot-only cleanup failures remain errors. Add real shared-history regressions for repair and cancellation racing preparation. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I1d2d46d3efa6ab60112c136ae0ac64c528607e9e --- .../agentSession.compactionShutdown.test.ts | 154 ++++++++++++++++++ src/node/services/agentSession.ts | 49 +++++- .../services/compactionCancellation.test.ts | 106 ++++++++++++ src/node/services/compactionCancellation.ts | 28 ++-- src/node/services/historyService.ts | 65 +++++++- 5 files changed, 387 insertions(+), 15 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index be68f255ca5..b04f4245e12 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1,6 +1,7 @@ import * as fs from "node:fs/promises"; import { readFile, readdir, writeFile } from "node:fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import type { CompactionMonitor } from "./compactionMonitor"; import type { ContinuousCompactor } from "./continuousCompactor"; import { afterEach, expect, mock, spyOn, test } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; @@ -1130,6 +1131,159 @@ test.each(["goal admission", "preparation", "row persistence"] as const)( } ); +test.each(["summary read", "preparation", "auto compaction preparation"] as const)( + "a foreign repair during %s prevents the captured follow-up from committing", + async (stage) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + if (stage === "auto compaction preparation") { + const monitor = (h.session as unknown as { compactionMonitor: CompactionMonitor }) + .compactionMonitor; + spyOn(monitor, "getThreshold").mockReturnValue(0.85); + spyOn(monitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + }); + } + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "summary read") { + const read = h.historyService.getLastMessages.bind(h.historyService); + spyOn(h.historyService, "getLastMessages").mockImplementationOnce(async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + }); + } else { + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const foreignStream = spyOn(foreign.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.session.runStartupRecovery(); + expect(await foreignHistory.readCompactionCancellation(workspaceId)).toBeNull(); + expect(foreignStream).not.toHaveBeenCalled(); + release.resolve(); + expect(await pending).toBe(false); + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( + "a skipped follow-up never accepts preparation snapshots (%s)", + async (failureMode) => { + const cleanupFails = failureMode === "cleanup failure"; + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const ownSnapshot = createMuxMessage("own-preparation", "assistant", "Snapshot", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: ownSnapshot, + materializedTokens: [], + }); + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + if (failureMode === "guard I/O failure") { + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValueOnce( + new Error("guard storage unavailable") + ); + } else { + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.session.runStartupRecovery(); + } + return result; + }); + if (cleanupFails) + spyOn(h.historyService, "deleteMessages").mockResolvedValueOnce( + Err("snapshot cleanup unavailable") + ); + let durableReceipts = 0; + const send = h.session.sendMessage.bind(h.session); + spyOn(h.session, "sendMessage").mockImplementation((message, sendOptions, internal) => + send(message, sendOptions, { + ...internal, + onRowsDurable: () => { + durableReceipts++; + internal?.onRowsDurable?.(); + }, + }) + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.internals.dispatchPendingFollowUp().catch((error: unknown) => error); + if (failureMode === "guard I/O failure") + expect(result).toHaveProperty( + "message", + "Failed to append history: guard storage unavailable" + ); + else if (cleanupFails) + expect(result).toHaveProperty( + "message", + "Failed to roll back preparation rows after compaction follow-up became stale" + ); + else expect(result).toBe(false); + expect(durableReceipts).toBe(0); + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); + expect(rows.success && rows.data.some((row) => row.id === ownSnapshot.id)).toBe(cleanupFails); + } finally { + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + test("cancellation storage-access failures still block automatic startup recovery", async () => { const h = await setup(); await h.historyService.appendToHistory(workspaceId, summary()); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 35d6e4ce5da..e47c9e12bdc 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -694,6 +694,8 @@ interface CompactionFollowUpDispatch { interface SendMessageInternalOptions { compactionHandoff?: CompactionToken; + /** Durable-summary handoffs revalidate their source inside the history append lock. */ + compactionHandoffSource?: { summary: MuxMessage; onSkipped: () => void }; preparation?: PreparationAttempt; /** A dequeued send keeps its admission owner through acceptance and startup failure. */ turnReservation?: TurnId; @@ -3404,6 +3406,19 @@ export class AgentSession { const cancelSignal = internal?.cancelSignal; const persistedCancelableMessageIds: string[] = []; + let compactionAppendSkipped = false; + const compactionAppendCondition = internal?.compactionHandoffSource + ? { + summary: internal.compactionHandoffSource.summary, + allowedTailMessageIds: persistedCancelableMessageIds, + isCurrent: () => + !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, + onSkipped: () => { + compactionAppendSkipped = true; + internal.compactionHandoffSource?.onSkipped(); + }, + } + : undefined; // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** @@ -3444,6 +3459,17 @@ export class AgentSession { ) ); }; + const refuseSkippedCompactionAppend = async (): Promise> => { + // Snapshot leftovers are not acceptance of a user/request row that never + // committed. Preserve the cleanup failure without publishing a durable receipt. + if (!(await rollbackPersistedTurnRows())) + throw new Error( + "Failed to roll back preparation rows after compaction follow-up became stale" + ); + return Err( + createUnknownSendMessageError("Compaction follow-up source became stale before append") + ); + }; const markRowsDurable = (): void => { if (attempt.durability !== "rollback-eligible") return; attempt.durability = "durable"; @@ -4109,11 +4135,14 @@ export class AgentSession { // Persist compaction request (NOT the user message — it's the follow-up) const appendCompactionResult = await this.historyService.appendToHistory( this.workspaceId, - autoCompactionMessage + autoCompactionMessage, + compactionAppendCondition ); if (!appendCompactionResult.success) { + if (compactionAppendCondition) throw new Error(appendCompactionResult.error); return Err(createUnknownSendMessageError(appendCompactionResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push(autoCompactionMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); @@ -4263,11 +4292,19 @@ export class AgentSession { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist // normally. - const appendResult = await this.historyService.appendToHistory(this.workspaceId, userMessage); + const appendResult = await this.historyService.appendToHistory( + this.workspaceId, + userMessage, + compactionAppendCondition + ); if (!appendResult.success) { await rollbackPersistedTurnRows(); + // Rollback can retire the local token; that must not disguise a real + // locked-read/write failure as an ordinary stale-source skip. + if (compactionAppendCondition) throw new Error(appendResult.error); return Err(createUnknownSendMessageError(appendResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push(userMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); @@ -8533,7 +8570,9 @@ export class AgentSession { : undefined; // Ordinary durable follow-ups reconstruct the interrupted request and precede // queued input. Optional/goal/requireIdle continuations yield to manual work. + let sourceAppendSkipped = false; const followUpAdmissionStale = () => + sourceAppendSkipped || repairRevision !== this.compactionCancellation.repairRevision || !this.coordinator.isCurrentCompaction(token) || idleRuleStale?.() === true || @@ -8646,6 +8685,12 @@ export class AgentSession { const sendResult = await this.sendMessage(finalText, options, { synthetic: true, compactionHandoff: token, + compactionHandoffSource: { + summary: lastMessage, + onSkipped: () => { + sourceAppendSkipped = true; + }, + }, // Goal sync and synchronous message observers run after rollback becomes // forbidden but before onAccepted. Stop cannot erase that durable handoff. onRowsDurable: () => { diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index 6af1dac4744..2a3cc6fa89e 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -340,3 +340,109 @@ test("a local Stop during corrupt repair prevents its obsolete history commit", await h.cleanup(); } }); + +test.each([ + "unchanged", + "summary text", + "pending payload", + "summary ID", + "summary sequence", + "foreign synthetic tail", + "preserved tail", + "own snapshot", + "unresolved Stop", + "exact Stop", + "other exact Stop", + "malformed cancellation", +] as const)("locked follow-up append revalidates %s", async (change) => { + const h = await createTestHistoryService(); + const workspaceId = "locked-follow-up"; + const source = createMuxMessage("summary", "assistant", "Earlier work", { + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + await h.historyService.appendToHistory(workspaceId, source); + const captured = structuredClone(source); + const foreign = new HistoryService(h.config); + const allowedTailMessageIds: string[] = []; + if (change === "summary text") { + await foreign.updateHistory(workspaceId, { + ...source, + parts: [{ type: "text", text: "Refined work" }], + }); + } else if (change === "pending payload") { + await foreign.updateHistory(workspaceId, { + ...source, + metadata: { + ...source.metadata, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Different request", model: "openai:gpt-4o", agentId: "exec" }, + }, + }, + }); + } else if (change === "summary ID") { + await foreign.updateHistory(workspaceId, { ...source, id: "replacement" }); + } else if (change === "summary sequence") { + await foreign.clearHistory(workspaceId); + await foreign.appendToHistory(workspaceId, createMuxMessage("earlier", "user", "Earlier")); + await foreign.appendToHistory(workspaceId, { + ...source, + metadata: { ...source.metadata, historySequence: undefined }, + }); + } else if ( + change === "foreign synthetic tail" || + change === "preserved tail" || + change === "own snapshot" + ) { + const tail = createMuxMessage("tail", "assistant", "Tail", { + synthetic: true, + ...(change === "preserved tail" ? { rlmPreservedTailCopy: true } : {}), + }); + await foreign.appendToHistory(workspaceId, tail); + if (change === "own snapshot") allowedTailMessageIds.push(tail.id); + } else if ( + change === "unresolved Stop" || + change === "exact Stop" || + change === "other exact Stop" + ) { + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected cancellation"); + if (change !== "unresolved Stop") + await cancellation.narrow( + record.nonce, + change === "exact Stop" ? source : { ...source, id: "other-summary" } + ); + } else if (change === "malformed cancellation") { + await writeFile(`${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, "{"); + } + const skipped = mock(() => undefined); + const candidate = createMuxMessage("follow-up", "user", "Continue", { synthetic: true }); + const allowed = [ + "unchanged", + "summary text", + "preserved tail", + "own snapshot", + "other exact Stop", + ].includes(change); + try { + const result = await h.historyService.appendToHistory(workspaceId, candidate, { + summary: captured, + allowedTailMessageIds, + isCurrent: () => true, + onSkipped: skipped, + }); + expect(result.success).toBe(change !== "malformed cancellation"); + expect(skipped).toHaveBeenCalledTimes(allowed || change === "malformed cancellation" ? 0 : 1); + const rows = await foreign.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.some((row) => row.id === candidate.id)).toBe(allowed); + if (change === "summary text") + expect(rows.success && rows.data[0].parts).toEqual([{ type: "text", text: "Refined work" }]); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 1e057fe76f4..9c2fb6df2b3 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -110,16 +110,7 @@ export class CompactionCancellation { } matches(record: CompactionCancellationRecord, summary: MuxMessage): boolean { - if (record.scope.kind === "unresolved") return true; - const metadata = summary.metadata?.muxMetadata; - return ( - record.scope.id === summary.id && - record.scope.sequence === summary.metadata?.historySequence && - isDeepStrictEqual( - record.scope.pendingFollowUp, - metadata && "pendingFollowUp" in metadata ? metadata.pendingFollowUp : undefined - ) - ); + return matchesCompactionCancellation(record, summary); } flush(): Promise { @@ -166,3 +157,20 @@ export class CompactionCancellation { return result; } } + +/** Shared by unlocked policy checks and the locked follow-up append admission. */ +export function matchesCompactionCancellation( + record: CompactionCancellationRecord, + summary: MuxMessage +): boolean { + if (record.scope.kind === "unresolved") return true; + const metadata = summary.metadata?.muxMetadata; + return ( + record.scope.id === summary.id && + record.scope.sequence === summary.metadata?.historySequence && + isDeepStrictEqual( + record.scope.pendingFollowUp, + metadata && "pendingFollowUp" in metadata ? metadata.pendingFollowUp : undefined + ) + ); +} diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 0c950106df8..fd78e65917a 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1,11 +1,13 @@ import * as path from "path"; import { createHash, randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; import { renameSync } from "node:fs"; import * as fs from "fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import { CompactionCancellationSchema, MalformedCompactionCancellationError, + matchesCompactionCancellation, type CompactionCancellationRecord, } from "./compactionCancellation"; import { ContinuousCompactionJournalStore } from "./continuousCompactionJournal"; @@ -63,6 +65,13 @@ import { */ const HISTORY_WRITE_LOCK_TIMEOUT_MS = 10_000; +interface CompactionFollowUpAppendCondition { + summary: MuxMessage; + allowedTailMessageIds: readonly string[]; + isCurrent: () => boolean; + onSkipped: () => void; +} + function hasDurableCompactionBoundary(metadata: MuxMetadata | undefined): boolean { if (metadata?.compactionBoundary !== true) { return false; @@ -2521,9 +2530,59 @@ export class HistoryService { } } - async appendToHistory(workspaceId: string, message: MuxMessage): Promise> { - return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to append history", () => - this.appendToHistoryUnderWriteLock(workspaceId, message) + async appendToHistory( + workspaceId: string, + message: MuxMessage, + compactionCondition?: CompactionFollowUpAppendCondition + ): Promise> { + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to append history", + async () => { + if (compactionCondition) { + // Another backend can consume/repair this summary while send preparation + // awaits. Validate its exact intent under the same lock as the new row. + const rows = await this.readChatHistory(workspaceId); + const expected = compactionCondition.summary; + const expectedMeta = expected.metadata?.muxMetadata; + const index = rows.findIndex( + (row) => + row.id === expected.id && + row.metadata?.historySequence === expected.metadata?.historySequence + ); + const current = rows[index]; + const currentMeta = current?.metadata?.muxMetadata; + const matches = + isNonNegativeInteger(expected.metadata?.historySequence) && + current?.role === "assistant" && + isCompactionSummaryMetadata(expectedMeta) && + expectedMeta.pendingFollowUp != null && + isCompactionSummaryMetadata(currentMeta) && + isDeepStrictEqual(currentMeta.pendingFollowUp, expectedMeta.pendingFollowUp) && + rows + .slice(index + 1) + .every( + (row) => + row.metadata?.rlmPreservedTailCopy === true || + compactionCondition.allowedTailMessageIds.includes(row.id) + ); + if (!matches || !compactionCondition.isCurrent()) { + compactionCondition.onSkipped(); + return Ok(undefined); + } + // Raw reads only: malformed/access failures must not authorize an append, + // and repair would reacquire this lock. A foreign Stop also wins here. + const cancellation = await this.readCompactionCancellation(workspaceId); + if ( + !compactionCondition.isCurrent() || + (cancellation && matchesCompactionCancellation(cancellation, current)) + ) { + compactionCondition.onSkipped(); + return Ok(undefined); + } + } + return this.appendToHistoryUnderWriteLock(workspaceId, message); + } ); } From 1fbb58db3850c145952295c188a81b90866ebb66 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 19:57:26 +0200 Subject: [PATCH 11/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20durable?= =?UTF-8?q?=20compaction=20cancellation=20and=20acceptance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recheck shared Stop for direct compaction handoffs, acknowledge Stop only after durable publication, and preserve accepted manual rows when witnessed cancellation cleanup fails. Restore absent heartbeat rollback state only when its captured predecessor identities survive. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I0e7f9c97d751d5197304584df113347eb20ca38b --- .../agentSession.compactionShutdown.test.ts | 274 +++++++++++++++--- src/node/services/agentSession.ts | 49 +++- .../services/compactionCancellation.test.ts | 127 ++++++++ src/node/services/compactionCancellation.ts | 40 ++- .../compactionHandler.continuous.test.ts | 35 ++- src/node/services/compactionHandler.ts | 18 +- src/node/services/historyService.ts | 94 ++++-- src/node/services/workspaceService.ts | 15 +- 8 files changed, 549 insertions(+), 103 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index b04f4245e12..c385e21ad26 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1,6 +1,7 @@ import * as fs from "node:fs/promises"; import { readFile, readdir, writeFile } from "node:fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; +import { CompactionCancellation } from "./compactionCancellation"; import type { CompactionMonitor } from "./compactionMonitor"; import type { ContinuousCompactor } from "./continuousCompactor"; import { afterEach, expect, mock, spyOn, test } from "bun:test"; @@ -185,7 +186,7 @@ test.each(["shutdown", "dispose"] as const)( let closing: Promise | undefined; try { await entered.promise; - await h.session.interruptStream({ abandonPartial: true }); + const stopping = h.session.interruptStream({ abandonPartial: true }); let closed = false; closing = (action === "shutdown" ? h.session.finishShutdown() : h.session.dispose()).then( () => { @@ -195,6 +196,7 @@ test.each(["shutdown", "dispose"] as const)( await Promise.resolve(); expect(closed).toBe(false); release.resolve(); + await stopping; expect(await pending).toBe(false); await closing; const rows = await h.historyService.getLastMessages(workspaceId, 1); @@ -271,7 +273,7 @@ test.each(["success", "goal rejection", "boundary publication rejection"] as con }, }); await entered.promise; - await h.session.interruptStream({ abandonPartial: true }); + const stopping = h.session.interruptStream({ abandonPartial: true }); let closed = false; closing = h.session.finishShutdown().then(() => { closed = true; @@ -279,6 +281,7 @@ test.each(["success", "goal rejection", "boundary publication rejection"] as con await Promise.resolve(); expect(closed).toBe(false); release.resolve(); + await stopping; const policy = consumer.mock.results.at(-1); if (policy?.type !== "return") throw new Error("Expected terminal consumer"); await policy.value; @@ -850,13 +853,14 @@ test("a delayed cancellation write survives failed manual preparation", async () return write(...args); }); try { - await h.session.interruptStream({ abandonPartial: true }); + const stopping = h.session.interruptStream({ abandonPartial: true }); await entered.promise; spyOn(h.historyService, "appendToHistory").mockResolvedValueOnce( Err("replacement append failed") ); expect((await h.session.sendMessage("replacement", options)).success).toBe(false); release.resolve(); + await stopping; await h.session.dispose(); const freshHistory = new HistoryService(h.config); expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); @@ -868,53 +872,71 @@ test("a delayed cancellation write survives failed manual preparation", async () } }); -test("a durable replacement witness survives a crash before cancellation retirement", async () => { - const h = await setup(); - await h.historyService.appendToHistory(workspaceId, summary()); - await h.session.interruptStream({ abandonPartial: true }); - const nonce = await h.session.getCompactionCancellationNonce(); - const write = h.historyService.writeCompactionCancellation.bind(h.historyService); - const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( - async (...args) => { - if (args[1] === null) throw new Error("cancellation unlink failed"); - return write(...args); - } - ); - const stream = spyOn(h.aiService, "streamMessage"); - try { - const result = await h.session - .sendMessage("replacement", options) - .catch((error: unknown) => error); - expect(result).toHaveProperty("message", "cancellation unlink failed"); - expect(stream).not.toHaveBeenCalled(); - const freshHistory = new HistoryService(h.config); - const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); - expect( - rows.success && - rows.data.find((row) => row.role === "user")?.metadata?.compactionCancellationNonce - ).toBe(nonce); - expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); - const restarted = await createAgentSessionHarness({ - workspaceId, - config: h.config, - historyService: freshHistory, +test.each([false, true])( + "durable replacement retirement failure preserves acceptance or its genuine goal error (goal error=%s)", + async (goalError) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + await h.session.interruptStream({ abandonPartial: true }); + const nonce = await h.session.getCompactionCancellationNonce(); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("cancellation unlink failed"); + return write(...args); + } + ); + const stream = spyOn(h.aiService, "streamMessage"); + const accepted = mock(() => undefined); + const visibleRows: string[] = []; + const detach = h.session.onChatEvent(({ message }) => { + if (message.type === "message" && message.role === "user") visibleRows.push(message.id); }); + if (goalError) + spyOn(h.goalService, "syncGoalModeWithChatTail").mockRejectedValueOnce( + new Error("genuine goal failure") + ); try { - await restarted.session.runStartupRecovery(); - expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); - const after = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + const result = await h.session + .sendMessage("replacement", options, { onAccepted: accepted }) + .catch((error: unknown) => error); + if (goalError) expect(result).toHaveProperty("message", "genuine goal failure"); + else expect(result).toHaveProperty("success", true); + expect(stream).toHaveBeenCalledTimes(goalError ? 0 : 1); + expect(accepted).toHaveBeenCalledTimes(goalError ? 0 : 1); + expect(visibleRows).toHaveLength(goalError ? 0 : 1); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + expect(h.session.hasBlockingCompactionCleanup).toBe(false); + const freshHistory = new HistoryService(h.config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); expect( - after.success && after.data.filter((row) => row.role === "user").map((row) => row.parts) - ).toMatchObject([[{ type: "text", text: "replacement" }]]); + rows.success && + rows.data.find((row) => row.role === "user")?.metadata?.compactionCancellationNonce + ).toBe(nonce); + expect(await freshHistory.readCompactionCancellation(workspaceId)).not.toBeNull(); + const restarted = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: freshHistory, + }); + try { + await restarted.session.runStartupRecovery(); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toBeNull(); + const after = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect( + after.success && after.data.filter((row) => row.role === "user").map((row) => row.parts) + ).toMatchObject([[{ type: "text", text: "replacement" }]]); + } finally { + await restarted.session.dispose(); + } } finally { - await restarted.session.dispose(); + detach(); + writes.mockRestore(); + await h.session.dispose().catch(() => undefined); + await h.cleanup(); } - } finally { - writes.mockRestore(); - await h.session.dispose().catch(() => undefined); - await h.cleanup(); } -}); +); test("cancellation persistence failures fail teardown until an explicit durable retry", async () => { const h = await setup(); @@ -1497,10 +1519,11 @@ test("a second Stop after the resume witness commits cannot be retired by the fi const resume = h.session.resumeStream(options); try { await entered.promise; - await h.session.interruptStream({ abandonPartial: true }); + const stopping = h.session.interruptStream({ abandonPartial: true }); const secondNonce = await h.session.getCompactionCancellationNonce(); expect(secondNonce).not.toBe(firstNonce); release.resolve(); + await stopping; expect(await resume).toEqual(Ok({ started: false })); await h.session.retryPendingCompactionCleanup(); expect(stream).not.toHaveBeenCalled(); @@ -1779,3 +1802,164 @@ test("failed cleanup cannot narrow a foreign replacement's newer Stop and preser await a.cleanup(); } }); + +test.each(["legacy", "failed apply", "failed apply compact"] as const)( + "direct %s handoff honors another backend's durable Stop during preparation", + async (kind) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original", "user", "Work") + ); + const context = { modelString: options.model, options, providersConfig: null }; + const direct = h.session as unknown as { + activeStreamContext: typeof context; + interruptForCompaction(): Promise; + finishContinuousCompaction( + applied: boolean, + capturedContext: typeof context, + token: NonNullable> + ): Promise; + compactionMonitor: CompactionMonitor; + }; + direct.activeStreamContext = context; + if (kind === "failed apply compact") + spyOn(direct.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const token = + kind === "legacy" + ? undefined + : h.internals.coordinator.beginCompactionObservation("continuous"); + if (token) h.internals.coordinator.setCompactionStage(token, "stopped"); + const pending = + kind === "legacy" + ? direct.interruptForCompaction() + : direct.finishContinuousCompaction(false, context, token!); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await entered.promise; + await new CompactionCancellation(new HistoryService(h.config), workspaceId).cancel(); + release.resolve(); + await pending; + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual(["original"]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + if (token) h.internals.coordinator.finishCompactionObservation(token); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("idle Stop reports cancellation publication failure instead of success", async () => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValue( + new Error("Stop publication failed") + ); + try { + const stopped = await h.session.interruptStream({ abandonPartial: true }); + expect(stopped).toEqual(Err("Stop publication failed")); + expect(h.session.hasPendingCompactionCleanup).toBe(true); + } finally { + write.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("idle Stop cannot acknowledge before its held publication commits", async () => { + const h = await setup(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + let committed = false; + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + await write(...args); + committed = true; + }); + const stopping = h.session + .interruptStream({ abandonPartial: true }) + .then((result) => { + expect(committed).toBe(true); + return result; + }) + .catch((error: unknown) => error); + try { + await entered.promise; + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + release.resolve(); + expect(await stopping).toEqual(Ok(undefined)); + } finally { + release.resolve(); + await stopping; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("overlapping Stops wait for the newest publication when the older write becomes a no-op", async () => { + const h = await setup(); + const firstEntered = Promise.withResolvers(); + const firstRelease = Promise.withResolvers(); + const secondEntered = Promise.withResolvers(); + const secondRelease = Promise.withResolvers(); + let secondCommitted = false; + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation") + .mockImplementationOnce(async (...args) => { + firstEntered.resolve(); + await firstRelease.promise; + return write(...args); + }) + .mockImplementationOnce(async (...args) => { + secondEntered.resolve(); + await secondRelease.promise; + await write(...args); + secondCommitted = true; + }); + const first = h.session + .interruptStream({ abandonPartial: true }) + .then((result) => { + expect(secondCommitted).toBe(true); + return result; + }) + .catch((error: unknown) => error); + let second: ReturnType | undefined; + try { + await firstEntered.promise; + second = h.session.interruptStream({ abandonPartial: true }); + firstRelease.resolve(); + await secondEntered.promise; + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toBeNull(); + secondRelease.resolve(); + expect(await first).toEqual(Ok(undefined)); + expect(await second).toEqual(Ok(undefined)); + } finally { + firstRelease.resolve(); + secondRelease.resolve(); + await first; + await second; + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e47c9e12bdc..0f8b82142c3 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1290,6 +1290,15 @@ export class AgentSession { ); } + get hasBlockingCompactionCleanup(): boolean { + return ( + this.compactionCancellation.blocksRecovery || + this.compactionCleanupRetry != null || + (this.deferredCompactionCleanup != null && + this.coordinator.canClearCompactionFollowUp(this.deferredCompactionCleanup.token)) + ); + } + retryPendingCompactionCleanup(): Promise { if (this.compactionCleanupRetry) return this.compactionCleanupRetry; if (this.compactionCancellation.needsPersistence) @@ -2576,12 +2585,27 @@ export class AgentSession { // A replacement witness is part of the row's atomic commit. The sidecar can // safely retire even if the previous process died before its unlink completed. if ( - history.data.length === 0 || - history.data.some((row) => row.metadata?.compactionCancellationNonce === cancellation.nonce) + await this.historyService.hasCompactionReplacementWitness( + this.workspaceId, + cancellation.nonce + ) ) + await this.retireWitnessedCompactionCancellation(cancellation.nonce); + else if (history.data.length === 0) await this.compactionCancellation.retire(cancellation.nonce); } + private async retireWitnessedCompactionCancellation(nonce: string): Promise { + try { + await this.compactionCancellation.retireReplacement(nonce); + } catch (error) { + log.warn("Deferred witnessed compaction cancellation cleanup", { + workspaceId: this.workspaceId, + error, + }); + } + } + getCompactionCancellationNonce(): Promise { return this.compactionCancellation.readForReplacement().then((record) => record?.nonce); } @@ -3399,6 +3423,7 @@ export class AgentSession { // Single admission-staleness predicate for all three turn-admission gates below. const isAdmissionStale = () => + compactionAppendSkipped || internal?.admissionEpochStale?.() === true || internal?.admissionStale?.() === true || !this.coordinator.isCurrentTurn(attempt.owner ?? attempt.expectedTurn) || @@ -3407,9 +3432,9 @@ export class AgentSession { const cancelSignal = internal?.cancelSignal; const persistedCancelableMessageIds: string[] = []; let compactionAppendSkipped = false; - const compactionAppendCondition = internal?.compactionHandoffSource + const compactionAppendCondition = internal?.compactionHandoff ? { - summary: internal.compactionHandoffSource.summary, + summary: internal.compactionHandoffSource?.summary, allowedTailMessageIds: persistedCancelableMessageIds, isCurrent: () => !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, @@ -4355,9 +4380,10 @@ export class AgentSession { // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. markRowsDurable(); - // No replacement provider/compaction may produce B before A's fence retires. + // The committed row witnesses replacement even if ancillary unlink fails. + // Continue acceptance so a visible, durable user send is never reported lost. if (compactionCancellationNonce) - await this.compactionCancellation.retire(compactionCancellationNonce); + await this.retireWitnessedCompactionCancellation(compactionCancellationNonce); try { await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); } catch (error) { @@ -5658,7 +5684,9 @@ export class AgentSession { // Startup edits must still preempt a blocked envelope; soft stop only requests // a future boundary, so neither joins policy here. const interruptedPolicy = this.coordinator.captureInterruptSettlement(options?.soft); - if (options?.abandonPartial || this.midStreamCompactionPending) { + const publishesCancellation = + options?.abandonPartial === true || this.midStreamCompactionPending; + if (publishesCancellation) { this.coordinator.invalidateCompaction(true); // Register durable cancellation before any Stop await. Its independent nonce // survives a later manual preparation that fails before committing its row. @@ -5692,6 +5720,13 @@ export class AgentSession { } await interruptedPolicy; + if (publishesCancellation) { + try { + await this.compactionCancellation.flush(); + } catch (error) { + return Err(getErrorMessage(error)); + } + } return Ok(undefined); } diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index 2a3cc6fa89e..7ecaba6d622 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -446,3 +446,130 @@ test.each([ await h.cleanup(); } }); + +test("witnessed deletion debt refreshes a foreign Stop without losing its own retry identity", async () => { + const h = await createTestHistoryService(); + const workspaceId = "witnessed-debt"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected Stop"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Replacement", { + compactionCancellationNonce: original.nonce, + }) + ); + const failure = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + await state.retireReplacement(original.nonce).catch(() => undefined); + try { + expect(await state.read()).toBeNull(); + await state.flush(); + expect(state.needsPersistence).toBe(true); + expect(state.blocksRecovery).toBe(false); + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const current = await foreign.read(); + expect(await state.read()).toEqual(current); + expect(state.needsPersistence).toBe(true); + failure.mockRestore(); + await state.retry(); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + current + ); + } finally { + await h.cleanup(); + } +}); + +test("an archived replacement witness authorizes B without masking a newer Stop", async () => { + const h = await createTestHistoryService(); + const workspaceId = "archived-witness"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const original = await state.read(); + if (!original) throw new Error("Expected Stop"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Replacement", { + compactionCancellationNonce: original.nonce, + }) + ); + const source = createMuxMessage("summary-b", "assistant", "B", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { text: "Continue B", model: "openai:gpt-4o", agentId: "exec" }, + }, + }); + await h.historyService.appendToHistory(workspaceId, source); + const foreign = new HistoryService(h.config); + const skipped = mock(() => undefined); + const condition = { + summary: source, + allowedTailMessageIds: [] as string[], + isCurrent: () => true, + onSkipped: skipped, + }; + try { + expect(await foreign.hasCompactionReplacementWitness(workspaceId, original.nonce)).toBe(true); + expect( + ( + await foreign.appendToHistory( + workspaceId, + createMuxMessage("b-user", "user", "Continue B"), + condition + ) + ).success + ).toBe(true); + expect(skipped).not.toHaveBeenCalled(); + await foreign.deleteMessage(workspaceId, "b-user"); + await new CompactionCancellation(foreign, workspaceId).cancel(); + expect( + ( + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("stopped", "user", "Continue B"), + condition + ) + ).success + ).toBe(true); + expect(skipped).toHaveBeenCalledTimes(1); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].id).toBe(source.id); + } finally { + await h.cleanup(); + } +}); + +test("a stale replacement receipt cannot make newer witnessed deletion debt blocking", async () => { + const h = await createTestHistoryService(); + const workspaceId = "stale-receipt"; + const state = new CompactionCancellation(h.historyService, workspaceId); + await state.cancel(); + const a = await state.read(); + await state.cancel(); + const b = await state.read(); + if (!a || !b) throw new Error("Expected distinct Stops"); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("replacement-b", "user", "B", { compactionCancellationNonce: b.nonce }) + ); + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + await state.retireReplacement(b.nonce).catch(() => undefined); + try { + await state.retireReplacement(a.nonce); + expect(await state.read()).toBeNull(); + expect(state.blocksRecovery).toBe(false); + expect(state.needsPersistence).toBe(true); + await state.flush(); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 9c2fb6df2b3..87308f3169f 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -31,6 +31,7 @@ export class CompactionCancellation { private current: CompactionCancellationRecord | null | undefined; private generation = 0; private repairedHistoryRevision = 0; + private replacementNonce?: string; private pending: Promise = Promise.resolve(); private unsettled = false; private mutation?: { record: CompactionCancellationRecord | null; retiredNonce?: string }; @@ -44,6 +45,18 @@ export class CompactionCancellation { return this.unsettled; } + get blocksRecovery(): boolean { + return this.unsettled && !this.isWitnessedRetirement(); + } + + private isWitnessedRetirement(): boolean { + return this.mutation?.record === null && this.mutation.retiredNonce === this.replacementNonce; + } + + private effectiveRecord(): CompactionCancellationRecord | null { + return this.current?.nonce === this.replacementNonce ? null : (this.current ?? null); + } + get repairRevision(): number { return this.repairedHistoryRevision; } @@ -51,7 +64,7 @@ export class CompactionCancellation { async read(): Promise { // Other backends can publish Stop after a previous read (including absence). // Local in-flight/failed mutations still own their conservative exclusion. - if (this.unsettled) return this.current ?? null; + if (this.unsettled && !this.isWitnessedRetirement()) return this.effectiveRecord(); const generation = this.generation; const mutation = this.mutation; const isCurrent = () => generation === this.generation && mutation === this.mutation; @@ -69,7 +82,7 @@ export class CompactionCancellation { // An obsolete read must not trigger explicit repair over a newer local Stop. if (isCurrent()) throw error; } - return this.current ?? null; + return this.effectiveRecord(); } cancel(): Promise { @@ -113,8 +126,27 @@ export class CompactionCancellation { return matchesCompactionCancellation(record, summary); } - flush(): Promise { - return this.pending; + async flush(): Promise { + // An obsolete publication may have become a no-op behind a newer Stop. + // Success acknowledges the latest mutation, never an absent superseded write. + for (;;) { + const pending = this.pending; + try { + await pending; + } catch (error) { + if (pending !== this.pending) continue; + if (!this.isWitnessedRetirement()) throw error; + } + if (pending === this.pending) return; + } + } + + retireReplacement(nonce: string): Promise { + if (this.current?.nonce !== nonce) return Promise.resolve(); + // Only callers holding an actual durable row witness may retire intent before + // unlink succeeds. The physical deletion mutation remains observable/retryable. + this.replacementNonce = nonce; + return this.retire(nonce); } retry(): Promise { diff --git a/src/node/services/compactionHandler.continuous.test.ts b/src/node/services/compactionHandler.continuous.test.ts index 953ed17dafd..27d984f00f3 100644 --- a/src/node/services/compactionHandler.continuous.test.ts +++ b/src/node/services/compactionHandler.continuous.test.ts @@ -6,6 +6,7 @@ import * as path from "node:path"; import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import assert from "@/common/utils/assert"; import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; +import { HistoryService } from "./historyService"; import { CompactionHandler } from "./compactionHandler"; import { prepareMessagesForProvider } from "./messagePipeline"; import { createTestHistoryService } from "./testHistoryService"; @@ -151,7 +152,13 @@ describe("continuous compaction provider replay", () => { } ); - it.each(["missing", "replacement sequence"] as const)( + it.each([ + "missing", + "replacement sequence", + "foreign clear", + "foreign replacement", + "empty predecessor", + ] as const)( "settles local heartbeat rollback after a shared-history change (%s)", async (state) => { const sessionDir = path.join(store.tempDir, "pending"); @@ -178,7 +185,8 @@ describe("continuous compaction provider replay", () => { output: { success: true, diff: "recent change" }, }, ]; - await store.historyService.appendToHistory(workspaceId, recent); + if (state !== "empty predecessor") + await store.historyService.appendToHistory(workspaceId, recent); const emitter = new EventEmitter(); const handler = new CompactionHandler({ workspaceId, @@ -195,9 +203,15 @@ describe("continuous compaction provider replay", () => { const boundary = rows.data[0]; // A second backend can commit the shared history deletion without touching // this handler's captured rollback or its post-reset memory. - expect((await store.historyService.deleteMessage(workspaceId, boundary.id)).success).toBe( - true - ); + const foreign = new HistoryService(store.config); + if (state === "foreign clear" || state === "foreign replacement") { + expect((await foreign.clearHistory(workspaceId)).success).toBe(true); + if (state === "foreign replacement") + await foreign.appendToHistory( + workspaceId, + createMuxMessage("new-context", "user", "New context") + ); + } else expect((await foreign.deleteMessage(workspaceId, boundary.id)).success).toBe(true); if (state === "replacement sequence") { await store.historyService.appendToHistory(workspaceId, { ...boundary, @@ -205,7 +219,8 @@ describe("continuous compaction provider replay", () => { }); } const before = await handler.peekPendingState(); - expect(before?.diffs.map((diff) => diff.path)).toContain("/tmp/recent.ts"); + if (state !== "empty predecessor") + expect(before?.diffs.map((diff) => diff.path)).toContain("/tmp/recent.ts"); const emit = spyOn(emitter, "emit"); const published = mock(() => { expect(handler.peekCachedFilePaths()).toEqual([priorDiff.path]); @@ -319,10 +334,12 @@ describe("continuous compaction provider replay", () => { } return unlink(file); }); - const internals = handler as unknown as { captureHeartbeatResetRollbackState(): void }; + const internals = handler as unknown as { + captureHeartbeatResetRollbackState(messages: MuxMessage[]): void; + }; const capture = internals.captureHeartbeatResetRollbackState.bind(handler); - spyOn(internals, "captureHeartbeatResetRollbackState").mockImplementation(() => { - capture(); + spyOn(internals, "captureHeartbeatResetRollbackState").mockImplementation((messages) => { + capture(messages); captured.resolve(); }); const rollingBack = handler.rollbackHeartbeatContextResetBoundary(first.data[0]); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 9638f26417d..d69b4207d9f 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -97,6 +97,7 @@ interface PersistedPostCompactionStateV1 { } interface HeartbeatResetRollbackState { + sourceRows: Array<{ id: string; sequence: number | undefined }>; postCompactionAttachmentsPending: boolean; cachedFileDiffs: FileEditDiff[]; cachedLoadedSkills: LoadedSkillSnapshot[]; @@ -618,8 +619,9 @@ export class CompactionHandler { } } - private captureHeartbeatResetRollbackState(): void { + private captureHeartbeatResetRollbackState(messages: MuxMessage[]): void { this.heartbeatResetRollbackState = { + sourceRows: messages.map((row) => ({ id: row.id, sequence: row.metadata?.historySequence })), postCompactionAttachmentsPending: this.postCompactionAttachmentsPending, cachedFileDiffs: [...this.cachedFileDiffs], cachedLoadedSkills: [...this.cachedLoadedSkills], @@ -824,7 +826,7 @@ export class CompactionHandler { const messages = historyResult.data; await this.loadPersistedPendingStateIfNeeded(); - this.captureHeartbeatResetRollbackState(); + this.captureHeartbeatResetRollbackState(messages); await this.preparePendingStateFromMessages(messages); const nextCompactionEpoch = getNextCompactionEpoch(messages); @@ -919,6 +921,7 @@ export class CompactionHandler { onCommitted?.(); } }; + const sourceRows = this.heartbeatResetRollbackState?.sourceRows ?? []; const deleteResult = await this.historyService.deleteMessage( this.workspaceId, summaryMessage.id, @@ -930,7 +933,16 @@ export class CompactionHandler { message.metadata?.historySequence === summaryMessage.metadata?.historySequence ), () => restore(true), - () => restore(false) + (remaining) => { + // Absence alone also describes a foreign clear/replacement. Restore only + // with positive evidence that this heartbeat's predecessor still survives. + const sequences = new Map(remaining.map((row) => [row.id, row.metadata?.historySequence])); + if ( + sourceRows.length > 0 && + sourceRows.every((row) => row.sequence != null && sequences.get(row.id) === row.sequence) + ) + restore(false); + } ); // Physical persistence outlives ownership. A later admission may not veto a // committed rollback; the existing write queue orders it before successor state. diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index fd78e65917a..8aeb9e57e8c 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -66,7 +66,7 @@ import { const HISTORY_WRITE_LOCK_TIMEOUT_MS = 10_000; interface CompactionFollowUpAppendCondition { - summary: MuxMessage; + summary?: MuxMessage; allowedTailMessageIds: readonly string[]; isCurrent: () => boolean; onSkipped: () => void; @@ -276,6 +276,27 @@ export class HistoryService { } } + async hasCompactionReplacementWitness(workspaceId: string, nonce: string): Promise { + return this.withRecoveredHistoryLock(workspaceId, () => + this.hasCompactionReplacementWitnessUnlocked(workspaceId, nonce) + ); + } + + private async hasCompactionReplacementWitnessUnlocked( + workspaceId: string, + nonce: string, + active?: MuxMessage[] + ): Promise { + const matches = (rows: MuxMessage[]) => + rows.some((row) => row.metadata?.compactionCancellationNonce === nonce); + // A later boundary may archive the receipt while ancillary unlink is still + // unavailable. Its exact nonce continues to distinguish a newer explicit Stop. + return ( + matches(active ?? (await this.readChatHistory(workspaceId))) || + matches(await this.readArchivedHistory(workspaceId)) + ); + } + async repairCompactionCancellation( workspaceId: string, isCurrent: () => boolean, @@ -2544,38 +2565,48 @@ export class HistoryService { // awaits. Validate its exact intent under the same lock as the new row. const rows = await this.readChatHistory(workspaceId); const expected = compactionCondition.summary; - const expectedMeta = expected.metadata?.muxMetadata; - const index = rows.findIndex( - (row) => - row.id === expected.id && - row.metadata?.historySequence === expected.metadata?.historySequence - ); - const current = rows[index]; - const currentMeta = current?.metadata?.muxMetadata; - const matches = - isNonNegativeInteger(expected.metadata?.historySequence) && - current?.role === "assistant" && - isCompactionSummaryMetadata(expectedMeta) && - expectedMeta.pendingFollowUp != null && - isCompactionSummaryMetadata(currentMeta) && - isDeepStrictEqual(currentMeta.pendingFollowUp, expectedMeta.pendingFollowUp) && - rows - .slice(index + 1) - .every( - (row) => - row.metadata?.rlmPreservedTailCopy === true || - compactionCondition.allowedTailMessageIds.includes(row.id) - ); - if (!matches || !compactionCondition.isCurrent()) { - compactionCondition.onSkipped(); - return Ok(undefined); + if (expected) { + const expectedMeta = expected.metadata?.muxMetadata; + const index = rows.findIndex( + (row) => + row.id === expected.id && + row.metadata?.historySequence === expected.metadata?.historySequence + ); + const current = rows[index]; + const currentMeta = current?.metadata?.muxMetadata; + const matches = + isNonNegativeInteger(expected.metadata?.historySequence) && + current?.role === "assistant" && + isCompactionSummaryMetadata(expectedMeta) && + expectedMeta.pendingFollowUp != null && + isCompactionSummaryMetadata(currentMeta) && + isDeepStrictEqual(currentMeta.pendingFollowUp, expectedMeta.pendingFollowUp) && + rows + .slice(index + 1) + .every( + (row) => + row.metadata?.rlmPreservedTailCopy === true || + compactionCondition.allowedTailMessageIds.includes(row.id) + ); + if (!matches || !compactionCondition.isCurrent()) { + compactionCondition.onSkipped(); + return Ok(undefined); + } } // Raw reads only: malformed/access failures must not authorize an append, // and repair would reacquire this lock. A foreign Stop also wins here. const cancellation = await this.readCompactionCancellation(workspaceId); if ( !compactionCondition.isCurrent() || - (cancellation && matchesCompactionCancellation(cancellation, current)) + (cancellation && + !(await this.hasCompactionReplacementWitnessUnlocked( + workspaceId, + cancellation.nonce, + rows + )) && + (expected + ? matchesCompactionCancellation(cancellation, expected) + : cancellation.scope.kind === "unresolved")) ) { compactionCondition.onSkipped(); return Ok(undefined); @@ -3072,7 +3103,7 @@ export class HistoryService { messageId: string, shouldDelete?: (messages: MuxMessage[]) => boolean, onCommitted?: () => void, - onAlreadyAbsent?: () => void + onAlreadyAbsent?: (remaining: MuxMessage[]) => void ): Promise> { assert( !(onCommitted ?? onAlreadyAbsent) || shouldDelete, @@ -3094,7 +3125,7 @@ export class HistoryService { messageId: string, shouldDelete?: (messages: MuxMessage[]) => boolean, onCommitted?: () => void, - onAlreadyAbsent?: () => void + onAlreadyAbsent?: (remaining: MuxMessage[]) => void ): Promise> { try { // Structural rewrite requires full file content @@ -3107,13 +3138,14 @@ export class HistoryService { if (shouldDelete) { // Another backend may already have deleted this exact active target. // An archived row is a replaced context, never an invitation to restore it. + const archived = onAlreadyAbsent ? await this.readArchivedHistory(workspaceId) : []; if ( onAlreadyAbsent && - !(await this.readArchivedHistory(workspaceId)).some((row) => row.id === messageId) && + !archived.some((row) => row.id === messageId) && shouldDelete(messages) ) { try { - onAlreadyAbsent(); + onAlreadyAbsent([...archived, ...messages]); } catch (error) { log.error("Absent history cleanup publication failed", { error: getErrorMessage(error), diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 669710888ae..853ec772d1d 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4119,7 +4119,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return; } - if (this.failedCompactionCleanups.has(trimmed)) { + if (this.hasBlockingCompactionCleanup(trimmed)) { this.retryFailedCompactionCleanup(trimmed).then( () => { if (!this.shuttingDown) this.startStartupRecovery(trimmed); @@ -4166,14 +4166,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); } + private hasBlockingCompactionCleanup(workspaceId: string): boolean { + return [...(this.failedCompactionCleanups.get(workspaceId)?.owners ?? [])].some( + (owner) => owner.hasBlockingCompactionCleanup + ); + } + private assertCompactionCleanupSettled(workspaceId: string): void { if (this.failedCompactionCleanups.has(workspaceId)) { this.retryFailedCompactionCleanup(workspaceId).catch((error: unknown) => log.warn("Stopped compaction cleanup retry failed", { workspaceId, error }) ); - throw new Error( - "Stopped compaction cleanup is still pending. Please retry opening this workspace." - ); + if (this.hasBlockingCompactionCleanup(workspaceId)) + throw new Error( + "Stopped compaction cleanup is still pending. Please retry opening this workspace." + ); } } From cd43821fc944bc844bbbe0df60aedf5c43ce4d2e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 20:23:30 +0200 Subject: [PATCH 12/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20shared?= =?UTF-8?q?=20cancellation=20for=20compaction=20and=20Retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject all unwitnessed cancellation scopes for handoffs without durable source identity. Compare explicit Retry's captured nonce or absence under the history write lock before committing acceptance, while preserving witnessed cleanup debt and local Stop guards. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I6cfad4dad94e65713e31948a5bf5435d148cf491 --- .../agentSession.compactionShutdown.test.ts | 163 +++++++++++++++++- src/node/services/agentSession.ts | 17 +- src/node/services/historyService.ts | 48 +++++- 3 files changed, 207 insertions(+), 21 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index c385e21ad26..d0669c85a43 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1554,7 +1554,7 @@ test.each(["history failure", "witness no-op", "automatic"] as const)( Err("resume history unavailable") ); if (outcome === "witness no-op") - spyOn(h.historyService, "updateHistory").mockResolvedValueOnce(Ok(undefined)); + spyOn(h.historyService, "acceptResumeCancellation").mockResolvedValueOnce(Ok(undefined)); const stream = spyOn(h.aiService, "streamMessage"); try { await h.session.resumeStream(options, { automatic: outcome === "automatic" }); @@ -1803,13 +1803,20 @@ test("failed cleanup cannot narrow a foreign replacement's newer Stop and preser } }); -test.each(["legacy", "failed apply", "failed apply compact"] as const)( - "direct %s handoff honors another backend's durable Stop during preparation", - async (kind) => { +test.each([ + ["legacy", false], + ["failed apply", false], + ["failed apply compact", false], + ["legacy", true], + ["failed apply", true], + ["failed apply compact", true], +] as const)( + "direct %s handoff honors another backend's durable Stop during preparation (narrowed=%s)", + async (kind, narrowed) => { const h = await setup(); await h.historyService.appendToHistory( workspaceId, - createMuxMessage("original", "user", "Work") + narrowed ? { ...summary(), id: "original" } : createMuxMessage("original", "user", "Work") ); const context = { modelString: options.model, options, providersConfig: null }; const direct = h.session as unknown as { @@ -1851,9 +1858,36 @@ test.each(["legacy", "failed apply", "failed apply compact"] as const)( ? direct.interruptForCompaction() : direct.finishContinuousCompaction(false, context, token!); const stream = spyOn(h.aiService, "streamMessage"); + let cleanupForeign: (() => Promise) | undefined; try { await entered.promise; - await new CompactionCancellation(new HistoryService(h.config), workspaceId).cancel(); + const foreignHistory = new HistoryService(h.config); + await new CompactionCancellation(foreignHistory, workspaceId).cancel(); + if (narrowed) { + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + const writes = foreignHistory as unknown as { + writeGuardedHistory( + path: string, + serialized: string, + guard: () => boolean + ): Promise; + }; + const failure = spyOn(writes, "writeGuardedHistory").mockRejectedValue( + new Error("foreign summary rewrite failed") + ); + cleanupForeign = async () => { + failure.mockRestore(); + await foreign.session.dispose(); + }; + await foreign.session.runStartupRecovery(); + expect((await foreignHistory.readCompactionCancellation(workspaceId))?.scope.kind).toBe( + "summary" + ); + } release.resolve(); await pending; expect(stream).not.toHaveBeenCalled(); @@ -1862,6 +1896,7 @@ test.each(["legacy", "failed apply", "failed apply compact"] as const)( } finally { release.resolve(); await pending.catch(() => undefined); + await cleanupForeign?.(); if (token) h.internals.coordinator.finishCompactionObservation(token); await h.session.dispose(); await h.cleanup(); @@ -1963,3 +1998,119 @@ test("overlapping Stops wait for the newest publication when the older write bec await h.cleanup(); } }); + +test.each([false, true])( + "explicit Retry rejects a foreign Stop after capture (initial Stop=%s)", + async (initialStop) => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, createMuxMessage("user", "user", "Work")); + if (initialStop) await h.session.interruptStream({ abandonPartial: true }); + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + entered.resolve(); + await release.promise; + return nonce; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.session.resumeStream(options); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const stopped = await foreign.read(); + release.resolve(); + expect(await pending).toEqual(Ok({ started: false })); + expect(stream).not.toHaveBeenCalled(); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata).not.toHaveProperty( + "compactionCancellationNonce" + ); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([ + "absent", + "witnessed debt", + "retirement failure", + "shared read failure", + "witness write failure", +] as const)("explicit Retry locked acceptance handles %s", async (state) => { + const h = await setup(); + const user = createMuxMessage("user", "user", "Work"); + await h.historyService.appendToHistory(workspaceId, user); + const cancellation = (h.session as unknown as { compactionCancellation: CompactionCancellation }) + .compactionCancellation; + const debt = state === "witnessed debt" || state === "retirement failure"; + let restoreWrites: (() => void) | undefined; + if (debt) { + await h.session.interruptStream({ abandonPartial: true }); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const writes = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + async (...args) => { + if (args[1] === null) throw new Error("unlink unavailable"); + return write(...args); + } + ); + restoreWrites = () => writes.mockRestore(); + if (state === "witnessed debt") { + await h.historyService.updateHistory(workspaceId, { + ...user, + metadata: { ...user.metadata, compactionCancellationNonce: record.nonce }, + }); + await cancellation.retireReplacement(record.nonce).catch(() => undefined); + expect(await cancellation.read()).toBeNull(); + } + } + if (state === "shared read failure") { + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + spyOn(h.historyService, "readCompactionCancellation").mockRejectedValueOnce( + new Error("shared storage unavailable") + ); + return nonce; + }); + } + if (state === "witness write failure") { + const writes = h.historyService as unknown as { + writeGuardedHistory(path: string, serialized: string, guard: () => boolean): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce( + new Error("witness storage unavailable") + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + try { + const result = await h.session.resumeStream(options); + const storageFailure = state === "shared read failure" || state === "witness write failure"; + if (storageFailure) { + expect(result.success).toBe(false); + expect(JSON.stringify(result)).toContain("storage unavailable"); + } else expect(result).toEqual(Ok({ started: true })); + expect(stream).toHaveBeenCalledTimes(storageFailure ? 0 : 1); + expect(cancellation.needsPersistence).toBe(debt); + if (debt) expect(h.session.hasBlockingCompactionCleanup).toBe(false); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect( + rows.success && rows.data.filter((row) => row.role === "user").map((row) => row.id) + ).toEqual([user.id]); + } finally { + restoreWrites?.(); + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0f8b82142c3..471d08709de 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -785,7 +785,7 @@ interface PreparationAttempt { failureAttempts?: number; failure?: SendMessageError; onFailure?: (error: SendMessageError) => Promise | void; - resumeCancellation?: { nonce: string; epoch: number }; + resumeCancellation?: { nonce: string | null; epoch: number }; } export class AgentSession { @@ -4716,7 +4716,7 @@ export class AgentSession { this.coordinator.closing ) return Ok({ started: false }); - if (nonce) attempt.resumeCancellation = { nonce, epoch }; + attempt.resumeCancellation = { nonce: nonce ?? null, epoch }; } this.setAutoRetryResumeState( optionsForStream, @@ -6055,27 +6055,22 @@ export class AgentSession { let committed = false; // Retry is explicit user intent but reuses its existing row. Persist that // acceptance before engine entry; failed preflight or a guarded no-op keeps Stop. - const witnessed = await this.historyService.updateHistory( + const witnessed = await this.historyService.acceptResumeCancellation( this.workspaceId, resumedUser, + resumedCancellation.nonce, (current) => !isStreamStartAborted() && this.coordinator.compactionIntent.epoch === resumedCancellation.epoch && current.id === resumedUser.id && current.role === "user" && current.metadata?.historySequence === resumedUser.metadata?.historySequence, - (current) => ({ - ...current, - metadata: { - ...current.metadata, - compactionCancellationNonce: resumedCancellation.nonce, - }, - }), () => { committed = true; } ); - if (committed) await this.compactionCancellation.retire(resumedCancellation.nonce); + if (committed && resumedCancellation.nonce !== null) + await this.retireWitnessedCompactionCancellation(resumedCancellation.nonce); if (!witnessed.success) return await fail(createUnknownSendMessageError(witnessed.error)); if ( !committed || diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 8aeb9e57e8c..4f45a97f6a6 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2594,7 +2594,8 @@ export class HistoryService { } } // Raw reads only: malformed/access failures must not authorize an append, - // and repair would reacquire this lock. A foreign Stop also wins here. + // and repair would reacquire this lock. Without a source identity, even + // a narrowed Stop must veto the handoff until a replacement is witnessed. const cancellation = await this.readCompactionCancellation(workspaceId); if ( !compactionCondition.isCurrent() || @@ -2604,9 +2605,7 @@ export class HistoryService { cancellation.nonce, rows )) && - (expected - ? matchesCompactionCancellation(cancellation, expected) - : cancellation.scope.kind === "unresolved")) + (!expected || matchesCompactionCancellation(cancellation, expected))) ) { compactionCondition.onSkipped(); return Ok(undefined); @@ -2769,6 +2768,47 @@ export class HistoryService { ); } + async acceptResumeCancellation( + workspaceId: string, + message: MuxMessage, + expectedNonce: string | null, + shouldUpdate: (current: MuxMessage) => boolean, + onCommitted: () => void + ): Promise> { + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to accept resume", + async () => { + // Retry must not stamp an obsolete receipt over a foreign Stop that arrived + // during preparation. Compare shared identity and commit under one lock. + const current = await this.readCompactionCancellation(workspaceId); + // Semantic absence can retain a physical file when witnessed unlink failed. + if ( + (current?.nonce ?? null) !== expectedNonce && + !( + expectedNonce === null && + current && + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce)) + ) + ) + return Ok(undefined); + return this.updateHistoryUnderWriteLock( + workspaceId, + message, + shouldUpdate, + (row) => + expectedNonce === null + ? row + : { + ...row, + metadata: { ...row.metadata, compactionCancellationNonce: expectedNonce }, + }, + onCommitted + ); + } + ); + } + private async updateHistoryUnderWriteLock( workspaceId: string, message: MuxMessage, From 3dad9e25a7441b6dbdcd02d038fd81a4ecb42be4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 21:26:08 +0200 Subject: [PATCH 13/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20fence=20repaired=20?= =?UTF-8?q?compaction=20journals=20across=20backends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain Stop until an exact durable replacement witness exists; empty provider history cannot authorize late work. Capture a durable journal generation before source preparation and validate publication, prefix consumption, and boundary folding under the shared history lock. Repair advances the generation without joining queued writers. Scope cleanup and recovery reads to their exact journal owner so reset cannot adopt a successor or leave an already-owned journal behind. Preserve settings and shutdown recovery, including idempotent cleanup after a completed fold. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I7bdae1a5f8c475f69f2ee07194a6702b77ae1173 --- .../orpc/schemas/continuousCompaction.ts | 2 + src/constants/continuousCompaction.ts | 3 + .../agentSession.compactionShutdown.test.ts | 39 +++- .../agentSession.scopedLifetimes.test.ts | 3 + src/node/services/agentSession.ts | 6 +- src/node/services/compactionHandler.ts | 5 +- .../services/continuousCompactionJournal.ts | 174 ++++++++++++--- src/node/services/continuousCompactor.test.ts | 190 +++++++++++++++- src/node/services/continuousCompactor.ts | 51 ++++- src/node/services/historyService.ts | 40 +++- ...streamManager.continuousCompaction.test.ts | 205 +++++++++++++++++- src/node/services/streamManager.ts | 40 ++-- 12 files changed, 690 insertions(+), 68 deletions(-) diff --git a/src/common/orpc/schemas/continuousCompaction.ts b/src/common/orpc/schemas/continuousCompaction.ts index fcc8c46250e..e139140ddc1 100644 --- a/src/common/orpc/schemas/continuousCompaction.ts +++ b/src/common/orpc/schemas/continuousCompaction.ts @@ -75,6 +75,8 @@ const attachment: z.ZodType = z.discriminatedUnion("ty export const ContinuousCompactionJournalSchema = z .object({ version: z.literal(1), + // Legacy absence is valid only before the first durable repair generation. + publicationGeneration: z.string().min(1).optional(), boundary: row, staticCopies: z.array(row), liveTailCopySpec: z.object({ diff --git a/src/constants/continuousCompaction.ts b/src/constants/continuousCompaction.ts index 3312bd56dcc..33a074701f9 100644 --- a/src/constants/continuousCompaction.ts +++ b/src/constants/continuousCompaction.ts @@ -4,3 +4,6 @@ export const TAIL_MIN_TOKENS = 4_000; export const TAIL_MAX_TOKENS = 60_000; export const MIN_HEAD_TOKENS = 8_000; export const SUMMARIZER_INPUT_FRACTION = 0.7; + +export const CONTINUOUS_COMPACTION_JOURNAL_FILE = "continuous-compaction.json"; +export const CONTINUOUS_COMPACTION_GENERATION_FILE = "continuous-compaction-generation.json"; diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index 67b9ed0eeb6..e6a10d8e3ed 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1355,7 +1355,7 @@ test.each(["quarantine", "journal", "history", "unlink"] as const)( return write(...args); }); } else if (step === "journal") { - spyOn(journal, "clear").mockRejectedValueOnce(failure); + spyOn(journal, "invalidateUnderHistoryLock").mockRejectedValueOnce(failure); } else if (step === "unlink") { const remove = fs.rm; let failed = false; @@ -2197,3 +2197,40 @@ test.each(["accepted", "foreign Stop", "raw reset", "append failure"] as const)( } } ); + +test.each(["empty", "raw reset", "journal only"])( + "startup retains a settled Stop on %s history against a late foreign summary", + async (state) => { + const h = await setup(); + try { + await h.session.interruptStream({ abandonPartial: true }); + if (state === "raw reset") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("private", "user", "Old context") + ); + await fs.appendFile( + `${h.config.sessionsDir}/${workspaceId}/chat.jsonl`, + '{"metadata":{"contextBoundaryKind":"reset"},broken\n' + ); + } + if (state === "journal only") + await writeFile( + h.historyService.getContinuousCompactionJournal(workspaceId).path, + "old journal" + ); + const stopped = await h.historyService.readCompactionCancellation(workspaceId); + await h.session.runStartupRecovery(); + const foreign = new HistoryService(h.config); + expect(await foreign.readCompactionCancellation(workspaceId)).toEqual(stopped); + expect(h.session.hasBlockingCompactionCleanup).toBe(false); + await foreign.appendToHistory(workspaceId, summary()); + await expectNoRecovery(h.config, foreign); + expect((await h.session.sendMessage("Explicit replacement", options)).success).toBe(true); + expect(await foreign.readCompactionCancellation(workspaceId)).toBeNull(); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); diff --git a/src/node/services/agentSession.scopedLifetimes.test.ts b/src/node/services/agentSession.scopedLifetimes.test.ts index 9b6326cfea8..2c8cf75320e 100644 --- a/src/node/services/agentSession.scopedLifetimes.test.ts +++ b/src/node/services/agentSession.scopedLifetimes.test.ts @@ -89,6 +89,7 @@ describe("AgentSession scoped turn lifetimes", () => { const h = await createAgentSessionHarness({ workspaceId }); const stream = spyOn(h.aiService, "streamMessage"); const release = Promise.withResolvers(); + const prepareEntered = Promise.withResolvers(); const { continuousCompactor } = h.session as unknown as { continuousCompactor: ContinuousCompactor; }; @@ -97,6 +98,7 @@ describe("AgentSession scoped turn lifetimes", () => { }; spyOn(deps, "prepare").mockImplementation(() => { h.session.beginShutdown(); + prepareEntered.resolve(); return release.promise; }); let closed = false; @@ -109,6 +111,7 @@ describe("AgentSession scoped turn lifetimes", () => { thresholdPercent: 80, phase: "on-send", }); + await prepareEntered.promise; // Reset detaches the job immediately; its original preparation still owns I/O. expect(h.session.closingSignal.aborted).toBe(true); expect(h.session.isBusy()).toBe(false); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 5f773d1ad46..f12af25d9a2 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2673,8 +2673,8 @@ export class AgentSession { private async reconcileCompactionCancellation(): Promise { const cancellation = await this.compactionCancellation.read(); if (!cancellation) return; - const history = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); - if (!history.success) throw new Error(history.error); + // Empty history is only a snapshot: a foreign producer can still publish + // canceled work later. Only explicit replacement evidence retires Stop. // A replacement witness is part of the row's atomic commit. The sidecar can // safely retire even if the previous process died before its unlink completed. if ( @@ -2684,8 +2684,6 @@ export class AgentSession { ) ) await this.retireWitnessedCompactionCancellation(cancellation.nonce); - else if (history.data.length === 0) - await this.compactionCancellation.retire(cancellation.nonce); } private async retireWitnessedCompactionCancellation(nonce: string): Promise { diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index d69b4207d9f..16b297f12f0 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1,3 +1,4 @@ +import type { ContinuousCompactionPublication } from "./continuousCompactionJournal"; import type { EventEmitter } from "events"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; @@ -1288,6 +1289,7 @@ export class CompactionHandler { params: Parameters[0] & { prepared?: { boundary: MuxMessage; copies: MuxMessage[] }; shouldPersist: (messages: MuxMessage[]) => boolean; + publication?: ContinuousCompactionPublication; } ): Promise { const canComplete = this.captureCompletionGuard?.(); @@ -1310,7 +1312,8 @@ export class CompactionHandler { boundary, copies, false, - params.shouldPersist + params.shouldPersist, + params.publication ); if (!result.success) { log.warn("[continuous-compaction] persist failed", result.error); diff --git a/src/node/services/continuousCompactionJournal.ts b/src/node/services/continuousCompactionJournal.ts index 237af1f1a3f..77cb40ebc70 100644 --- a/src/node/services/continuousCompactionJournal.ts +++ b/src/node/services/continuousCompactionJournal.ts @@ -2,6 +2,9 @@ import { prepareProviderRequestMessages } from "./turnContextAssembler"; import { addInterruptedSentinel } from "@/browser/utils/messages/modelMessageTransform"; import { applyCacheControl } from "@/common/utils/ai/cacheStrategy"; import { promises as fs } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import * as path from "node:path"; +import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; import { isDeepStrictEqual } from "node:util"; import { modelMessageSchema, type ModelMessage } from "ai"; import writeFileAtomic from "write-file-atomic"; @@ -79,24 +82,94 @@ export async function rebuildContinuousPrefix( ]; } -/** Serialized journal I/O plus a synchronous invalidation fence shared by reset and prepareStep. */ +/** Presence distinguishes a captured legacy generation from an unguarded history mutation. */ +export interface ContinuousCompactionPublication { + generation: string | undefined; +} + +interface JournalReadOwnership { + isCurrent: () => boolean; + shouldDiscard: () => boolean; + onRead: (journal: ContinuousCompactionJournal) => void; +} + +/** Journal ownership and publication share the history lock across backend processes. */ export class ContinuousCompactionJournalStore { - private generation = 0; private pending: Promise = Promise.resolve(); constructor( readonly path: string, - private readonly workspaceId: string + private readonly workspaceId: string, + private readonly withHistoryLock: (operation: () => Promise) => Promise, + private readonly canPublishUnderHistoryLock: () => Promise ) {} private enqueue(operation: () => Promise): Promise { - const result = this.pending.then(operation); + const result = this.pending.then(() => this.withHistoryLock(operation)); this.pending = result.catch(() => undefined); return result; } - clear(): Promise { - this.generation++; - return this.enqueue(() => fs.rm(this.path, { force: true })); + private async readGenerationUnderHistoryLock(): Promise { + try { + // The bytes are an opaque version, not semantic configuration. Damaged + // bytes fence older work while fresh capture can still make progress. + const bytes = await fs.readFile( + path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE) + ); + return createHash("sha256").update(bytes).digest("hex"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + captureGeneration(): Promise { + return this.enqueue(() => this.readGenerationUnderHistoryLock()); + } + + /** Caller already owns the history lock; never join the queue of writers waiting for it. */ + async isPublicationCurrentUnderHistoryLock( + publication: ContinuousCompactionPublication + ): Promise { + return ( + publication.generation === (await this.readGenerationUnderHistoryLock()) && + (await this.canPublishUnderHistoryLock()) + ); + } + + /** Authoritative repair, unlike an old compactor's identity-scoped cleanup. */ + async invalidateUnderHistoryLock(): Promise { + await writeFileAtomic( + path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE), + randomUUID(), + { mode: 0o600 } + ); + await fs.rm(this.path, { force: true }); + } + + private async clearOwnedUnderHistoryLock(expected: ContinuousCompactionJournal): Promise { + let current: ContinuousCompactionJournal; + try { + current = ContinuousCompactionJournalSchema.parse( + JSON.parse(await fs.readFile(this.path, "utf8")) + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + current.boundary.id === expected.boundary.id && + current.publicationGeneration === expected.publicationGeneration + ) { + await fs.rm(this.path, { force: true }); + } + } + + clear(expected: ContinuousCompactionJournal | undefined): Promise { + // No captured owner means no authority to adopt/delete a foreign journal. + // Still join this store's existing I/O without acquiring a lock or recreating a directory. + if (!expected) return this.pending.then(() => undefined); + return this.enqueue(() => this.clearOwnedUnderHistoryLock(expected)); } exists(): Promise { @@ -112,23 +185,48 @@ export class ContinuousCompactionJournalStore { }); } - read(): Promise { + read(ownership?: JournalReadOwnership): Promise { return this.enqueue(async () => { + // A queued stale read never owned the journal now on disk (possibly B). + if (ownership && !ownership.isCurrent()) return null; + let contents: string; try { - return ContinuousCompactionJournalSchema.parse( - JSON.parse(await fs.readFile(this.path, "utf8")) - ); + contents = await fs.readFile(this.path, "utf8"); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - log.warn("[continuous-compaction] discarded invalid journal", error); - await fs - .rm(this.path, { force: true }) - .catch((cleanupError: unknown) => - log.warn("[continuous-compaction] invalid journal cleanup failed", cleanupError) - ); - } + if ((error as NodeJS.ErrnoException).code !== "ENOENT") + log.warn("[continuous-compaction] journal unavailable", error); + return null; + } + // Settings changes and shutdown preserve recovery even if the read began. + if (ownership && !ownership.isCurrent() && !ownership.shouldDiscard()) return null; + let journal: ContinuousCompactionJournal; + try { + journal = ContinuousCompactionJournalSchema.parse(JSON.parse(contents)); + } catch (error) { + // The shared lock keeps a malformed A cleanup from unlinking a new B. + log.warn("[continuous-compaction] discarded invalid journal", error); + await fs.rm(this.path, { force: true }); + return null; + } + if (ownership && !ownership.isCurrent()) { + if (ownership.shouldDiscard()) await this.clearOwnedUnderHistoryLock(journal); return null; } + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) { + await this.clearOwnedUnderHistoryLock(journal); + return null; + } + if (ownership && !ownership.isCurrent()) { + if (ownership.shouldDiscard()) await this.clearOwnedUnderHistoryLock(journal); + return null; + } + // Transfer exact ownership before releasing the lock, through later fold awaits. + ownership?.onRead(journal); + return journal; }); } @@ -140,16 +238,21 @@ export class ContinuousCompactionJournalStore { providerOptions?: Record; system?: string | ModelMessage; }, - isCurrent: () => boolean + isCurrent: () => boolean, + onCommitted?: (journal: ContinuousCompactionJournal) => void ): Promise { - const generation = this.generation; return this.enqueue(async () => { try { + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) + return null; const current = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) ); - if (generation !== this.generation || !isCurrent() || !isDeepStrictEqual(current, journal)) - return null; + if (!isCurrent() || !isDeepStrictEqual(current, journal)) return null; const prefix = request.prefix.map(exactJson); const parsedPrefix = prefix.map((message) => modelMessageSchema.parse(message)); assert( @@ -165,7 +268,7 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(updated), payload), "Fallback journal dropped request fields" ); - if (generation !== this.generation || !isCurrent()) return null; + if (!isCurrent()) return null; await writeFileAtomic(this.path, JSON.stringify(payload), { mode: 0o600 }); const reread = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) @@ -174,7 +277,9 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(reread), payload), "Fallback journal round-trip mismatch" ); - return generation === this.generation && isCurrent() ? reread : null; + if (!isCurrent()) return null; + onCommitted?.(reread); + return reread; } catch (error) { // Unlike the initial write, this record already describes a consumed request. // Failure must keep it available for P1's durable fold or startup recovery. @@ -190,11 +295,17 @@ export class ContinuousCompactionJournalStore { write( journal: ContinuousCompactionJournal, prefix: ModelMessage[], - isCurrent: () => boolean + isCurrent: () => boolean, + onCommitted?: (journal: ContinuousCompactionJournal) => void ): Promise { - const generation = this.generation; return this.enqueue(async () => { try { + if ( + !(await this.isPublicationCurrentUnderHistoryLock({ + generation: journal.publicationGeneration, + })) + ) + return null; let wire: ContinuousCompactionJournal["prefix"]; try { wire = z.array(z.json()).parse(exactJson(prefix)); @@ -218,20 +329,21 @@ export class ContinuousCompactionJournalStore { isDeepStrictEqual(exactJson(parsed), payload), "Journal schema dropped request fields" ); - if (generation !== this.generation || !isCurrent()) return null; + if (!isCurrent()) return null; await writeFileAtomic(this.path, JSON.stringify(payload), { mode: 0o600 }); const reread = ContinuousCompactionJournalSchema.parse( JSON.parse(await fs.readFile(this.path, "utf8")) ); assert(isDeepStrictEqual(exactJson(reread), payload), "Journal round-trip mismatch"); - if (generation !== this.generation || !isCurrent()) { - await fs.rm(this.path, { force: true }); + if (!isCurrent()) { + await this.clearOwnedUnderHistoryLock(journal); return null; } + onCommitted?.(reread); return reread; } catch (error) { log.warn("[continuous-compaction] prefix not swapped: journal failed", error); - await fs.rm(this.path, { force: true }).catch(() => undefined); + await this.clearOwnedUnderHistoryLock(journal).catch(() => undefined); return null; } }); diff --git a/src/node/services/continuousCompactor.test.ts b/src/node/services/continuousCompactor.test.ts index 2ca295a0f35..eef837729bd 100644 --- a/src/node/services/continuousCompactor.test.ts +++ b/src/node/services/continuousCompactor.test.ts @@ -1,3 +1,6 @@ +import { CompactionCancellation } from "./compactionCancellation"; +import { HistoryService } from "./historyService"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import type { ContinuousPrefixSwap } from "./continuousCompactionJournal"; import { writeFile } from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; @@ -684,6 +687,98 @@ describe("ContinuousCompactor", () => { return { answer, journal, journalStore, dependencies, swap }; } + async function repairForeignCancellation() { + await writeFile( + `${store.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(store.config); + expect(await new CompactionCancellation(foreign, workspaceId).read()).toBeNull(); + return foreign; + } + + it("foreign repair fences a durable fold that already read its journal", async () => { + const { answer, journal } = await activateJournaledSwap(); + assert(live, "Expected live source"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + streaming = false; + live = undefined; + const entered = deferred(); + const release = deferred(); + const persist = store.historyService.persistBoundaryWithTailCopies.bind(store.historyService); + spyOn(store.historyService, "persistBoundaryWithTailCopies").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return persist(...args); + } + ); + const pending = compactor.observe(context.thresholdPercent, context); + try { + await entered.promise; + await repairForeignCancellation(); + release.resolve(); + expect(await pending).not.toBe("applied"); + expect((await rows()).some((row) => row.id === journal.boundary.id)).toBe(false); + expect(completed).not.toHaveBeenCalled(); + } finally { + release.resolve(); + await pending; + } + }); + + it("foreign repair fences non-journal work already awaiting preparation", async () => { + await seedConversation(); + const entered = deferred(); + const release = deferred(); + prepare.mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + }); + const { job } = await start(); + try { + await entered.promise; + await repairForeignCancellation(); + release.resolve(); + await job; + expect(await compactor.observe(context.thresholdPercent, context)).not.toBe("applied"); + expect((await rows())[0].id).toBe("old-user"); + expect(completed).not.toHaveBeenCalled(); + // A newly captured generation may compact the still-valid source normally. + const freshJob = eagerJob(compactor); + jobs.push(freshJob); + await freshJob; + expect(await compactor.observe(context.thresholdPercent, context)).toBe("applied"); + expect(completed).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await job; + } + }); + + it("a failed repair generation write keeps staged durable publication fenced", async () => { + await seedConversation(); + await stage(); + await writeFile( + `${store.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const original = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementationOnce( + Object.assign(() => Promise.reject(new Error("epoch storage unavailable")), { + sync: original.sync, + }) + ); + const failed = await new CompactionCancellation(new HistoryService(store.config), workspaceId) + .read() + .catch((error: unknown) => error); + expect(failed).toBeInstanceOf(Error); + expect(await compactor.observe(context.thresholdPercent, context)).not.toBe("applied"); + expect((await rows())[0].id).toBe("old-user"); + expect(completed).not.toHaveBeenCalled(); + }); + for (const reason of ["disabled", "threshold-changed", "context-changed"]) { it(`preserves consumed journal through ${reason} and finalizes once even after tracker retirement`, async () => { const { answer, journal, journalStore } = await activateJournaledSwap(); @@ -841,6 +936,97 @@ describe("ContinuousCompactor", () => { expect(fastApply).not.toHaveBeenCalled(); }); + it.each( + ["read lock", "read return", "partial commit"].flatMap((stage) => + ["user-interrupt", "shutdown", "disabled"].map((reason) => [stage, reason] as const) + ) + )("recovery ownership during %s honors %s", async (stage, reason) => { + const { answer, journalStore, dependencies } = await activateJournaledSwap(); + assert(live, "Expected live source"); + answer.parts = live.parts; + await store.historyService.writePartial(workspaceId, answer); + compactor.reset("shutdown"); + streaming = false; + live = undefined; + compactor = new ContinuousCompactor(dependencies); + const entered = deferred(); + const release = deferred(); + if (stage === "read lock") { + const validate = journalStore.isPublicationCurrentUnderHistoryLock.bind(journalStore); + spyOn(journalStore, "isPublicationCurrentUnderHistoryLock").mockImplementationOnce( + async (...args) => { + const result = await validate(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const read = journalStore.read.bind(journalStore); + spyOn(journalStore, "read").mockImplementationOnce(async (...args) => { + const result = await read(...args); + if (stage === "read return") { + entered.resolve(); + await release.promise; + } else { + const commit = store.historyService.commitPartial.bind(store.historyService); + spyOn(store.historyService, "commitPartial").mockImplementationOnce( + async (...commitArgs) => { + const committed = await commit(...commitArgs); + entered.resolve(); + await release.promise; + return committed; + } + ); + } + return result; + }); + } + const pending = compactor.recover(); + try { + await entered.promise; + compactor.reset(reason); + release.resolve(); + expect(await pending).toBe(false); + expect(await journalStore.exists()).toBe(reason !== "user-interrupt"); + if (reason !== "user-interrupt") { + compactor = new ContinuousCompactor(dependencies); + expect(await compactor.recover()).toBe(true); + } + } finally { + release.resolve(); + await pending; + } + }); + + it("a reset before a queued recovery read starts cannot adopt a foreign journal", async () => { + const { journalStore, dependencies, swap } = await activateJournaledSwap(); + compactor.reset("shutdown"); + streaming = false; + live = undefined; + compactor = new ContinuousCompactor(dependencies); + const release = deferred(); + // Hold only the local queue; B must remain able to publish under the real shared lock. + (journalStore as unknown as { pending: Promise }).pending = release.promise; + const pending = ( + compactor as unknown as { finalizeJournal(): Promise } + ).finalizeJournal(); + try { + compactor.reset("user-interrupt"); + const foreign = new HistoryService(store.config).getContinuousCompactionJournal(workspaceId); + const b = structuredClone(swap.journal); + b.boundary.id = "foreign-after-reset"; + const published = await foreign.write(b, swap.prefix, () => true); + expect(published).not.toBeNull(); + release.resolve(); + expect(await pending).toBe(false); + expect(await foreign.read()).toEqual(published); + } finally { + release.resolve(); + await pending; + } + }); + it("recovers growing crash partials verbatim and retries recovery idempotently after boundary persistence", async () => { const { answer, journal, journalStore, dependencies } = await activateJournaledSwap(); assert(live, "Live fixture missing"); @@ -943,8 +1129,8 @@ describe("ContinuousCompactor", () => { const entered = deferred(); const release = deferred(); const clear = journalStore.clear.bind(journalStore); - spyOn(journalStore, "clear").mockImplementationOnce(async () => { - await clear(); + spyOn(journalStore, "clear").mockImplementationOnce(async (expected) => { + await clear(expected); entered.resolve(); await release.promise; }); diff --git a/src/node/services/continuousCompactor.ts b/src/node/services/continuousCompactor.ts index a21339f674f..df6e2c655d0 100644 --- a/src/node/services/continuousCompactor.ts +++ b/src/node/services/continuousCompactor.ts @@ -76,7 +76,13 @@ interface Dependencies { apply: (pendingFollowUp?: CompactionFollowUpRequest) => Promise ): Promise; } +interface JournalReadAttempt { + discard: boolean; + journal?: ContinuousCompactionJournal; +} + interface StagedSummary { + publicationGeneration: string | undefined; generation: number; epoch: number; boundarySequence?: number; @@ -135,6 +141,7 @@ export class ContinuousCompactor { private applying: Promise | null = null; private swapAttempted: StagedSummary | null = null; private activeSwap: ContinuousPrefixSwap | null = null; + private journalRead?: JournalReadAttempt; constructor(private readonly deps: Dependencies) { assert(deps.workspaceId.length > 0, "ContinuousCompactor requires a workspace"); @@ -149,6 +156,7 @@ export class ContinuousCompactor { // Hydrating settings must not erase a previous process's journal before recovery. // It also keeps the disabled hot path free of journal I/O when no swap ever activated. const discardJournal = !settingsOnly || this.activeSwap !== null || this.swapAttempted !== null; + const ownedJournal = this.activeSwap?.journal ?? this.journalRead?.journal; this.generation++; const job = this.job; this.job = null; @@ -157,7 +165,8 @@ export class ContinuousCompactor { this.activeSwap = null; // Graceful shutdown retains the write-ahead record for ordinary startup recovery. if (discardJournal && reason !== "shutdown") { - this.clearJournal().catch((error: unknown) => + if (this.journalRead) this.journalRead.discard = true; + this.clearJournal(ownedJournal).catch((error: unknown) => log.warn("[continuous-compaction] journal clear failed", error) ); } @@ -171,9 +180,11 @@ export class ContinuousCompactor { log.debug("[continuous-compaction] reset", { workspaceId: this.deps.workspaceId, reason }); } - private async clearJournal(): Promise { + private async clearJournal(journal: ContinuousCompactionJournal | undefined): Promise { using _execution = this.deps.enterExecution?.(); - await this.deps.historyService.getContinuousCompactionJournal(this.deps.workspaceId).clear(); + await this.deps.historyService + .getContinuousCompactionJournal(this.deps.workspaceId) + .clear(journal); } hasConsumedSwap(): boolean { @@ -318,6 +329,12 @@ export class ContinuousCompactor { job: NonNullable, context: ContinuousCompactionContext ): Promise { + // Capture before any source preparation: a later repair must fence this job + // even when the transcript fingerprint itself remains unchanged. + const publicationGeneration = await this.deps.historyService + .getContinuousCompactionJournal(this.deps.workspaceId) + .captureGeneration(); + if (job.generation !== this.generation) return; await this.deps.prepare(); if (job.generation !== this.generation) return; const rows = await this.readSnapshot(); @@ -351,6 +368,7 @@ export class ContinuousCompactor { "Rolling head must have a durable sequence" ); const stagedBase = { + publicationGeneration, generation: job.generation, ...boundaryIdentity(rows), cut, @@ -507,6 +525,7 @@ export class ContinuousCompactor { try { const journal: ContinuousCompactionJournal = { version: 1, + publicationGeneration: staged.publicationGeneration, boundary, staticCopies, liveTailCopySpec: { @@ -581,7 +600,20 @@ export class ContinuousCompactor { private async finalizeJournal(pendingFollowUp?: CompactionFollowUpRequest): Promise { const generation = this.generation; const store = this.deps.historyService.getContinuousCompactionJournal(this.deps.workspaceId); - const journal = await store.read(); + const readAttempt: JournalReadAttempt = { discard: false }; + this.journalRead = readAttempt; + using _readOwnership = { + [Symbol.dispose]: () => { + if (this.journalRead === readAttempt) this.journalRead = undefined; + }, + }; + const journal = await store.read({ + isCurrent: () => generation === this.generation, + shouldDiscard: () => readAttempt.discard, + onRead: (journal) => { + readAttempt.journal = journal; + }, + }); if (generation !== this.generation) return false; if (!journal) { this.activeSwap = null; @@ -596,7 +628,7 @@ export class ContinuousCompactor { journal.staticCopies.every((copy) => rows.some((row) => row.id === copy.id)) && rows.some((row) => row.id === journal.liveTailCopySpec.copyId); if (rows.some((row) => row.id === journal.boundary.id) && copiesPresent) { - await store.clear(); + await store.clear(journal); if (generation !== this.generation) return false; this.activeSwap = null; return true; @@ -633,7 +665,7 @@ export class ContinuousCompactor { log.warn("[continuous-compaction] discarded mismatched journal", { workspaceId: this.deps.workspaceId, }); - await store.clear(); + await store.clear(journal); if (generation !== this.generation) return false; this.activeSwap = null; return false; @@ -677,6 +709,7 @@ export class ContinuousCompactor { journal.postCompactionAttachments ).reduce((sum, row) => sum + estimateMuxMessageTokens(row), 0), prepared: { boundary, copies: [...journal.staticCopies, liveCopy] }, + publication: { generation: journal.publicationGeneration }, shouldPersist: (current) => generation === this.generation && !this.deps.streamManager.isStreaming(this.deps.workspaceId) && @@ -686,7 +719,10 @@ export class ContinuousCompactor { boundary.id ); if (applied && generation === this.generation) { - await store.clear(); + await store.clear(journal); + // This owner has completed cleanup; reset must not queue another unlink + // after recovery returns (or race a subsequent recovery of the same journal). + readAttempt.journal = undefined; // Persistence may finish after an edit/reset. Its old clear is already ordered // by the store; a new reset here would enqueue another clear behind replacement work. if (generation === this.generation) this.reset("applied"); @@ -736,6 +772,7 @@ export class ContinuousCompactor { fingerprint(current) === snapshotFingerprint ); }, + publication: { generation: staged.publicationGeneration }, messages: rows, text: staged.text, model: staged.model, diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 7bffca7aad7..e668f69151c 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -28,7 +28,11 @@ import { matchesCompactionCancellation, type CompactionCancellationRecord, } from "./compactionCancellation"; -import { ContinuousCompactionJournalStore } from "./continuousCompactionJournal"; +import { + ContinuousCompactionJournalStore, + type ContinuousCompactionPublication, +} from "./continuousCompactionJournal"; +import { CONTINUOUS_COMPACTION_JOURNAL_FILE } from "@/constants/continuousCompaction"; import writeFileAtomic from "write-file-atomic"; import assert from "node:assert"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; @@ -339,8 +343,24 @@ export class HistoryService { let journal = this.continuousJournals.get(workspaceId); if (!journal) { journal = new ContinuousCompactionJournalStore( - path.join(this.getSessionDir(workspaceId), "continuous-compaction.json"), - workspaceId + path.join(this.getSessionDir(workspaceId), CONTINUOUS_COMPACTION_JOURNAL_FILE), + workspaceId, + (operation) => + this.withHistoryWriteFileLock(workspaceId, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) + throw new Error(`workspace ${workspaceId} was removed; refusing journal mutation`); + await ensurePrivateDir(this.getSessionDir(workspaceId)); + return operation(); + }), + async () => { + // Raw reads only under the held lock. Even a failed epoch publication + // leaves malformed/unresolved Stop authoritative until exact replacement. + const cancellation = await this.readCompactionCancellation(workspaceId); + return ( + !cancellation || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, cancellation.nonce)) + ); + } ); this.continuousJournals.set(workspaceId, journal); } @@ -416,7 +436,7 @@ export class HistoryService { flag: "wx", }); if (!isCurrent()) return null; - await this.getContinuousCompactionJournal(workspaceId).clear(); + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); const historyPath = this.getChatHistoryPath(workspaceId); const { rows } = await this.readHistoryForRewrite(historyPath); let changed = false; @@ -3323,13 +3343,23 @@ export class HistoryService { summaryMessage: MuxMessage, tailCopies: readonly MuxMessage[], updateExisting: boolean, - shouldPersist?: (messages: MuxMessage[]) => boolean + shouldPersist?: (messages: MuxMessage[]) => boolean, + publication?: ContinuousCompactionPublication ): Promise> { // Continuous compaction may intentionally keep no tail when no complete turn fits. return this.withRecoveredHistoryWriteResultLock( workspaceId, "Failed to persist compaction boundary with tail copies", async () => { + // A prepared summary or already-read journal can outlive foreign repair. + // Validate its captured generation under this lock before touching sequences. + if ( + publication && + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); invalidateHistoryAppendProvenance(); try { // r52: this path assigns fresh sequences (appended summary + every diff --git a/src/node/services/streamManager.continuousCompaction.test.ts b/src/node/services/streamManager.continuousCompaction.test.ts index 691c246e086..1a2c1132b10 100644 --- a/src/node/services/streamManager.continuousCompaction.test.ts +++ b/src/node/services/streamManager.continuousCompaction.test.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { CompactionCancellation } from "./compactionCancellation"; +import { HistoryService } from "./historyService"; +import { removeSessionDirUnderMemoryLocks } from "./workspaceRemoval"; +import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; import { assemblePromptPayload } from "./turnContextAssembler"; import type { ActiveTurnThinkingOverride } from "./thinkingOverride"; import { prepareMessagesForProvider } from "./messagePipeline"; @@ -7,6 +11,8 @@ import * as ai from "ai"; import * as atomicWrite from "write-file-atomic"; import { createAnthropic } from "@ai-sdk/anthropic"; import { readFile, writeFile } from "node:fs/promises"; +import { promises as journalFs } from "node:fs"; +import { CONTINUOUS_COMPACTION_GENERATION_FILE } from "@/constants/continuousCompaction"; import assert from "@/common/utils/assert"; import { createMuxMessage } from "@/common/types/message"; import { @@ -172,6 +178,201 @@ describe("continuous prefix prepareStep and journal", () => { return { ...harness, store, tracker, swap, manager }; } + it("a foreign repair rejects a prefix prepared before its journal publication", async () => { + const { run, tracker, store } = await setup(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const write = store.write.bind(store); + spyOn(store, "write").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const pending = run(); + try { + await entered.promise; + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(history.config); + expect(await new CompactionCancellation(foreign, workspaceId).read()).toBeNull(); + release.resolve(); + const result = await pending; + expect(tracker.consumedPrefixSwap).toBeUndefined(); + expect(result?.messages).toBeUndefined(); + expect(await foreign.getContinuousCompactionJournal(workspaceId).exists()).toBe(false); + } finally { + release.resolve(); + await pending; + } + }); + + it("old identity cleanup cannot invalidate a queued newer journal in the same store", async () => { + const { store, swap } = await setup(); + const a = await store.write(swap.journal, swap.prefix, () => true); + assert(a, "Expected A"); + await store.clear(a); + const b = structuredClone(swap.journal); + b.boundary.id = "newer-boundary"; + const writingB = store.write(b, swap.prefix, () => true); + await store.clear(a); + expect(await writingB).not.toBeNull(); + expect((await store.read())?.boundary.id).toBe(b.boundary.id); + }); + + it.each(["generation write", "journal clear"])( + "failed repair at %s still rejects old prefix publication", + async (failure) => { + const { run, tracker, store } = await setup(); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreign = new HistoryService(history.config); + if (failure === "generation write") { + const original = atomicWrite.default; + spyOn(atomicWrite, "default").mockImplementationOnce( + Object.assign(() => Promise.reject(new Error("generation unavailable")), { + sync: original.sync, + }) + ); + } else { + const remove = journalFs.rm; + let failed = false; + spyOn(journalFs, "rm").mockImplementation(async (...args) => { + if (!failed && args[0] === store.path) { + failed = true; + throw new Error("clear unavailable"); + } + return remove(...args); + }); + } + const repair = await new CompactionCancellation(foreign, workspaceId) + .read() + .catch((error: unknown) => error); + expect(repair).toBeInstanceOf(Error); + expect( + await readFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "utf8" + ) + ).toBe("{"); + await run(); + expect(tracker.consumedPrefixSwap).toBeUndefined(); + expect(await store.exists()).toBe(false); + } + ); + + it("opaque corrupt generation bytes fence old work and allow a fresh publication", async () => { + const { store, swap } = await setup(); + const generationPath = `${history.config.sessionsDir}/${workspaceId}/${CONTINUOUS_COMPACTION_GENERATION_FILE}`; + await writeFile(generationPath, Buffer.from([0, 255, 123])); + expect(await store.write(swap.journal, swap.prefix, () => true)).toBeNull(); + const fresh = journalFixture(); + fresh.publicationGeneration = await store.captureGeneration(); + const prefix = await rebuildContinuousPrefix(fresh, workspaceId); + expect(await store.write(fresh, prefix, () => true)).not.toBeNull(); + const foreign = new HistoryService(history.config).getContinuousCompactionJournal(workspaceId); + expect((await foreign.read())?.publicationGeneration).toBe(fresh.publicationGeneration); + }); + + it("old cleanup and fallback cannot replace a fresh foreign journal after repair", async () => { + const { store, swap } = await setup(); + const original = await store.write(swap.journal, swap.prefix, () => true); + assert(original, "Expected original journal"); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const foreignHistory = new HistoryService(history.config); + expect(await new CompactionCancellation(foreignHistory, workspaceId).read()).toBeNull(); + const foreign = foreignHistory.getContinuousCompactionJournal(workspaceId); + const fresh = journalFixture(); + fresh.publicationGeneration = await foreign.captureGeneration(); + fresh.boundary.id = "foreign-boundary"; + const prefix = await rebuildContinuousPrefix(fresh, workspaceId); + const published = await foreign.write(fresh, prefix, () => true); + assert(published, "Expected fresh publication"); + await store.clear(original); + expect( + await store.recordFallbackPrefix( + original, + { modelString: original.parentModel, prefix: swap.prefix }, + () => true + ) + ).toBeNull(); + expect(await store.write(original, swap.prefix, () => true)).toBeNull(); + // A failed stale write/cleanup must not adopt the newer journal it sees. + await store.clear(undefined); + expect(await foreign.read()).toEqual(published); + expect( + await foreign.recordFallbackPrefix( + published, + { modelString: published.parentModel, prefix }, + () => true + ) + ).not.toBeNull(); + }); + + it("repair does not join a journal writer queued behind its held history lock", async () => { + const { swap } = await setup(); + const foreignHistory = new HistoryService(history.config); + const foreign = foreignHistory.getContinuousCompactionJournal(workspaceId); + await writeFile( + `${history.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const invalidate = foreign.invalidateUnderHistoryLock.bind(foreign); + spyOn(foreign, "invalidateUnderHistoryLock").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return invalidate(); + }); + const repair = new CompactionCancellation(foreignHistory, workspaceId).read(); + let queued: ReturnType | undefined; + try { + await entered.promise; + queued = foreign.write(swap.journal, swap.prefix, () => true); + release.resolve(); + expect(await repair).toBeNull(); + expect(await queued).toBeNull(); + expect(await foreign.exists()).toBe(false); + } finally { + release.resolve(); + await repair; + await queued; + } + }); + + it("journal capture, publication and old cleanup cannot recreate a removed workspace", async () => { + const { store, swap } = await setup(); + const original = await store.write(swap.journal, swap.prefix, () => true); + assert(original, "Expected journal"); + const sessionDir = `${history.config.sessionsDir}/${workspaceId}`; + await removeSessionDirUnderMemoryLocks({ + rootDir: history.config.rootDir, + sessionDir, + workspaceId, + attemptId: "remove-journal-test", + }); + for (const operation of [ + () => store.captureGeneration(), + () => store.write(original, swap.prefix, () => true), + () => store.clear(original), + ]) { + const result = await operation().catch((error: unknown) => error); + expect(result).toHaveProperty("message", expect.stringContaining("removed")); + } + await store.clear(undefined); + expect(await journalFs.stat(sessionDir).catch((error: unknown) => error)).toHaveProperty( + "code", + "ENOENT" + ); + }); + it("journals before returning, swaps once by content identity, and strips retained cache markers", async () => { const { run, tracker, store, swap } = await setup(); const result = await run(); @@ -461,7 +662,7 @@ describe("continuous prefix prepareStep and journal", () => { it.each(["reset-before-write", "abort-after-write"] as const)( "%s fences the swap return and deletes the stale journal", async (mode) => { - const { run, tracker, store, controller } = await setup(); + const { run, tracker, store, controller, swap } = await setup(); let release!: () => void; const gate = new Promise((resolve) => { release = resolve; @@ -487,7 +688,7 @@ describe("continuous prefix prepareStep and journal", () => { let cleared = Promise.resolve(); if (mode === "reset-before-write") { tracker.pendingPrefixSwap = undefined; - cleared = store.clear(); + cleared = store.clear(swap.journal); } else { controller.abort(); } diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 9e2d078e9aa..d956751e586 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -2624,19 +2624,28 @@ export class StreamManager { !abortController.signal.aborted && request.thinkingOverrideState?.pending == null && request.thinkingOverrideState?.applied === thinkingLevel; + const unswappedMessages = effectiveMessages; const journal = await store.write( { ...swap.journal, stepNumber }, swap.prefix, - isCurrent + isCurrent, + (journal) => { + // Consume in the publication lock, so repair orders before or after + // this receipt instead of racing the return from journal.write(). + swap.journal = journal; + swap.consumed = true; + stepTracker.consumedPrefixSwap = swap; + effectiveMessages = swapped; + } ); - if (journal && isCurrent()) { - swap.journal = journal; - swap.consumed = true; - stepTracker.consumedPrefixSwap = swap; - effectiveMessages = swapped; - } else if (journal && stepTracker.pendingPrefixSwap === swap) { - // A thinking change can also arrive after write()'s last fence. - await store.clear(); + if (journal && !isCurrent()) { + // Local thinking/abort changes still win before provider entry. + // Cleanup cannot delete a foreign journal published after our receipt. + effectiveMessages = unswappedMessages; + swap.consumed = false; + if (stepTracker.consumedPrefixSwap === swap) + stepTracker.consumedPrefixSwap = undefined; + await store.clear(journal); } } if (stepTracker.pendingPrefixSwap === swap) stepTracker.pendingPrefixSwap = undefined; @@ -3622,7 +3631,7 @@ export class StreamManager { !streamInfo.abortController.signal.aborted && nextRequest.thinkingOverrideState?.pending == null && nextRequest.thinkingOverrideState?.applied === appliedThinking; - const journal = await this.historyService + await this.historyService .getContinuousCompactionJournal(workspaceId) .recordFallbackPrefix( consumedSwap.journal, @@ -3632,12 +3641,13 @@ export class StreamManager { providerOptions: nextRequest.providerOptions, system: nextRequest.system, }, - isCurrent + isCurrent, + (journal) => { + consumedSwap.journal = journal; + messages = swapped; + } ); - if (journal && isCurrent()) { - consumedSwap.journal = journal; - messages = swapped; - } + if (!isCurrent()) messages = null; } } if (messages) nextRequest.messages = messages; From 6a66d64b2e743bb6b8952e9ac2394d44bd0e2c1b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 08:31:41 +0200 Subject: [PATCH 14/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20guard=20durable=20c?= =?UTF-8?q?ompaction=20and=20manual-send=20acceptance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compare captured Stop identity under every manual row-acceptance lock, sharing explicit Retry semantics. Commit internal handoff snapshots and their user row atomically, and reconcile live replacement witnesses before cancellation-driven cleanup. Revalidate the exact unwitnessed nonce under the summary update lock so stale cleanup cannot erase newer intent. Cover foreign Stop ordering, snapshot crash recovery, active and archived witnesses, failed unlink, and retained batches after Stop and failed rollback with real history services. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I9f650056769eb78fdcf8ffdae9b98a1e95694ff8 --- .../agentSession.compactionShutdown.test.ts | 459 +++++++++++++++++- src/node/services/agentSession.ts | 129 +++-- src/node/services/historyService.ts | 69 ++- 3 files changed, 571 insertions(+), 86 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index 505ca2ad949..ff92d2d1e4a 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1238,11 +1238,13 @@ test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( materializeFileAtMentionsSnapshot(text: string): Promise<{ snapshotMessage: ReturnType; materializedTokens: string[]; + fileStates: []; } | null>; }; spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ snapshotMessage: ownSnapshot, materializedTokens: [], + fileStates: [], }); const foreignHistory = new HistoryService(h.config); const foreign = await createAgentSessionHarness({ @@ -1250,9 +1252,8 @@ test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( config: h.config, historyService: foreignHistory, }); - const append = h.historyService.appendToHistory.bind(h.historyService); - spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { - const result = await append(...args); + const append = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { if (failureMode === "guard I/O failure") { spyOn(h.historyService, "readCompactionCancellation").mockRejectedValueOnce( new Error("guard storage unavailable") @@ -1264,7 +1265,7 @@ test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( ); await foreign.session.runStartupRecovery(); } - return result; + return append(...args); }); if (cleanupFails) spyOn(h.historyService, "deleteMessages").mockResolvedValueOnce( @@ -1287,19 +1288,14 @@ test.each(["clean", "cleanup failure", "guard I/O failure"] as const)( if (failureMode === "guard I/O failure") expect(result).toHaveProperty( "message", - "Failed to append history: guard storage unavailable" - ); - else if (cleanupFails) - expect(result).toHaveProperty( - "message", - "Failed to roll back preparation rows after compaction follow-up became stale" + "Failed to append to history: guard storage unavailable" ); else expect(result).toBe(false); expect(durableReceipts).toBe(0); expect(stream).not.toHaveBeenCalled(); const rows = await foreignHistory.getHistoryFromLatestBoundary(workspaceId); expect(rows.success && rows.data.some((row) => row.role === "user")).toBe(false); - expect(rows.success && rows.data.some((row) => row.id === ownSnapshot.id)).toBe(cleanupFails); + expect(rows.success && rows.data.some((row) => row.id === ownSnapshot.id)).toBe(false); } finally { await foreign.session.dispose(); await h.session.dispose(); @@ -2124,16 +2120,23 @@ test.each([ } }); -test.each(["accepted", "foreign Stop", "raw reset", "append failure"] as const)( - "token-budget handoff batch preserves locked admission (%s)", - async (action) => { +test.each( + [false, true].flatMap((tokenBudget) => + ["accepted", "foreign Stop", "raw reset", "append failure"].map((action) => ({ + tokenBudget, + action, + })) + ) +)( + "handoff batch preserves locked admission ($action, tokenBudget=$tokenBudget)", + async ({ action, tokenBudget }) => { const h = await setup(); const source = summary(); source.metadata = { ...source.metadata, muxMetadata: { type: "compaction-summary", - pendingFollowUp: { text: "Continue", ...options, experiments: { tokenBudget: true } }, + pendingFollowUp: { text: "Continue", ...options, experiments: { tokenBudget } }, }, }; await h.historyService.appendToHistory(workspaceId, source); @@ -2307,3 +2310,429 @@ test("cancelable acceptance precedes blocked cancellation retirement and runs on await h.cleanup(); } }); + +test.each( + [false, true].flatMap((initialStop) => + [ + "single", + "token-budget single", + "token-budget batch", + "pre-turn batch", + "on-send compaction", + ].map((branch) => ({ initialStop, branch })) + ) +)( + "manual $branch rejects a foreign Stop after capture (initial Stop=$initialStop)", + async ({ initialStop, branch }) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original", "user", "Earlier work") + ); + if (initialStop) await h.session.interruptStream({ abandonPartial: true }); + if (branch === "on-send compaction") { + const monitor = (h.session as unknown as { compactionMonitor: CompactionMonitor }) + .compactionMonitor; + spyOn(monitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + } + if (branch === "token-budget batch") { + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: createMuxMessage("snapshot", "assistant", "Context", { synthetic: true }), + materializedTokens: [], + fileStates: [], + }); + } + const capture = h.session.getCompactionCancellationNonce.bind(h.session); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(h.session, "getCompactionCancellationNonce").mockImplementationOnce(async () => { + const nonce = await capture(); + entered.resolve(); + await release.promise; + return nonce; + }); + const stream = spyOn(h.aiService, "streamMessage"); + const durable = mock(() => undefined); + const pending = h.session.sendMessage( + "Manual replacement", + { + ...options, + ...(branch.startsWith("token-budget") ? { experiments: { tokenBudget: true } } : {}), + }, + { + onRowsDurable: durable, + ...(branch === "pre-turn batch" + ? { + preTurnMessages: [ + createMuxMessage("payload", "assistant", "Payload", { synthetic: true }), + ], + } + : {}), + } + ); + try { + await entered.promise; + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + const stopped = await foreign.read(); + release.resolve(); + expect((await pending).success).toBe(false); + expect(stream).not.toHaveBeenCalled(); + expect(durable).not.toHaveBeenCalled(); + const rows = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual(["original"]); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + stopped + ); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each([false, true])( + "live dispatch reconciles a foreign replacement witness (archived=%s)", + async (archived) => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const stop = new CompactionCancellation(foreign, workspaceId); + await stop.cancel(); + const record = await stop.read(); + if (!record) throw new Error("Expected Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Fresh work", { + compactionCancellationNonce: record.nonce, + }) + ); + spyOn(foreign, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("unlink unavailable") + ); + expect( + await stop.retireReplacement(record.nonce).catch((error: unknown) => error) + ).toHaveProperty("message", "unlink unavailable"); + expect(await foreign.readCompactionCancellation(workspaceId)).toEqual(record); + const boundary = summary(); + if (archived) + boundary.metadata = { + ...boundary.metadata, + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }; + await foreign.appendToHistory(workspaceId, boundary); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(true); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].role).toBe("user"); + } finally { + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("interrupted handoff snapshot persistence leaves a fresh service able to recover", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const snapshot = createMuxMessage("interrupted-snapshot", "user", "Expanded context", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeAgentSkillSnapshots(): Promise>>; + materializeMcpPromptSnapshots(): Promise>>; + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: snapshot, + materializedTokens: [], + fileStates: [], + }); + spyOn(materializer, "materializeAgentSkillSnapshots").mockResolvedValue([ + createMuxMessage("skill-prelude", "user", "Skill context", { synthetic: true }), + ]); + spyOn(materializer, "materializeMcpPromptSnapshots").mockResolvedValue([ + createMuxMessage("mcp-prelude", "user", "MCP context", { synthetic: true }), + ]); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + entered.resolve(); + await release.promise; + return result; + }); + const batch = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { + // Simulate process loss before publication by observing disk from an independent service. + entered.resolve(); + await release.promise; + return batch(...args); + }); + const pending = h.internals.dispatchPendingFollowUp(); + let fresh: Awaited> | undefined; + try { + await entered.promise; + fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const recoveredStream = spyOn(fresh.aiService, "streamMessage"); + await fresh.session.runStartupRecovery(); + expect(recoveredStream).toHaveBeenCalledTimes(1); + const recovered = await fresh.historyService.getLastMessages(workspaceId, 1); + expect(recovered.success && recovered.data[0].parts).toMatchObject([ + { type: "text", text: "Continue" }, + ]); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await fresh?.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("Stop survives failed rollback of a committed handoff snapshot batch across restart", async () => { + const h = await setup(); + await h.historyService.appendToHistory(workspaceId, summary()); + const snapshot = createMuxMessage("retained-snapshot", "user", "Expanded context", { + synthetic: true, + }); + const materializer = h.session as unknown as { + materializeFileAtMentionsSnapshot(text: string): Promise<{ + snapshotMessage: ReturnType; + materializedTokens: string[]; + fileStates: []; + } | null>; + }; + spyOn(materializer, "materializeFileAtMentionsSnapshot").mockResolvedValue({ + snapshotMessage: snapshot, + materializedTokens: [], + fileStates: [], + }); + const append = h.historyService.appendManyToHistory.bind(h.historyService); + spyOn(h.historyService, "appendManyToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + await h.session.interruptStream({ abandonPartial: true }); + return result; + }); + spyOn(h.historyService, "deleteMessages").mockResolvedValue(Err("rollback unavailable")); + const stream = spyOn(h.aiService, "streamMessage"); + try { + await h.internals.dispatchPendingFollowUp(); + expect(stream).not.toHaveBeenCalled(); + const foreign = new HistoryService(h.config); + expect(await foreign.readCompactionCancellation(workspaceId)).not.toBeNull(); + const rows = await foreign.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.filter((row) => row.role === "user")).toHaveLength(2); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreign, + }); + try { + const recovered = spyOn(fresh.aiService, "streamMessage"); + await fresh.session.runStartupRecovery(); + expect(recovered).not.toHaveBeenCalled(); + expect(await foreign.readCompactionCancellation(workspaceId)).not.toBeNull(); + } finally { + await fresh.session.dispose(); + } + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["witness read", "retirement"] as const)( + "live witness reconciliation preserves newer Stop during %s", + async (stage) => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("replacement", "user", "Work", { compactionCancellationNonce: record.nonce }) + ); + await foreign.appendToHistory(workspaceId, summary()); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage === "witness read") { + const read = h.historyService.hasCompactionReplacementWitness.bind(h.historyService); + spyOn(h.historyService, "hasCompactionReplacementWitness").mockImplementationOnce( + async (...args) => { + const result = await read(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + } else { + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce( + async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + } + ); + } + const stream = spyOn(h.aiService, "streamMessage"); + const pending = h.internals.dispatchPendingFollowUp(); + try { + await entered.promise; + await cancellation.cancel(); + const newer = await cancellation.read(); + // Keep the newer cancellation observable through a failed durable cleanup. + const writes = h.historyService as unknown as { + writeGuardedHistory( + path: string, + serialized: string | Buffer, + guard: () => boolean + ): Promise; + }; + spyOn(writes, "writeGuardedHistory").mockRejectedValueOnce(new Error("cleanup unavailable")); + release.resolve(); + expect(await pending.catch((error: unknown) => error)).toHaveProperty( + "message", + "Failed to clear skipped pending follow-up: Failed to update history: cleanup unavailable" + ); + expect(stream).not.toHaveBeenCalled(); + expect((await foreign.readCompactionCancellation(workspaceId))?.nonce).toBe(newer?.nonce); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test("cancellation cleanup rechecks a replacement witness under the summary write lock", async () => { + const h = await setup(); + const foreign = new HistoryService(h.config); + const user = createMuxMessage("replacement", "user", "Work"); + await foreign.appendToHistory(workspaceId, user); + const boundary = summary(); + await foreign.appendToHistory(workspaceId, boundary); + const cancellation = new CompactionCancellation(foreign, workspaceId); + await cancellation.cancel(); + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + await foreign.updateHistory(workspaceId, { + ...user, + metadata: { ...user.metadata, compactionCancellationNonce: record.nonce }, + }); + return update(...args); + }); + try { + expect(await h.internals.dispatchPendingFollowUp()).toBe(false); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); + } finally { + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each(["absence", "witnessed debt", "Stop after commit"] as const)( + "manual locked acceptance handles %s", + async (state) => { + const h = await setup(); + const cancellation = ( + h.session as unknown as { compactionCancellation: CompactionCancellation } + ).compactionCancellation; + let captured: string | undefined; + let newer: string | undefined; + let restoreWrite: (() => void) | undefined; + if (state !== "absence") { + await h.session.interruptStream({ abandonPartial: true }); + captured = await h.session.getCompactionCancellationNonce(); + if (!captured) throw new Error("Expected Stop"); + if (state === "witnessed debt") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("accepted-earlier", "user", "Work", { + compactionCancellationNonce: captured, + }) + ); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const spy = spyOn(h.historyService, "writeCompactionCancellation").mockImplementation( + (...args) => + args[1] === null ? Promise.reject(new Error("unlink unavailable")) : write(...args) + ); + restoreWrite = () => spy.mockRestore(); + await cancellation.retireReplacement(captured).catch(() => undefined); + expect(await h.session.getCompactionCancellationNonce()).toBeUndefined(); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + const result = await append(...args); + const foreign = new CompactionCancellation(new HistoryService(h.config), workspaceId); + await foreign.cancel(); + newer = (await foreign.read())?.nonce; + return result; + }); + } + } + const durable = mock(() => undefined); + const accepted = mock(() => undefined); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect( + ( + await h.session.sendMessage("Accepted work", options, { + onRowsDurable: durable, + onAccepted: accepted, + }) + ).success + ).toBe(true); + expect(durable).toHaveBeenCalledTimes(1); + expect(accepted).toHaveBeenCalledTimes(1); + expect(stream).toHaveBeenCalledTimes(1); + const foreign = new HistoryService(h.config); + const rows = await foreign.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].metadata?.compactionCancellationNonce).toBe( + state === "Stop after commit" ? captured : undefined + ); + if (state === "Stop after commit") + expect((await foreign.readCompactionCancellation(workspaceId))?.nonce).toBe(newer); + if (state === "witnessed debt") expect(cancellation.blocksRecovery).toBe(false); + } finally { + restoreWrite?.(); + await h.session.dispose(); + await h.cleanup(); + } + } +); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7dbe1d8b557..24a482e8bb2 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -33,7 +33,10 @@ import * as path from "path"; import assert from "@/common/utils/assert"; import { EventEmitter } from "events"; import { Effect, Fiber } from "effect"; -import { CompactionCancellation } from "./compactionCancellation"; +import { + CompactionCancellation, + type CompactionCancellationRecord, +} from "./compactionCancellation"; import { StartupRecovery, type StartupRecoveryOutcome } from "./startupRecovery"; import { mkdir, readdir, readFile, unlink, writeFile } from "fs/promises"; import type { Dirent } from "fs"; @@ -2718,20 +2721,23 @@ export class AgentSession { return retryRequest?.model ?? null; } - private async reconcileCompactionCancellation(): Promise { - const cancellation = await this.compactionCancellation.read(); - if (!cancellation) return; - // Empty history is only a snapshot: a foreign producer can still publish - // canceled work later. Only explicit replacement evidence retires Stop. - // A replacement witness is part of the row's atomic commit. The sidecar can - // safely retire even if the previous process died before its unlink completed. - if ( - await this.historyService.hasCompactionReplacementWitness( + private async reconcileCompactionCancellation(): Promise { + let cancellation = await this.compactionCancellation.read(); + // Empty history cannot retire Stop: only an exact active/archive row witness can. + // Live dispatch uses the same reconciliation as startup because another backend + // can commit a replacement while its ancillary sidecar unlink keeps failing. + while ( + cancellation && + (await this.historyService.hasCompactionReplacementWitness( this.workspaceId, cancellation.nonce - ) - ) + )) + ) { await this.retireWitnessedCompactionCancellation(cancellation.nonce); + // Re-read shared identity after every await; retiring witnessed A cannot hide B. + cancellation = await this.compactionCancellation.read(); + } + return cancellation; } private async retireWitnessedCompactionCancellation(nonce: string): Promise { @@ -3569,18 +3575,6 @@ export class AgentSession { const cancelSignal = internal?.cancelSignal; const persistedCancelableMessageIds: string[] = []; let compactionAppendSkipped = false; - const compactionAppendCondition = internal?.compactionHandoff - ? { - summary: internal.compactionHandoffSource?.summary, - allowedTailMessageIds: persistedCancelableMessageIds, - isCurrent: () => - !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, - onSkipped: () => { - compactionAppendSkipped = true; - internal.compactionHandoffSource?.onSkipped(); - }, - } - : undefined; // Roll back synthetic snapshots if the invoking user row fails to persist, or // later provider requests could consume orphaned context. /** @@ -3629,7 +3623,11 @@ export class AgentSession { "Failed to roll back preparation rows after compaction follow-up became stale" ); return Err( - createUnknownSendMessageError("Compaction follow-up source became stale before append") + createUnknownSendMessageError( + isManualUserMessage + ? "Send superseded by a newer Stop before append" + : "Compaction follow-up source became stale before append" + ) ); }; const markRowsDurable = (): void => { @@ -4102,6 +4100,22 @@ export class AgentSession { const compactionCancellationNonce = isManualUserMessage ? await this.getCompactionCancellationNonce() : undefined; + const compactionAppendCondition = + isManualUserMessage || internal?.compactionHandoff + ? { + ...(isManualUserMessage + ? { replacementNonce: compactionCancellationNonce ?? null } + : {}), + summary: internal?.compactionHandoffSource?.summary, + allowedTailMessageIds: persistedCancelableMessageIds, + isCurrent: () => + !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, + onSkipped: () => { + compactionAppendSkipped = true; + internal?.compactionHandoffSource?.onSkipped(); + }, + } + : undefined; const userMessage = createMuxMessage( messageId, "user", @@ -4344,7 +4358,7 @@ export class AgentSession { compactionAppendCondition ); if (!appendCompactionResult.success) { - if (compactionAppendCondition) throw new Error(appendCompactionResult.error); + if (internal?.compactionHandoff) throw new Error(appendCompactionResult.error); return Err(createUnknownSendMessageError(appendCompactionResult.error)); } if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); @@ -4391,6 +4405,9 @@ export class AgentSession { // Persist snapshots only when this turn will be sent immediately. // On on-send compaction paths, snapshots are deferred with the follow-up turn. const shouldPersistTurnSnapshots = autoCompactionMessage === null; + // A handoff snapshot without its trigger strands pendingFollowUp after a crash. + // Commit their exact materialized rows together, including with token budgets disabled. + const batchTurnSnapshots = tokenBudgetActive || internal?.compactionHandoff != null; let skillSnapshotMessages: MuxMessage[] = []; let mcpPromptSnapshotMessages: MuxMessage[] = []; @@ -4414,7 +4431,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && snapshotResult?.snapshotMessage) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && snapshotResult?.snapshotMessage) { const snapshotAppendResult = await this.historyService.appendToHistory( this.workspaceId, snapshotResult.snapshotMessage @@ -4428,7 +4445,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && skillSnapshotMessages.length > 0) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && skillSnapshotMessages.length > 0) { for (const snapshotMessage of skillSnapshotMessages) { const skillSnapshotAppendResult = await this.historyService.appendToHistory( this.workspaceId, @@ -4445,7 +4462,7 @@ export class AgentSession { } } - if (shouldPersistTurnSnapshots && !tokenBudgetActive && mcpPromptSnapshotMessages.length > 0) { + if (shouldPersistTurnSnapshots && !batchTurnSnapshots && mcpPromptSnapshotMessages.length > 0) { for (const snapshotMessage of mcpPromptSnapshotMessages) { const appendResult = await this.historyService.appendToHistory( this.workspaceId, @@ -4478,7 +4495,7 @@ export class AgentSession { "sendMessage: preTurnMessages must be synthetic assistant rows" ); } - if (tokenBudgetActive) { + if (batchTurnSnapshots && !autoCompactionMessage) { const requestPrelude = [ ...(snapshotResult?.snapshotMessage ? [snapshotResult.snapshotMessage] : []), ...skillSnapshotMessages, @@ -4489,17 +4506,19 @@ export class AgentSession { // before clearing context state or publishing a reset. Reuse these rows below: // skill directives and MCP prompt expansion must not execute a second time. if (requestPrelude.length > 0) { - const freshBudget = await this.checkFreshContextBudget( - userMessage, - optionsForStream.model, - optionsForStream, - [...contextBudgetPrefix, ...requestPrelude] - ); - if (await cancelBeforeAcceptance()) return Ok(undefined); - if (isAdmissionStale() || this.coordinator.admissionBlocked || this.coordinator.closing) { - return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + if (tokenBudgetActive) { + const freshBudget = await this.checkFreshContextBudget( + userMessage, + optionsForStream.model, + optionsForStream, + [...contextBudgetPrefix, ...requestPrelude] + ); + if (await cancelBeforeAcceptance()) return Ok(undefined); + if (isAdmissionStale() || this.coordinator.admissionBlocked || this.coordinator.closing) { + return Err(createUnknownSendMessageError(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE)); + } + if (!freshBudget.success) return await rejectBudgetSend(freshBudget.error); } - if (!freshBudget.success) return await rejectBudgetSend(freshBudget.error); userMessage.metadata = { ...userMessage.metadata, requestPreludeMessageIds: requestPrelude.map((row) => row.id), @@ -4559,11 +4578,11 @@ export class AgentSession { compactionAppendCondition ); if (!appended.success) { - if (compactionAppendCondition) throw new Error(appended.error); + if (internal?.compactionHandoff) throw new Error(appended.error); return Err(createUnknownSendMessageError(appended.error)); } } catch (error) { - if (compactionAppendCondition) throw error; + if (internal?.compactionHandoff) throw error; return Err(createUnknownSendMessageError(getErrorMessage(error))); } if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); @@ -4583,14 +4602,16 @@ export class AgentSession { } if (await cancelBeforeAcceptance()) return Ok(undefined); } else if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { - const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ - ...internal.preTurnMessages, - userMessage, - ]); + const batchAppendResult = await this.historyService.appendManyToHistory( + this.workspaceId, + [...internal.preTurnMessages, userMessage], + compactionAppendCondition + ); if (!batchAppendResult.success) { await rollbackPersistedTurnRows(); return Err(createUnknownSendMessageError(batchAppendResult.error)); } + if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); persistedCancelableMessageIds.push( ...internal.preTurnMessages.map((message) => message.id), userMessage.id @@ -4611,7 +4632,7 @@ export class AgentSession { await rollbackPersistedTurnRows(); // Rollback can retire the local token; that must not disguise a real // locked-read/write failure as an ordinary stale-source skip. - if (compactionAppendCondition) throw new Error(appendResult.error); + if (internal?.compactionHandoff) throw new Error(appendResult.error); return Err(createUnknownSendMessageError(appendResult.error)); } if (compactionAppendSkipped) return refuseSkippedCompactionAppend(); @@ -9841,7 +9862,7 @@ export class AgentSession { return false; } - const cancellation = await this.compactionCancellation.read(); + const cancellation = await this.reconcileCompactionCancellation(); if (!this.coordinator.isCurrentCompaction(token)) { if (this.coordinator.canClearCompactionFollowUp(token)) await this.clearPendingFollowUpFromSummary(lastMessage, token); @@ -9851,7 +9872,7 @@ export class AgentSession { // Its now-absent fence cannot authorize the stale request we read before repair. if (repairRevision !== this.compactionCancellation.repairRevision) return false; if (cancellation && this.compactionCancellation.matches(cancellation, lastMessage)) { - await this.clearPendingFollowUpFromSummary(lastMessage, token); + await this.clearPendingFollowUpFromSummary(lastMessage, token, cancellation.nonce); return false; } @@ -10221,7 +10242,8 @@ export class AgentSession { private async clearPendingFollowUpFromSummary( summaryMessage: MuxMessage, - token: CompactionToken + token: CompactionToken, + cancellationNonce?: string ): Promise { assert( summaryMessage.role === "assistant", @@ -10239,10 +10261,12 @@ export class AgentSession { } if (!this.coordinator.canClearCompactionFollowUp(token)) return; - const cancellation = await this.compactionCancellation.read(); + const cancellation = await this.reconcileCompactionCancellation(); if (!this.coordinator.canClearCompactionFollowUp(token)) return; const canceled = cancellation != null && this.compactionCancellation.matches(cancellation, summaryMessage); + if (cancellationNonce !== undefined && (!canceled || cancellation?.nonce !== cancellationNonce)) + return; let matched = false; let committed = false; const updateResult = await this.historyService.updateHistory( @@ -10273,7 +10297,8 @@ export class AgentSession { }, () => { committed = true; - } + }, + cancellationNonce ); if (!updateResult.success) { // Only a target verified under the write lock may narrow an unresolved Stop. diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index e668f69151c..5d4d96f0b72 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -89,6 +89,8 @@ import { const HISTORY_WRITE_LOCK_TIMEOUT_MS = 10_000; interface CompactionFollowUpAppendCondition { + /** Presence selects manual replacement admission; null captures semantic absence. */ + replacementNonce?: string | null; summary?: MuxMessage; allowedTailMessageIds: readonly string[]; isCurrent: () => boolean; @@ -2876,6 +2878,14 @@ export class HistoryService { workspaceId: string, condition: CompactionFollowUpAppendCondition ): Promise { + if (condition.replacementNonce !== undefined) { + return ( + (await this.matchesReplacementCancellationUnlocked( + workspaceId, + condition.replacementNonce + )) && condition.isCurrent() + ); + } // Another backend can consume/repair this summary while send preparation // awaits. Validate its exact intent under the same lock as the new row. const rows = await this.readChatHistory(workspaceId); @@ -3078,17 +3088,47 @@ export class HistoryService { message: MuxMessage, shouldUpdate?: (current: MuxMessage) => boolean, updateFromCurrent?: (current: MuxMessage) => MuxMessage, - onCommitted?: () => void + onCommitted?: () => void, + cancellationNonce?: string ): Promise> { assert(!onCommitted || shouldUpdate, "Update commit observers require a conditional mutation"); - return this.withRecoveredHistoryWriteResultLock(workspaceId, "Failed to update history", () => - this.updateHistoryUnderWriteLock( - workspaceId, - message, - shouldUpdate, - updateFromCurrent, - onCommitted - ) + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to update history", + async () => { + // A live cancellation read can age across another backend's accepted replacement. + // Only the exact still-unwitnessed Stop may erase its captured pending summary. + if (cancellationNonce !== undefined) { + const cancellation = await this.readCompactionCancellation(workspaceId); + if ( + cancellation?.nonce !== cancellationNonce || + !matchesCompactionCancellation(cancellation, message) || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, cancellationNonce)) + ) + return Ok(undefined); + } + return this.updateHistoryUnderWriteLock( + workspaceId, + message, + shouldUpdate, + updateFromCurrent, + onCommitted + ); + } + ); + } + + private async matchesReplacementCancellationUnlocked( + workspaceId: string, + expectedNonce: string | null + ): Promise { + const current = await this.readCompactionCancellation(workspaceId); + // Semantic absence can retain a physical file when witnessed unlink failed. + return ( + (current?.nonce ?? null) === expectedNonce || + (expectedNonce === null && + current != null && + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce))) ); } @@ -3105,16 +3145,7 @@ export class HistoryService { async () => { // Retry must not stamp an obsolete receipt over a foreign Stop that arrived // during preparation. Compare shared identity and commit under one lock. - const current = await this.readCompactionCancellation(workspaceId); - // Semantic absence can retain a physical file when witnessed unlink failed. - if ( - (current?.nonce ?? null) !== expectedNonce && - !( - expectedNonce === null && - current && - (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce)) - ) - ) + if (!(await this.matchesReplacementCancellationUnlocked(workspaceId, expectedNonce))) return Ok(undefined); return this.updateHistoryUnderWriteLock( workspaceId, From ba66021f32ec7518e5ec87c51d0a6302dbe5ae51 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 09:44:35 +0200 Subject: [PATCH 15/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20Stop=20a?= =?UTF-8?q?cross=20retry=20and=20foreign=20compaction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fence automatic recovery and late compaction publication with durable Stop ownership. Recover failed Stop writes before manual replacement while preserving newer publications and journals. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I98df522ce2552318bdd2b51b1cd50965b22640f3 --- .../agentSession.compactionShutdown.test.ts | 511 +++++++++++++++++- src/node/services/agentSession.ts | 50 ++ .../services/compactionCancellation.test.ts | 154 +++++- src/node/services/compactionCancellation.ts | 87 ++- src/node/services/compactionHandler.ts | 144 +++-- .../services/continuousCompactionJournal.ts | 12 +- src/node/services/historyService.ts | 120 +++- src/node/services/workspaceService.test.ts | 154 +++++- src/node/services/workspaceService.ts | 14 +- 9 files changed, 1172 insertions(+), 74 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index ff92d2d1e4a..259e64913bf 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -1,3 +1,5 @@ +import { makeTestEffectRunner } from "./di/testEffectRunner"; +import { calculateBackoffDelay } from "@/common/utils/messages/retryState"; import * as fs from "node:fs/promises"; import { readFile, readdir, writeFile } from "node:fs/promises"; import { COMPACTION_CANCELLATION_FILE } from "@/common/constants/compactionCancellation"; @@ -15,7 +17,7 @@ import { log } from "./log"; import { ExtensionMetadataService } from "./ExtensionMetadataService"; import { WorkspaceGoalService } from "./workspaceGoalService"; import { createTestHistoryService } from "./testHistoryService"; -import { createAgentSessionHarness } from "./agentSession.testHarness"; +import { createAgentSessionHarness, createStreamLifecycleMocks } from "./agentSession.testHarness"; const workspaceId = "compaction-shutdown"; const options = { model: "openai:gpt-4o", agentId: "exec" }; @@ -858,8 +860,9 @@ test("a delayed cancellation write survives failed manual preparation", async () spyOn(h.historyService, "appendToHistory").mockResolvedValueOnce( Err("replacement append failed") ); - expect((await h.session.sendMessage("replacement", options)).success).toBe(false); + const replacement = h.session.sendMessage("replacement", options); release.resolve(); + expect((await replacement).success).toBe(false); await stopping; await h.session.dispose(); const freshHistory = new HistoryService(h.config); @@ -1518,7 +1521,11 @@ test("a second Stop after the resume witness commits cannot be retired by the fi try { await entered.promise; const stopping = h.session.interruptStream({ abandonPartial: true }); - const secondNonce = await h.session.getCompactionCancellationNonce(); + const secondNonce = ( + await ( + h.session as unknown as { compactionCancellation: CompactionCancellation } + ).compactionCancellation.read() + )?.nonce; expect(secondNonce).not.toBe(firstNonce); release.resolve(); await stopping; @@ -1559,7 +1566,7 @@ test.each(["history failure", "witness no-op", "automatic"] as const)( expect( (await new HistoryService(h.config).readCompactionCancellation(workspaceId))?.nonce ).toBe(nonce); - expect(stream).toHaveBeenCalledTimes(outcome === "automatic" ? 1 : 0); + expect(stream).not.toHaveBeenCalled(); } finally { await h.session.dispose(); await h.cleanup(); @@ -2520,6 +2527,7 @@ test("interrupted handoff snapshot persistence leaves a fresh service able to re test("Stop survives failed rollback of a committed handoff snapshot batch across restart", async () => { const h = await setup(); + const clock = makeTestEffectRunner(); await h.historyService.appendToHistory(workspaceId, summary()); const snapshot = createMuxMessage("retained-snapshot", "user", "Expanded context", { synthetic: true, @@ -2555,10 +2563,19 @@ test("Stop survives failed rollback of a committed handoff snapshot batch across workspaceId, config: h.config, historyService: foreign, + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, }); try { const recovered = spyOn(fresh.aiService, "streamMessage"); + const retries = fresh.session as unknown as { retryActiveStream(): Promise }; + const deliver = spyOn(retries, "retryActiveStream"); await fresh.session.runStartupRecovery(); + await clock.adjust(calculateBackoffDelay(6) * 2); + // Deliver any armed retry and join its actual callback before checking provider entry. + await Promise.all(deliver.mock.results.map((result) => result.value)); + expect(deliver).not.toHaveBeenCalled(); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); expect(recovered).not.toHaveBeenCalled(); expect(await foreign.readCompactionCancellation(workspaceId)).not.toBeNull(); } finally { @@ -2567,6 +2584,7 @@ test("Stop survives failed rollback of a committed handoff snapshot batch across } finally { await h.session.dispose(); await h.cleanup(); + await clock.dispose(); } }); @@ -2736,3 +2754,488 @@ test.each(["absence", "witnessed debt", "Stop after commit"] as const)( } } ); + +test.each([false, true])( + "manual replacement retries a failed Stop publication (storage still fails=%s)", + async (persistent) => { + const h = await setup(); + const write = spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("Stop disk unavailable") + ); + const stream = spyOn(h.aiService, "streamMessage"); + try { + expect((await h.session.interruptStream({ abandonPartial: true })).success).toBe(false); + if (persistent) write.mockRejectedValue(new Error("Stop disk unavailable")); + const result = await h.session + .sendMessage("Fresh explicit work", options) + .catch((error: unknown) => error); + if (persistent) { + expect(result).toHaveProperty("message", "Stop disk unavailable"); + expect(stream).not.toHaveBeenCalled(); + expect(h.session.hasBlockingCompactionCleanup).toBe(true); + } else { + expect(result).toEqual(Ok(undefined)); + expect(stream).toHaveBeenCalledTimes(1); + const rows = await new HistoryService(h.config).getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { type: "text", text: "Fresh explicit work" }, + ]); + } + } finally { + write.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each( + ["legacy", "failed apply", "failed apply compact"].flatMap((kind) => + [false, true].map((unlinkDebt) => ({ kind, unlinkDebt })) + ) +)( + "captured source-less $kind cannot follow a foreign replacement (unlink debt=$unlinkDebt)", + async ({ kind, unlinkDebt }) => { + const h = await setup(); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("original", "user", "Old work") + ); + const context = { + modelString: options.model, + options, + providersConfig: null, + compactionPublication: { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }, + }; + const direct = h.session as unknown as { + activeStreamContext: typeof context; + interruptForCompaction(): Promise; + finishContinuousCompaction( + applied: boolean, + streamContext: typeof context, + token: NonNullable> + ): Promise; + compactionMonitor: CompactionMonitor; + }; + direct.activeStreamContext = context; + if (kind === "failed apply compact") + spyOn(direct.compactionMonitor, "checkBeforeSend").mockReturnValue({ + shouldShowWarning: true, + shouldForceCompact: true, + usagePercentage: 99, + thresholdPercentage: 85, + contextTokens: 99_000, + maxTokens: 100_000, + }); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const pricing = h.goalService.assertPricedModelForBudgetedGoal.bind(h.goalService); + spyOn(h.goalService, "assertPricedModelForBudgetedGoal").mockImplementationOnce( + async (...args) => { + const result = await pricing(...args); + entered.resolve(); + await release.promise; + return result; + } + ); + const token = + kind === "legacy" + ? undefined + : h.internals.coordinator.beginCompactionObservation("continuous"); + if (token) h.internals.coordinator.setCompactionStage(token, "stopped"); + const pending = + kind === "legacy" + ? direct.interruptForCompaction() + : direct.finishContinuousCompaction(false, context, token!); + const stream = spyOn(h.aiService, "streamMessage"); + const foreignHistory = new HistoryService(h.config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreignHistory, + }); + let restoreWrite: (() => void) | undefined; + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + if (unlinkDebt) { + const write = foreignHistory.writeCompactionCancellation.bind(foreignHistory); + const spy = spyOn(foreignHistory, "writeCompactionCancellation").mockImplementation( + (...args) => + args[1] === null ? Promise.reject(new Error("unlink unavailable")) : write(...args) + ); + restoreWrite = () => spy.mockRestore(); + } + expect((await foreign.session.sendMessage("New accepted work", options)).success).toBe(true); + release.resolve(); + await pending; + expect(stream).not.toHaveBeenCalled(); + const rows = await foreignHistory.getLastMessages(workspaceId, 10); + expect( + rows.success && rows.data.filter((row) => row.role === "user").map((row) => row.parts) + ).toMatchObject([[{ text: "Old work" }], [{ text: "New accepted work" }]]); + if (token) h.internals.coordinator.finishCompactionObservation(token); + direct.activeStreamContext = { + ...context, + compactionPublication: { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }, + }; + await direct.interruptForCompaction(); + expect(stream).toHaveBeenCalledTimes(1); + } finally { + release.resolve(); + await pending.catch(() => undefined); + restoreWrite?.(); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["unresolved", "absent", "witnessed", "Stop during backoff", "Stop during preparation"])( + "startup retry actually delivers retained user rows only without unresolved Stop (%s)", + async (state) => { + const h = await setup(); + const clock = makeTestEffectRunner(); + const foreign = new HistoryService(h.config); + const cancellation = new CompactionCancellation(foreign, workspaceId); + if (state === "unresolved" || state === "witnessed") await cancellation.cancel(); + const record = await cancellation.read(); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("retained-trigger", "user", "Retained explicit work", { + synthetic: true, + uiVisible: true, + retrySendOptions: options, + ...(state === "witnessed" ? { compactionCancellationNonce: record?.nonce } : {}), + }) + ); + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: foreign, + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, + }); + const deliver = spyOn( + fresh.session as unknown as { retryActiveStream(): Promise }, + "retryActiveStream" + ); + const stream = spyOn(fresh.aiService, "streamMessage"); + try { + await fresh.session.runStartupRecovery(); + if (state === "Stop during backoff") await cancellation.cancel(); + if (state === "Stop during preparation") { + const read = foreign.getHistoryFromLatestBoundary.bind(foreign); + spyOn(foreign, "getHistoryFromLatestBoundary").mockImplementationOnce(async (...args) => { + const result = await read(...args); + await cancellation.cancel(); + return result; + }); + } + await clock.adjust(calculateBackoffDelay(6) * 2); + await Promise.all(deliver.mock.results.map((result) => result.value)); + expect(deliver).toHaveBeenCalledTimes(state === "unresolved" ? 0 : 1); + expect(stream).toHaveBeenCalledTimes(state === "absent" || state === "witnessed" ? 1 : 0); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe( + state !== "unresolved" + ); + } finally { + await fresh.session.dispose(); + await h.session.dispose(); + await clock.dispose(); + await h.cleanup(); + } + } +); + +test.each(["before capture", "after capture"])( + "a real stream pins handoff publication when foreign Stop wins %s", + async (stage) => { + const h = await setup(); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + if (stage === "before capture") { + entered.resolve(); + await release.promise; + } + const generation = await capture(); + if (stage === "after capture") { + entered.resolve(); + await release.promise; + } + return generation; + }); + const completed = Promise.withResolvers(); + const stream = spyOn(h.aiService, "streamMessage").mockResolvedValueOnce( + Ok({ messageId: "original-live-stream", completion: completed.promise }) + ); + spyOn(h.aiService, "stopStream").mockImplementation(() => { + completed.resolve({ status: "aborted", abortReason: "system" }); + return Promise.resolve(Ok(undefined)); + }); + const sending = h.session.sendMessage("Original work", options); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("Replacement work", options)).success).toBe(true); + release.resolve(); + expect((await sending).success).toBe(true); + const direct = h.session as unknown as { interruptForCompaction(): Promise }; + await direct.interruptForCompaction(); + // Capturing after Stop precedes this stream's first context read/provider call; + // capturing before it keeps the older epoch even through delayed provider entry. + expect(stream).toHaveBeenCalledTimes(stage === "before capture" ? 2 : 1); + } finally { + release.resolve(); + completed.resolve({ status: "aborted", abortReason: "system" }); + await sending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["replacement", "dispose"])( + "held stream generation capture cannot mutate its %s successor", + async (action) => { + const h = await setup(); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + const generation = await capture(); + entered.resolve(); + await release.promise; + return generation; + }); + const pending = h.session.sendMessage("Old held preparation", options); + let closing: Promise | undefined; + try { + await entered.promise; + if (action === "replacement") { + await h.session.interruptStream({ abandonPartial: true }); + // This fixture has no engine startup-abort event; finish its logical turn + // while the old physical capture remains held, then admit the successor. + h.internals.coordinator.finishTurn(h.internals.coordinator.turnId); + expect(await h.session.sendMessage("New successor", options)).toEqual(Ok(undefined)); + const rows = await h.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([{ text: "New successor" }]); + } else closing = h.session.dispose(); + const state = h.session as unknown as { + activeStreamContext: unknown; + activeStreamUserMessageId: string | undefined; + }; + const context = state.activeStreamContext; + const userId = state.activeStreamUserMessageId; + await h.historyService.writePartial( + workspaceId, + createMuxMessage("successor-partial", "assistant", "Successor owned partial") + ); + const commit = spyOn(h.historyService, "commitPartial"); + release.resolve(); + await pending; + await closing; + expect(commit).not.toHaveBeenCalled(); + expect(state.activeStreamContext).toBe(context); + expect(state.activeStreamUserMessageId).toBe(userId); + expect((await h.historyService.readPartial(workspaceId))?.id).toBe("successor-partial"); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await closing; + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each(["heartbeat", "legacy"])( + "invalidated %s publication preserves a successor partial and pending context", + async (producer) => { + const h = await setup(); + const followUp = { text: "Continue old work", ...options }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const capture = journal.captureGeneration.bind(journal); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + spyOn(journal, "captureGeneration").mockImplementationOnce(async () => { + const generation = await capture(); + entered.resolve(); + await release.promise; + return generation; + }); + const pending = + producer === "heartbeat" + ? h.internals.compactionHandler.appendHeartbeatContextResetBoundary({ + boundaryText: "Old context", + pendingFollowUp: followUp, + }) + : h.internals.compactionHandler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Old context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("B replacement", options)).success).toBe(true); + await foreign.historyService.writePartial( + workspaceId, + createMuxMessage("b-partial", "assistant", "B live partial") + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + await writeFile(pendingPath, "foreign B pending state"); + release.resolve(); + await pending; + expect((await foreign.historyService.readPartial(workspaceId))?.id).toBe("b-partial"); + expect(await readFile(pendingPath, "utf8")).toBe("foreign B pending state"); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); + +test.each( + ["heartbeat", "legacy"].flatMap((producer) => + ["prepare write", "failed cleanup", "invalid read cleanup"].map((stage) => ({ + producer, + stage, + })) + ) +)( + "retired $producer $stage cannot change successor pending-state bytes", + async ({ producer, stage }) => { + const h = await setup(); + const followUp = { text: "Continue old work", ...options }; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + if (producer === "legacy") + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-stream", "assistant", "Streamed old summary") + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + if (stage === "invalid read cleanup") await writeFile(pendingPath, "malformed old state"); + const handler = h.internals.compactionHandler; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (stage !== "failed cleanup") { + const writes = handler as unknown as { + enqueuePendingStateWrite(write: () => Promise): Promise; + }; + const enqueue = writes.enqueuePendingStateWrite.bind(writes); + spyOn(writes, "enqueuePendingStateWrite").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return enqueue(...args); + }); + } else if (producer === "heartbeat") { + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async () => { + entered.resolve(); + await release.promise; + return Err("held heartbeat append failure"); + }); + } else { + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + } + const pending = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Old context", + pendingFollowUp: followUp, + }) + : handler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Old context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const successor = JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/foreign/successor.ts"], + }); + try { + await entered.promise; + await foreign.session.interruptStream({ abandonPartial: true }); + expect((await foreign.session.sendMessage("B replacement", options)).success).toBe(true); + await writeFile(pendingPath, successor); + release.resolve(); + await pending; + expect(await readFile(pendingPath, "utf8").catch(() => "missing")).toBe(successor); + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } + } +); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 24a482e8bb2..84e6c8c8ec2 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1,3 +1,4 @@ +import type { ContinuousCompactionPublication } from "./continuousCompactionJournal"; import { AsyncLocalStorage } from "node:async_hooks"; import type { GoalRecordV1 } from "@/common/types/goal"; import type { PreparedStreamMessage } from "./turnRequestBuilder"; @@ -765,6 +766,7 @@ interface CompactionFollowUpDispatch { interface SendMessageInternalOptions { compactionHandoff?: CompactionToken; + compactionHandoffPublication?: ContinuousCompactionPublication; /** Durable-summary handoffs revalidate their source inside the history append lock. */ compactionHandoffSource?: { summary: MuxMessage; onSkipped: () => void }; preparation?: PreparationAttempt; @@ -867,6 +869,7 @@ interface PreparationAttempt { failure?: SendMessageError; onFailure?: (error: SendMessageError) => Promise | void; resumeCancellation?: { nonce: string | null; epoch: number }; + automaticResume?: boolean; } export class AgentSession { @@ -1126,6 +1129,7 @@ export class AgentSession { /** Context needed to retry the current stream (cleared on stream end/abort/error). */ private activeStreamContext?: { + compactionPublication?: ContinuousCompactionPublication; modelString: string; contextBudgetRetried?: boolean; requestAssemblySnapshot?: RequestAssemblySnapshot; @@ -1210,6 +1214,7 @@ export class AgentSession { this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, + getCompactionPublication: () => this.activeStreamContext?.compactionPublication, captureCompletionGuard: () => { const epoch = this.coordinator.compactionIntent.epoch; return () => !this.coordinator.closing && this.coordinator.compactionIntent.epoch === epoch; @@ -1761,6 +1766,14 @@ export class AgentSession { ): Promise { if (this.coordinator.closing || !isCurrent()) return; using _execution = this.coordinator.enterExecution(); + // A foreign Stop can arrive while backoff sleeps. Automatic delivery never + // replaces that intent; only an accepted explicit row can witness replacement. + if ( + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || + !isCurrent() || + this.coordinator.closing + ) + return; const request = this.lastAutoRetryResumeRequest; if (!request) { this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "missing_retry_options" }); @@ -1786,6 +1799,11 @@ export class AgentSession { if (this.coordinator.closing || !isCurrent()) return; if (result.success) { if (!result.data.started) { + if ( + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || + !isCurrent() + ) + return; // resumeStream can defer when a turn is still PREPARING/COMPLETING. // Treat this as retriable so auto-retry keeps progressing instead of // stalling after the "auto-retry-starting" status event. @@ -2870,6 +2888,10 @@ export class AgentSession { }); return "completed"; } + // A retained handoff row may look interrupted after failed rollback. The + // unresolved Stop still owns it even when no live session remembers the abort. + if ((await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" || !isCurrent()) + return "completed"; await this.handleStreamFailureForAutoRetry( { type: "unknown", @@ -4107,6 +4129,7 @@ export class AgentSession { ? { replacementNonce: compactionCancellationNonce ?? null } : {}), summary: internal?.compactionHandoffSource?.summary, + publication: internal?.compactionHandoffPublication, allowedTailMessageIds: persistedCancelableMessageIds, isCurrent: () => !isAdmissionStale() && !this.coordinator.closing && cancelSignal?.aborted !== true, @@ -5070,6 +5093,7 @@ export class AgentSession { } const attempt: PreparationAttempt = { + automaticResume: internal?.automatic === true, expectedTurn: expectedTurnId, outcome: "preparing", durability: "accepted", @@ -6712,6 +6736,7 @@ export class AgentSession { { synthetic: true, compactionHandoff: token, + compactionHandoffPublication: context.compactionPublication ?? { generation: undefined }, agentInitiated: fallback?.agentInitiated ?? context.agentInitiated, goalKind: fallback ? undefined : context.goalKind, goalId: fallback ? undefined : context.goalId, @@ -6805,6 +6830,9 @@ export class AgentSession { synthetic: true, agentInitiated: autoCompactionRequest.agentInitiated, compactionHandoff: token, + compactionHandoffPublication: streamContext.compactionPublication ?? { + generation: undefined, + }, admissionStale: () => !this.coordinator.isCurrentCompaction(token), } ); @@ -7049,6 +7077,15 @@ export class AgentSession { const operation = this.coordinator.registerOperation(turn); let completionTransferred = false; try { + // Capture before the original provider can produce old-context work. + // Publish no shared startup state if this await outlives its operation. + const compactionPublication = { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; + if (isStreamStartAborted() || !this.coordinator.isCurrentOperation(operation)) + return Ok(undefined); // Reset per-stream flags (used for retries / crash-safe bookkeeping). this.compactionMonitor.resetForNewStream(); this.clearLiveUsageState(); @@ -7057,6 +7094,7 @@ export class AgentSession { this.activeStreamHadPostCompactionInjection = false; const providersConfig = this.getProvidersConfigSafe(); this.activeStreamContext = { + compactionPublication, modelString, contextBudgetRetried, requestAssemblySnapshot, @@ -7258,6 +7296,12 @@ export class AgentSession { // collect them so the Err path resolves each exactly once. const preStartErrors: StreamErrorPayload[] = []; this.coordinator.configureOperation(operation, this.activeCompactionRequest != null); + if ( + preparation?.automaticResume && + (await this.reconcileCompactionCancellation())?.scope.kind === "unresolved" + ) + return Ok(undefined); + if (isStreamStartAborted()) return Ok(undefined); const resumedCancellation = preparation?.resumeCancellation; if (resumedCancellation) { const resumedUser = @@ -9018,6 +9062,12 @@ export class AgentSession { return this.coordinator.reserve("admission"); } + async fenceCompactionForContextClear(): Promise { + // Empty history is no replacement witness: foreign legacy/heartbeat writers + // can still publish captured summaries after the clear has finished. + await this.compactionCancellation.cancel({ retainUntilReplacement: true }); + } + async contextMutationCommitted(cancellationNonce?: string | null): Promise { this.coordinator.invalidateCompaction(false); this.continuousCompactor.reset("context-mutation"); diff --git a/src/node/services/compactionCancellation.test.ts b/src/node/services/compactionCancellation.test.ts index 0bb877b1596..04b788c5cda 100644 --- a/src/node/services/compactionCancellation.test.ts +++ b/src/node/services/compactionCancellation.test.ts @@ -330,7 +330,7 @@ test("a local Stop during corrupt repair prevents its obsolete history commit", expect(await pending).toEqual(newer); await stop; const freshHistory = new HistoryService(h.config); - expect(await freshHistory.readCompactionCancellation(workspaceId)).toEqual(newer); + expect(await freshHistory.readCompactionCancellation(workspaceId)).toMatchObject(newer!); const rows = await freshHistory.getLastMessages(workspaceId, 1); expect(rows.success && rows.data[0].metadata?.muxMetadata).toHaveProperty("pendingFollowUp"); } finally { @@ -636,3 +636,155 @@ test("cancellation repair preserves raw reset privacy and invalidates an earlier await h.cleanup(); } }); + +test.each([false, true])( + "concurrent replacement readers share the latest Stop publication (initial failure=%s)", + async (failed) => { + const h = await createTestHistoryService(); + const workspaceId = "shared-replacement-read"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const writes = spyOn(h.historyService, "writeCompactionCancellation"); + if (failed) writes.mockRejectedValueOnce(new Error("initial publication failed")); + writes.mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return write(...args); + }); + const stopped = state.cancel().catch(() => undefined); + if (failed) await stopped; + const a = state.readForReplacement(); + const b = state.readForReplacement(); + try { + await entered.promise; + release.resolve(); + const [first, second] = await Promise.all([a, b]); + expect(first).toEqual(second); + expect(first).not.toBeNull(); + expect(writes).toHaveBeenCalledTimes(failed ? 2 : 1); + expect(await new HistoryService(h.config).readCompactionCancellation(workspaceId)).toEqual( + first + ); + } finally { + release.resolve(); + await Promise.all([stopped, a.catch(() => undefined), b.catch(() => undefined)]); + await h.cleanup(); + } + } +); + +test.each([ + "before lock", + "witnessed before lock", + "before advance", + "after advance", + "foreign Stop", + "foreign repair", + "witnessed acknowledgement failure", +])("failed Stop retry preserves monotonic publication ownership (%s)", async (failure) => { + const h = await createTestHistoryService(); + const workspaceId = "failed-publication-generation"; + const state = new CompactionCancellation(h.historyService, workspaceId); + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const initial = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + if (failure === "before lock" || failure === "witnessed before lock") + spyOn(h.historyService, "writeCompactionCancellation").mockRejectedValueOnce( + new Error("publication unavailable before lock") + ); + else if (failure === "before advance") + spyOn(journal, "invalidateUnderHistoryLock").mockRejectedValueOnce( + new Error("generation unavailable") + ); + else if (failure !== "witnessed acknowledgement failure") + spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce(async (...args) => { + await invalidate(...args); + throw new Error("sidecar unavailable after advancement"); + }); + else { + const write = h.historyService.writeCompactionCancellation.bind(h.historyService); + spyOn(h.historyService, "writeCompactionCancellation").mockImplementationOnce( + async (...args) => { + await write(...args); + throw new Error("acknowledgement unavailable"); + } + ); + } + try { + expect( + await state.cancel().then( + () => false, + () => true + ) + ).toBe(true); + const attempted = await state.read(); + const advanced = await journal.captureGeneration(); + expect(advanced === initial).toBe( + failure === "before advance" || + failure === "before lock" || + failure === "witnessed before lock" + ); + const foreign = new HistoryService(h.config); + let successor = await foreign.readCompactionCancellation(workspaceId); + if ( + failure === "foreign Stop" || + failure === "before lock" || + failure === "witnessed before lock" + ) { + await new CompactionCancellation(foreign, workspaceId).cancel(); + successor = await foreign.readCompactionCancellation(workspaceId); + if (failure === "witnessed before lock") { + const replacement = new CompactionCancellation(foreign, workspaceId); + const stopped = await replacement.read(); + if (!stopped) throw new Error("Expected foreign Stop"); + await foreign.appendToHistory( + workspaceId, + createMuxMessage("b-accepted", "user", "B accepted replacement", { + compactionCancellationNonce: stopped.nonce, + }) + ); + await replacement.retireReplacement(stopped.nonce); + successor = null; + } + } else if (failure === "foreign repair") { + await writeFile( + `${h.config.sessionsDir}/${workspaceId}/${COMPACTION_CANCELLATION_FILE}`, + "{" + ); + await foreign.repairCompactionCancellation( + workspaceId, + () => true, + () => undefined + ); + successor = null; + } else if (failure === "witnessed acknowledgement failure") { + await foreign.appendToHistory( + workspaceId, + createMuxMessage("accepted", "user", "New accepted work", { + compactionCancellationNonce: attempted?.nonce, + }) + ); + } + const epochBeforeRetry = await journal.captureGeneration(); + const preserveForeign = + failure.startsWith("foreign") || + failure === "witnessed acknowledgement failure" || + failure === "before lock" || + failure === "witnessed before lock"; + if (preserveForeign) await writeFile(journal.path, "newer journal must survive stale cleanup"); + const result = await state.readForReplacement(); + if (preserveForeign) { + expect(result).toEqual(successor); + expect(await journal.captureGeneration()).toBe(epochBeforeRetry); + expect(await readFile(journal.path, "utf8")).toBe("newer journal must survive stale cleanup"); + } else { + expect(result?.nonce).toBe(attempted?.nonce); + expect(await journal.captureGeneration()).not.toBe(advanced); + } + expect(state.needsPersistence).toBe(false); + } finally { + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionCancellation.ts b/src/node/services/compactionCancellation.ts index 87308f3169f..01725b35150 100644 --- a/src/node/services/compactionCancellation.ts +++ b/src/node/services/compactionCancellation.ts @@ -7,6 +7,7 @@ import type { HistoryService } from "./historyService"; export const CompactionCancellationSchema = z.object({ version: z.literal(1), nonce: z.string().min(1), + retainUntilReplacement: z.boolean().optional(), scope: z.discriminatedUnion("kind", [ z.object({ kind: z.literal("unresolved") }), z.object({ @@ -19,6 +20,12 @@ export const CompactionCancellationSchema = z.object({ }); export type CompactionCancellationRecord = z.infer; +/** Process-local retry ownership; durable nonce + epoch prevent adopting foreign mutations. */ +export interface CompactionCancellationPublication { + attempts: number; + predecessor?: { nonce: string | null | undefined; generation: string | undefined }; +} + /** Only successfully read bytes with invalid JSON/schema may enter automatic repair. */ export class MalformedCompactionCancellationError extends Error { constructor(readonly contents: Uint8Array) { @@ -34,7 +41,11 @@ export class CompactionCancellation { private replacementNonce?: string; private pending: Promise = Promise.resolve(); private unsettled = false; - private mutation?: { record: CompactionCancellationRecord | null; retiredNonce?: string }; + private mutation?: { + record: CompactionCancellationRecord | null; + retiredNonce?: string; + publication?: CompactionCancellationPublication; + }; constructor( private readonly history: HistoryService, @@ -85,29 +96,64 @@ export class CompactionCancellation { return this.effectiveRecord(); } - cancel(): Promise { + cancel(options?: { retainUntilReplacement?: boolean }): Promise { // Each explicit Stop is new intent, even during a previous retirement's // post-commit await. Only retry() may reuse publication identity. - this.current = { version: 1, nonce: randomUUID(), scope: { kind: "unresolved" } }; + this.current = { + version: 1, + nonce: randomUUID(), + scope: { kind: "unresolved" }, + ...(options?.retainUntilReplacement || this.current?.retainUntilReplacement + ? { retainUntilReplacement: true } + : {}), + }; this.generation++; - return this.persist(this.current); + return this.persist(this.current, undefined, { attempts: 0 }); } async readForReplacement(): Promise { - try { - return await this.read(); - } catch { - // Explicit user intervention may repair corrupt state. First publish a - // conservative fence; failed writes still refuse the replacement safely. - await this.cancel(); - return this.read(); + for (;;) { + // An unpublished nonce cannot pass the shared append CAS. Repair only the + // latest blocking publication; witnessed unlink debt remains ancillary. + if (this.blocksRecovery) { + const pending = this.pending; + try { + await pending; + } catch { + // Join first, then retry only the exact failed publication. Concurrent + // readers share the retry instead of continually superseding one another. + if (pending !== this.pending) continue; + const retried = this.retry(); + try { + await retried; + } catch (error) { + if (retried === this.pending) throw error; + } + } + continue; + } + let record: CompactionCancellationRecord | null; + try { + record = await this.read(); + } catch { + // Explicit intervention can replace corrupt/unreadable state with a + // conservative fence, but failed publication must remain visible. + await this.cancel(); + continue; + } + if (!this.blocksRecovery) return record; } } async narrow(nonce: string, summary: MuxMessage): Promise { // Exact CAS cannot turn a failed initial publication into apparent success. await this.pending; - if (this.current?.nonce !== nonce || this.current.scope.kind !== "unresolved") return; + if ( + this.current?.nonce !== nonce || + this.current.scope.kind !== "unresolved" || + this.current.retainUntilReplacement + ) + return; const metadata = summary.metadata?.muxMetadata; if (!metadata || !("pendingFollowUp" in metadata) || !metadata.pendingFollowUp) return; this.current = { @@ -151,12 +197,16 @@ export class CompactionCancellation { retry(): Promise { return this.unsettled && this.mutation - ? this.persist(this.mutation.record, this.mutation.retiredNonce) + ? this.persist(this.mutation.record, this.mutation.retiredNonce, this.mutation.publication) : this.pending; } retire(nonce: string): Promise { - if (this.current?.nonce !== nonce) return Promise.resolve(); + if ( + this.current?.nonce !== nonce || + (this.current.retainUntilReplacement && this.replacementNonce !== nonce) + ) + return Promise.resolve(); this.generation++; // Keep conservative exclusion in memory until deletion really commits. The // retry payload remains a deletion, not a republication of that read state. @@ -165,20 +215,23 @@ export class CompactionCancellation { private persist( snapshot: CompactionCancellationRecord | null, - retiredNonce?: string + retiredNonce?: string, + publication?: CompactionCancellationPublication ): Promise { const generation = this.generation; - const mutation = { record: snapshot, retiredNonce }; + const mutation = { record: snapshot, retiredNonce, publication }; this.mutation = mutation; this.unsettled = true; const result = this.pending .catch(() => undefined) .then(async () => { + if (publication) publication.attempts++; await this.history.writeCompactionCancellation( this.workspaceId, snapshot, () => this.generation === generation, - retiredNonce + retiredNonce, + publication ); if (this.generation === generation && this.mutation === mutation) { this.unsettled = false; diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 16b297f12f0..cae5cbe4dce 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -386,6 +386,7 @@ interface CompactionHandlerOptions { onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; /** Capture semantic ownership before I/O; durable completion must not retarget a successor. */ captureCompletionGuard?: () => () => boolean; + getCompactionPublication?: () => ContinuousCompactionPublication | undefined; /** * Called with the terminal outcome of an idle compaction (source === "idle-compaction"), * after the summary is actually persisted (success) or a post-stream persistence failure @@ -416,6 +417,7 @@ export class CompactionHandler { private readonly onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; private readonly captureCompletionGuard?: () => () => boolean; + private readonly getCompactionPublication?: () => ContinuousCompactionPublication | undefined; private readonly onIdleCompactionOutcome?: (success: boolean) => void; /** Flag indicating post-compaction attachments should be generated on next turn */ @@ -426,8 +428,15 @@ export class CompactionHandler { private heartbeatResetRollbackState: HeartbeatResetRollbackState | null = null; private pendingStateWrites: Promise = Promise.resolve(); - private enqueuePendingStateWrite(write: () => Promise): Promise { - const result = this.pendingStateWrites.then(write); + private enqueuePendingStateWrite( + write: () => Promise, + publication?: ContinuousCompactionPublication + ): Promise { + const result = this.pendingStateWrites.then(() => + publication + ? this.historyService.withCompactionPublicationWrite(this.workspaceId, publication, write) + : write() + ); this.pendingStateWrites = result.catch(() => undefined); return result; } @@ -450,10 +459,13 @@ export class CompactionHandler { this.emitter = options.emitter; this.onCompactionComplete = options.onCompactionComplete; this.captureCompletionGuard = options.captureCompletionGuard; + this.getCompactionPublication = options.getCompactionPublication; this.onIdleCompactionOutcome = options.onIdleCompactionOutcome; } - private async loadPersistedPendingStateIfNeeded(): Promise { + private async loadPersistedPendingStateIfNeeded( + publication?: ContinuousCompactionPublication + ): Promise { if (this.persistedPendingStateLoaded || this.postCompactionAttachmentsPending) { return; } @@ -472,14 +484,14 @@ export class CompactionHandler { parsed = JSON.parse(raw); } catch { log.warn("Invalid post-compaction state JSON; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication); return; } let state = coercePersistedPostCompactionState(parsed); if (!state) { log.warn("Invalid post-compaction state schema; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication); return; } @@ -495,7 +507,7 @@ export class CompactionHandler { // nor lose an older, still-pending attachment snapshot. state = state.previousState ?? null; if (!state || (state.boundaryMessageId && state.boundaryMessageId !== boundaryId)) { - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication); return; } } @@ -612,15 +624,20 @@ export class CompactionHandler { } } - private async deletePersistedPendingStateBestEffort(): Promise { + private async deletePersistedPendingStateBestEffort( + publication?: ContinuousCompactionPublication + ): Promise { try { - await this.enqueuePendingStateWrite(() => fsPromises.unlink(this.postCompactionStatePath)); + await this.enqueuePendingStateWrite( + () => fsPromises.unlink(this.postCompactionStatePath), + publication + ); } catch { // ignore } } - private captureHeartbeatResetRollbackState(messages: MuxMessage[]): void { + private captureHeartbeatResetRollbackState(messages: MuxMessage[]): HeartbeatResetRollbackState { this.heartbeatResetRollbackState = { sourceRows: messages.map((row) => ({ id: row.id, sequence: row.metadata?.historySequence })), postCompactionAttachmentsPending: this.postCompactionAttachmentsPending, @@ -629,11 +646,14 @@ export class CompactionHandler { cachedReadFilePaths: [...this.cachedReadFilePaths], persistedPendingStateLoaded: this.persistedPendingStateLoaded, }; + return this.heartbeatResetRollbackState; } - private restoreHeartbeatResetRollbackState(): Promise { - const rollbackState = this.heartbeatResetRollbackState; - if (!rollbackState) { + private restoreHeartbeatResetRollbackState( + publication?: ContinuousCompactionPublication, + rollbackState = this.heartbeatResetRollbackState + ): Promise { + if (!rollbackState || rollbackState !== this.heartbeatResetRollbackState) { return Promise.resolve(); } @@ -651,10 +671,13 @@ export class CompactionHandler { return this.persistPendingStateBestEffort( this.cachedFileDiffs, this.cachedLoadedSkills, - this.cachedReadFilePaths + this.cachedReadFilePaths, + undefined, + undefined, + publication ); } else { - return this.deletePersistedPendingStateBestEffort(); + return this.deletePersistedPendingStateBestEffort(publication); } } @@ -663,7 +686,8 @@ export class CompactionHandler { loadedSkills: LoadedSkillSnapshot[], readFiles: string[], boundaryMessageId?: string, - previousState?: PersistedPostCompactionStateV1 + previousState?: PersistedPostCompactionStateV1, + publication?: ContinuousCompactionPublication ): Promise { try { for (const snapshot of loadedSkills) { @@ -685,7 +709,7 @@ export class CompactionHandler { await this.enqueuePendingStateWrite(async () => { await fsPromises.mkdir(this.sessionDir, { recursive: true }); await fsPromises.writeFile(this.postCompactionStatePath, serialized); - }); + }, publication); } catch (error) { log.warn("Failed to persist post-compaction state", { workspaceId: this.workspaceId, @@ -697,9 +721,10 @@ export class CompactionHandler { async preparePendingStateFromMessages( messages: MuxMessage[], boundaryMessageId?: string, - previousState?: PersistedPostCompactionStateV1 + previousState?: PersistedPostCompactionStateV1, + publication?: ContinuousCompactionPublication ): Promise { - await this.loadPersistedPendingStateIfNeeded(); + await this.loadPersistedPendingStateIfNeeded(publication); this.pendingStateBoundaryMessageId = boundaryMessageId; const latestCompactionEpochMessages = sliceMessagesFromLatestCompactionBoundary(messages); @@ -726,7 +751,8 @@ export class CompactionHandler { this.cachedLoadedSkills, this.cachedReadFilePaths, boundaryMessageId, - previousState + previousState, + publication ); } @@ -813,11 +839,20 @@ export class CompactionHandler { "appendHeartbeatContextResetBoundary requires non-empty boundary text" ); - const deletePartialResult = await this.historyService.deletePartial(this.workspaceId); + const publication = { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; + const deletePartialResult = await this.historyService.deletePartial( + this.workspaceId, + publication + ); if (!deletePartialResult.success) { log.warn( `Failed to delete partial before heartbeat reset boundary: ${deletePartialResult.error}` ); + return Err(deletePartialResult.error); } const historyResult = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); @@ -826,9 +861,9 @@ export class CompactionHandler { } const messages = historyResult.data; - await this.loadPersistedPendingStateIfNeeded(); - this.captureHeartbeatResetRollbackState(messages); - await this.preparePendingStateFromMessages(messages); + await this.loadPersistedPendingStateIfNeeded(publication); + const rollbackState = this.captureHeartbeatResetRollbackState(messages); + await this.preparePendingStateFromMessages(messages, undefined, undefined, publication); const nextCompactionEpoch = getNextCompactionEpoch(messages); assert( @@ -868,12 +903,22 @@ export class CompactionHandler { ); const maxExistingHistorySequence = this.getMaxExistingHistorySequence(messages); + let skipped = false; const persistenceResult = await this.historyService.appendToHistory( this.workspaceId, - summaryMessage + summaryMessage, + { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + skipped = true; + }, + } ); + if (skipped) return Err("Heartbeat publication was invalidated"); if (!persistenceResult.success) { - await this.restoreHeartbeatResetRollbackState(); + await this.restoreHeartbeatResetRollbackState(publication, rollbackState); return Err(`Failed to append heartbeat reset boundary: ${persistenceResult.error}`); } @@ -980,6 +1025,13 @@ export class CompactionHandler { event: StreamEndEvent, compactionRequestMessageId?: string ): Promise { + // A streamed summary retains its original stream epoch across foreign Stop. + // Standalone callers capture before reading the context used for this completion. + const publication = this.getCompactionPublication?.() ?? { + generation: await this.historyService + .getContinuousCompactionJournal(this.workspaceId) + .captureGeneration(), + }; const canComplete = this.captureCompletionGuard?.(); // The current stream identifies its request when available. Synthetic prompt snapshots can // follow that request in history, so the last user row is not always the compaction request. @@ -1077,7 +1129,8 @@ export class CompactionHandler { event.messageId, compactionRequestMessage.id, isIdleCompaction, - pendingFollowUp + pendingFollowUp, + publication ); if (!result.success) { log.error("Compaction failed:", result.error); @@ -1365,7 +1418,8 @@ export class CompactionHandler { streamedSummaryMessageId: string, compactionRequestMessageId: string, isIdleCompaction = false, - pendingFollowUp?: CompactionFollowUpRequest + pendingFollowUp?: CompactionFollowUpRequest, + publication?: ContinuousCompactionPublication ): Promise> { assert(summary.trim().length > 0, "performCompaction requires a non-empty summary"); assert(metadata.model.trim().length > 0, "Compaction summary requires a model"); @@ -1380,17 +1434,20 @@ export class CompactionHandler { // 2. sendQueuedMessages triggers commitPartial // 3. commitPartial finds stale partial.json and appends it to history // By deleting partial first, commitPartial becomes a no-op - const deletePartialResult = await this.historyService.deletePartial(this.workspaceId); + const deletePartialResult = await this.historyService.deletePartial( + this.workspaceId, + publication + ); if (!deletePartialResult.success) { log.warn(`Failed to delete partial before compaction: ${deletePartialResult.error}`); - // Continue anyway - the partial may not exist, which is fine + if (publication) return Err(deletePartialResult.error); } // Extract diffs from the latest compaction epoch only, so append-only history // does not re-inject stale pre-boundary edits after subsequent compactions. // If boundary markers are malformed, slicing self-heals by falling back to // full history instead of crashing or dropping all diffs. - await this.preparePendingStateFromMessages(messages); + await this.preparePendingStateFromMessages(messages, undefined, undefined, publication); const nextCompactionEpoch = getNextCompactionEpoch(messages); assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); @@ -1505,21 +1562,40 @@ export class CompactionHandler { summaryMessage.id ); + let skipped = false; const persistenceResult = preservedTailCopies.length > 0 ? await this.historyService.persistBoundaryWithTailCopies( this.workspaceId, summaryMessage, preservedTailCopies, - persistedStreamSummary !== null + persistedStreamSummary !== null, + undefined, + publication ) : persistedStreamSummary - ? await this.historyService.updateHistory(this.workspaceId, summaryMessage) - : await this.historyService.appendToHistory(this.workspaceId, summaryMessage); + ? await this.historyService.updateHistory( + this.workspaceId, + summaryMessage, + undefined, + undefined, + undefined, + undefined, + publication + ) + : await this.historyService.appendToHistory(this.workspaceId, summaryMessage, { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + skipped = true; + }, + }); + if (skipped) return Err("Compaction publication was invalidated"); if (!persistenceResult.success) { this.cachedFileDiffs = []; this.cachedLoadedSkills = []; - await this.deletePersistedPendingStateBestEffort(); + await this.deletePersistedPendingStateBestEffort(publication); const operation = preservedTailCopies.length > 0 ? "commit boundary with preserved tail" diff --git a/src/node/services/continuousCompactionJournal.ts b/src/node/services/continuousCompactionJournal.ts index 77cb40ebc70..90b4abb2f10 100644 --- a/src/node/services/continuousCompactionJournal.ts +++ b/src/node/services/continuousCompactionJournal.ts @@ -109,7 +109,7 @@ export class ContinuousCompactionJournalStore { return result; } - private async readGenerationUnderHistoryLock(): Promise { + async captureGenerationUnderHistoryLock(): Promise { try { // The bytes are an opaque version, not semantic configuration. Damaged // bytes fence older work while fresh capture can still make progress. @@ -124,7 +124,7 @@ export class ContinuousCompactionJournalStore { } captureGeneration(): Promise { - return this.enqueue(() => this.readGenerationUnderHistoryLock()); + return this.enqueue(() => this.captureGenerationUnderHistoryLock()); } /** Caller already owns the history lock; never join the queue of writers waiting for it. */ @@ -132,18 +132,20 @@ export class ContinuousCompactionJournalStore { publication: ContinuousCompactionPublication ): Promise { return ( - publication.generation === (await this.readGenerationUnderHistoryLock()) && + publication.generation === (await this.captureGenerationUnderHistoryLock()) && (await this.canPublishUnderHistoryLock()) ); } /** Authoritative repair, unlike an old compactor's identity-scoped cleanup. */ - async invalidateUnderHistoryLock(): Promise { + async invalidateUnderHistoryLock(onAdvanced?: (generation: string) => void): Promise { + const generation = randomUUID(); await writeFileAtomic( path.join(path.dirname(this.path), CONTINUOUS_COMPACTION_GENERATION_FILE), - randomUUID(), + generation, { mode: 0o600 } ); + onAdvanced?.(createHash("sha256").update(generation).digest("hex")); await fs.rm(this.path, { force: true }); } diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 5d4d96f0b72..da2fa0eb9d2 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -27,6 +27,7 @@ import { MalformedCompactionCancellationError, matchesCompactionCancellation, type CompactionCancellationRecord, + type CompactionCancellationPublication, } from "./compactionCancellation"; import { ContinuousCompactionJournalStore, @@ -89,6 +90,7 @@ import { const HISTORY_WRITE_LOCK_TIMEOUT_MS = 10_000; interface CompactionFollowUpAppendCondition { + publication?: ContinuousCompactionPublication; /** Presence selects manual replacement admission; null captures semantic absence. */ replacementNonce?: string | null; summary?: MuxMessage; @@ -472,7 +474,8 @@ export class HistoryService { workspaceId: string, record: CompactionCancellationRecord | null, isCurrent: () => boolean, - retiredNonce?: string + retiredNonce?: string, + publication?: CompactionCancellationPublication ): Promise { // This file supplements legacy pendingFollowUp removal when transcript I/O // fails. Older builds only honor successful cleanup of that legacy marker. @@ -487,19 +490,67 @@ export class HistoryService { ); if (record === null) { const current = await this.readCompactionCancellation(workspaceId); + if (!isCurrent() || current?.nonce !== retiredNonce) return; + if ( + current?.retainUntilReplacement && + !(await this.hasCompactionReplacementWitnessUnlocked(workspaceId, current.nonce)) + ) + return; if (isCurrent() && current?.nonce === retiredNonce) await fs.rm(cancellationPath, { force: true }); return; } + const current = await this.readCompactionCancellation(workspaceId).catch(() => undefined); if (record.scope.kind === "summary") { - const current = await this.readCompactionCancellation(workspaceId); - if (current?.nonce !== record.nonce) return; + if (current?.nonce !== record.nonce || current.retainUntilReplacement) return; + } else if (current?.retainUntilReplacement || current === undefined) { + // Carry clear's conservative fence through a foreign Stop. An unreadable + // old sidecar cannot safely prove that this obligation was absent. + record = { ...record, retainUntilReplacement: true }; } await ensurePrivateDir(this.getSessionDir(workspaceId)); + if (record.scope.kind === "unresolved") { + // A lost acknowledgement must not erase fresh work after this exact Stop + // already published or was replaced. Transcript failure still permits Stop. + if ( + current?.nonce === record.nonce || + (await this.hasCompactionReplacementWitnessUnlocked(workspaceId, record.nonce).catch( + () => false + )) + ) + return; + const journal = this.getContinuousCompactionJournal(workspaceId); + const predecessor = { + nonce: current === undefined ? undefined : (current?.nonce ?? null), + generation: await journal.captureGenerationUnderHistoryLock(), + }; + // An unobserved failed attempt cannot adopt any established frontier, + // including a generation whose Stop was already witnessed and unlinked. + if ( + publication && + publication.attempts > 1 && + !publication.predecessor && + (current || predecessor.generation !== undefined) + ) + return; + // Failed A may retry only its own publication frontier. A foreign Stop + // or repair supersedes it; refresh shared state instead of overwriting B. + if (publication?.predecessor && !isDeepStrictEqual(publication.predecessor, predecessor)) + return; + if (publication) publication.predecessor = predecessor; + // Publish a never-reused epoch BEFORE the sidecar. Record advancement + // before journal cleanup so partial failure still has an exact retry CAS. + await journal.invalidateUnderHistoryLock((generation) => { + if (publication) publication.predecessor = { ...predecessor, generation }; + }); + } const stagedPath = `${cancellationPath}.${randomUUID()}`; try { await writeFileAtomic(stagedPath, JSON.stringify(record), { mode: 0o600 }); - if (isCurrent()) renameSync(stagedPath, cancellationPath); + if (isCurrent()) { + renameSync(stagedPath, cancellationPath); + if (publication?.predecessor) publication.predecessor.nonce = record.nonce; + } } finally { await fs.rm(stagedPath, { force: true }); } @@ -2341,9 +2392,49 @@ export class HistoryService { } /** - * Delete the partial message file for a workspace. + * Pending-state writes must retain their original publication epoch across Stop. + * The callback performs filesystem I/O only; it must not reacquire history or journal queues. */ - async deletePartial(workspaceId: string): Promise> { + async withCompactionPublicationWrite( + workspaceId: string, + publication: ContinuousCompactionPublication, + write: () => Promise + ): Promise { + await this.fileLocks.withLock(workspaceId, () => + this.withHistoryWriteFileLock(workspaceId, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return; + if ( + await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication) + ) { + await write(); + } + }) + ); + } + + /** Delete the partial message file for a workspace. */ + async deletePartial( + workspaceId: string, + publication?: ContinuousCompactionPublication + ): Promise> { + if (publication) + return this.withRecoveredHistoryWriteResultLock( + workspaceId, + "Failed to delete partial", + async () => { + // Compaction cleanup must share the publication lock: an old producer + // cannot unlink the replacement stream's partial while awaiting its epoch. + if ( + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); + return this.deletePartialUnlocked(workspaceId); + } + ); return this.fileLocks.withLock(workspaceId, () => this.deletePartialUnlocked(workspaceId)); } @@ -2878,6 +2969,13 @@ export class HistoryService { workspaceId: string, condition: CompactionFollowUpAppendCondition ): Promise { + if ( + condition.publication && + !(await this.getContinuousCompactionJournal(workspaceId).isPublicationCurrentUnderHistoryLock( + condition.publication + )) + ) + return false; if (condition.replacementNonce !== undefined) { return ( (await this.matchesReplacementCancellationUnlocked( @@ -3089,13 +3187,21 @@ export class HistoryService { shouldUpdate?: (current: MuxMessage) => boolean, updateFromCurrent?: (current: MuxMessage) => MuxMessage, onCommitted?: () => void, - cancellationNonce?: string + cancellationNonce?: string, + publication?: ContinuousCompactionPublication ): Promise> { assert(!onCommitted || shouldUpdate, "Update commit observers require a conditional mutation"); return this.withRecoveredHistoryWriteResultLock( workspaceId, "Failed to update history", async () => { + if ( + publication && + !(await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ) + return Err("Compaction publication was invalidated"); // A live cancellation read can age across another backend's accepted replacement. // Only the exact still-unwitnessed Stop may erase its captured pending summary. if (cancellationNonce !== undefined) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 54c4d63d0d6..58e88916a43 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,3 +1,7 @@ +import type { CompactionHandler } from "./compactionHandler"; +import { CompactionCancellation } from "./compactionCancellation"; +import { makeTestEffectRunner } from "./di/testEffectRunner"; +import { calculateBackoffDelay } from "@/common/utils/messages/retryState"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -7378,6 +7382,147 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { return { aiService, config, historyService, workspaceService, goalService, cleanup }; } + test.each( + ["legacy", "heartbeat"].flatMap((producer) => + [ + "absent", + "unresolved", + "narrowed", + "manual before publication", + "newer Stop during clear", + ].map((initial) => ({ producer, initial })) + ) + )( + "full clear fences captured foreign $producer publication ($initial Stop)", + async ({ producer, initial }) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "clear-foreign-compaction"; + const options = { model: "openai:gpt-4o", agentId: "exec" }; + await config.addWorkspace("/tmp/clear-foreign-project", { + id: workspaceId, + name: workspaceId, + projectName: "clear-foreign-project", + projectPath: "/tmp/clear-foreign-project", + runtimeConfig: { type: "local" }, + }); + const followUp = { text: "Continue discarded work", ...options }; + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: followUp }, + }, + }) + ); + const cancellation = new CompactionCancellation(historyService, workspaceId); + const foreignHistory = new HistoryService(config); + const foreign = await createAgentSessionHarness({ + workspaceId, + config, + historyService: foreignHistory, + }); + const handler = (foreign.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const append = foreignHistory.appendToHistory.bind(foreignHistory); + spyOn(foreignHistory, "appendToHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return append(...args); + }); + const publication = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Captured old context", + pendingFollowUp: followUp, + }) + : handler.handleCompletion({ + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Captured old context" }], + metadata: { model: options.model, duration: 1 }, + }); + const clock = makeTestEffectRunner(); + let fresh: Awaited> | undefined; + try { + await entered.promise; + if (initial === "unresolved" || initial === "narrowed") { + await cancellation.cancel(); + if (initial === "narrowed") { + const record = await cancellation.read(); + if (!record) throw new Error("Expected Stop"); + await cancellation.narrow( + record.nonce, + createMuxMessage("older-summary", "assistant", "Older work", { + historySequence: 10, + muxMetadata: { type: "compaction-summary", pendingFollowUp: followUp }, + }) + ); + } + } + expect((await workspaceService.truncateHistory(workspaceId)).success).toBe(true); + if (initial === "newer Stop during clear") + await new CompactionCancellation(new HistoryService(config), workspaceId).cancel(); + if (initial === "manual before publication") { + const replacement = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + }); + try { + expect( + (await replacement.session.sendMessage("Fresh explicit work", options)).success + ).toBe(true); + } finally { + await replacement.session.dispose(); + } + } + release.resolve(); + await publication.catch(() => undefined); + const published = await foreignHistory.getLastMessages(workspaceId, 10); + expect( + published.success && + published.data.some((row) => row.metadata?.muxMetadata?.type === "compaction-summary") + ).toBe(false); + fresh = await createAgentSessionHarness({ + workspaceId, + config, + historyService: new HistoryService(config), + streamManager: { ...createStreamLifecycleMocks(), effectRunner: clock.runner }, + captureEvents: true, + }); + const stream = spyOn(fresh.aiService, "streamMessage"); + await fresh.session.runStartupRecovery(); + await clock.adjust(calculateBackoffDelay(6) * 2); + if (initial === "manual before publication") { + const rows = await fresh.historyService.getLastMessages(workspaceId, 1); + expect(rows.success && rows.data[0].parts).toMatchObject([ + { text: "Fresh explicit work" }, + ]); + } else { + expect(stream).not.toHaveBeenCalled(); + expect(fresh.events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + expect((await fresh.session.sendMessage("Fresh explicit work", options)).success).toBe( + true + ); + expect(stream).toHaveBeenCalledTimes(1); + } + } finally { + release.resolve(); + await publication.catch(() => undefined); + await fresh?.session.dispose(); + await foreign.session.dispose(); + await workspaceService.disposeSession(workspaceId); + await clock.dispose(); + await cleanup(); + } + } + ); + test("requireIdle sends carry a live idle-admission probe re-evaluated at session gates", async () => { // Codex P1 (PRRT_kwDOPxxmWM6cJ6NI): the preflight count check at // sendMessage entry is a one-shot snapshot — a manual send can enter @@ -8571,8 +8716,13 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { workspaceId, createMuxMessage("replacement", "assistant", "new context") ); - expect(result.success).toBe(false); - expect(!result.success && result.error).toContain("cancel unlink unavailable"); + expect(result.success).toBe(operation === "clear"); + if (operation === "clear") + expect(await historyService.readCompactionCancellation(workspaceId)).toMatchObject({ + retainUntilReplacement: true, + scope: { kind: "unresolved" }, + }); + else expect(!result.success && result.error).toContain("cancel unlink unavailable"); expect(epochs.get(workspaceId)).toBe(before + 1); if (operation === "replace") { const persisted = await historyService.getHistoryFromLatestBoundary(workspaceId); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6a1e05ffa83..fdbbdb44f18 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12644,9 +12644,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { refuseRowRemoval: truncationScope === "none", requireFullDelete: truncationScope === "all", }); - const cancellationNonce = isFullClear - ? ((await this.getOrCreateSession(workspaceId).getCompactionCancellationNonce()) ?? null) - : null; + if (isFullClear) { + try { + await this.getOrCreateSession(workspaceId).fenceCompactionForContextClear(); + } catch (error) { + return Err( + `Cannot clear history: compaction cancellation could not be persisted (${getErrorMessage(error)})` + ); + } + } const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { @@ -12660,7 +12666,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). const cancellationRetirement = isFullClear - ? await this.advanceContextMutationEpoch(workspaceId, cancellationNonce) + ? await this.advanceContextMutationEpoch(workspaceId, null) : Ok(undefined); // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the From 48048946224dff899afc8623954f6d0decc0ca5b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 10:33:13 +0200 Subject: [PATCH 16/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20fence=20context=20r?= =?UTF-8?q?eplacement=20across=20backends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalidate captured compaction publications under the same history lock as reset and destructive context writes. Preserve rejected and non-destructive operations, and refuse unfenced writes when generation publication fails. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I57ef548fbdb7ab24962f29d00a986cfcfa15c0bd --- src/node/services/historyService.test.ts | 116 +++++++++++++++++ src/node/services/historyService.ts | 27 ++++ src/node/services/workspaceService.test.ts | 143 +++++++++++++++++++-- 3 files changed, 275 insertions(+), 11 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 3b2d4f7f976..6b2378b5886 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -991,6 +991,112 @@ describe("HistoryService", () => { }); }); + describe("context publication fencing", () => { + it.each([false, true])( + "reset batches validate their original publication before advancing it (admitted=%s)", + async (admitted) => { + const workspaceId = "reset-batch-publication"; + await service.appendToHistory(workspaceId, createMuxMessage("old", "user", "Old context")); + const journal = service.getContinuousCompactionJournal(workspaceId); + const publication = { generation: await journal.captureGeneration() }; + await fs.writeFile(journal.path, "existing journal"); + let skipped = false; + const result = await service.appendManyToHistory( + workspaceId, + [ + createMuxMessage("reset", "assistant", "", { + contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + }), + createMuxMessage("replacement", "user", "Fresh request"), + ], + { + publication, + allowedTailMessageIds: [], + isCurrent: () => admitted, + onSkipped: () => { + skipped = true; + }, + } + ); + expect(result.success).toBe(true); + expect(skipped).toBe(!admitted); + expect((await journal.captureGeneration()) === publication.generation).toBe(!admitted); + expect((await collectFullHistory(service, workspaceId)).map((row) => row.id)).toEqual( + admitted ? ["old", "reset", "replacement"] : ["old"] + ); + if (admitted) { + // The admitted batch succeeds with its old receipt; subsequent foreign + // handoffs carrying that same receipt must now be refused. + let foreignSkipped = false; + await new HistoryService(config).appendToHistory( + workspaceId, + createMuxMessage("stale", "user", "Discarded follow-up"), + { + publication, + allowedTailMessageIds: [], + isCurrent: () => true, + onSkipped: () => { + foreignSkipped = true; + }, + } + ); + expect(foreignSkipped).toBe(true); + } else { + expect(await fs.readFile(journal.path, "utf8")).toBe("existing journal"); + } + } + ); + + it("refuses a reset batch when its publication generation cannot be written", async () => { + const workspaceId = "reset-batch-generation-failure"; + await service.appendToHistory(workspaceId, createMuxMessage("old", "user", "Old context")); + const failure = spyOn( + service.getContinuousCompactionJournal(workspaceId), + "invalidateUnderHistoryLock" + ).mockRejectedValueOnce(new Error("generation unavailable")); + try { + const result = await service.appendManyToHistory(workspaceId, [ + createMuxMessage("reset", "assistant", "", { + contextBoundaryKind: CONTEXT_BOUNDARY_KINDS.RESET, + }), + createMuxMessage("replacement", "user", "Fresh request"), + ]); + expect(!result.success && result.error).toContain("generation unavailable"); + expect((await collectFullHistory(service, workspaceId)).map((row) => row.id)).toEqual([ + "old", + ]); + } finally { + failure.mockRestore(); + } + }); + + it.each(["full", "rounded full", "refused full", "no rows", "zero"])( + "only actual full deletions retire the publication generation (%s)", + async (operation) => { + const workspaceId = "full-delete-publication"; + await fs.mkdir(path.join(config.sessionsDir, workspaceId), { recursive: true }); + if (operation !== "no rows") { + await service.appendToHistory( + workspaceId, + createMuxMessage("old", "user", "Old context") + ); + } + const journal = service.getContinuousCompactionJournal(workspaceId); + await fs.writeFile(journal.path, "existing journal"); + const generation = await journal.captureGeneration(); + const result = await service.truncateHistory( + workspaceId, + operation === "zero" ? 0 : operation === "full" || operation === "no rows" ? 1 : 0.5, + { refuseFullDelete: operation === "refused full" } + ); + const removed = operation === "full" || operation === "rounded full"; + expect(result.success).toBe(operation !== "refused full"); + expect((await journal.captureGeneration()) === generation).toBe(!removed); + if (!removed) expect(await fs.readFile(journal.path, "utf8")).toBe("existing journal"); + } + ); + }); + describe("clearHistory", () => { it("should delete chat.jsonl file", async () => { const workspaceId = "workspace1"; @@ -2577,7 +2683,12 @@ describe("HistoryService", () => { }) ); + const journal = service.getContinuousCompactionJournal(wsId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "active publication journal"); expect((await service.truncateHistory(wsId, 0.5)).success).toBe(true); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("active publication journal"); const active = await service.getHistoryFromLatestBoundary(wsId); expect(active.success).toBe(true); @@ -2606,7 +2717,12 @@ describe("HistoryService", () => { }) ); + const journal = service.getContinuousCompactionJournal(wsId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "active publication journal"); expect((await service.truncateHistory(wsId, 0.2)).success).toBe(true); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("active publication journal"); const active = await service.getHistoryFromLatestBoundary(wsId); expect(active.success).toBe(true); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index da2fa0eb9d2..34b83627802 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2712,6 +2712,7 @@ export class HistoryService { `[HISTORY APPEND] Assigned historySequence=${message.metadata.historySequence ?? "unknown"} role=${message.role}` ); + await this.fenceContextResetUnderHistoryLock(workspaceId, [message]); await this.getAppendProvenance(workspaceId).appendChat( Buffer.from(JSON.stringify(historyEntry) + "\n") ); @@ -2722,6 +2723,20 @@ export class HistoryService { } } + private async fenceContextResetUnderHistoryLock( + workspaceId: string, + messages: readonly MuxMessage[] + ): Promise { + if ( + messages.some((message) => getContextBoundaryKind(message) === CONTEXT_BOUNDARY_KINDS.RESET) + ) { + // A reset discards foreign producers' captured context even without a Stop. + // Fence after admission checks and before the write, under this same lock; + // a failed epoch write must never leave a committed reset unfenced. + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } + } + /** Serialize messages as JSONL rows tagged with workspace context. */ private serializeHistoryEntries(messages: readonly MuxMessage[], workspaceId: string): string { return messages.map((msg) => JSON.stringify({ ...msg, workspaceId }) + "\n").join(""); @@ -3114,6 +3129,7 @@ export class HistoryService { // temp-and-rename helper the other history mutations use, under the // cross-process append lock (r50) so a foreign backend's row cannot // land between this read and the replace and be silently deleted. + await this.fenceContextResetUnderHistoryLock(workspaceId, messages); await this.getAppendProvenance(workspaceId).appendChat( Buffer.from(this.serializeHistoryEntries(messages, workspaceId)), true @@ -4100,6 +4116,11 @@ export class HistoryService { .filter((s): s is number => isNonNegativeInteger(s)); if (percentage >= 1.0) { + // Full deletion also retires captured publications when no Stop exists. + // Keep the epoch advance and destructive rewrite in one locked operation. + if (archiveRows.length > 0 || chatRows.length > 0) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); @@ -4156,6 +4177,7 @@ export class HistoryService { "Truncation would remove every remaining message; retry to run it as a full clear." ); } + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); @@ -4183,6 +4205,11 @@ export class HistoryService { retainedMessages, sanitize ); + // A prefix cut retires captured publications only when it removes active + // provider context; trimming sealed or display-only rows preserves them. + if (activeContextChanged) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked( workspaceId, remainingArchive.length > 0 ? remainingArchive : null, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 48b78611875..da8c064f828 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7385,17 +7385,21 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { test.each( ["legacy", "heartbeat"].flatMap((producer) => - [ - "absent", - "unresolved", - "narrowed", - "manual before publication", - "newer Stop during clear", - ].map((initial) => ({ producer, initial })) + ["absent", "unresolved", "narrowed", "manual before publication", "newer Stop during clear"] + .map((initial) => ({ producer, initial, operation: "full clear" })) + .concat( + ["reset", "destructive replacement", "active prefix trim", "sealed prefix trim"].map( + (operation) => ({ + producer, + initial: "absent", + operation, + }) + ) + ) ) )( - "full clear fences captured foreign $producer publication ($initial Stop)", - async ({ producer, initial }) => { + "$operation checks captured foreign $producer publication ($initial Stop)", + async ({ producer, initial, operation }) => { const { config, historyService, workspaceService, cleanup } = await createServices(); const workspaceId = "clear-foreign-compaction"; const options = { model: "openai:gpt-4o", agentId: "exec" }; @@ -7407,6 +7411,20 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { runtimeConfig: { type: "local" }, }); const followUp = { text: "Continue discarded work", ...options }; + if (operation === "sealed prefix trim") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("sealed", "user", "Sealed context") + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("sealed-boundary", "assistant", "Preserved summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + } await historyService.appendToHistory( workspaceId, createMuxMessage("old-request", "user", "Compact", { @@ -7418,6 +7436,13 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }) ); const cancellation = new CompactionCancellation(historyService, workspaceId); + if (operation === "active prefix trim") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("survivor", "assistant", "Surviving context") + ); + expect(await historyService.classifyTruncationRemoval(workspaceId, 0.25)).toBe("partial"); + } const foreignHistory = new HistoryService(config); const foreign = await createAgentSessionHarness({ workspaceId, @@ -7465,7 +7490,34 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { ); } } - expect((await workspaceService.truncateHistory(workspaceId)).success).toBe(true); + const mutation = + operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "destructive replacement" + ? await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "Fresh context") + ) + : await workspaceService.truncateHistory( + workspaceId, + operation === "active prefix trim" + ? 0.25 + : operation === "sealed prefix trim" + ? 0.01 + : undefined + ); + expect(mutation.success).toBe(true); + if (operation === "active prefix trim") { + const trimmed = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(trimmed.success && trimmed.data.map((row) => row.id)).toEqual(["survivor"]); + } + if (operation === "sealed prefix trim") { + const retainedIds: string[] = []; + await historyService.iterateFullHistory(workspaceId, "forward", (rows) => { + retainedIds.push(...rows.map((row) => row.id)); + }); + expect(retainedIds).toEqual(["sealed-boundary", "old-request"]); + } if (initial === "newer Stop during clear") await new CompactionCancellation(new HistoryService(config), workspaceId).cancel(); if (initial === "manual before publication") { @@ -7488,7 +7540,8 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { expect( published.success && published.data.some((row) => row.metadata?.muxMetadata?.type === "compaction-summary") - ).toBe(false); + ).toBe(operation === "sealed prefix trim"); + if (operation === "sealed prefix trim") return; fresh = await createAgentSessionHarness({ workspaceId, config, @@ -7524,6 +7577,74 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } ); + test.each( + ["reset", "replace", "active prefix trim"].flatMap((operation) => + [false, true].map((afterAdvance) => ({ operation, afterAdvance })) + ) + )( + "$operation refuses history mutation when publication fencing fails (after advance=$afterAdvance)", + async ({ operation, afterAdvance }) => { + const { config, historyService, workspaceService, cleanup } = await createServices(); + const workspaceId = "context-publication-failure"; + await config.addWorkspace("/tmp/context-publication-project", { + id: workspaceId, + name: workspaceId, + projectName: "context-publication-project", + projectPath: "/tmp/context-publication-project", + runtimeConfig: { type: "local" }, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("old-user", "user", "Original context") + ); + if (operation === "active prefix trim") { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("survivor", "assistant", "Surviving context") + ); + } + const journal = historyService.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + const failure = spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce( + async (...args) => { + if (afterAdvance) await invalidate(...args); + throw new Error("context publication unavailable"); + } + ); + const mutate = () => + operation === "reset" + ? workspaceService.resetContext(workspaceId) + : operation === "active prefix trim" + ? workspaceService.truncateHistory(workspaceId, 0.25) + : workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "Fresh context") + ); + try { + const result = await mutate(); + expect(!result.success && result.error).toContain("context publication unavailable"); + const freshHistory = new HistoryService(config); + const rows = await freshHistory.getHistoryFromLatestBoundary(workspaceId); + expect(rows.success && rows.data.map((row) => row.id)).toEqual( + operation === "active prefix trim" ? ["old-user", "survivor"] : ["old-user"] + ); + const failedGeneration = await freshHistory + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(); + expect(failedGeneration === generation).toBe(!afterAdvance); + // A partial epoch advance stays retired. Retry publishes a fresh epoch rather + // than restoring an old producer's authority after storage becomes writable. + expect((await mutate()).success).toBe(true); + expect(await journal.captureGeneration()).not.toBe(failedGeneration); + } finally { + failure.mockRestore(); + await workspaceService.disposeSession(workspaceId); + await cleanup(); + } + } + ); + test("requireIdle sends carry a live idle-admission probe re-evaluated at session gates", async () => { // Codex P1 (PRRT_kwDOPxxmWM6cJ6NI): the preflight count check at // sendMessage entry is a one-shot snapshot — a manual send can enter From 4e95ef2c9be204d595dbb1562d231e3ddb1f9b18 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 11:13:37 +0200 Subject: [PATCH 17/17] =?UTF-8?q?=F0=9F=A4=96=20fix:=20fence=20history=20e?= =?UTF-8?q?dits=20and=20provisional=20compaction=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalidate foreign compaction publications inside edit truncation writes. Tie provisional attachments to their intended durable boundary and clean exact owned writes after cancellation, preserving newer preparations and valid prior state. Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: I5748fc35360ba556cf5f444bbc0433663b175144 --- .../agentSession.compactionShutdown.test.ts | 434 ++++++++++++++++++ src/node/services/compactionHandler.ts | 191 ++++++-- src/node/services/continuousCompactor.test.ts | 3 +- src/node/services/historyService.test.ts | 79 ++++ src/node/services/historyService.ts | 32 +- src/node/services/workspaceService.test.ts | 72 ++- 6 files changed, 739 insertions(+), 72 deletions(-) diff --git a/src/node/services/agentSession.compactionShutdown.test.ts b/src/node/services/agentSession.compactionShutdown.test.ts index 259e64913bf..919fb453721 100644 --- a/src/node/services/agentSession.compactionShutdown.test.ts +++ b/src/node/services/agentSession.compactionShutdown.test.ts @@ -3239,3 +3239,437 @@ test.each( } } ); + +test.each( + ["heartbeat", "legacy append", "legacy update"].flatMap((producer) => + ["restart before cleanup", "cleanup", "reset", "edit", "successor"].map((stage) => ({ + producer, + stage, + })) + ) +)("uncommitted $producer pending state is owned across $stage", async ({ producer, stage }) => { + const h = await setup(); + const readMessage = createMuxMessage("read", "assistant", ""); + readMessage.parts = [ + { + type: "dynamic-tool", + toolCallId: "read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/same-logical-read.ts" }, + output: { success: true }, + }, + ]; + await h.historyService.appendToHistory(workspaceId, readMessage); + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-request", "user", "Compact", { + muxMetadata: { + type: "compaction-request", + rawCommand: "/compact", + parsed: { followUpContent: { text: "Continue", ...options } }, + }, + }) + ); + if (producer === "legacy update") + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-stream", "assistant", "Uncommitted summary") + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + if (producer === "legacy update") { + const update = h.historyService.updateHistory.bind(h.historyService); + spyOn(h.historyService, "updateHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return update(...args); + }); + } else { + const append = h.historyService.appendToHistory.bind(h.historyService); + spyOn(h.historyService, "appendToHistory").mockImplementationOnce(async (...args) => { + entered.resolve(); + await release.promise; + return append(...args); + }); + } + const handler = h.internals.compactionHandler; + const pending = + producer === "heartbeat" + ? handler.appendHeartbeatContextResetBoundary({ + boundaryText: "Captured context", + pendingFollowUp: { text: "Continue", ...options }, + }) + : handler.handleCompletion( + { + type: "stream-end", + workspaceId, + messageId: "old-stream", + parts: [{ type: "text", text: "Captured context" }], + metadata: { model: options.model, duration: 1 }, + }, + "old-request" + ); + const foreign = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + const foreignHandler = (foreign.session as unknown as { compactionHandler: CompactionHandler }) + .compactionHandler; + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + try { + await entered.promise; + expect(JSON.parse(await readFile(pendingPath, "utf8"))).toHaveProperty("readFiles", [ + "/same-logical-read.ts", + ]); + if (stage === "reset") { + await foreign.historyService.appendToHistory( + workspaceId, + createMuxMessage("reset", "assistant", "", { contextBoundaryKind: "reset" }) + ); + } else if (stage === "edit") { + await foreign.historyService.truncateAfterMessage(workspaceId, "read"); + } else { + await foreign.session.interruptStream({ abandonPartial: true }); + } + if (stage === "restart before cleanup") + expect(await foreignHandler.peekPendingState()).toBeNull(); + let successor: string | undefined; + if (stage === "successor") { + await foreignHandler.preparePendingStateFromMessages([readMessage]); + successor = await readFile(pendingPath, "utf8"); + } + release.resolve(); + await pending; + if (successor !== undefined) { + expect(await readFile(pendingPath, "utf8")).toBe(successor); + } else { + expect(await foreignHandler.peekPendingState()).toBeNull(); + expect(await readFile(pendingPath, "utf8").catch(() => null)).toBeNull(); + } + } finally { + release.resolve(); + await pending.catch(() => undefined); + await foreign.session.dispose(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test.each([ + "same-generation successor", + "new-generation successor", + "write rejected", + "write acknowledgement lost", + "cleanup failure", + "current prior state", +])("provisional pending receipt handles %s without claiming successor bytes", async (scenario) => { + const h = await setup(); + const handler = h.internals.compactionHandler; + const journal = h.historyService.getContinuousCompactionJournal(workspaceId); + const publication = { generation: await journal.captureGeneration() }; + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + const cleanup = ( + handler as unknown as { + cleanupProvisionalPendingState( + receipt: Awaited>, + capturedPublication: typeof publication + ): Promise; + } + ).cleanupProvisionalPendingState.bind(handler); + let prior: string | undefined; + if (scenario === "write rejected" || scenario === "current prior state") { + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("committed", "assistant", "Prior context", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + prior = JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/prior.ts"], + boundaryMessageId: "committed", + }); + await writeFile(pendingPath, prior); + } + if (scenario === "write rejected" || scenario === "write acknowledgement lost") { + const write = h.historyService.withCompactionPublicationWrite.bind(h.historyService); + spyOn(h.historyService, "withCompactionPublicationWrite").mockImplementationOnce( + async (...args) => { + if (scenario === "write acknowledgement lost") await write(...args); + throw new Error("pending write unavailable"); + } + ); + } + const now = spyOn(Date, "now").mockReturnValue(1234); + try { + const read = createMuxMessage("a-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "a-read", + toolName: "file_read", + state: "output-available", + input: { path: "/provisional.ts" }, + output: { success: true }, + }, + ]; + const receipt = await handler.preparePendingStateFromMessages( + [read], + "provisional", + undefined, + publication + ); + if (scenario === "write rejected") expect(receipt.write).toBeUndefined(); + else expect(receipt.write).toBeDefined(); + if ( + scenario === "new-generation successor" || + scenario === "write acknowledgement lost" || + scenario === "cleanup failure" + ) { + await new CompactionCancellation(new HistoryService(h.config), workspaceId).cancel(); + } + if (scenario === "new-generation successor") { + expect((await h.session.sendMessage("Fresh successor", options)).success).toBe(true); + } + let successor: string | undefined; + if (scenario === "same-generation successor" || scenario === "new-generation successor") { + await handler.preparePendingStateFromMessages([read], "provisional", undefined, { + generation: await journal.captureGeneration(), + }); + successor = await readFile(pendingPath, "utf8"); + expect(successor).not.toBe(receipt.write?.serialized); + } + if (scenario === "cleanup failure") { + const unlink = fs.unlink; + let failed = false; + spyOn(fs, "unlink").mockImplementation(async (...args) => { + if (!failed && args[0] === pendingPath) { + failed = true; + throw new Error("cleanup unavailable"); + } + return unlink(...args); + }); + } + await cleanup(receipt, publication); + const expected = successor ?? prior; + if (expected !== undefined) { + expect(await readFile(pendingPath, "utf8")).toBe(expected); + if (prior !== undefined) + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/prior.ts"]); + } else { + const fresh = await createAgentSessionHarness({ + workspaceId, + config: h.config, + historyService: new HistoryService(h.config), + }); + try { + expect( + await ( + fresh.session as unknown as { compactionHandler: CompactionHandler } + ).compactionHandler.peekPendingState() + ).toBeNull(); + expect(await readFile(pendingPath, "utf8").catch(() => null)).toBeNull(); + } finally { + await fresh.session.dispose(); + } + } + } finally { + now.mockRestore(); + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("provisional cleanup cannot clear memory prepared by a successor waiting for its lock", async () => { + const h = await setup(); + const handler = h.internals.compactionHandler; + const publication = { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }; + const receipt = await handler.preparePendingStateFromMessages( + [], + "a-boundary", + undefined, + publication + ); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const locked = h.historyService.withCompactionPublicationLock.bind(h.historyService); + spyOn(h.historyService, "withCompactionPublicationLock").mockImplementationOnce( + (id, captured, operation) => + locked(id, captured, async (current) => { + entered.resolve(); + await release.promise; + await operation(current); + }) + ); + const cleanup = ( + handler as unknown as { + cleanupProvisionalPendingState( + capturedReceipt: typeof receipt, + captured: typeof publication + ): Promise; + } + ).cleanupProvisionalPendingState(receipt, publication); + await entered.promise; + const queued = Promise.withResolvers(); + const writes = handler as unknown as { + enqueuePendingStateWrite(write: () => Promise): Promise; + }; + const enqueue = writes.enqueuePendingStateWrite.bind(writes); + spyOn(writes, "enqueuePendingStateWrite").mockImplementationOnce((...args) => { + queued.resolve(); + return enqueue(...args); + }); + const read = createMuxMessage("b-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "b-read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/b.ts" }, + output: { success: true }, + }, + ]; + const preparation = handler.preparePendingStateFromMessages( + [read], + "b-boundary", + undefined, + publication + ); + try { + await queued.promise; + release.resolve(); + await cleanup; + await preparation; + expect( + await handler.persistContinuousCompaction({ + boundaryMessageId: "b-boundary", + messages: [read], + text: "B summary", + model: options.model, + tail: [], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + publication, + }) + ).toBe(true); + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/b.ts"]); + expect(handler as unknown as { pendingStateBoundaryMessageId: string }).toHaveProperty( + "pendingStateBoundaryMessageId", + "b-boundary" + ); + } finally { + release.resolve(); + await cleanup; + await preparation; + await h.session.dispose(); + await h.cleanup(); + } +}); + +test("held pending-state load cannot overwrite a newer preparation", async () => { + const h = await setup(); + const handler = h.internals.compactionHandler; + await h.historyService.appendToHistory( + workspaceId, + createMuxMessage("prior", "assistant", "Prior context", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + const pendingPath = `${h.config.sessionsDir}/${workspaceId}/post-compaction.json`; + await writeFile( + pendingPath, + JSON.stringify({ + version: 1, + createdAt: 1, + diffs: [], + loadedSkills: [], + readFiles: ["/old.ts"], + boundaryMessageId: "prior", + }) + ); + const publication = { + generation: await h.historyService + .getContinuousCompactionJournal(workspaceId) + .captureGeneration(), + }; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const readHistory = h.historyService.getHistoryFromLatestBoundary.bind(h.historyService); + spyOn(h.historyService, "getHistoryFromLatestBoundary").mockImplementationOnce( + async (...args) => { + const history = await readHistory(...args); + entered.resolve(); + await release.promise; + return history; + } + ); + const oldPreparation = handler.preparePendingStateFromMessages( + [], + "a-boundary", + undefined, + publication + ); + try { + await entered.promise; + const read = createMuxMessage("b-read", "assistant", ""); + read.parts = [ + { + type: "dynamic-tool", + toolCallId: "b-read-call", + toolName: "file_read", + state: "output-available", + input: { path: "/b.ts" }, + output: { success: true }, + }, + ]; + const successor = await handler.preparePendingStateFromMessages( + [read], + "b-boundary", + undefined, + publication + ); + const serialized = successor.write?.serialized; + if (serialized === undefined) throw new Error("Expected successor pending write"); + release.resolve(); + expect((await oldPreparation).write).toBeUndefined(); + expect(await fs.readFile(pendingPath, "utf8")).toBe(serialized); + expect( + await handler.persistContinuousCompaction({ + boundaryMessageId: "b-boundary", + messages: [read], + text: "B summary", + model: options.model, + tail: [], + systemMessageTokens: 0, + attachmentTokens: 0, + shouldPersist: () => true, + publication, + }) + ).toBe(true); + expect(await handler.peekPendingState()).toHaveProperty("readFiles", ["/b.ts"]); + expect(handler as unknown as { pendingStateBoundaryMessageId: string }).toHaveProperty( + "pendingStateBoundaryMessageId", + "b-boundary" + ); + } finally { + release.resolve(); + await oldPreparation; + await h.session.dispose(); + await h.cleanup(); + } +}); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index cae5cbe4dce..01bbae95975 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -4,6 +4,8 @@ import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { isNonNegativeInteger, isPositiveInteger } from "@/common/utils/numbers"; import * as path from "path"; +import { randomUUID } from "node:crypto"; +import writeFileAtomic from "write-file-atomic"; import type { HistoryService } from "./historyService"; @@ -92,9 +94,20 @@ interface PersistedPostCompactionStateV1 { * model when RLM mode is on. Absent in files written by older builds. */ readFiles: string[]; - /** Only continuous preparation is provisional until this boundary is durable. */ + /** Provisional preparation is usable only after this exact boundary is durable. */ boundaryMessageId?: string; previousState?: PersistedPostCompactionStateV1; + /** Unique per actual write, so equal attachment contents cannot cause ABA cleanup. */ + writeId?: string; +} + +interface PendingStateWriteReceipt { + serialized: string; + previous?: string; +} + +interface PendingStatePreparation { + write?: PendingStateWriteReceipt; } interface HeartbeatResetRollbackState { @@ -427,6 +440,7 @@ export class CompactionHandler { /** Rollback snapshot for synthetic heartbeat reset boundaries that get skipped before dispatch. */ private heartbeatResetRollbackState: HeartbeatResetRollbackState | null = null; private pendingStateWrites: Promise = Promise.resolve(); + private pendingStatePreparation?: PendingStatePreparation; private enqueuePendingStateWrite( write: () => Promise, @@ -470,6 +484,7 @@ export class CompactionHandler { return; } + const preparation = this.pendingStatePreparation; this.persistedPendingStateLoaded = true; let raw: string; @@ -484,21 +499,21 @@ export class CompactionHandler { parsed = JSON.parse(raw); } catch { log.warn("Invalid post-compaction state JSON; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(publication); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } let state = coercePersistedPostCompactionState(parsed); if (!state) { log.warn("Invalid post-compaction state schema; ignoring", { workspaceId: this.workspaceId }); - await this.deletePersistedPendingStateBestEffort(publication); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } if (state.boundaryMessageId) { const history = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); if (!history.success) { - this.persistedPendingStateLoaded = false; + if (this.pendingStatePreparation === preparation) this.persistedPendingStateLoaded = false; return; } const boundaryId = history.data.findLast(isDurableContextBoundaryMarker)?.id; @@ -507,11 +522,12 @@ export class CompactionHandler { // nor lose an older, still-pending attachment snapshot. state = state.previousState ?? null; if (!state || (state.boundaryMessageId && state.boundaryMessageId !== boundaryId)) { - await this.deletePersistedPendingStateBestEffort(publication); + await this.deletePersistedPendingStateBestEffort(publication, raw); return; } } } + if (this.pendingStatePreparation !== preparation) return; this.pendingStateBoundaryMessageId = state.boundaryMessageId; this.cachedFileDiffs = state.diffs; this.cachedLoadedSkills = state.loadedSkills; @@ -625,9 +641,26 @@ export class CompactionHandler { } private async deletePersistedPendingStateBestEffort( - publication?: ContinuousCompactionPublication + publication?: ContinuousCompactionPublication, + expectedBytes?: string ): Promise { try { + if (expectedBytes !== undefined) { + await this.enqueuePendingStateWrite(() => + this.historyService.withCompactionPublicationLock( + this.workspaceId, + publication, + async () => { + if ( + (await fsPromises.readFile(this.postCompactionStatePath, "utf8")) === expectedBytes + ) { + await fsPromises.unlink(this.postCompactionStatePath); + } + } + ) + ); + return; + } await this.enqueuePendingStateWrite( () => fsPromises.unlink(this.postCompactionStatePath), publication @@ -651,7 +684,8 @@ export class CompactionHandler { private restoreHeartbeatResetRollbackState( publication?: ContinuousCompactionPublication, - rollbackState = this.heartbeatResetRollbackState + rollbackState = this.heartbeatResetRollbackState, + persist = true ): Promise { if (!rollbackState || rollbackState !== this.heartbeatResetRollbackState) { return Promise.resolve(); @@ -667,6 +701,8 @@ export class CompactionHandler { this.cachedReadFilePaths = [...rollbackState.cachedReadFilePaths]; this.persistedPendingStateLoaded = rollbackState.persistedPendingStateLoaded; + if (!persist) return Promise.resolve(); + if (rollbackState.postCompactionAttachmentsPending) { return this.persistPendingStateBestEffort( this.cachedFileDiffs, @@ -675,7 +711,7 @@ export class CompactionHandler { undefined, undefined, publication - ); + ).then(() => undefined); } else { return this.deletePersistedPendingStateBestEffort(publication); } @@ -688,7 +724,8 @@ export class CompactionHandler { boundaryMessageId?: string, previousState?: PersistedPostCompactionStateV1, publication?: ContinuousCompactionPublication - ): Promise { + ): Promise { + let receipt: PendingStateWriteReceipt | undefined; try { for (const snapshot of loadedSkills) { assert(snapshot.name.trim().length > 0, "loaded skill snapshot name must not be empty"); @@ -705,10 +742,17 @@ export class CompactionHandler { // Freeze the admitted snapshot, then order writes AND unlinks. A held rollback // must not erase a newer compaction's pending state after that compaction completes. - const serialized = JSON.stringify(persisted); + const serialized = JSON.stringify({ ...persisted, writeId: randomUUID() }); await this.enqueuePendingStateWrite(async () => { await fsPromises.mkdir(this.sessionDir, { recursive: true }); - await fsPromises.writeFile(this.postCompactionStatePath, serialized); + const previous = await fsPromises + .readFile(this.postCompactionStatePath, "utf8") + .catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + }); + receipt = { serialized, previous }; + await writeFileAtomic(this.postCompactionStatePath, serialized); }, publication); } catch (error) { log.warn("Failed to persist post-compaction state", { @@ -716,6 +760,61 @@ export class CompactionHandler { error: getErrorMessage(error), }); } + return receipt; + } + + private async cleanupProvisionalPendingState( + preparation: PendingStatePreparation | undefined, + publication: ContinuousCompactionPublication | undefined, + rollbackState?: HeartbeatResetRollbackState + ): Promise { + if (!preparation) return; + const receipt = preparation.write; + try { + await this.enqueuePendingStateWrite(() => + this.historyService.withCompactionPublicationLock( + this.workspaceId, + publication, + async (current) => { + const exactWrite = + receipt !== undefined && + (await fsPromises.readFile(this.postCompactionStatePath, "utf8").then( + (raw) => raw === receipt.serialized, + () => false + )); + if (this.pendingStatePreparation === preparation) { + this.pendingStatePreparation = undefined; + if ((!receipt || exactWrite) && current && rollbackState) { + // This variant restores memory only; no nested pending/history lock. + await this.restoreHeartbeatResetRollbackState(publication, rollbackState, false); + } else { + this.postCompactionAttachmentsPending = false; + this.cachedFileDiffs = []; + this.cachedLoadedSkills = []; + this.cachedReadFilePaths = []; + this.pendingStateBoundaryMessageId = undefined; + this.persistedPendingStateLoaded = false; + if (this.heartbeatResetRollbackState === rollbackState) + this.heartbeatResetRollbackState = null; + } + } + if (!receipt || !exactWrite) return; + // Cleanup owns the exact write even after Stop. Only a still-current + // epoch may restore prior context; reset/edit must never resurrect it. + if (current && receipt.previous !== undefined) { + await writeFileAtomic(this.postCompactionStatePath, receipt.previous); + } else { + await fsPromises.unlink(this.postCompactionStatePath); + } + } + ) + ); + } catch (error) { + log.warn("Failed to clean provisional post-compaction state", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } } async preparePendingStateFromMessages( @@ -723,8 +822,13 @@ export class CompactionHandler { boundaryMessageId?: string, previousState?: PersistedPostCompactionStateV1, publication?: ContinuousCompactionPublication - ): Promise { + ): Promise { + // Memory ownership starts before the queued write: B may prepare while A's + // exact-file cleanup still holds the lock that B's write is waiting for. + const preparation: PendingStatePreparation = {}; + this.pendingStatePreparation = preparation; await this.loadPersistedPendingStateIfNeeded(publication); + if (this.pendingStatePreparation !== preparation) return preparation; this.pendingStateBoundaryMessageId = boundaryMessageId; const latestCompactionEpochMessages = sliceMessagesFromLatestCompactionBoundary(messages); @@ -746,7 +850,7 @@ export class CompactionHandler { // Persist pending state before append so pre-boundary diffs survive crashes/restarts. // Best-effort: boundary creation must not fail just because persistence fails. - await this.persistPendingStateBestEffort( + preparation.write = await this.persistPendingStateBestEffort( this.cachedFileDiffs, this.cachedLoadedSkills, this.cachedReadFilePaths, @@ -754,6 +858,7 @@ export class CompactionHandler { previousState, publication ); + return preparation; } async withContinuousPendingState( @@ -863,7 +968,13 @@ export class CompactionHandler { const messages = historyResult.data; await this.loadPersistedPendingStateIfNeeded(publication); const rollbackState = this.captureHeartbeatResetRollbackState(messages); - await this.preparePendingStateFromMessages(messages, undefined, undefined, publication); + const summaryMessageId = createCompactionSummaryMessageId(); + const pendingStateReceipt = await this.preparePendingStateFromMessages( + messages, + summaryMessageId, + undefined, + publication + ); const nextCompactionEpoch = getNextCompactionEpoch(messages); assert( @@ -871,23 +982,18 @@ export class CompactionHandler { "heartbeat reset boundary must compute a positive compaction epoch" ); - const summaryMessage = createMuxMessage( - createCompactionSummaryMessageId(), - "assistant", - params.boundaryText, - { - timestamp: Date.now(), - synthetic: true, - uiVisible: true, - compacted: "heartbeat", - compactionEpoch: nextCompactionEpoch, - compactionBoundary: true, - muxMetadata: { - type: "compaction-summary", - pendingFollowUp: params.pendingFollowUp, - }, - } - ); + const summaryMessage = createMuxMessage(summaryMessageId, "assistant", params.boundaryText, { + timestamp: Date.now(), + synthetic: true, + uiVisible: true, + compacted: "heartbeat", + compactionEpoch: nextCompactionEpoch, + compactionBoundary: true, + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: params.pendingFollowUp, + }, + }); assert( summaryMessage.metadata?.compacted === "heartbeat", @@ -916,9 +1022,9 @@ export class CompactionHandler { }, } ); - if (skipped) return Err("Heartbeat publication was invalidated"); - if (!persistenceResult.success) { - await this.restoreHeartbeatResetRollbackState(publication, rollbackState); + if (skipped || !persistenceResult.success) { + await this.cleanupProvisionalPendingState(pendingStateReceipt, publication, rollbackState); + if (skipped || persistenceResult.success) return Err("Heartbeat publication was invalidated"); return Err(`Failed to append heartbeat reset boundary: ${persistenceResult.error}`); } @@ -1447,8 +1553,6 @@ export class CompactionHandler { // does not re-inject stale pre-boundary edits after subsequent compactions. // If boundary markers are malformed, slicing self-heals by falling back to // full history instead of crashing or dropping all diffs. - await this.preparePendingStateFromMessages(messages, undefined, undefined, publication); - const nextCompactionEpoch = getNextCompactionEpoch(messages); assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); @@ -1524,6 +1628,12 @@ export class CompactionHandler { muxMetadata: summaryMuxMetadata, } ); + const pendingStateReceipt = await this.preparePendingStateFromMessages( + messages, + summaryMessage.id, + undefined, + publication + ); if (persistedSummaryHistorySequence !== undefined) { summaryMessage.metadata = { ...(summaryMessage.metadata ?? {}), @@ -1591,11 +1701,10 @@ export class CompactionHandler { skipped = true; }, }); - if (skipped) return Err("Compaction publication was invalidated"); - if (!persistenceResult.success) { - this.cachedFileDiffs = []; - this.cachedLoadedSkills = []; - await this.deletePersistedPendingStateBestEffort(publication); + if (skipped || !persistenceResult.success) { + await this.cleanupProvisionalPendingState(pendingStateReceipt, publication); + if (skipped || persistenceResult.success) + return Err("Compaction publication was invalidated"); const operation = preservedTailCopies.length > 0 ? "commit boundary with preserved tail" diff --git a/src/node/services/continuousCompactor.test.ts b/src/node/services/continuousCompactor.test.ts index eef837729bd..7885552607a 100644 --- a/src/node/services/continuousCompactor.test.ts +++ b/src/node/services/continuousCompactor.test.ts @@ -424,9 +424,10 @@ describe("ContinuousCompactor", () => { const release = deferred(); const original = handler.preparePendingStateFromMessages.bind(handler); spyOn(handler, "preparePendingStateFromMessages").mockImplementation(async (...args) => { - await original(...args); + const preparation = await original(...args); entered.resolve(); await release.promise; + return preparation; }); const applying = compactor.observe(context.thresholdPercent, context); await entered.promise; diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 6b2378b5886..16077ce83a6 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -992,6 +992,85 @@ describe("HistoryService", () => { }); describe("context publication fencing", () => { + it.each( + ["active", "archived"].flatMap((target) => + [false, true].map((afterAdvance) => ({ target, afterAdvance })) + ) + )( + "edit truncation refuses unfenced writes ($target, after advance=$afterAdvance)", + async ({ target, afterAdvance }) => { + const workspaceId = "edit-generation-failure"; + await service.appendToHistory( + workspaceId, + createMuxMessage("target", "user", "Original context") + ); + if (target === "archived") { + await service.appendToHistory( + workspaceId, + createMuxMessage("boundary", "assistant", "Summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + }) + ); + } + await service.appendToHistory( + workspaceId, + createMuxMessage("tail", "assistant", "Original answer") + ); + const before = await collectFullHistory(service, workspaceId); + const journal = service.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + const invalidate = journal.invalidateUnderHistoryLock.bind(journal); + const failure = spyOn(journal, "invalidateUnderHistoryLock").mockImplementationOnce( + async (...args) => { + if (afterAdvance) await invalidate(...args); + throw new Error("edit generation unavailable"); + } + ); + try { + const result = await service.truncateAfterMessage(workspaceId, "target"); + expect(!result.success && result.error).toContain("edit generation unavailable"); + expect(await collectFullHistory(new HistoryService(config), workspaceId)).toEqual(before); + expect((await journal.captureGeneration()) === generation).toBe(!afterAdvance); + expect((await service.truncateAfterMessage(workspaceId, "target")).success).toBe(true); + expect(await collectFullHistory(service, workspaceId)).toEqual([]); + } finally { + failure.mockRestore(); + } + } + ); + + it.each(["missing", "keep tail", "display tail"])( + "edit truncation preserves publication for %s", + async (cut) => { + const workspaceId = "edit-no-context-cut"; + await service.appendToHistory( + workspaceId, + createMuxMessage("target", "user", "Current context") + ); + if (cut === "display tail") { + await service.appendToHistory( + workspaceId, + createMuxMessage("display", "user", "Display only", { + muxMetadata: { type: "workflow-trigger-display", rawCommand: "/wf", runId: "run" }, + }) + ); + } + const journal = service.getContinuousCompactionJournal(workspaceId); + const generation = await journal.captureGeneration(); + await fs.writeFile(journal.path, "current journal"); + const result = await service.truncateAfterMessage( + workspaceId, + cut === "missing" ? "missing" : "target", + { keepTargetMessage: true } + ); + expect(result.success).toBe(cut !== "missing"); + expect(await journal.captureGeneration()).toBe(generation); + expect(await fs.readFile(journal.path, "utf8")).toBe("current journal"); + } + ); + it.each([false, true])( "reset batches validate their original publication before advancing it (admitted=%s)", async (admitted) => { diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 34b83627802..ae41b9fbdfd 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2399,17 +2399,27 @@ export class HistoryService { workspaceId: string, publication: ContinuousCompactionPublication, write: () => Promise + ): Promise { + return this.withCompactionPublicationLock(workspaceId, publication, async (current) => { + if (current) await write(); + }); + } + + /** Exact pending-write cleanup may run after invalidation, but may restore only current state. */ + async withCompactionPublicationLock( + workspaceId: string, + publication: ContinuousCompactionPublication | undefined, + operation: (current: boolean) => Promise ): Promise { await this.fileLocks.withLock(workspaceId, () => this.withHistoryWriteFileLock(workspaceId, async () => { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return; - if ( - await this.getContinuousCompactionJournal( - workspaceId - ).isPublicationCurrentUnderHistoryLock(publication) - ) { - await write(); - } + await operation( + !publication || + (await this.getContinuousCompactionJournal( + workspaceId + ).isPublicationCurrentUnderHistoryLock(publication)) + ); }) ); } @@ -3899,6 +3909,11 @@ export class HistoryService { const archiveMaxSeq = await this.getArchiveTailMaxSequence(workspaceId); + // Edit/fork cuts discard captured provider context across backend instances. + // A declined target or keep-target-at-tail no-op must retain its publication. + if (hasProviderEligibleMessages(filterWorkflowDisplayOnlyMessages(removedMessages))) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } // Atomic write prevents corruption if app crashes mid-write await writeFileAtomic(historyPath, historyEntries); @@ -3979,6 +3994,9 @@ export class HistoryService { if (lastArchiveRow && lastArchiveRow.raw.at(-1) !== 10 && activeEpochRows.length > 0) { archiveRows.push({ raw: Buffer.from("\n"), message: undefined }); } + if (hasProviderEligibleMessages(filterWorkflowDisplayOnlyMessages(removedMessages))) { + await this.getContinuousCompactionJournal(workspaceId).invalidateUnderHistoryLock(); + } await this.rewriteHistoryFilesUnlocked( workspaceId, null, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index da8c064f828..799b3e6ce06 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7388,13 +7388,18 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { ["absent", "unresolved", "narrowed", "manual before publication", "newer Stop during clear"] .map((initial) => ({ producer, initial, operation: "full clear" })) .concat( - ["reset", "destructive replacement", "active prefix trim", "sealed prefix trim"].map( - (operation) => ({ - producer, - initial: "absent", - operation, - }) - ) + [ + "reset", + "destructive replacement", + "active prefix trim", + "sealed prefix trim", + "active edit", + "archived edit", + ].map((operation) => ({ + producer, + initial: "absent", + operation, + })) ) ) )( @@ -7411,7 +7416,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { runtimeConfig: { type: "local" }, }); const followUp = { text: "Continue discarded work", ...options }; - if (operation === "sealed prefix trim") { + if (operation === "sealed prefix trim" || operation === "archived edit") { await historyService.appendToHistory( workspaceId, createMuxMessage("sealed", "user", "Sealed context") @@ -7491,21 +7496,37 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { } } const mutation = - operation === "reset" - ? await workspaceService.resetContext(workspaceId) - : operation === "destructive replacement" - ? await workspaceService.replaceHistory( - workspaceId, - createMuxMessage("replacement", "assistant", "Fresh context") - ) - : await workspaceService.truncateHistory( + operation === "active edit" || operation === "archived edit" + ? await (async () => { + const editor = await createAgentSessionHarness({ workspaceId, - operation === "active prefix trim" - ? 0.25 - : operation === "sealed prefix trim" - ? 0.01 - : undefined - ); + config, + historyService: new HistoryService(config), + }); + try { + return await editor.session.sendMessage("Edited context", { + ...options, + editMessageId: operation === "active edit" ? "old-request" : "sealed", + }); + } finally { + await editor.session.dispose(); + } + })() + : operation === "reset" + ? await workspaceService.resetContext(workspaceId) + : operation === "destructive replacement" + ? await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement", "assistant", "Fresh context") + ) + : await workspaceService.truncateHistory( + workspaceId, + operation === "active prefix trim" + ? 0.25 + : operation === "sealed prefix trim" + ? 0.01 + : undefined + ); expect(mutation.success).toBe(true); if (operation === "active prefix trim") { const trimmed = await historyService.getHistoryFromLatestBoundary(workspaceId); @@ -7541,7 +7562,12 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { published.success && published.data.some((row) => row.metadata?.muxMetadata?.type === "compaction-summary") ).toBe(operation === "sealed prefix trim"); - if (operation === "sealed prefix trim") return; + if ( + operation === "sealed prefix trim" || + operation === "active edit" || + operation === "archived edit" + ) + return; fresh = await createAgentSessionHarness({ workspaceId, config,