From 75b9cb8c506af9b2410addf5999a16f51d8605b9 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:31:46 +0500 Subject: [PATCH 01/38] test: add RED durable retry cancellation regressions --- .../durable-retry-cancellation.test.ts | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 test/application/durable-retry-cancellation.test.ts diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts new file mode 100644 index 0000000..c538360 --- /dev/null +++ b/test/application/durable-retry-cancellation.test.ts @@ -0,0 +1,300 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { Agent, Goal, Lease, Session, Task } from '@mindrail/contracts'; +import { describe, expect, it } from 'vitest'; + +import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts'; +import type { ApplicationDispatcher } from '../../src/application/ports.ts'; +import type { + CancelGoalResult, + CancelTaskResult, + FailTaskResult, +} from '../../src/runtime/in-memory-control-plane.ts'; +import { workspace } from '../persistence/fixtures.ts'; +import { openPersistence } from '../persistence/setup.ts'; +import { canonicalDomainValidator } from '../runtime/canonical-domain-validator.ts'; + +type CommandResponse = Awaited>; + +function databasePath(): string { + return join(mkdtempSync(join(tmpdir(), 'mindrail-durable-retry-cancel-')), 'runtime.sqlite'); +} + +async function openDispatcher(path: string, prefix: string, now: Date) { + const opened = await openPersistence(path); + let sequence = 0; + return { + ...opened, + dispatcher: createDurableApplicationDispatcher({ + persistence: opened.persistence, + now: () => new Date(now), + idFactory: (kind) => `${prefix}-${kind}-${++sequence}`, + leaseDurationMs: 120_000, + sessionTimeoutMs: 300_000, + validateCanonicalDomainRecord: canonicalDomainValidator, + }), + }; +} + +function success(response: CommandResponse): T { + expect(response).not.toHaveProperty('error'); + if ('error' in response) throw new Error(`Expected success, got ${response.error.code}.`); + return response.result as T; +} + +async function seedClaimedTask(dispatcher: ApplicationDispatcher, prefix: string) { + const systemActor = { type: 'system' as const, id: 'system-1' }; + const agent = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RegisterAgent', + commandId: `${prefix}-register`, + workspaceId: 'ws-a', + actor: systemActor, + displayName: 'Cancellation worker', + capabilities: ['repo.write'], + }), + ); + const agentActor = { type: 'agent' as const, id: agent.id }; + const session = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'StartSession', + commandId: `${prefix}-session`, + workspaceId: 'ws-a', + actor: systemActor, + agentId: agent.id, + }), + ); + const goal = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateGoal', + commandId: `${prefix}-goal`, + workspaceId: 'ws-a', + actor: systemActor, + title: 'Retry cancellation goal', + objective: 'Exercise durable controller transitions.', + successCriteria: ['Controller transitions survive restart.'], + }), + ); + const task = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: `${prefix}-task`, + workspaceId: 'ws-a', + actor: systemActor, + goalId: goal.id, + title: 'Retry cancellation task', + objective: 'Exercise retry and cancellation.', + acceptanceCriteria: ['State is durable.'], + requiredCapabilities: ['repo.write'], + dependencyTaskIds: [], + }), + ); + const claim = success<{ task: Task; lease: Lease }>( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: `${prefix}-claim`, + workspaceId: 'ws-a', + actor: agentActor, + taskId: task.id, + sessionId: session.id, + expectedTaskRevision: task.revision, + }), + ); + return { systemActor, agentActor, agent, session, goal, task, claim }; +} + +describe('durable retry and cancellation', () => { + it('retries a durably failed Task through controller authority and replays after restart', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T18:00:00.000Z'); + let app = await openDispatcher(path, 'retry-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'retry'); + const failed = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'FailTask', + commandId: 'retry-fail', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'execution.failed', summary: 'Retryable deterministic failure.' }, + summary: 'Retry this work.', + evidence: [], + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'retry-after', now); + const retryResponse = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-command', + correlationId: 'retry-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + const retried = success(retryResponse); + expect(retried).toMatchObject({ status: 'ready', revision: failed.task.revision + 1 }); + expect(retried).not.toHaveProperty('statusReason'); + app.database.close(); + + app = await openDispatcher(path, 'retry-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-command', + correlationId: 'retry-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'retry-replay' }); + expect(success(replay)).toEqual(retried); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(retried); + app.database.close(); + }); + + it('cancels a running Task with its active Lease in one durable controller mutation', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T19:00:00.000Z'); + let app = await openDispatcher(path, 'cancel-task-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-task'); + app.database.close(); + + app = await openDispatcher(path, 'cancel-task-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-task-command', + correlationId: 'cancel-task-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Work is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.task).toMatchObject({ + status: 'cancelled', + revision: seeded.claim.task.revision + 1, + }); + expect(cancelled.lease).toMatchObject({ + status: 'revoked', + revision: seeded.claim.lease.revision + 1, + fencingToken: seeded.claim.lease.fencingToken, + }); + app.database.close(); + + app = await openDispatcher(path, 'cancel-task-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-task-command', + correlationId: 'cancel-task-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Work is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-task-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual(cancelled.lease); + app.database.close(); + }); + + it('cancels a Goal, its cancellable Tasks, active Lease, and receipt atomically', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T20:00:00.000Z'); + let app = await openDispatcher(path, 'cancel-goal-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal'); + const secondTask = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-goal-second-task', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Second cancellable task', + objective: 'Remain unclaimed until cancellation.', + acceptanceCriteria: ['Cancellation is durable.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-command', + correlationId: 'cancel-goal-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'The goal is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.goal).toMatchObject({ + status: 'cancelled', + revision: seeded.goal.revision + 1, + }); + expect(cancelled.tasks).toHaveLength(2); + expect(cancelled.tasks.every((task) => task.status === 'cancelled')).toBe(true); + expect(cancelled.leases).toEqual([ + expect.objectContaining({ + id: seeded.claim.lease.id, + status: 'revoked', + fencingToken: seeded.claim.lease.fencingToken, + }), + ]); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-command', + correlationId: 'cancel-goal-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'The goal is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-goal-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(cancelled.goal); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual( + cancelled.tasks.find((task) => task.id === seeded.task.id), + ); + expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual( + cancelled.tasks.find((task) => task.id === secondTask.id), + ); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual( + cancelled.leases[0], + ); + app.database.close(); + }); +}); From a5957d8118f66cbb46f55c4cc0068ba65aca3ec2 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:36:40 +0500 Subject: [PATCH 02/38] chore: add one-time durable retry cancellation patcher --- .../apply-durable-retry-cancellation-green.py | 664 ++++++++++++++++++ 1 file changed, 664 insertions(+) create mode 100644 scripts/apply-durable-retry-cancellation-green.py diff --git a/scripts/apply-durable-retry-cancellation-green.py b/scripts/apply-durable-retry-cancellation-green.py new file mode 100644 index 0000000..175a449 --- /dev/null +++ b/scripts/apply-durable-retry-cancellation-green.py @@ -0,0 +1,664 @@ +from pathlib import Path + + +def replace_once(path: str, before: str, after: str) -> None: + file = Path(path) + text = file.read_text() + count = text.count(before) + if count != 1: + raise RuntimeError(f'{path}: expected one marker, found {count}') + file.write_text(text.replace(before, after, 1)) + + +replace_once( + 'src/persistence/ports.ts', + '''export interface TaskOutcomeCommitValue { + task: Task; + lease: Lease; + checkpoint: Checkpoint; +} + +export interface DurableRuntimePersistence {''', + '''export interface TaskOutcomeCommitValue { + task: Task; + lease: Lease; + checkpoint: Checkpoint; +} + +export interface CancelTaskCommitInput { + workspaceId: string; + task: Task; + lease?: Lease; + expectedTaskRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelTaskCommitValue { + task: Task; + lease?: Lease; +} + +export interface CancelGoalCommitInput { + workspaceId: string; + goal: Goal; + tasks: Task[]; + leases: Lease[]; + expectedGoalRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelGoalCommitValue { + goal: Goal; + tasks: Task[]; + leases: Lease[]; +} + +export interface DurableRuntimePersistence {''', +) + +replace_once( + 'src/persistence/ports.ts', + ''' resumeTask(input: { + task: Task; + expectedRevision: number; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise>; + appendAuditEvent(input: { auditEvent: AuditEvent }): Promise;''', + ''' resumeTask(input: { + task: Task; + expectedRevision: number; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise>; + retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise>; + cancelTask(input: CancelTaskCommitInput): Promise>; + cancelGoal(input: CancelGoalCommitInput): Promise>; + appendAuditEvent(input: { auditEvent: AuditEvent }): Promise;''', +) + +replace_once( + 'src/persistence/cloudflare/d1-runtime-persistence.ts', + ''' type ClaimTaskCommitInput, + type ClaimTaskCommitValue, + type CommandReceiptInput,''', + ''' type CancelGoalCommitInput, + type CancelGoalCommitValue, + type CancelTaskCommitInput, + type CancelTaskCommitValue, + type ClaimTaskCommitInput, + type ClaimTaskCommitValue, + type CommandReceiptInput,''', +) + +methods = r''' async retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise> { + const { task } = input; + this.assertCanonical('Task', task); + this.assertRelatedAudit(task.workspaceId, input.auditEvent); + this.assertReceipt(task.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'retry Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); + + return this.coordinator.runSerialized(task.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(task.workspaceId, task.id); + if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); + if (current.revision !== input.expectedRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, + ); + } + const goal = await this.getGoal(task.workspaceId, current.goalId); + if (!goal) { + throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); + } + if (goal.status !== 'active' || current.status !== 'failed') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${task.id} cannot be retried from current durable state.`, + ); + } + if ( + await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs) + ) { + throw new PersistenceError('CONFLICT', `Task ${task.id} still has active execution authority.`); + } + + const expected: Task = { + ...clone(current), + status: 'ready', + revision: input.expectedRevision + 1, + updatedAt: input.now, + }; + delete expected.statusReason; + if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { + throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed'`, + ) + .bind( + task.revision, + task.status, + timestampMs(task.updatedAt, 'Task.updatedAt'), + serializeJson(task, 'Task'), + task.workspaceId, + task.id, + input.expectedRevision, + ), + ]; + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'retry Task'); + if (changes(results[0]!) !== 1) { + throw new PersistenceError('REVISION_MISMATCH', `Task ${task.id} lost its retry race.`); + } + return { kind: 'committed', value: clone(task) }; + }); + } + + async cancelTask( + input: CancelTaskCommitInput, + ): Promise> { + this.assertCanonical('Task', input.task); + if (input.lease) this.assertCanonical('Lease', input.lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(input.workspaceId, input.task.id); + if (!current) { + throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); + } + if (current.revision !== input.expectedTaskRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, + ); + } + if (!isCancellableTaskStatus(current.status)) { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${current.id} cannot be cancelled from ${current.status}.`, + ); + } + if (!input.task.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); + } + const expectedTask: Task = { + ...clone(current), + status: 'cancelled', + statusReason: clone(input.task.statusReason), + revision: input.expectedTaskRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); + } + + const effectiveLease = await this.getEffectiveActiveLease( + input.workspaceId, + current.id, + nowMs, + cutoffMs, + ); + if ((effectiveLease === undefined) !== (input.lease === undefined)) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); + } + if (effectiveLease && input.lease) { + const expectedLease: Lease = { + ...clone(effectiveLease), + status: 'revoked', + revision: effectiveLease.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.lease, 'Lease') !== + serializeJson(expectedLease, 'expected CancelTask Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + input.task.revision, + input.task.status, + timestampMs(input.task.updatedAt, 'Task.updatedAt'), + serializeJson(input.task, 'Task'), + input.workspaceId, + input.task.id, + input.expectedTaskRevision, + current.status, + ), + ]; + if (effectiveLease && input.lease) { + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + input.lease.revision, + input.lease.status, + timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), + serializeJson(input.lease, 'Lease'), + input.workspaceId, + input.lease.id, + effectiveLease.revision, + effectiveLease.fencingToken, + ), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'cancel Task'); + if (changes(results[0]!) !== 1) { + throw new PersistenceError('REVISION_MISMATCH', `Task ${current.id} lost its cancel race.`); + } + if (effectiveLease && changes(results[1]!) !== 1) { + throw new PersistenceError('STALE_AUTHORITY', `Task ${current.id} Lease lost its cancel race.`); + } + return { + kind: 'committed', + value: { + task: clone(input.task), + ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), + }, + }; + }); + } + + async cancelGoal( + input: CancelGoalCommitInput, + ): Promise> { + this.assertCanonical('Goal', input.goal); + for (const task of input.tasks) this.assertCanonical('Task', task); + for (const lease of input.leases) this.assertCanonical('Lease', lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Goal now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); + if (!currentGoal) { + throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); + } + if (currentGoal.revision !== input.expectedGoalRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, + ); + } + if (currentGoal.status !== 'active') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Goal ${currentGoal.id} is already terminal.`, + ); + } + const expectedGoal: Goal = { + ...clone(currentGoal), + status: 'cancelled', + revision: input.expectedGoalRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); + } + + const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); + if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); + } + const effectiveLeases: Lease[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id); + if (!output || !output.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); + } + const expectedTask: Task = { + ...clone(currentTask), + status: 'cancelled', + statusReason: clone(output.statusReason), + revision: currentTask.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); + } + const effective = await this.getEffectiveActiveLease( + input.workspaceId, + currentTask.id, + nowMs, + cutoffMs, + ); + if (effective) effectiveLeases.push(effective); + } + + const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); + if (outputLeases.size !== input.leases.length || input.leases.length !== effectiveLeases.length) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id); + if (!output) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); + } + const expectedLease: Lease = { + ...clone(effective), + status: 'revoked', + revision: effective.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Lease') !== serializeJson(expectedLease, 'expected CancelGoal Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + input.goal.revision, + input.goal.status, + timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), + serializeJson(input.goal, 'Goal'), + input.workspaceId, + input.goal.id, + input.expectedGoalRevision, + ), + ]; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Task.updatedAt'), + serializeJson(output, 'Task'), + input.workspaceId, + output.id, + currentTask.revision, + currentTask.status, + ), + ); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id)!; + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Lease.updatedAt'), + serializeJson(output, 'Lease'), + input.workspaceId, + output.id, + effective.revision, + effective.fencingToken, + ), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'cancel Goal'); + const mutationCount = 1 + cancellable.length + effectiveLeases.length; + for (let index = 0; index < mutationCount; index += 1) { + if (changes(results[index]!) !== 1) { + throw new PersistenceError('CONFLICT', `Goal ${currentGoal.id} lost its cancel race.`); + } + } + return { + kind: 'committed', + value: { + goal: clone(input.goal), + tasks: clone(input.tasks), + leases: clone(input.leases), + }, + }; + }); + } + +''' +replace_once( + 'src/persistence/cloudflare/d1-runtime-persistence.ts', + ' private async commitTaskOutcome(', + methods + ' private async commitTaskOutcome(', +) + +replace_once( + 'src/persistence/cloudflare/d1-runtime-persistence.ts', + ''' private async getActiveLease(workspaceId: string, taskId: string): Promise { + return this.readRecord(''', + ''' private async getEffectiveActiveLease( + workspaceId: string, + taskId: string, + nowMs: number, + sessionCutoffMs: number, + ): Promise { + const lease = await this.getActiveLease(workspaceId, taskId); + if (!lease || timestampMs(lease.expiresAt, 'Lease.expiresAt') <= nowMs) return undefined; + const session = await this.getSession(workspaceId, lease.sessionId); + if ( + !session || + session.status !== 'active' || + timestampMs(session.lastSeenAt, 'Session.lastSeenAt') <= sessionCutoffMs + ) { + return undefined; + } + return lease; + } + + private async getActiveLease(workspaceId: string, taskId: string): Promise { + return this.readRecord(''', +) + +replace_once( + 'src/persistence/cloudflare/d1-runtime-persistence.ts', + '''function serializeJson(value: unknown, label: string, maxBytes?: number): string {''', + '''function isCancellableTaskStatus(status: Task['status']): boolean { + return status === 'pending' || status === 'ready' || status === 'running' || status === 'blocked'; +} + +function serializeJson(value: unknown, label: string, maxBytes?: number): string {''', +) + +replace_once( + 'src/application/durable-dispatcher.ts', + ''' type BlockTaskResult, + type ClaimTaskResult, + type CompleteTaskResult, + type EndSessionResult, + type FailTaskResult,''', + ''' type BlockTaskResult, + type CancelGoalResult, + type CancelTaskResult, + type ClaimTaskResult, + type CompleteTaskResult, + type EndSessionResult, + type FailTaskResult,''', +) + +replace_once( + 'src/application/durable-dispatcher.ts', + ''' command.command === 'FailTask' || + command.command === 'BlockTask' || + command.command === 'ResumeTask' + );''', + ''' command.command === 'FailTask' || + command.command === 'BlockTask' || + command.command === 'ResumeTask' || + command.command === 'RetryTask' || + command.command === 'CancelTask' || + command.command === 'CancelGoal' + );''', +) + +replace_once( + 'src/application/durable-dispatcher.ts', + ''' case 'ResumeTask': { + const result = semanticResponse.result as Task; + return resolveMutationResult( + command, + await options.persistence.resumeTask({ + task: result, + expectedRevision: command.expectedTaskRevision, + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + default:''', + ''' case 'ResumeTask': { + const result = semanticResponse.result as Task; + return resolveMutationResult( + command, + await options.persistence.resumeTask({ + task: result, + expectedRevision: command.expectedTaskRevision, + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'RetryTask': { + const result = semanticResponse.result as Task; + const now = result.updatedAt; + return resolveMutationResult( + command, + await options.persistence.retryTask({ + task: result, + expectedRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelTask': { + const result = semanticResponse.result as CancelTaskResult; + const now = result.task.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelTask({ + workspaceId: command.workspaceId, + task: result.task, + ...(result.lease === undefined ? {} : { lease: result.lease }), + expectedTaskRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelGoal': { + const result = semanticResponse.result as CancelGoalResult; + const now = result.goal.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelGoal({ + workspaceId: command.workspaceId, + goal: result.goal, + tasks: result.tasks, + leases: result.leases, + expectedGoalRevision: command.expectedGoalRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + default:''', +) From 3804cf12589c518485bfdc874d3ec5ab4c90bd9e Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:36:52 +0500 Subject: [PATCH 03/38] chore: add one-time durable retry cancellation green workflow --- ...-durable-retry-cancellation-green-once.yml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/apply-durable-retry-cancellation-green-once.yml diff --git a/.github/workflows/apply-durable-retry-cancellation-green-once.yml b/.github/workflows/apply-durable-retry-cancellation-green-once.yml new file mode 100644 index 0000000..38e3b15 --- /dev/null +++ b/.github/workflows/apply-durable-retry-cancellation-green-once.yml @@ -0,0 +1,36 @@ +name: Apply durable retry cancellation GREEN once + +on: + push: + branches: + - feature/durable-retry-cancellation + +permissions: + contents: write + +jobs: + apply: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: feature/durable-retry-cancellation + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/apply-durable-retry-cancellation-green.py + - run: pnpm exec prettier --write src/application/durable-dispatcher.ts src/persistence/ports.ts src/persistence/cloudflare/d1-runtime-persistence.ts + - run: git rm .github/workflows/apply-durable-retry-cancellation-green-once.yml scripts/apply-durable-retry-cancellation-green.py + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts + - run: pnpm check + - run: pnpm test:coverage + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/application/durable-dispatcher.ts src/persistence/ports.ts src/persistence/cloudflare/d1-runtime-persistence.ts + git commit -m "feat: add durable retry cancellation" + git push origin HEAD:feature/durable-retry-cancellation From c86bbb20f9c8b0fb6bcc5f962516a13cb684f43c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:37:38 +0000 Subject: [PATCH 04/38] feat: add durable retry cancellation --- ...-durable-retry-cancellation-green-once.yml | 36 - .../apply-durable-retry-cancellation-green.py | 664 ------------------ src/application/durable-dispatcher.ts | 69 +- .../cloudflare/d1-runtime-persistence.ts | 419 +++++++++++ src/persistence/ports.ts | 44 ++ 5 files changed, 531 insertions(+), 701 deletions(-) delete mode 100644 .github/workflows/apply-durable-retry-cancellation-green-once.yml delete mode 100644 scripts/apply-durable-retry-cancellation-green.py diff --git a/.github/workflows/apply-durable-retry-cancellation-green-once.yml b/.github/workflows/apply-durable-retry-cancellation-green-once.yml deleted file mode 100644 index 38e3b15..0000000 --- a/.github/workflows/apply-durable-retry-cancellation-green-once.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Apply durable retry cancellation GREEN once - -on: - push: - branches: - - feature/durable-retry-cancellation - -permissions: - contents: write - -jobs: - apply: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - ref: feature/durable-retry-cancellation - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/apply-durable-retry-cancellation-green.py - - run: pnpm exec prettier --write src/application/durable-dispatcher.ts src/persistence/ports.ts src/persistence/cloudflare/d1-runtime-persistence.ts - - run: git rm .github/workflows/apply-durable-retry-cancellation-green-once.yml scripts/apply-durable-retry-cancellation-green.py - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts - - run: pnpm check - - run: pnpm test:coverage - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/application/durable-dispatcher.ts src/persistence/ports.ts src/persistence/cloudflare/d1-runtime-persistence.ts - git commit -m "feat: add durable retry cancellation" - git push origin HEAD:feature/durable-retry-cancellation diff --git a/scripts/apply-durable-retry-cancellation-green.py b/scripts/apply-durable-retry-cancellation-green.py deleted file mode 100644 index 175a449..0000000 --- a/scripts/apply-durable-retry-cancellation-green.py +++ /dev/null @@ -1,664 +0,0 @@ -from pathlib import Path - - -def replace_once(path: str, before: str, after: str) -> None: - file = Path(path) - text = file.read_text() - count = text.count(before) - if count != 1: - raise RuntimeError(f'{path}: expected one marker, found {count}') - file.write_text(text.replace(before, after, 1)) - - -replace_once( - 'src/persistence/ports.ts', - '''export interface TaskOutcomeCommitValue { - task: Task; - lease: Lease; - checkpoint: Checkpoint; -} - -export interface DurableRuntimePersistence {''', - '''export interface TaskOutcomeCommitValue { - task: Task; - lease: Lease; - checkpoint: Checkpoint; -} - -export interface CancelTaskCommitInput { - workspaceId: string; - task: Task; - lease?: Lease; - expectedTaskRevision: number; - now: string; - sessionCutoff: string; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; -} - -export interface CancelTaskCommitValue { - task: Task; - lease?: Lease; -} - -export interface CancelGoalCommitInput { - workspaceId: string; - goal: Goal; - tasks: Task[]; - leases: Lease[]; - expectedGoalRevision: number; - now: string; - sessionCutoff: string; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; -} - -export interface CancelGoalCommitValue { - goal: Goal; - tasks: Task[]; - leases: Lease[]; -} - -export interface DurableRuntimePersistence {''', -) - -replace_once( - 'src/persistence/ports.ts', - ''' resumeTask(input: { - task: Task; - expectedRevision: number; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise>; - appendAuditEvent(input: { auditEvent: AuditEvent }): Promise;''', - ''' resumeTask(input: { - task: Task; - expectedRevision: number; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise>; - retryTask(input: { - task: Task; - expectedRevision: number; - now: string; - sessionCutoff: string; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise>; - cancelTask(input: CancelTaskCommitInput): Promise>; - cancelGoal(input: CancelGoalCommitInput): Promise>; - appendAuditEvent(input: { auditEvent: AuditEvent }): Promise;''', -) - -replace_once( - 'src/persistence/cloudflare/d1-runtime-persistence.ts', - ''' type ClaimTaskCommitInput, - type ClaimTaskCommitValue, - type CommandReceiptInput,''', - ''' type CancelGoalCommitInput, - type CancelGoalCommitValue, - type CancelTaskCommitInput, - type CancelTaskCommitValue, - type ClaimTaskCommitInput, - type ClaimTaskCommitValue, - type CommandReceiptInput,''', -) - -methods = r''' async retryTask(input: { - task: Task; - expectedRevision: number; - now: string; - sessionCutoff: string; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise> { - const { task } = input; - this.assertCanonical('Task', task); - this.assertRelatedAudit(task.workspaceId, input.auditEvent); - this.assertReceipt(task.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'retry Task now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); - - return this.coordinator.runSerialized(task.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const current = await this.getTask(task.workspaceId, task.id); - if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); - if (current.revision !== input.expectedRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, - ); - } - const goal = await this.getGoal(task.workspaceId, current.goalId); - if (!goal) { - throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); - } - if (goal.status !== 'active' || current.status !== 'failed') { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Task ${task.id} cannot be retried from current durable state.`, - ); - } - if ( - await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs) - ) { - throw new PersistenceError('CONFLICT', `Task ${task.id} still has active execution authority.`); - } - - const expected: Task = { - ...clone(current), - status: 'ready', - revision: input.expectedRevision + 1, - updatedAt: input.now, - }; - delete expected.statusReason; - if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { - throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); - } - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed'`, - ) - .bind( - task.revision, - task.status, - timestampMs(task.updatedAt, 'Task.updatedAt'), - serializeJson(task, 'Task'), - task.workspaceId, - task.id, - input.expectedRevision, - ), - ]; - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'retry Task'); - if (changes(results[0]!) !== 1) { - throw new PersistenceError('REVISION_MISMATCH', `Task ${task.id} lost its retry race.`); - } - return { kind: 'committed', value: clone(task) }; - }); - } - - async cancelTask( - input: CancelTaskCommitInput, - ): Promise> { - this.assertCanonical('Task', input.task); - if (input.lease) this.assertCanonical('Lease', input.lease); - this.assertRelatedAudit(input.workspaceId, input.auditEvent); - this.assertReceipt(input.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'cancel Task now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); - - return this.coordinator.runSerialized(input.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const current = await this.getTask(input.workspaceId, input.task.id); - if (!current) { - throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); - } - if (current.revision !== input.expectedTaskRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, - ); - } - if (!isCancellableTaskStatus(current.status)) { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Task ${current.id} cannot be cancelled from ${current.status}.`, - ); - } - if (!input.task.statusReason) { - throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); - } - const expectedTask: Task = { - ...clone(current), - status: 'cancelled', - statusReason: clone(input.task.statusReason), - revision: input.expectedTaskRevision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); - } - - const effectiveLease = await this.getEffectiveActiveLease( - input.workspaceId, - current.id, - nowMs, - cutoffMs, - ); - if ((effectiveLease === undefined) !== (input.lease === undefined)) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); - } - if (effectiveLease && input.lease) { - const expectedLease: Lease = { - ...clone(effectiveLease), - status: 'revoked', - revision: effectiveLease.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.lease, 'Lease') !== - serializeJson(expectedLease, 'expected CancelTask Lease') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); - } - } - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) - .bind( - input.task.revision, - input.task.status, - timestampMs(input.task.updatedAt, 'Task.updatedAt'), - serializeJson(input.task, 'Task'), - input.workspaceId, - input.task.id, - input.expectedTaskRevision, - current.status, - ), - ]; - if (effectiveLease && input.lease) { - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - AND fencing_token = ?`, - ) - .bind( - input.lease.revision, - input.lease.status, - timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), - serializeJson(input.lease, 'Lease'), - input.workspaceId, - input.lease.id, - effectiveLease.revision, - effectiveLease.fencingToken, - ), - ); - } - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'cancel Task'); - if (changes(results[0]!) !== 1) { - throw new PersistenceError('REVISION_MISMATCH', `Task ${current.id} lost its cancel race.`); - } - if (effectiveLease && changes(results[1]!) !== 1) { - throw new PersistenceError('STALE_AUTHORITY', `Task ${current.id} Lease lost its cancel race.`); - } - return { - kind: 'committed', - value: { - task: clone(input.task), - ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), - }, - }; - }); - } - - async cancelGoal( - input: CancelGoalCommitInput, - ): Promise> { - this.assertCanonical('Goal', input.goal); - for (const task of input.tasks) this.assertCanonical('Task', task); - for (const lease of input.leases) this.assertCanonical('Lease', lease); - this.assertRelatedAudit(input.workspaceId, input.auditEvent); - this.assertReceipt(input.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'cancel Goal now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); - - return this.coordinator.runSerialized(input.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); - if (!currentGoal) { - throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); - } - if (currentGoal.revision !== input.expectedGoalRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, - ); - } - if (currentGoal.status !== 'active') { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Goal ${currentGoal.id} is already terminal.`, - ); - } - const expectedGoal: Goal = { - ...clone(currentGoal), - status: 'cancelled', - revision: input.expectedGoalRevision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); - } - - const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); - const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); - const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); - if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); - } - const effectiveLeases: Lease[] = []; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id); - if (!output || !output.statusReason) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); - } - const expectedTask: Task = { - ...clone(currentTask), - status: 'cancelled', - statusReason: clone(output.statusReason), - revision: currentTask.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); - } - const effective = await this.getEffectiveActiveLease( - input.workspaceId, - currentTask.id, - nowMs, - cutoffMs, - ); - if (effective) effectiveLeases.push(effective); - } - - const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); - if (outputLeases.size !== input.leases.length || input.leases.length !== effectiveLeases.length) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); - } - for (const effective of effectiveLeases) { - const output = outputLeases.get(effective.id); - if (!output) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); - } - const expectedLease: Lease = { - ...clone(effective), - status: 'revoked', - revision: effective.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(output, 'Lease') !== serializeJson(expectedLease, 'expected CancelGoal Lease') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); - } - } - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE goals - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, - ) - .bind( - input.goal.revision, - input.goal.status, - timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), - serializeJson(input.goal, 'Goal'), - input.workspaceId, - input.goal.id, - input.expectedGoalRevision, - ), - ]; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id)!; - statements.push( - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) - .bind( - output.revision, - output.status, - timestampMs(output.updatedAt, 'Task.updatedAt'), - serializeJson(output, 'Task'), - input.workspaceId, - output.id, - currentTask.revision, - currentTask.status, - ), - ); - } - for (const effective of effectiveLeases) { - const output = outputLeases.get(effective.id)!; - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - AND fencing_token = ?`, - ) - .bind( - output.revision, - output.status, - timestampMs(output.updatedAt, 'Lease.updatedAt'), - serializeJson(output, 'Lease'), - input.workspaceId, - output.id, - effective.revision, - effective.fencingToken, - ), - ); - } - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'cancel Goal'); - const mutationCount = 1 + cancellable.length + effectiveLeases.length; - for (let index = 0; index < mutationCount; index += 1) { - if (changes(results[index]!) !== 1) { - throw new PersistenceError('CONFLICT', `Goal ${currentGoal.id} lost its cancel race.`); - } - } - return { - kind: 'committed', - value: { - goal: clone(input.goal), - tasks: clone(input.tasks), - leases: clone(input.leases), - }, - }; - }); - } - -''' -replace_once( - 'src/persistence/cloudflare/d1-runtime-persistence.ts', - ' private async commitTaskOutcome(', - methods + ' private async commitTaskOutcome(', -) - -replace_once( - 'src/persistence/cloudflare/d1-runtime-persistence.ts', - ''' private async getActiveLease(workspaceId: string, taskId: string): Promise { - return this.readRecord(''', - ''' private async getEffectiveActiveLease( - workspaceId: string, - taskId: string, - nowMs: number, - sessionCutoffMs: number, - ): Promise { - const lease = await this.getActiveLease(workspaceId, taskId); - if (!lease || timestampMs(lease.expiresAt, 'Lease.expiresAt') <= nowMs) return undefined; - const session = await this.getSession(workspaceId, lease.sessionId); - if ( - !session || - session.status !== 'active' || - timestampMs(session.lastSeenAt, 'Session.lastSeenAt') <= sessionCutoffMs - ) { - return undefined; - } - return lease; - } - - private async getActiveLease(workspaceId: string, taskId: string): Promise { - return this.readRecord(''', -) - -replace_once( - 'src/persistence/cloudflare/d1-runtime-persistence.ts', - '''function serializeJson(value: unknown, label: string, maxBytes?: number): string {''', - '''function isCancellableTaskStatus(status: Task['status']): boolean { - return status === 'pending' || status === 'ready' || status === 'running' || status === 'blocked'; -} - -function serializeJson(value: unknown, label: string, maxBytes?: number): string {''', -) - -replace_once( - 'src/application/durable-dispatcher.ts', - ''' type BlockTaskResult, - type ClaimTaskResult, - type CompleteTaskResult, - type EndSessionResult, - type FailTaskResult,''', - ''' type BlockTaskResult, - type CancelGoalResult, - type CancelTaskResult, - type ClaimTaskResult, - type CompleteTaskResult, - type EndSessionResult, - type FailTaskResult,''', -) - -replace_once( - 'src/application/durable-dispatcher.ts', - ''' command.command === 'FailTask' || - command.command === 'BlockTask' || - command.command === 'ResumeTask' - );''', - ''' command.command === 'FailTask' || - command.command === 'BlockTask' || - command.command === 'ResumeTask' || - command.command === 'RetryTask' || - command.command === 'CancelTask' || - command.command === 'CancelGoal' - );''', -) - -replace_once( - 'src/application/durable-dispatcher.ts', - ''' case 'ResumeTask': { - const result = semanticResponse.result as Task; - return resolveMutationResult( - command, - await options.persistence.resumeTask({ - task: result, - expectedRevision: command.expectedTaskRevision, - receipt: receiptFor( - command, - fingerprint, - successResponse(command, result), - options.now(), - ), - }), - ); - } - default:''', - ''' case 'ResumeTask': { - const result = semanticResponse.result as Task; - return resolveMutationResult( - command, - await options.persistence.resumeTask({ - task: result, - expectedRevision: command.expectedTaskRevision, - receipt: receiptFor( - command, - fingerprint, - successResponse(command, result), - options.now(), - ), - }), - ); - } - case 'RetryTask': { - const result = semanticResponse.result as Task; - const now = result.updatedAt; - return resolveMutationResult( - command, - await options.persistence.retryTask({ - task: result, - expectedRevision: command.expectedTaskRevision, - now, - sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), - receipt: receiptFor( - command, - fingerprint, - successResponse(command, result), - options.now(), - ), - }), - ); - } - case 'CancelTask': { - const result = semanticResponse.result as CancelTaskResult; - const now = result.task.updatedAt; - return resolveMutationResult( - command, - await options.persistence.cancelTask({ - workspaceId: command.workspaceId, - task: result.task, - ...(result.lease === undefined ? {} : { lease: result.lease }), - expectedTaskRevision: command.expectedTaskRevision, - now, - sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), - receipt: receiptFor( - command, - fingerprint, - successResponse(command, result), - options.now(), - ), - }), - ); - } - case 'CancelGoal': { - const result = semanticResponse.result as CancelGoalResult; - const now = result.goal.updatedAt; - return resolveMutationResult( - command, - await options.persistence.cancelGoal({ - workspaceId: command.workspaceId, - goal: result.goal, - tasks: result.tasks, - leases: result.leases, - expectedGoalRevision: command.expectedGoalRevision, - now, - sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), - receipt: receiptFor( - command, - fingerprint, - successResponse(command, result), - options.now(), - ), - }), - ); - } - default:''', -) diff --git a/src/application/durable-dispatcher.ts b/src/application/durable-dispatcher.ts index 345d0d7..0223d56 100644 --- a/src/application/durable-dispatcher.ts +++ b/src/application/durable-dispatcher.ts @@ -22,6 +22,8 @@ import type { CanonicalDomainValidator } from '../runtime/domain-validation.ts'; import { InMemoryControlPlane, type BlockTaskResult, + type CancelGoalResult, + type CancelTaskResult, type ClaimTaskResult, type CompleteTaskResult, type EndSessionResult, @@ -258,7 +260,10 @@ function isFirstDurableLoopCommand(command: ProtocolCommand): boolean { command.command === 'CompleteTask' || command.command === 'FailTask' || command.command === 'BlockTask' || - command.command === 'ResumeTask' + command.command === 'ResumeTask' || + command.command === 'RetryTask' || + command.command === 'CancelTask' || + command.command === 'CancelGoal' ); } @@ -543,6 +548,68 @@ async function commitDurableSuccess( }), ); } + case 'RetryTask': { + const result = semanticResponse.result as Task; + const now = result.updatedAt; + return resolveMutationResult( + command, + await options.persistence.retryTask({ + task: result, + expectedRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelTask': { + const result = semanticResponse.result as CancelTaskResult; + const now = result.task.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelTask({ + workspaceId: command.workspaceId, + task: result.task, + ...(result.lease === undefined ? {} : { lease: result.lease }), + expectedTaskRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelGoal': { + const result = semanticResponse.result as CancelGoalResult; + const now = result.goal.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelGoal({ + workspaceId: command.workspaceId, + goal: result.goal, + tasks: result.tasks, + leases: result.leases, + expectedGoalRevision: command.expectedGoalRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } default: return commandFailure(command, 'UNSUPPORTED_OPERATION', 'Command is not durable yet.'); } diff --git a/src/persistence/cloudflare/d1-runtime-persistence.ts b/src/persistence/cloudflare/d1-runtime-persistence.ts index ff0cc23..b954008 100644 --- a/src/persistence/cloudflare/d1-runtime-persistence.ts +++ b/src/persistence/cloudflare/d1-runtime-persistence.ts @@ -13,6 +13,10 @@ import type { import { PersistenceError, + type CancelGoalCommitInput, + type CancelGoalCommitValue, + type CancelTaskCommitInput, + type CancelTaskCommitValue, type ClaimTaskCommitInput, type ClaimTaskCommitValue, type CommandReceiptInput, @@ -1035,6 +1039,398 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { }); } + async retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise> { + const { task } = input; + this.assertCanonical('Task', task); + this.assertRelatedAudit(task.workspaceId, input.auditEvent); + this.assertReceipt(task.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'retry Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); + + return this.coordinator.runSerialized(task.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(task.workspaceId, task.id); + if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); + if (current.revision !== input.expectedRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, + ); + } + const goal = await this.getGoal(task.workspaceId, current.goalId); + if (!goal) { + throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); + } + if (goal.status !== 'active' || current.status !== 'failed') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${task.id} cannot be retried from current durable state.`, + ); + } + if (await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs)) { + throw new PersistenceError( + 'CONFLICT', + `Task ${task.id} still has active execution authority.`, + ); + } + + const expected: Task = { + ...clone(current), + status: 'ready', + revision: input.expectedRevision + 1, + updatedAt: input.now, + }; + delete expected.statusReason; + if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { + throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed'`, + ) + .bind( + task.revision, + task.status, + timestampMs(task.updatedAt, 'Task.updatedAt'), + serializeJson(task, 'Task'), + task.workspaceId, + task.id, + input.expectedRevision, + ), + ]; + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'retry Task'); + if (changes(results[0]!) !== 1) { + throw new PersistenceError('REVISION_MISMATCH', `Task ${task.id} lost its retry race.`); + } + return { kind: 'committed', value: clone(task) }; + }); + } + + async cancelTask( + input: CancelTaskCommitInput, + ): Promise> { + this.assertCanonical('Task', input.task); + if (input.lease) this.assertCanonical('Lease', input.lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(input.workspaceId, input.task.id); + if (!current) { + throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); + } + if (current.revision !== input.expectedTaskRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, + ); + } + if (!isCancellableTaskStatus(current.status)) { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${current.id} cannot be cancelled from ${current.status}.`, + ); + } + if (!input.task.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); + } + const expectedTask: Task = { + ...clone(current), + status: 'cancelled', + statusReason: clone(input.task.statusReason), + revision: input.expectedTaskRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); + } + + const effectiveLease = await this.getEffectiveActiveLease( + input.workspaceId, + current.id, + nowMs, + cutoffMs, + ); + if ((effectiveLease === undefined) !== (input.lease === undefined)) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); + } + if (effectiveLease && input.lease) { + const expectedLease: Lease = { + ...clone(effectiveLease), + status: 'revoked', + revision: effectiveLease.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.lease, 'Lease') !== + serializeJson(expectedLease, 'expected CancelTask Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + input.task.revision, + input.task.status, + timestampMs(input.task.updatedAt, 'Task.updatedAt'), + serializeJson(input.task, 'Task'), + input.workspaceId, + input.task.id, + input.expectedTaskRevision, + current.status, + ), + ]; + if (effectiveLease && input.lease) { + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + input.lease.revision, + input.lease.status, + timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), + serializeJson(input.lease, 'Lease'), + input.workspaceId, + input.lease.id, + effectiveLease.revision, + effectiveLease.fencingToken, + ), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'cancel Task'); + if (changes(results[0]!) !== 1) { + throw new PersistenceError('REVISION_MISMATCH', `Task ${current.id} lost its cancel race.`); + } + if (effectiveLease && changes(results[1]!) !== 1) { + throw new PersistenceError( + 'STALE_AUTHORITY', + `Task ${current.id} Lease lost its cancel race.`, + ); + } + return { + kind: 'committed', + value: { + task: clone(input.task), + ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), + }, + }; + }); + } + + async cancelGoal( + input: CancelGoalCommitInput, + ): Promise> { + this.assertCanonical('Goal', input.goal); + for (const task of input.tasks) this.assertCanonical('Task', task); + for (const lease of input.leases) this.assertCanonical('Lease', lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Goal now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); + if (!currentGoal) { + throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); + } + if (currentGoal.revision !== input.expectedGoalRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, + ); + } + if (currentGoal.status !== 'active') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Goal ${currentGoal.id} is already terminal.`, + ); + } + const expectedGoal: Goal = { + ...clone(currentGoal), + status: 'cancelled', + revision: input.expectedGoalRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); + } + + const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); + if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); + } + const effectiveLeases: Lease[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id); + if (!output || !output.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); + } + const expectedTask: Task = { + ...clone(currentTask), + status: 'cancelled', + statusReason: clone(output.statusReason), + revision: currentTask.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); + } + const effective = await this.getEffectiveActiveLease( + input.workspaceId, + currentTask.id, + nowMs, + cutoffMs, + ); + if (effective) effectiveLeases.push(effective); + } + + const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); + if ( + outputLeases.size !== input.leases.length || + input.leases.length !== effectiveLeases.length + ) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id); + if (!output) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); + } + const expectedLease: Lease = { + ...clone(effective), + status: 'revoked', + revision: effective.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Lease') !== + serializeJson(expectedLease, 'expected CancelGoal Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + input.goal.revision, + input.goal.status, + timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), + serializeJson(input.goal, 'Goal'), + input.workspaceId, + input.goal.id, + input.expectedGoalRevision, + ), + ]; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Task.updatedAt'), + serializeJson(output, 'Task'), + input.workspaceId, + output.id, + currentTask.revision, + currentTask.status, + ), + ); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id)!; + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Lease.updatedAt'), + serializeJson(output, 'Lease'), + input.workspaceId, + output.id, + effective.revision, + effective.fencingToken, + ), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + const results = await this.batch(statements, 'cancel Goal'); + const mutationCount = 1 + cancellable.length + effectiveLeases.length; + for (let index = 0; index < mutationCount; index += 1) { + if (changes(results[index]!) !== 1) { + throw new PersistenceError('CONFLICT', `Goal ${currentGoal.id} lost its cancel race.`); + } + } + return { + kind: 'committed', + value: { + goal: clone(input.goal), + tasks: clone(input.tasks), + leases: clone(input.leases), + }, + }; + }); + } + private async commitTaskOutcome( input: TaskOutcomeCommitInput, expectedTaskStatus: 'failed' | 'blocked', @@ -2105,6 +2501,25 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { }); } + private async getEffectiveActiveLease( + workspaceId: string, + taskId: string, + nowMs: number, + sessionCutoffMs: number, + ): Promise { + const lease = await this.getActiveLease(workspaceId, taskId); + if (!lease || timestampMs(lease.expiresAt, 'Lease.expiresAt') <= nowMs) return undefined; + const session = await this.getSession(workspaceId, lease.sessionId); + if ( + !session || + session.status !== 'active' || + timestampMs(session.lastSeenAt, 'Session.lastSeenAt') <= sessionCutoffMs + ) { + return undefined; + } + return lease; + } + private async getActiveLease(workspaceId: string, taskId: string): Promise { return this.readRecord( `SELECT record_json FROM leases @@ -2183,6 +2598,10 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { } } +function isCancellableTaskStatus(status: Task['status']): boolean { + return status === 'pending' || status === 'ready' || status === 'running' || status === 'blocked'; +} + function serializeJson(value: unknown, label: string, maxBytes?: number): string { let json: string | undefined; try { diff --git a/src/persistence/ports.ts b/src/persistence/ports.ts index 09b7dc6..7754ff6 100644 --- a/src/persistence/ports.ts +++ b/src/persistence/ports.ts @@ -162,6 +162,40 @@ export interface TaskOutcomeCommitValue { checkpoint: Checkpoint; } +export interface CancelTaskCommitInput { + workspaceId: string; + task: Task; + lease?: Lease; + expectedTaskRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelTaskCommitValue { + task: Task; + lease?: Lease; +} + +export interface CancelGoalCommitInput { + workspaceId: string; + goal: Goal; + tasks: Task[]; + leases: Lease[]; + expectedGoalRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelGoalCommitValue { + goal: Goal; + tasks: Task[]; + leases: Lease[]; +} + export interface DurableRuntimePersistence { bootstrapWorkspace(workspace: Workspace): Promise; createAgent(input: { @@ -234,6 +268,16 @@ export interface DurableRuntimePersistence { receipt?: CommandReceiptInput; auditEvent?: AuditEvent; }): Promise>; + retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise>; + cancelTask(input: CancelTaskCommitInput): Promise>; + cancelGoal(input: CancelGoalCommitInput): Promise>; appendAuditEvent(input: { auditEvent: AuditEvent }): Promise; appendPermissionRequestWithInitialDecision(input: { request: PermissionRequest; From 68f97ce9a04c4a2a6d96ce30caa951658ea6ce11 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:40:15 +0500 Subject: [PATCH 05/38] test: add one-time retry cancellation review hardening patcher --- ...arden-durable-retry-cancellation-review.py | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 scripts/harden-durable-retry-cancellation-review.py diff --git a/scripts/harden-durable-retry-cancellation-review.py b/scripts/harden-durable-retry-cancellation-review.py new file mode 100644 index 0000000..426f866 --- /dev/null +++ b/scripts/harden-durable-retry-cancellation-review.py @@ -0,0 +1,295 @@ +from pathlib import Path + + +def replace_once(path: str, before: str, after: str) -> None: + file = Path(path) + text = file.read_text() + count = text.count(before) + if count != 1: + raise RuntimeError(f'{path}: expected one marker, found {count}') + file.write_text(text.replace(before, after, 1)) + + +replace_once( + 'test/persistence/setup.ts', + "import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts';\n", + "import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts';\nimport type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts';\n", +) +replace_once( + 'test/persistence/setup.ts', + '''export async function openPersistence(path: string): Promise<{ + database: SqliteD1Database; + persistence: D1RuntimePersistence; +}> {''', + '''export async function openPersistence( + path: string, + coordinator: WorkspaceMutationCoordinator = new WorkspaceDurableObjectCoordinator(), +): Promise<{ + database: SqliteD1Database; + persistence: D1RuntimePersistence; +}> {''', +) +replace_once( + 'test/persistence/setup.ts', + ''' const persistence = new D1RuntimePersistence({ + database, + coordinator: new WorkspaceDurableObjectCoordinator(), + validateCanonicalDomainRecord: persistenceCanonicalValidator, + });''', + ''' const persistence = new D1RuntimePersistence({ + database, + coordinator, + validateCanonicalDomainRecord: persistenceCanonicalValidator, + });''', +) + +replace_once( + 'test/application/durable-retry-cancellation.test.ts', + "import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts';\n", + "import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts';\nimport { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts';\nimport type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts';\n", +) +replace_once( + 'test/application/durable-retry-cancellation.test.ts', + '''async function openDispatcher(path: string, prefix: string, now: Date) { + const opened = await openPersistence(path);''', + '''async function openDispatcher( + path: string, + prefix: string, + now: Date, + coordinator?: WorkspaceMutationCoordinator, +) { + const opened = await openPersistence(path, coordinator);''', +) + +class_code = r''' +class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { + readonly firstArrived: Promise; + private firstArrivedResolve!: () => void; + private secondArrived: Promise; + private secondArrivedResolve!: () => void; + private firstQueued: Promise; + private firstQueuedResolve!: () => void; + private arrivals = 0; + + constructor(private readonly inner: WorkspaceMutationCoordinator) { + this.firstArrived = new Promise((resolve) => { + this.firstArrivedResolve = resolve; + }); + this.secondArrived = new Promise((resolve) => { + this.secondArrivedResolve = resolve; + }); + this.firstQueued = new Promise((resolve) => { + this.firstQueuedResolve = resolve; + }); + } + + runSerialized(workspaceId: string, operation: () => Promise): Promise { + this.arrivals += 1; + const position = this.arrivals; + if (position === 1) { + this.firstArrivedResolve(); + return this.queueFirst(workspaceId, operation); + } + if (position === 2) { + this.secondArrivedResolve(); + return this.queueSecond(workspaceId, operation); + } + return this.inner.runSerialized(workspaceId, operation); + } + + private async queueFirst(workspaceId: string, operation: () => Promise): Promise { + await this.secondArrived; + const result = this.inner.runSerialized(workspaceId, operation); + this.firstQueuedResolve(); + return result; + } + + private async queueSecond(workspaceId: string, operation: () => Promise): Promise { + await this.firstQueued; + return this.inner.runSerialized(workspaceId, operation); + } +} +''' +replace_once( + 'test/application/durable-retry-cancellation.test.ts', + "\nfunction success(response: CommandResponse): T {", + class_code + "\nfunction success(response: CommandResponse): T {", +) + +new_tests = r''' + + it('cancels a recoverable running Task without requiring an effective Lease', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T19:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-no-lease-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-no-lease'); + success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-no-lease-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-no-lease-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); + app.database.close(); + }); + + it('rolls back Goal, Tasks, Lease, and receipt when CancelGoal batch fails mid-transaction', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T20:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-goal-atomic-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal-atomic'); + const secondTask = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-goal-atomic-second-task', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Second atomic cancellation task', + objective: 'Remain ready before injected cancellation failure.', + acceptanceCriteria: ['Rollback preserves this Task.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }), + ); + + app.database.failNextBatchAfterStatements(2); + const failed = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + expect(failed).toHaveProperty('error'); + expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(seeded.goal); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(seeded.claim.task); + expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual(secondTask); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual(seeded.claim.lease); + expect(await app.persistence.getCommandReceipt('ws-a', 'cancel-goal-atomic-command')).toBeUndefined(); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-atomic-retry', now); + const retry = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + const committed = success(retry); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + expect(committed.leases).toHaveLength(1); + app.database.close(); + }); + + it('serializes stale CreateTask persistence behind CancelGoal across independent dispatchers', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-race'); + seed.database.close(); + + const sharedAuthority = new WorkspaceDurableObjectCoordinator('ws-a'); + const rendezvous = new OrderedTwoPartyCoordinator(sharedAuthority); + const cancelApp = await openDispatcher(path, 'cancel-race-controller', now, rendezvous); + const createApp = await openDispatcher(path, 'cancel-race-creator', now, rendezvous); + + const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Cancel before stale Task commit.' }, + }); + await rendezvous.firstArrived; + + const createPromise = createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-race-create', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Stale observed Task', + objective: 'Must not survive beneath a cancelled Goal.', + acceptanceCriteria: ['Persistence rejects stale creation.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + + const [cancelledResponse, createResponse] = await Promise.all([cancelPromise, createPromise]); + expect(success(cancelledResponse).goal.status).toBe('cancelled'); + expect('error' in createResponse && createResponse.error.code).toBe('INVALID_STATE_TRANSITION'); + + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + const goalTasks = snapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(goalTasks).toHaveLength(1); + expect(goalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + createApp.database.close(); + }); +''' + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +marker = '\n});\n' +if not text.endswith(marker): + raise RuntimeError('unexpected retry cancellation test suffix') +path.write_text(text[:-len(marker)] + new_tests + marker) From 84b950a9e3307d16c37644a8031cd8de281f57ca Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:25:20 +0500 Subject: [PATCH 06/38] test: strengthen retry cancellation race regressions --- ...arden-durable-retry-cancellation-review.py | 208 +++++++++++++++++- 1 file changed, 205 insertions(+), 3 deletions(-) diff --git a/scripts/harden-durable-retry-cancellation-review.py b/scripts/harden-durable-retry-cancellation-review.py index 426f866..c74f889 100644 --- a/scripts/harden-durable-retry-cancellation-review.py +++ b/scripts/harden-durable-retry-cancellation-review.py @@ -43,6 +43,46 @@ def replace_once(path: str, before: str, after: str) -> None: });''', ) +replace_once( + 'test/persistence/d1-sqlite-harness.ts', + ''' private failNextBatchAfterStatementCount: number | undefined; +''', + ''' private failNextBatchAfterStatementCount: number | undefined; + private beforeNextBatchHook: (() => Promise) | undefined; +''', +) +replace_once( + 'test/persistence/d1-sqlite-harness.ts', + ''' async batch(statements: D1PreparedStatementLike[]): Promise { + this.database.exec('BEGIN IMMEDIATE;');''', + ''' async batch(statements: D1PreparedStatementLike[]): Promise { + const beforeBatch = this.beforeNextBatchHook; + this.beforeNextBatchHook = undefined; + if (beforeBatch) await beforeBatch(); + this.database.exec('BEGIN IMMEDIATE;');''', +) +replace_once( + 'test/persistence/d1-sqlite-harness.ts', + ''' failNextBatchAfterStatements(statementCount: number): void { + if (!Number.isSafeInteger(statementCount) || statementCount < 0) { + throw new TypeError('statementCount must be a non-negative safe integer.'); + } + this.failNextBatchAfterStatementCount = statementCount; + } +''', + ''' failNextBatchAfterStatements(statementCount: number): void { + if (!Number.isSafeInteger(statementCount) || statementCount < 0) { + throw new TypeError('statementCount must be a non-negative safe integer.'); + } + this.failNextBatchAfterStatementCount = statementCount; + } + + beforeNextBatch(hook: () => Promise): void { + this.beforeNextBatchHook = hook; + } +''', +) + replace_once( 'test/application/durable-retry-cancellation.test.ts', "import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts';\n", @@ -234,7 +274,7 @@ class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { app.database.close(); }); - it('serializes stale CreateTask persistence behind CancelGoal across independent dispatchers', async () => { + it('serializes stale CreateTask persistence behind CancelGoal through one Workspace authority', async () => { const path = databasePath(); const now = new Date('2026-08-30T21:00:00.000Z'); let seed = await openDispatcher(path, 'cancel-race-seed', now); @@ -250,7 +290,7 @@ class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { const cancelPromise = cancelApp.dispatcher.dispatchCommand({ protocolVersion: '0.1', command: 'CancelGoal', - commandId: 'cancel-race-goal', + commandId: 'cancel-race-goal-shared', workspaceId: 'ws-a', actor: seeded.systemActor, goalId: seeded.goal.id, @@ -262,7 +302,7 @@ class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { const createPromise = createApp.dispatcher.dispatchCommand({ protocolVersion: '0.1', command: 'CreateTask', - commandId: 'cancel-race-create', + commandId: 'cancel-race-create-shared', workspaceId: 'ws-a', actor: seeded.systemActor, goalId: seeded.goal.id, @@ -285,6 +325,168 @@ class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { cancelApp.database.close(); createApp.database.close(); }); + + it('does not retain a success receipt when RetryTask loses its database CAS race', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:30:00.000Z'); + let seed = await openDispatcher(path, 'retry-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'retry-race'); + const failed = success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'FailTask', + commandId: 'retry-race-fail', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'execution.failed', summary: 'Prepare a retry race.' }, + summary: 'Race retries.', + evidence: [], + }), + ); + seed.database.close(); + + const first = await openDispatcher(path, 'retry-race-first', now); + const second = await openDispatcher(path, 'retry-race-second', now); + let firstArrivedResolve!: () => void; + let secondArrivedResolve!: () => void; + let releaseFirst!: () => void; + let releaseSecond!: () => void; + const firstArrived = new Promise((resolve) => (firstArrivedResolve = resolve)); + const secondArrived = new Promise((resolve) => (secondArrivedResolve = resolve)); + const firstRelease = new Promise((resolve) => (releaseFirst = resolve)); + const secondRelease = new Promise((resolve) => (releaseSecond = resolve)); + first.database.beforeNextBatch(async () => { + firstArrivedResolve(); + await firstRelease; + }); + second.database.beforeNextBatch(async () => { + secondArrivedResolve(); + await secondRelease; + }); + + const firstPromise = first.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-first-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await firstArrived; + const secondPromise = second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await secondArrived; + + releaseFirst(); + success(await firstPromise); + releaseSecond(); + const lost = await secondPromise; + expect('error' in lost && lost.error.code).toBe('REVISION_MISMATCH'); + expect(await second.persistence.getCommandReceipt('ws-a', 'retry-race-second-command')).toBeUndefined(); + + const replayAttempt = await second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + correlationId: 'retry-race-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + expect('error' in replayAttempt && replayAttempt.error.code).toBe('REVISION_MISMATCH'); + expect(replayAttempt).not.toHaveProperty('replayed', true); + first.database.close(); + second.database.close(); + }); + + it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const createApp = await openDispatcher(path, 'cancel-db-race-creator', now); + let cancelArrivedResolve!: () => void; + let releaseCancel!: () => void; + const cancelArrived = new Promise((resolve) => (cancelArrivedResolve = resolve)); + const cancelRelease = new Promise((resolve) => (releaseCancel = resolve)); + cancelApp.database.beforeNextBatch(async () => { + cancelArrivedResolve(); + await cancelRelease; + }); + + const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + await cancelArrived; + + const createdResponse = await createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-db-race-create', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Either commit before cancellation or be rejected.', + acceptanceCriteria: ['Never survive under a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + const created = success(createdResponse); + releaseCancel(); + const staleCancellation = await cancelPromise; + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === created.id)?.status).toBe('ready'); + + const freshCancel = await createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await createApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + createApp.database.close(); + }); ''' path = Path('test/application/durable-retry-cancellation.test.ts') From 3148ee9d20ecbb2b5114c9b06adf542c6b695eb7 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:25:37 +0500 Subject: [PATCH 07/38] chore: add one-time retry cancellation review test workflow --- ...e-retry-cancellation-review-tests-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml diff --git a/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml b/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml new file mode 100644 index 0000000..606138c --- /dev/null +++ b/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml @@ -0,0 +1,33 @@ +name: Apply durable retry cancellation review tests once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/harden-durable-retry-cancellation-review.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts test/persistence/setup.ts test/persistence/d1-sqlite-harness.ts + - run: git rm .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml scripts/harden-durable-retry-cancellation-review.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts test/persistence/setup.ts test/persistence/d1-sqlite-harness.ts + git commit -m "test: harden durable retry cancellation races" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From 05e8026836f78c1ad558b363896b9fc5bb93b206 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:25:55 +0000 Subject: [PATCH 08/38] test: harden durable retry cancellation races --- ...e-retry-cancellation-review-tests-once.yml | 33 -- ...arden-durable-retry-cancellation-review.py | 497 ------------------ .../durable-retry-cancellation.test.ts | 396 +++++++++++++- test/persistence/d1-sqlite-harness.ts | 8 + test/persistence/setup.ts | 8 +- 5 files changed, 408 insertions(+), 534 deletions(-) delete mode 100644 .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml delete mode 100644 scripts/harden-durable-retry-cancellation-review.py diff --git a/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml b/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml deleted file mode 100644 index 606138c..0000000 --- a/.github/workflows/apply-durable-retry-cancellation-review-tests-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Apply durable retry cancellation review tests once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/harden-durable-retry-cancellation-review.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts test/persistence/setup.ts test/persistence/d1-sqlite-harness.ts - - run: git rm .github/workflows/apply-durable-retry-cancellation-review-tests-once.yml scripts/harden-durable-retry-cancellation-review.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts test/persistence/setup.ts test/persistence/d1-sqlite-harness.ts - git commit -m "test: harden durable retry cancellation races" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/harden-durable-retry-cancellation-review.py b/scripts/harden-durable-retry-cancellation-review.py deleted file mode 100644 index c74f889..0000000 --- a/scripts/harden-durable-retry-cancellation-review.py +++ /dev/null @@ -1,497 +0,0 @@ -from pathlib import Path - - -def replace_once(path: str, before: str, after: str) -> None: - file = Path(path) - text = file.read_text() - count = text.count(before) - if count != 1: - raise RuntimeError(f'{path}: expected one marker, found {count}') - file.write_text(text.replace(before, after, 1)) - - -replace_once( - 'test/persistence/setup.ts', - "import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts';\n", - "import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts';\nimport type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts';\n", -) -replace_once( - 'test/persistence/setup.ts', - '''export async function openPersistence(path: string): Promise<{ - database: SqliteD1Database; - persistence: D1RuntimePersistence; -}> {''', - '''export async function openPersistence( - path: string, - coordinator: WorkspaceMutationCoordinator = new WorkspaceDurableObjectCoordinator(), -): Promise<{ - database: SqliteD1Database; - persistence: D1RuntimePersistence; -}> {''', -) -replace_once( - 'test/persistence/setup.ts', - ''' const persistence = new D1RuntimePersistence({ - database, - coordinator: new WorkspaceDurableObjectCoordinator(), - validateCanonicalDomainRecord: persistenceCanonicalValidator, - });''', - ''' const persistence = new D1RuntimePersistence({ - database, - coordinator, - validateCanonicalDomainRecord: persistenceCanonicalValidator, - });''', -) - -replace_once( - 'test/persistence/d1-sqlite-harness.ts', - ''' private failNextBatchAfterStatementCount: number | undefined; -''', - ''' private failNextBatchAfterStatementCount: number | undefined; - private beforeNextBatchHook: (() => Promise) | undefined; -''', -) -replace_once( - 'test/persistence/d1-sqlite-harness.ts', - ''' async batch(statements: D1PreparedStatementLike[]): Promise { - this.database.exec('BEGIN IMMEDIATE;');''', - ''' async batch(statements: D1PreparedStatementLike[]): Promise { - const beforeBatch = this.beforeNextBatchHook; - this.beforeNextBatchHook = undefined; - if (beforeBatch) await beforeBatch(); - this.database.exec('BEGIN IMMEDIATE;');''', -) -replace_once( - 'test/persistence/d1-sqlite-harness.ts', - ''' failNextBatchAfterStatements(statementCount: number): void { - if (!Number.isSafeInteger(statementCount) || statementCount < 0) { - throw new TypeError('statementCount must be a non-negative safe integer.'); - } - this.failNextBatchAfterStatementCount = statementCount; - } -''', - ''' failNextBatchAfterStatements(statementCount: number): void { - if (!Number.isSafeInteger(statementCount) || statementCount < 0) { - throw new TypeError('statementCount must be a non-negative safe integer.'); - } - this.failNextBatchAfterStatementCount = statementCount; - } - - beforeNextBatch(hook: () => Promise): void { - this.beforeNextBatchHook = hook; - } -''', -) - -replace_once( - 'test/application/durable-retry-cancellation.test.ts', - "import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts';\n", - "import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts';\nimport { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts';\nimport type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts';\n", -) -replace_once( - 'test/application/durable-retry-cancellation.test.ts', - '''async function openDispatcher(path: string, prefix: string, now: Date) { - const opened = await openPersistence(path);''', - '''async function openDispatcher( - path: string, - prefix: string, - now: Date, - coordinator?: WorkspaceMutationCoordinator, -) { - const opened = await openPersistence(path, coordinator);''', -) - -class_code = r''' -class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { - readonly firstArrived: Promise; - private firstArrivedResolve!: () => void; - private secondArrived: Promise; - private secondArrivedResolve!: () => void; - private firstQueued: Promise; - private firstQueuedResolve!: () => void; - private arrivals = 0; - - constructor(private readonly inner: WorkspaceMutationCoordinator) { - this.firstArrived = new Promise((resolve) => { - this.firstArrivedResolve = resolve; - }); - this.secondArrived = new Promise((resolve) => { - this.secondArrivedResolve = resolve; - }); - this.firstQueued = new Promise((resolve) => { - this.firstQueuedResolve = resolve; - }); - } - - runSerialized(workspaceId: string, operation: () => Promise): Promise { - this.arrivals += 1; - const position = this.arrivals; - if (position === 1) { - this.firstArrivedResolve(); - return this.queueFirst(workspaceId, operation); - } - if (position === 2) { - this.secondArrivedResolve(); - return this.queueSecond(workspaceId, operation); - } - return this.inner.runSerialized(workspaceId, operation); - } - - private async queueFirst(workspaceId: string, operation: () => Promise): Promise { - await this.secondArrived; - const result = this.inner.runSerialized(workspaceId, operation); - this.firstQueuedResolve(); - return result; - } - - private async queueSecond(workspaceId: string, operation: () => Promise): Promise { - await this.firstQueued; - return this.inner.runSerialized(workspaceId, operation); - } -} -''' -replace_once( - 'test/application/durable-retry-cancellation.test.ts', - "\nfunction success(response: CommandResponse): T {", - class_code + "\nfunction success(response: CommandResponse): T {", -) - -new_tests = r''' - - it('cancels a recoverable running Task without requiring an effective Lease', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T19:30:00.000Z'); - let app = await openDispatcher(path, 'cancel-no-lease-before', now); - await app.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(app.dispatcher, 'cancel-no-lease'); - success( - await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'ReleaseLease', - commandId: 'cancel-no-lease-release', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - leaseId: seeded.claim.lease.id, - fencingToken: seeded.claim.lease.fencingToken, - expectedLeaseRevision: seeded.claim.lease.revision, - }), - ); - app.database.close(); - - app = await openDispatcher(path, 'cancel-no-lease-after', now); - const response = await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelTask', - commandId: 'cancel-no-lease-command', - correlationId: 'cancel-no-lease-first', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: seeded.claim.task.revision, - reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, - }); - const cancelled = success(response); - expect(cancelled.task.status).toBe('cancelled'); - expect(cancelled).not.toHaveProperty('lease'); - app.database.close(); - - app = await openDispatcher(path, 'cancel-no-lease-replay', now); - const replay = await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelTask', - commandId: 'cancel-no-lease-command', - correlationId: 'cancel-no-lease-replay', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: seeded.claim.task.revision, - reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, - }); - expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-no-lease-replay' }); - expect(success(replay)).toEqual(cancelled); - expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); - app.database.close(); - }); - - it('rolls back Goal, Tasks, Lease, and receipt when CancelGoal batch fails mid-transaction', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T20:30:00.000Z'); - let app = await openDispatcher(path, 'cancel-goal-atomic-before', now); - await app.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal-atomic'); - const secondTask = success( - await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CreateTask', - commandId: 'cancel-goal-atomic-second-task', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - title: 'Second atomic cancellation task', - objective: 'Remain ready before injected cancellation failure.', - acceptanceCriteria: ['Rollback preserves this Task.'], - requiredCapabilities: [], - dependencyTaskIds: [], - }), - ); - - app.database.failNextBatchAfterStatements(2); - const failed = await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-goal-atomic-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, - }); - expect(failed).toHaveProperty('error'); - expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(seeded.goal); - expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(seeded.claim.task); - expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual(secondTask); - expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual(seeded.claim.lease); - expect(await app.persistence.getCommandReceipt('ws-a', 'cancel-goal-atomic-command')).toBeUndefined(); - app.database.close(); - - app = await openDispatcher(path, 'cancel-goal-atomic-retry', now); - const retry = await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-goal-atomic-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, - }); - const committed = success(retry); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - expect(committed.leases).toHaveLength(1); - app.database.close(); - }); - - it('serializes stale CreateTask persistence behind CancelGoal through one Workspace authority', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T21:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-race'); - seed.database.close(); - - const sharedAuthority = new WorkspaceDurableObjectCoordinator('ws-a'); - const rendezvous = new OrderedTwoPartyCoordinator(sharedAuthority); - const cancelApp = await openDispatcher(path, 'cancel-race-controller', now, rendezvous); - const createApp = await openDispatcher(path, 'cancel-race-creator', now, rendezvous); - - const cancelPromise = cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-race-goal-shared', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Cancel before stale Task commit.' }, - }); - await rendezvous.firstArrived; - - const createPromise = createApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CreateTask', - commandId: 'cancel-race-create-shared', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - title: 'Stale observed Task', - objective: 'Must not survive beneath a cancelled Goal.', - acceptanceCriteria: ['Persistence rejects stale creation.'], - requiredCapabilities: [], - dependencyTaskIds: [], - }); - - const [cancelledResponse, createResponse] = await Promise.all([cancelPromise, createPromise]); - expect(success(cancelledResponse).goal.status).toBe('cancelled'); - expect('error' in createResponse && createResponse.error.code).toBe('INVALID_STATE_TRANSITION'); - - const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); - const goalTasks = snapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(goalTasks).toHaveLength(1); - expect(goalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - createApp.database.close(); - }); - - it('does not retain a success receipt when RetryTask loses its database CAS race', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T21:30:00.000Z'); - let seed = await openDispatcher(path, 'retry-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'retry-race'); - const failed = success( - await seed.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'FailTask', - commandId: 'retry-race-fail', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - leaseId: seeded.claim.lease.id, - fencingToken: seeded.claim.lease.fencingToken, - expectedTaskRevision: seeded.claim.task.revision, - reason: { code: 'execution.failed', summary: 'Prepare a retry race.' }, - summary: 'Race retries.', - evidence: [], - }), - ); - seed.database.close(); - - const first = await openDispatcher(path, 'retry-race-first', now); - const second = await openDispatcher(path, 'retry-race-second', now); - let firstArrivedResolve!: () => void; - let secondArrivedResolve!: () => void; - let releaseFirst!: () => void; - let releaseSecond!: () => void; - const firstArrived = new Promise((resolve) => (firstArrivedResolve = resolve)); - const secondArrived = new Promise((resolve) => (secondArrivedResolve = resolve)); - const firstRelease = new Promise((resolve) => (releaseFirst = resolve)); - const secondRelease = new Promise((resolve) => (releaseSecond = resolve)); - first.database.beforeNextBatch(async () => { - firstArrivedResolve(); - await firstRelease; - }); - second.database.beforeNextBatch(async () => { - secondArrivedResolve(); - await secondRelease; - }); - - const firstPromise = first.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'RetryTask', - commandId: 'retry-race-first-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: failed.task.revision, - }); - await firstArrived; - const secondPromise = second.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'RetryTask', - commandId: 'retry-race-second-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: failed.task.revision, - }); - await secondArrived; - - releaseFirst(); - success(await firstPromise); - releaseSecond(); - const lost = await secondPromise; - expect('error' in lost && lost.error.code).toBe('REVISION_MISMATCH'); - expect(await second.persistence.getCommandReceipt('ws-a', 'retry-race-second-command')).toBeUndefined(); - - const replayAttempt = await second.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'RetryTask', - commandId: 'retry-race-second-command', - correlationId: 'retry-race-replay', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: failed.task.revision, - }); - expect('error' in replayAttempt && replayAttempt.error.code).toBe('REVISION_MISMATCH'); - expect(replayAttempt).not.toHaveProperty('replayed', true); - first.database.close(); - second.database.close(); - }); - - it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const createApp = await openDispatcher(path, 'cancel-db-race-creator', now); - let cancelArrivedResolve!: () => void; - let releaseCancel!: () => void; - const cancelArrived = new Promise((resolve) => (cancelArrivedResolve = resolve)); - const cancelRelease = new Promise((resolve) => (releaseCancel = resolve)); - cancelApp.database.beforeNextBatch(async () => { - cancelArrivedResolve(); - await cancelRelease; - }); - - const cancelPromise = cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, - }); - await cancelArrived; - - const createdResponse = await createApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CreateTask', - commandId: 'cancel-db-race-create', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Either commit before cancellation or be rejected.', - acceptanceCriteria: ['Never survive under a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - }); - const created = success(createdResponse); - releaseCancel(); - const staleCancellation = await cancelPromise; - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === created.id)?.status).toBe('ready'); - - const freshCancel = await createApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await createApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - createApp.database.close(); - }); -''' - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -marker = '\n});\n' -if not text.endswith(marker): - raise RuntimeError('unexpected retry cancellation test suffix') -path.write_text(text[:-len(marker)] + new_tests + marker) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index c538360..ded7b43 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -6,6 +6,8 @@ import type { Agent, Goal, Lease, Session, Task } from '@mindrail/contracts'; import { describe, expect, it } from 'vitest'; import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts'; +import { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts'; +import type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts'; import type { ApplicationDispatcher } from '../../src/application/ports.ts'; import type { CancelGoalResult, @@ -22,8 +24,13 @@ function databasePath(): string { return join(mkdtempSync(join(tmpdir(), 'mindrail-durable-retry-cancel-')), 'runtime.sqlite'); } -async function openDispatcher(path: string, prefix: string, now: Date) { - const opened = await openPersistence(path); +async function openDispatcher( + path: string, + prefix: string, + now: Date, + coordinator?: WorkspaceMutationCoordinator, +) { + const opened = await openPersistence(path, coordinator); let sequence = 0; return { ...opened, @@ -38,6 +45,54 @@ async function openDispatcher(path: string, prefix: string, now: Date) { }; } +class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { + readonly firstArrived: Promise; + private firstArrivedResolve!: () => void; + private secondArrived: Promise; + private secondArrivedResolve!: () => void; + private firstQueued: Promise; + private firstQueuedResolve!: () => void; + private arrivals = 0; + + constructor(private readonly inner: WorkspaceMutationCoordinator) { + this.firstArrived = new Promise((resolve) => { + this.firstArrivedResolve = resolve; + }); + this.secondArrived = new Promise((resolve) => { + this.secondArrivedResolve = resolve; + }); + this.firstQueued = new Promise((resolve) => { + this.firstQueuedResolve = resolve; + }); + } + + runSerialized(workspaceId: string, operation: () => Promise): Promise { + this.arrivals += 1; + const position = this.arrivals; + if (position === 1) { + this.firstArrivedResolve(); + return this.queueFirst(workspaceId, operation); + } + if (position === 2) { + this.secondArrivedResolve(); + return this.queueSecond(workspaceId, operation); + } + return this.inner.runSerialized(workspaceId, operation); + } + + private async queueFirst(workspaceId: string, operation: () => Promise): Promise { + await this.secondArrived; + const result = this.inner.runSerialized(workspaceId, operation); + this.firstQueuedResolve(); + return result; + } + + private async queueSecond(workspaceId: string, operation: () => Promise): Promise { + await this.firstQueued; + return this.inner.runSerialized(workspaceId, operation); + } +} + function success(response: CommandResponse): T { expect(response).not.toHaveProperty('error'); if ('error' in response) throw new Error(`Expected success, got ${response.error.code}.`); @@ -297,4 +352,341 @@ describe('durable retry and cancellation', () => { ); app.database.close(); }); + + it('cancels a recoverable running Task without requiring an effective Lease', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T19:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-no-lease-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-no-lease'); + success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-no-lease-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-no-lease-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); + app.database.close(); + }); + + it('rolls back Goal, Tasks, Lease, and receipt when CancelGoal batch fails mid-transaction', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T20:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-goal-atomic-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal-atomic'); + const secondTask = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-goal-atomic-second-task', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Second atomic cancellation task', + objective: 'Remain ready before injected cancellation failure.', + acceptanceCriteria: ['Rollback preserves this Task.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }), + ); + + app.database.failNextBatchAfterStatements(2); + const failed = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + expect(failed).toHaveProperty('error'); + expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(seeded.goal); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(seeded.claim.task); + expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual(secondTask); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual( + seeded.claim.lease, + ); + expect( + await app.persistence.getCommandReceipt('ws-a', 'cancel-goal-atomic-command'), + ).toBeUndefined(); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-atomic-retry', now); + const retry = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + const committed = success(retry); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + expect(committed.leases).toHaveLength(1); + app.database.close(); + }); + + it('serializes stale CreateTask persistence behind CancelGoal through one Workspace authority', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-race'); + seed.database.close(); + + const sharedAuthority = new WorkspaceDurableObjectCoordinator('ws-a'); + const rendezvous = new OrderedTwoPartyCoordinator(sharedAuthority); + const cancelApp = await openDispatcher(path, 'cancel-race-controller', now, rendezvous); + const createApp = await openDispatcher(path, 'cancel-race-creator', now, rendezvous); + + const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-race-goal-shared', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Cancel before stale Task commit.' }, + }); + await rendezvous.firstArrived; + + const createPromise = createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-race-create-shared', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Stale observed Task', + objective: 'Must not survive beneath a cancelled Goal.', + acceptanceCriteria: ['Persistence rejects stale creation.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + + const [cancelledResponse, createResponse] = await Promise.all([cancelPromise, createPromise]); + expect(success(cancelledResponse).goal.status).toBe('cancelled'); + expect('error' in createResponse && createResponse.error.code).toBe('INVALID_STATE_TRANSITION'); + + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + const goalTasks = snapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(goalTasks).toHaveLength(1); + expect(goalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + createApp.database.close(); + }); + + it('does not retain a success receipt when RetryTask loses its database CAS race', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:30:00.000Z'); + let seed = await openDispatcher(path, 'retry-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'retry-race'); + const failed = success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'FailTask', + commandId: 'retry-race-fail', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'execution.failed', summary: 'Prepare a retry race.' }, + summary: 'Race retries.', + evidence: [], + }), + ); + seed.database.close(); + + const first = await openDispatcher(path, 'retry-race-first', now); + const second = await openDispatcher(path, 'retry-race-second', now); + let firstArrivedResolve!: () => void; + let secondArrivedResolve!: () => void; + let releaseFirst!: () => void; + let releaseSecond!: () => void; + const firstArrived = new Promise((resolve) => (firstArrivedResolve = resolve)); + const secondArrived = new Promise((resolve) => (secondArrivedResolve = resolve)); + const firstRelease = new Promise((resolve) => (releaseFirst = resolve)); + const secondRelease = new Promise((resolve) => (releaseSecond = resolve)); + first.database.beforeNextBatch(async () => { + firstArrivedResolve(); + await firstRelease; + }); + second.database.beforeNextBatch(async () => { + secondArrivedResolve(); + await secondRelease; + }); + + const firstPromise = first.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-first-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await firstArrived; + const secondPromise = second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await secondArrived; + + releaseFirst(); + success(await firstPromise); + releaseSecond(); + const lost = await secondPromise; + expect('error' in lost && lost.error.code).toBe('REVISION_MISMATCH'); + expect( + await second.persistence.getCommandReceipt('ws-a', 'retry-race-second-command'), + ).toBeUndefined(); + + const replayAttempt = await second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + correlationId: 'retry-race-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + expect('error' in replayAttempt && replayAttempt.error.code).toBe('REVISION_MISMATCH'); + expect(replayAttempt).not.toHaveProperty('replayed', true); + first.database.close(); + second.database.close(); + }); + + it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const createApp = await openDispatcher(path, 'cancel-db-race-creator', now); + let cancelArrivedResolve!: () => void; + let releaseCancel!: () => void; + const cancelArrived = new Promise((resolve) => (cancelArrivedResolve = resolve)); + const cancelRelease = new Promise((resolve) => (releaseCancel = resolve)); + cancelApp.database.beforeNextBatch(async () => { + cancelArrivedResolve(); + await cancelRelease; + }); + + const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + await cancelArrived; + + const createdResponse = await createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-db-race-create', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Either commit before cancellation or be rejected.', + acceptanceCriteria: ['Never survive under a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + const created = success(createdResponse); + releaseCancel(); + const staleCancellation = await cancelPromise; + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === created.id)?.status).toBe('ready'); + + const freshCancel = await createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await createApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = + finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + createApp.database.close(); + }); }); diff --git a/test/persistence/d1-sqlite-harness.ts b/test/persistence/d1-sqlite-harness.ts index 899ec70..c80d73e 100644 --- a/test/persistence/d1-sqlite-harness.ts +++ b/test/persistence/d1-sqlite-harness.ts @@ -9,6 +9,7 @@ import type { export class SqliteD1Database implements D1DatabaseLike { private readonly database: DatabaseSync; private failNextBatchAfterStatementCount: number | undefined; + private beforeNextBatchHook: (() => Promise) | undefined; constructor(path: string) { this.database = new DatabaseSync(path); @@ -20,6 +21,9 @@ export class SqliteD1Database implements D1DatabaseLike { } async batch(statements: D1PreparedStatementLike[]): Promise { + const beforeBatch = this.beforeNextBatchHook; + this.beforeNextBatchHook = undefined; + if (beforeBatch) await beforeBatch(); this.database.exec('BEGIN IMMEDIATE;'); try { const results: D1ResultLike[] = []; @@ -52,6 +56,10 @@ export class SqliteD1Database implements D1DatabaseLike { this.failNextBatchAfterStatementCount = statementCount; } + beforeNextBatch(hook: () => Promise): void { + this.beforeNextBatchHook = hook; + } + async exec(sql: string): Promise { this.database.exec(sql); } diff --git a/test/persistence/setup.ts b/test/persistence/setup.ts index bdd4675..eca52d8 100644 --- a/test/persistence/setup.ts +++ b/test/persistence/setup.ts @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts'; +import type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts'; import { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts'; import { persistenceCanonicalValidator } from './canonical-domain-validator.ts'; import { SqliteD1Database } from './d1-sqlite-harness.ts'; @@ -14,7 +15,10 @@ const migrations = readdirSync(migrationDirectory) .sort() .map((name) => readFileSync(join(migrationDirectory, name), 'utf8')); -export async function openPersistence(path: string): Promise<{ +export async function openPersistence( + path: string, + coordinator: WorkspaceMutationCoordinator = new WorkspaceDurableObjectCoordinator(), +): Promise<{ database: SqliteD1Database; persistence: D1RuntimePersistence; }> { @@ -24,7 +28,7 @@ export async function openPersistence(path: string): Promise<{ } const persistence = new D1RuntimePersistence({ database, - coordinator: new WorkspaceDurableObjectCoordinator(), + coordinator, validateCanonicalDomainRecord: persistenceCanonicalValidator, }); return { database, persistence }; From 6ed5504944ecf04b2db5aa96c9f1ff0728ea459b Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:27:09 +0500 Subject: [PATCH 09/38] test: prepare deterministic cancel goal race regression --- .../replace-cancel-goal-race-regression.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 scripts/replace-cancel-goal-race-regression.py diff --git a/scripts/replace-cancel-goal-race-regression.py b/scripts/replace-cancel-goal-race-regression.py new file mode 100644 index 0000000..5bc65dd --- /dev/null +++ b/scripts/replace-cancel-goal-race-regression.py @@ -0,0 +1,73 @@ +from pathlib import Path + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +start = text.index(" it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators'") +end = text.index('\n });\n});', start) + len('\n });') +replacement = r''' it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const injector = await openPersistence(path); + const concurrentTask: Task = { + workspaceId: 'ws-a', + id: 'cancel-db-race-concurrent-task', + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Commit after the cancellation snapshot but before its durable batch.', + acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + status: 'ready', + revision: 1, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; + cancelApp.database.beforeNextBatch(async () => { + const injected = await injector.persistence.createTask({ task: concurrentTask }); + expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); + }); + + const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); + + const freshCancel = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = + finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + injector.database.close(); + });''' +path.write_text(text[:start] + replacement + text[end:]) From 9386d6de9dc3be3444d89873248837cd6711f3b7 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:27:18 +0500 Subject: [PATCH 10/38] chore: add one-time cancel goal race regression workflow --- ...place-cancel-goal-race-regression-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/replace-cancel-goal-race-regression-once.yml diff --git a/.github/workflows/replace-cancel-goal-race-regression-once.yml b/.github/workflows/replace-cancel-goal-race-regression-once.yml new file mode 100644 index 0000000..c5ab6f4 --- /dev/null +++ b/.github/workflows/replace-cancel-goal-race-regression-once.yml @@ -0,0 +1,33 @@ +name: Replace cancel goal race regression once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/replace-cancel-goal-race-regression-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/replace-cancel-goal-race-regression.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/replace-cancel-goal-race-regression-once.yml scripts/replace-cancel-goal-race-regression.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts + git commit -m "test: make cancel goal race regression deterministic" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From 9318feb87d34d97e02b57f84db2018d290a995fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:27:34 +0000 Subject: [PATCH 11/38] test: make cancel goal race regression deterministic --- ...place-cancel-goal-race-regression-once.yml | 33 --------- .../replace-cancel-goal-race-regression.py | 73 ------------------- .../durable-retry-cancellation.test.ts | 54 ++++++-------- 3 files changed, 23 insertions(+), 137 deletions(-) delete mode 100644 .github/workflows/replace-cancel-goal-race-regression-once.yml delete mode 100644 scripts/replace-cancel-goal-race-regression.py diff --git a/.github/workflows/replace-cancel-goal-race-regression-once.yml b/.github/workflows/replace-cancel-goal-race-regression-once.yml deleted file mode 100644 index c5ab6f4..0000000 --- a/.github/workflows/replace-cancel-goal-race-regression-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Replace cancel goal race regression once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/replace-cancel-goal-race-regression-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/replace-cancel-goal-race-regression.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/replace-cancel-goal-race-regression-once.yml scripts/replace-cancel-goal-race-regression.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts - git commit -m "test: make cancel goal race regression deterministic" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/replace-cancel-goal-race-regression.py b/scripts/replace-cancel-goal-race-regression.py deleted file mode 100644 index 5bc65dd..0000000 --- a/scripts/replace-cancel-goal-race-regression.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -start = text.index(" it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators'") -end = text.index('\n });\n});', start) + len('\n });') -replacement = r''' it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const injector = await openPersistence(path); - const concurrentTask: Task = { - workspaceId: 'ws-a', - id: 'cancel-db-race-concurrent-task', - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Commit after the cancellation snapshot but before its durable batch.', - acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - status: 'ready', - revision: 1, - createdAt: now.toISOString(), - updatedAt: now.toISOString(), - }; - cancelApp.database.beforeNextBatch(async () => { - const injected = await injector.persistence.createTask({ task: concurrentTask }); - expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); - }); - - const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, - }); - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); - - const freshCancel = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = - finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - injector.database.close(); - });''' -path.write_text(text[:start] + replacement + text[end:]) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index ded7b43..2c1c0b4 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -615,7 +615,7 @@ describe('durable retry and cancellation', () => { second.database.close(); }); - it('prevents a stale CreateTask batch from surviving beneath CancelGoal with independent coordinators', async () => { + it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { const path = databasePath(); const now = new Date('2026-08-30T22:00:00.000Z'); let seed = await openDispatcher(path, 'cancel-db-race-seed', now); @@ -624,17 +624,27 @@ describe('durable retry and cancellation', () => { seed.database.close(); const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const createApp = await openDispatcher(path, 'cancel-db-race-creator', now); - let cancelArrivedResolve!: () => void; - let releaseCancel!: () => void; - const cancelArrived = new Promise((resolve) => (cancelArrivedResolve = resolve)); - const cancelRelease = new Promise((resolve) => (releaseCancel = resolve)); + const injector = await openPersistence(path); + const concurrentTask: Task = { + workspaceId: 'ws-a', + id: 'cancel-db-race-concurrent-task', + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Commit after the cancellation snapshot but before its durable batch.', + acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + status: 'ready', + revision: 1, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; cancelApp.database.beforeNextBatch(async () => { - cancelArrivedResolve(); - await cancelRelease; + const injected = await injector.persistence.createTask({ task: concurrentTask }); + expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); }); - const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ protocolVersion: '0.1', command: 'CancelGoal', commandId: 'cancel-db-race-goal', @@ -644,31 +654,13 @@ describe('durable retry and cancellation', () => { expectedGoalRevision: seeded.goal.revision, reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, }); - await cancelArrived; - - const createdResponse = await createApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CreateTask', - commandId: 'cancel-db-race-create', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Either commit before cancellation or be rejected.', - acceptanceCriteria: ['Never survive under a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - }); - const created = success(createdResponse); - releaseCancel(); - const staleCancellation = await cancelPromise; expect(staleCancellation).toHaveProperty('error'); const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === created.id)?.status).toBe('ready'); + expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); - const freshCancel = await createApp.dispatcher.dispatchCommand({ + const freshCancel = await cancelApp.dispatcher.dispatchCommand({ protocolVersion: '0.1', command: 'CancelGoal', commandId: 'cancel-db-race-goal-fresh', @@ -681,12 +673,12 @@ describe('durable retry and cancellation', () => { const committed = success(freshCancel); expect(committed.goal.status).toBe('cancelled'); expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await createApp.persistence.loadWorkspaceState('ws-a'); + const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); const finalGoalTasks = finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; expect(finalGoalTasks).toHaveLength(2); expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); cancelApp.database.close(); - createApp.database.close(); + injector.database.close(); }); }); From 7e1049024ac5327ee6896100c3376564dca685a6 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:28:24 +0500 Subject: [PATCH 12/38] test: prepare database-local cancel goal race --- scripts/make-cancel-goal-race-db-local.py | 159 ++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 scripts/make-cancel-goal-race-db-local.py diff --git a/scripts/make-cancel-goal-race-db-local.py b/scripts/make-cancel-goal-race-db-local.py new file mode 100644 index 0000000..9722e3b --- /dev/null +++ b/scripts/make-cancel-goal-race-db-local.py @@ -0,0 +1,159 @@ +from pathlib import Path + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +before = r''' it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const injector = await openPersistence(path); + const concurrentTask: Task = { + workspaceId: 'ws-a', + id: 'cancel-db-race-concurrent-task', + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Commit after the cancellation snapshot but before its durable batch.', + acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + status: 'ready', + revision: 1, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; + cancelApp.database.beforeNextBatch(async () => { + const injected = await injector.persistence.createTask({ task: concurrentTask }); + expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); + }); + + const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); + + const freshCancel = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = + finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + injector.database.close(); + });''' +after = r''' it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const concurrentTask: Task = { + workspaceId: 'ws-a', + id: 'cancel-db-race-concurrent-task', + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Commit after the cancellation snapshot but before its durable batch.', + acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + status: 'ready', + revision: 1, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; + cancelApp.database.beforeNextBatch(async () => { + await cancelApp.database + .prepare( + `INSERT INTO tasks( + workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + concurrentTask.workspaceId, + concurrentTask.id, + concurrentTask.goalId, + concurrentTask.revision, + concurrentTask.status, + now.getTime(), + now.getTime(), + JSON.stringify(concurrentTask), + ) + .run(); + await cancelApp.database + .prepare( + `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) + VALUES (?, ?, 0)`, + ) + .bind(concurrentTask.workspaceId, concurrentTask.id) + .run(); + }); + + const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); + + const freshCancel = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = + finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + });''' +if text.count(before) != 1: + raise RuntimeError('expected one race block') +path.write_text(text.replace(before, after, 1)) From 434dda619559c57261ab7986d485e765b2f0ac79 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:28:32 +0500 Subject: [PATCH 13/38] chore: add one-time database-local cancel goal race workflow --- .../make-cancel-goal-race-db-local-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/make-cancel-goal-race-db-local-once.yml diff --git a/.github/workflows/make-cancel-goal-race-db-local-once.yml b/.github/workflows/make-cancel-goal-race-db-local-once.yml new file mode 100644 index 0000000..425bdfe --- /dev/null +++ b/.github/workflows/make-cancel-goal-race-db-local-once.yml @@ -0,0 +1,33 @@ +name: Make cancel goal race database-local once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/make-cancel-goal-race-db-local-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/make-cancel-goal-race-db-local.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/make-cancel-goal-race-db-local-once.yml scripts/make-cancel-goal-race-db-local.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts + git commit -m "test: make cancel goal race database-local" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From 00dde3b70b598240568aec0f8195d2fa349d55f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:28:50 +0000 Subject: [PATCH 14/38] test: make cancel goal race database-local --- .../make-cancel-goal-race-db-local-once.yml | 33 ---- scripts/make-cancel-goal-race-db-local.py | 159 ------------------ .../durable-retry-cancellation.test.ts | 30 +++- 3 files changed, 25 insertions(+), 197 deletions(-) delete mode 100644 .github/workflows/make-cancel-goal-race-db-local-once.yml delete mode 100644 scripts/make-cancel-goal-race-db-local.py diff --git a/.github/workflows/make-cancel-goal-race-db-local-once.yml b/.github/workflows/make-cancel-goal-race-db-local-once.yml deleted file mode 100644 index 425bdfe..0000000 --- a/.github/workflows/make-cancel-goal-race-db-local-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Make cancel goal race database-local once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/make-cancel-goal-race-db-local-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/make-cancel-goal-race-db-local.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/make-cancel-goal-race-db-local-once.yml scripts/make-cancel-goal-race-db-local.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts - git commit -m "test: make cancel goal race database-local" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/make-cancel-goal-race-db-local.py b/scripts/make-cancel-goal-race-db-local.py deleted file mode 100644 index 9722e3b..0000000 --- a/scripts/make-cancel-goal-race-db-local.py +++ /dev/null @@ -1,159 +0,0 @@ -from pathlib import Path - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -before = r''' it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const injector = await openPersistence(path); - const concurrentTask: Task = { - workspaceId: 'ws-a', - id: 'cancel-db-race-concurrent-task', - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Commit after the cancellation snapshot but before its durable batch.', - acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - status: 'ready', - revision: 1, - createdAt: now.toISOString(), - updatedAt: now.toISOString(), - }; - cancelApp.database.beforeNextBatch(async () => { - const injected = await injector.persistence.createTask({ task: concurrentTask }); - expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); - }); - - const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, - }); - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); - - const freshCancel = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = - finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - injector.database.close(); - });''' -after = r''' it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const concurrentTask: Task = { - workspaceId: 'ws-a', - id: 'cancel-db-race-concurrent-task', - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Commit after the cancellation snapshot but before its durable batch.', - acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - status: 'ready', - revision: 1, - createdAt: now.toISOString(), - updatedAt: now.toISOString(), - }; - cancelApp.database.beforeNextBatch(async () => { - await cancelApp.database - .prepare( - `INSERT INTO tasks( - workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - ) - .bind( - concurrentTask.workspaceId, - concurrentTask.id, - concurrentTask.goalId, - concurrentTask.revision, - concurrentTask.status, - now.getTime(), - now.getTime(), - JSON.stringify(concurrentTask), - ) - .run(); - await cancelApp.database - .prepare( - `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) - VALUES (?, ?, 0)`, - ) - .bind(concurrentTask.workspaceId, concurrentTask.id) - .run(); - }); - - const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, - }); - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); - - const freshCancel = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = - finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - });''' -if text.count(before) != 1: - raise RuntimeError('expected one race block') -path.write_text(text.replace(before, after, 1)) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index 2c1c0b4..dee3ba7 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -615,7 +615,7 @@ describe('durable retry and cancellation', () => { second.database.close(); }); - it('rejects a stale CancelGoal snapshot when an independent persistence handle admits a Task before its batch', async () => { + it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch', async () => { const path = databasePath(); const now = new Date('2026-08-30T22:00:00.000Z'); let seed = await openDispatcher(path, 'cancel-db-race-seed', now); @@ -624,7 +624,6 @@ describe('durable retry and cancellation', () => { seed.database.close(); const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const injector = await openPersistence(path); const concurrentTask: Task = { workspaceId: 'ws-a', id: 'cancel-db-race-concurrent-task', @@ -640,8 +639,30 @@ describe('durable retry and cancellation', () => { updatedAt: now.toISOString(), }; cancelApp.database.beforeNextBatch(async () => { - const injected = await injector.persistence.createTask({ task: concurrentTask }); - expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); + await cancelApp.database + .prepare( + `INSERT INTO tasks( + workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind( + concurrentTask.workspaceId, + concurrentTask.id, + concurrentTask.goalId, + concurrentTask.revision, + concurrentTask.status, + now.getTime(), + now.getTime(), + JSON.stringify(concurrentTask), + ) + .run(); + await cancelApp.database + .prepare( + `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) + VALUES (?, ?, 0)`, + ) + .bind(concurrentTask.workspaceId, concurrentTask.id) + .run(); }); const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ @@ -679,6 +700,5 @@ describe('durable retry and cancellation', () => { expect(finalGoalTasks).toHaveLength(2); expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); cancelApp.database.close(); - injector.database.close(); }); }); From 3f85fd3a33758213f2abee4d1968af2362f28501 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:29:55 +0500 Subject: [PATCH 15/38] test: prepare clean independent cancel race harness --- scripts/fix-cancel-race-harness.py | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 scripts/fix-cancel-race-harness.py diff --git a/scripts/fix-cancel-race-harness.py b/scripts/fix-cancel-race-harness.py new file mode 100644 index 0000000..9557c7a --- /dev/null +++ b/scripts/fix-cancel-race-harness.py @@ -0,0 +1,73 @@ +from pathlib import Path + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +start = text.index(" it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch'") +end = text.index('\n });\n});', start) + len('\n });') +replacement = r''' it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + let seed = await openDispatcher(path, 'cancel-db-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); + seed.database.close(); + + const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const injector = await openPersistence(path); + const concurrentTask: Task = { + workspaceId: 'ws-a', + id: 'cancel-db-race-concurrent-task', + goalId: seeded.goal.id, + title: 'Concurrent Task', + objective: 'Commit after the cancellation snapshot but before its durable batch.', + acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], + requiredCapabilities: [], + dependencyTaskIds: [], + status: 'ready', + revision: 1, + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; + cancelApp.database.beforeNextBatch(async () => { + const injected = await injector.persistence.createTask({ task: concurrentTask }); + expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); + injector.database.close(); + }); + + const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + }); + expect(staleCancellation).toHaveProperty('error'); + + const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); + expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); + + const freshCancel = await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-db-race-goal-fresh', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, + }); + const committed = success(freshCancel); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + const finalGoalTasks = + finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(finalGoalTasks).toHaveLength(2); + expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + });''' +path.write_text(text[:start] + replacement + text[end:]) From cf6f38d68de9b2030ec9f48c0d98e8a65e36ee10 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:30:05 +0500 Subject: [PATCH 16/38] chore: add one-time clean cancel race harness workflow --- .../fix-cancel-race-harness-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/fix-cancel-race-harness-once.yml diff --git a/.github/workflows/fix-cancel-race-harness-once.yml b/.github/workflows/fix-cancel-race-harness-once.yml new file mode 100644 index 0000000..dd48362 --- /dev/null +++ b/.github/workflows/fix-cancel-race-harness-once.yml @@ -0,0 +1,33 @@ +name: Fix cancel race harness once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/fix-cancel-race-harness-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/fix-cancel-race-harness.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/fix-cancel-race-harness-once.yml scripts/fix-cancel-race-harness.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts + git commit -m "test: clean cancel goal race harness" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From b19b435bc69c3aa86f63026959e3f39925615513 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:30:17 +0000 Subject: [PATCH 17/38] test: clean cancel goal race harness --- .../fix-cancel-race-harness-once.yml | 33 --------- scripts/fix-cancel-race-harness.py | 73 ------------------- .../durable-retry-cancellation.test.ts | 30 ++------ 3 files changed, 5 insertions(+), 131 deletions(-) delete mode 100644 .github/workflows/fix-cancel-race-harness-once.yml delete mode 100644 scripts/fix-cancel-race-harness.py diff --git a/.github/workflows/fix-cancel-race-harness-once.yml b/.github/workflows/fix-cancel-race-harness-once.yml deleted file mode 100644 index dd48362..0000000 --- a/.github/workflows/fix-cancel-race-harness-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Fix cancel race harness once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/fix-cancel-race-harness-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/fix-cancel-race-harness.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/fix-cancel-race-harness-once.yml scripts/fix-cancel-race-harness.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts - git commit -m "test: clean cancel goal race harness" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/fix-cancel-race-harness.py b/scripts/fix-cancel-race-harness.py deleted file mode 100644 index 9557c7a..0000000 --- a/scripts/fix-cancel-race-harness.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -start = text.index(" it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch'") -end = text.index('\n });\n});', start) + len('\n });') -replacement = r''' it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const injector = await openPersistence(path); - const concurrentTask: Task = { - workspaceId: 'ws-a', - id: 'cancel-db-race-concurrent-task', - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Commit after the cancellation snapshot but before its durable batch.', - acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - status: 'ready', - revision: 1, - createdAt: now.toISOString(), - updatedAt: now.toISOString(), - }; - cancelApp.database.beforeNextBatch(async () => { - const injected = await injector.persistence.createTask({ task: concurrentTask }); - expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); - injector.database.close(); - }); - - const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, - }); - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); - - const freshCancel = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = - finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); - });''' -path.write_text(text[:start] + replacement + text[end:]) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index dee3ba7..e0870a7 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -615,7 +615,7 @@ describe('durable retry and cancellation', () => { second.database.close(); }); - it('rejects a stale CancelGoal snapshot when durable Task membership changes before its batch', async () => { + it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch', async () => { const path = databasePath(); const now = new Date('2026-08-30T22:00:00.000Z'); let seed = await openDispatcher(path, 'cancel-db-race-seed', now); @@ -624,6 +624,7 @@ describe('durable retry and cancellation', () => { seed.database.close(); const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); + const injector = await openPersistence(path); const concurrentTask: Task = { workspaceId: 'ws-a', id: 'cancel-db-race-concurrent-task', @@ -639,30 +640,9 @@ describe('durable retry and cancellation', () => { updatedAt: now.toISOString(), }; cancelApp.database.beforeNextBatch(async () => { - await cancelApp.database - .prepare( - `INSERT INTO tasks( - workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - ) - .bind( - concurrentTask.workspaceId, - concurrentTask.id, - concurrentTask.goalId, - concurrentTask.revision, - concurrentTask.status, - now.getTime(), - now.getTime(), - JSON.stringify(concurrentTask), - ) - .run(); - await cancelApp.database - .prepare( - `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) - VALUES (?, ?, 0)`, - ) - .bind(concurrentTask.workspaceId, concurrentTask.id) - .run(); + const injected = await injector.persistence.createTask({ task: concurrentTask }); + expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); + injector.database.close(); }); const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ From f1411fdc473d46fb768f04cda54a39cf35a56bf3 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:33:39 +0500 Subject: [PATCH 18/38] test: prepare stale goal create task red --- scripts/replace-race-with-create-task-red.py | 59 ++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 scripts/replace-race-with-create-task-red.py diff --git a/scripts/replace-race-with-create-task-red.py b/scripts/replace-race-with-create-task-red.py new file mode 100644 index 0000000..54af91e --- /dev/null +++ b/scripts/replace-race-with-create-task-red.py @@ -0,0 +1,59 @@ +from pathlib import Path + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +start = text.index(" it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch'") +end = text.index('\n });\n});', start) + len('\n });') +replacement = r''' it('rejects CreateTask when its Goal becomes terminal after precheck but before the durable batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + const app = await openDispatcher(path, 'create-goal-race', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'create-goal-race'); + const cancelledGoal: Goal = { + ...seeded.goal, + revision: seeded.goal.revision + 1, + status: 'cancelled', + updatedAt: now.toISOString(), + }; + app.database.beforeNextBatch(async () => { + await app.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + cancelledGoal.revision, + cancelledGoal.status, + now.getTime(), + JSON.stringify(cancelledGoal), + cancelledGoal.workspaceId, + cancelledGoal.id, + seeded.goal.revision, + ) + .run(); + }); + + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'create-goal-race-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Stale Goal Task', + objective: 'Must not commit beneath a terminal Goal.', + acceptanceCriteria: ['Database predicate rejects stale admission.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + expect(response).toHaveProperty('error'); + + const snapshot = await app.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)).toEqual(cancelledGoal); + expect(snapshot?.tasks.some((task) => task.title === 'Stale Goal Task')).toBe(false); + expect(await app.persistence.getCommandReceipt('ws-a', 'create-goal-race-command')).toBeUndefined(); + app.database.close(); + });''' +path.write_text(text[:start] + replacement + text[end:]) From 14904ba4092c995505d625b53729415d1e493a00 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:33:49 +0500 Subject: [PATCH 19/38] chore: add one-time stale goal create task red workflow --- ...replace-race-with-create-task-red-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/replace-race-with-create-task-red-once.yml diff --git a/.github/workflows/replace-race-with-create-task-red-once.yml b/.github/workflows/replace-race-with-create-task-red-once.yml new file mode 100644 index 0000000..e285e45 --- /dev/null +++ b/.github/workflows/replace-race-with-create-task-red-once.yml @@ -0,0 +1,33 @@ +name: Replace race with CreateTask RED once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/replace-race-with-create-task-red-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/replace-race-with-create-task-red.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/replace-race-with-create-task-red-once.yml scripts/replace-race-with-create-task-red.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts + git commit -m "test: cover stale Goal CreateTask race" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From cdaaf8560214d7f62f9e2924aaed326ffda4692f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:34:03 +0000 Subject: [PATCH 20/38] test: cover stale Goal CreateTask race --- ...replace-race-with-create-task-red-once.yml | 33 ------- scripts/replace-race-with-create-task-red.py | 59 ------------ .../durable-retry-cancellation.test.ts | 94 ++++++++----------- 3 files changed, 41 insertions(+), 145 deletions(-) delete mode 100644 .github/workflows/replace-race-with-create-task-red-once.yml delete mode 100644 scripts/replace-race-with-create-task-red.py diff --git a/.github/workflows/replace-race-with-create-task-red-once.yml b/.github/workflows/replace-race-with-create-task-red-once.yml deleted file mode 100644 index e285e45..0000000 --- a/.github/workflows/replace-race-with-create-task-red-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Replace race with CreateTask RED once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/replace-race-with-create-task-red-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/replace-race-with-create-task-red.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/replace-race-with-create-task-red-once.yml scripts/replace-race-with-create-task-red.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts - git commit -m "test: cover stale Goal CreateTask race" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/replace-race-with-create-task-red.py b/scripts/replace-race-with-create-task-red.py deleted file mode 100644 index 54af91e..0000000 --- a/scripts/replace-race-with-create-task-red.py +++ /dev/null @@ -1,59 +0,0 @@ -from pathlib import Path - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -start = text.index(" it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch'") -end = text.index('\n });\n});', start) + len('\n });') -replacement = r''' it('rejects CreateTask when its Goal becomes terminal after precheck but before the durable batch', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:00:00.000Z'); - const app = await openDispatcher(path, 'create-goal-race', now); - await app.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(app.dispatcher, 'create-goal-race'); - const cancelledGoal: Goal = { - ...seeded.goal, - revision: seeded.goal.revision + 1, - status: 'cancelled', - updatedAt: now.toISOString(), - }; - app.database.beforeNextBatch(async () => { - await app.database - .prepare( - `UPDATE goals - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, - ) - .bind( - cancelledGoal.revision, - cancelledGoal.status, - now.getTime(), - JSON.stringify(cancelledGoal), - cancelledGoal.workspaceId, - cancelledGoal.id, - seeded.goal.revision, - ) - .run(); - }); - - const response = await app.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CreateTask', - commandId: 'create-goal-race-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - title: 'Stale Goal Task', - objective: 'Must not commit beneath a terminal Goal.', - acceptanceCriteria: ['Database predicate rejects stale admission.'], - requiredCapabilities: [], - dependencyTaskIds: [], - }); - expect(response).toHaveProperty('error'); - - const snapshot = await app.persistence.loadWorkspaceState('ws-a'); - expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)).toEqual(cancelledGoal); - expect(snapshot?.tasks.some((task) => task.title === 'Stale Goal Task')).toBe(false); - expect(await app.persistence.getCommandReceipt('ws-a', 'create-goal-race-command')).toBeUndefined(); - app.database.close(); - });''' -path.write_text(text[:start] + replacement + text[end:]) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index e0870a7..7092669 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -615,70 +615,58 @@ describe('durable retry and cancellation', () => { second.database.close(); }); - it('rejects a stale CancelGoal snapshot when an independent Task commit changes membership before its batch', async () => { + it('rejects CreateTask when its Goal becomes terminal after precheck but before the durable batch', async () => { const path = databasePath(); const now = new Date('2026-08-30T22:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-db-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-db-race'); - seed.database.close(); - - const cancelApp = await openDispatcher(path, 'cancel-db-race-controller', now); - const injector = await openPersistence(path); - const concurrentTask: Task = { - workspaceId: 'ws-a', - id: 'cancel-db-race-concurrent-task', - goalId: seeded.goal.id, - title: 'Concurrent Task', - objective: 'Commit after the cancellation snapshot but before its durable batch.', - acceptanceCriteria: ['Never survive beneath a cancelled Goal.'], - requiredCapabilities: [], - dependencyTaskIds: [], - status: 'ready', - revision: 1, - createdAt: now.toISOString(), + const app = await openDispatcher(path, 'create-goal-race', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'create-goal-race'); + const cancelledGoal: Goal = { + ...seeded.goal, + revision: seeded.goal.revision + 1, + status: 'cancelled', updatedAt: now.toISOString(), }; - cancelApp.database.beforeNextBatch(async () => { - const injected = await injector.persistence.createTask({ task: concurrentTask }); - expect(injected).toMatchObject({ kind: 'committed', value: concurrentTask }); - injector.database.close(); + app.database.beforeNextBatch(async () => { + await app.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + cancelledGoal.revision, + cancelledGoal.status, + now.getTime(), + JSON.stringify(cancelledGoal), + cancelledGoal.workspaceId, + cancelledGoal.id, + seeded.goal.revision, + ) + .run(); }); - const staleCancellation = await cancelApp.dispatcher.dispatchCommand({ + const response = await app.dispatcher.dispatchCommand({ protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal', + command: 'CreateTask', + commandId: 'create-goal-race-command', workspaceId: 'ws-a', actor: seeded.systemActor, goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Race database task admission.' }, + title: 'Stale Goal Task', + objective: 'Must not commit beneath a terminal Goal.', + acceptanceCriteria: ['Database predicate rejects stale admission.'], + requiredCapabilities: [], + dependencyTaskIds: [], }); - expect(staleCancellation).toHaveProperty('error'); - - const afterRace = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(afterRace?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('active'); - expect(afterRace?.tasks.find((task) => task.id === concurrentTask.id)).toEqual(concurrentTask); + expect(response).toHaveProperty('error'); - const freshCancel = await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'cancel-db-race-goal-fresh', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Retry with a fresh Task set.' }, - }); - const committed = success(freshCancel); - expect(committed.goal.status).toBe('cancelled'); - expect(committed.tasks).toHaveLength(2); - const finalSnapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - const finalGoalTasks = - finalSnapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; - expect(finalGoalTasks).toHaveLength(2); - expect(finalGoalTasks.every((task) => task.status === 'cancelled')).toBe(true); - cancelApp.database.close(); + const snapshot = await app.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)).toEqual(cancelledGoal); + expect(snapshot?.tasks.some((task) => task.title === 'Stale Goal Task')).toBe(false); + expect( + await app.persistence.getCommandReceipt('ws-a', 'create-goal-race-command'), + ).toBeUndefined(); + app.database.close(); }); }); From 43b300afb9dd41fa4b2207e2040901f4b0ed7b04 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:34:52 +0500 Subject: [PATCH 21/38] feat: add D1 mutation batch guards --- migrations/0003_mutation_batch_guards.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 migrations/0003_mutation_batch_guards.sql diff --git a/migrations/0003_mutation_batch_guards.sql b/migrations/0003_mutation_batch_guards.sql new file mode 100644 index 0000000..b10ea82 --- /dev/null +++ b/migrations/0003_mutation_batch_guards.sql @@ -0,0 +1,5 @@ +CREATE TABLE mutation_batch_guards ( + workspace_id TEXT NOT NULL, + ok INTEGER NOT NULL, + CONSTRAINT mutation_batch_guard_ok CHECK (ok = 1) +); From 94426a6d308ff5d919e31fcb9d5597cbe522e027 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:35:52 +0500 Subject: [PATCH 22/38] feat: prepare D1 transaction guard fix --- scripts/apply-d1-transaction-guards.py | 550 +++++++++++++++++++++++++ 1 file changed, 550 insertions(+) create mode 100644 scripts/apply-d1-transaction-guards.py diff --git a/scripts/apply-d1-transaction-guards.py b/scripts/apply-d1-transaction-guards.py new file mode 100644 index 0000000..cbcb591 --- /dev/null +++ b/scripts/apply-d1-transaction-guards.py @@ -0,0 +1,550 @@ +from pathlib import Path + +path = Path('src/persistence/cloudflare/d1-runtime-persistence.ts') +text = path.read_text() + + +def replace_region(start_marker: str, end_marker: str, replacement: str) -> None: + global text + start = text.index(start_marker) + end = text.index(end_marker, start) + text = text[:start] + replacement + '\n\n' + text[end:] + + +create_task = r''' async createTask(input: { + task: Task; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise> { + const { task } = input; + this.assertCanonical('Task', task); + this.assertRelatedAudit(task.workspaceId, input.auditEvent); + this.assertReceipt(task.workspaceId, input.receipt); + + return this.coordinator.runSerialized(task.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + await this.requireWorkspace(task.workspaceId); + const parentGoal = await this.getGoal(task.workspaceId, task.goalId); + if (!parentGoal) { + throw new PersistenceError('NOT_FOUND', `Goal ${task.goalId} was not found.`); + } + if (parentGoal.status !== 'active') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Goal ${parentGoal.id} is terminal and cannot accept Task ${task.id}.`, + ); + } + await this.assertDependencies(task); + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `INSERT INTO tasks( + workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ? + WHERE EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, + ) + .bind( + task.workspaceId, + task.id, + task.goalId, + task.revision, + task.status, + timestampMs(task.createdAt, 'Task.createdAt'), + timestampMs(task.updatedAt, 'Task.updatedAt'), + serializeJson(task, 'Task'), + task.workspaceId, + task.goalId, + parentGoal.revision, + ), + this.mutationChangesGuardStatement(task.workspaceId), + ...task.requiredCapabilities.map((capability) => + this.database + .prepare( + `INSERT INTO task_required_capabilities(workspace_id, task_id, capability) + VALUES (?, ?, ?)`, + ) + .bind(task.workspaceId, task.id, capability), + ), + ...task.dependencyTaskIds.map((dependencyTaskId) => + this.database + .prepare( + `INSERT INTO task_dependencies( + workspace_id, goal_id, task_id, dependency_task_id + ) VALUES (?, ?, ?, ?)`, + ) + .bind(task.workspaceId, task.goalId, task.id, dependencyTaskId), + ), + this.database + .prepare( + `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) + VALUES (?, ?, 0)`, + ) + .bind(task.workspaceId, task.id), + ]; + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); + await this.batch(statements, 'create Task'); + return { kind: 'committed', value: clone(task) }; + }); + }''' +replace_region(' async createTask(input: {', ' async claimTask(', create_task) + +retry_task = r''' async retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise> { + const { task } = input; + this.assertCanonical('Task', task); + this.assertRelatedAudit(task.workspaceId, input.auditEvent); + this.assertReceipt(task.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'retry Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); + + return this.coordinator.runSerialized(task.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(task.workspaceId, task.id); + if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); + if (current.revision !== input.expectedRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, + ); + } + const goal = await this.getGoal(task.workspaceId, current.goalId); + if (!goal) { + throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); + } + if (goal.status !== 'active' || current.status !== 'failed') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${task.id} cannot be retried from current durable state.`, + ); + } + if (await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs)) { + throw new PersistenceError( + 'CONFLICT', + `Task ${task.id} still has active execution authority.`, + ); + } + + const expected: Task = { + ...clone(current), + status: 'ready', + revision: input.expectedRevision + 1, + updatedAt: input.now, + }; + delete expected.statusReason; + if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { + throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed' + AND EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, + ) + .bind( + task.revision, + task.status, + timestampMs(task.updatedAt, 'Task.updatedAt'), + serializeJson(task, 'Task'), + task.workspaceId, + task.id, + input.expectedRevision, + task.workspaceId, + current.goalId, + goal.revision, + ), + this.mutationChangesGuardStatement(task.workspaceId), + ]; + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); + await this.batch(statements, 'retry Task'); + return { kind: 'committed', value: clone(task) }; + }); + }''' +replace_region(' async retryTask(input: {', ' async cancelTask(', retry_task) + +cancel_task = r''' async cancelTask( + input: CancelTaskCommitInput, + ): Promise> { + this.assertCanonical('Task', input.task); + if (input.lease) this.assertCanonical('Lease', input.lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(input.workspaceId, input.task.id); + if (!current) { + throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); + } + if (current.revision !== input.expectedTaskRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, + ); + } + if (!isCancellableTaskStatus(current.status)) { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${current.id} cannot be cancelled from ${current.status}.`, + ); + } + if (!input.task.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); + } + const expectedTask: Task = { + ...clone(current), + status: 'cancelled', + statusReason: clone(input.task.statusReason), + revision: input.expectedTaskRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); + } + + const effectiveLease = await this.getEffectiveActiveLease( + input.workspaceId, + current.id, + nowMs, + cutoffMs, + ); + if ((effectiveLease === undefined) !== (input.lease === undefined)) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); + } + if (effectiveLease && input.lease) { + const expectedLease: Lease = { + ...clone(effectiveLease), + status: 'revoked', + revision: effectiveLease.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.lease, 'Lease') !== + serializeJson(expectedLease, 'expected CancelTask Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + input.task.revision, + input.task.status, + timestampMs(input.task.updatedAt, 'Task.updatedAt'), + serializeJson(input.task, 'Task'), + input.workspaceId, + input.task.id, + input.expectedTaskRevision, + current.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ]; + if (effectiveLease && input.lease) { + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + input.lease.revision, + input.lease.status, + timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), + serializeJson(input.lease, 'Lease'), + input.workspaceId, + input.lease.id, + effectiveLease.revision, + effectiveLease.fencingToken, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Task'); + return { + kind: 'committed', + value: { + task: clone(input.task), + ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), + }, + }; + }); + }''' +replace_region(' async cancelTask(', ' async cancelGoal(', cancel_task) + +cancel_goal = r''' async cancelGoal( + input: CancelGoalCommitInput, + ): Promise> { + this.assertCanonical('Goal', input.goal); + for (const task of input.tasks) this.assertCanonical('Task', task); + for (const lease of input.leases) this.assertCanonical('Lease', lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Goal now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); + if (!currentGoal) { + throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); + } + if (currentGoal.revision !== input.expectedGoalRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, + ); + } + if (currentGoal.status !== 'active') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Goal ${currentGoal.id} is already terminal.`, + ); + } + const expectedGoal: Goal = { + ...clone(currentGoal), + status: 'cancelled', + revision: input.expectedGoalRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); + } + + const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); + if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); + } + const effectiveLeases: Lease[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id); + if (!output || !output.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); + } + const expectedTask: Task = { + ...clone(currentTask), + status: 'cancelled', + statusReason: clone(output.statusReason), + revision: currentTask.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); + } + const effective = await this.getEffectiveActiveLease( + input.workspaceId, + currentTask.id, + nowMs, + cutoffMs, + ); + if (effective) effectiveLeases.push(effective); + } + + const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); + if ( + outputLeases.size !== input.leases.length || + input.leases.length !== effectiveLeases.length + ) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id); + if (!output) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); + } + const expectedLease: Lease = { + ...clone(effective), + status: 'revoked', + revision: effective.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Lease') !== + serializeJson(expectedLease, 'expected CancelGoal Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Task.updatedAt'), + serializeJson(output, 'Task'), + input.workspaceId, + output.id, + currentTask.revision, + currentTask.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id)!; + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Lease.updatedAt'), + serializeJson(output, 'Lease'), + input.workspaceId, + output.id, + effective.revision, + effective.fencingToken, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + statements.push( + this.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND NOT EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND goal_id = ? + AND status IN ('pending', 'ready', 'running', 'blocked') + )`, + ) + .bind( + input.goal.revision, + input.goal.status, + timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), + serializeJson(input.goal, 'Goal'), + input.workspaceId, + input.goal.id, + input.expectedGoalRevision, + input.workspaceId, + input.goal.id, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Goal'); + return { + kind: 'committed', + value: { + goal: clone(input.goal), + tasks: clone(input.tasks), + leases: clone(input.leases), + }, + }; + }); + }''' +replace_region(' async cancelGoal(', ' private async commitTaskOutcome(', cancel_goal) + +helper_marker = ''' private async readRecord(sql: string, ...values: unknown[]): Promise {''' +helpers = r''' private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + VALUES (?, changes())`, + ) + .bind(workspaceId); + } + + private clearMutationBatchGuardsStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare(`DELETE FROM mutation_batch_guards WHERE workspace_id = ?`) + .bind(workspaceId); + } + +''' +if text.count(helper_marker) != 1: + raise RuntimeError('readRecord marker mismatch') +text = text.replace(helper_marker, helpers + helper_marker, 1) + +old_batch = r''' private async batch( + statements: D1PreparedStatementLike[], + context: string, + ): Promise { + try { + return await this.database.batch(statements); + } catch (error) { + throw wrapDatabaseError(context, error); + } + }''' +new_batch = r''' private async batch( + statements: D1PreparedStatementLike[], + context: string, + ): Promise { + try { + return await this.database.batch(statements); + } catch (error) { + const message = errorMessage(error); + if (message.includes('mutation_batch_guard_ok')) { + const code = context === 'retry Task' ? 'REVISION_MISMATCH' : 'CONFLICT'; + throw new PersistenceError(code, `${context} lost its durable mutation race.`); + } + throw wrapDatabaseError(context, error); + } + }''' +if text.count(old_batch) != 1: + raise RuntimeError('batch method marker mismatch') +text = text.replace(old_batch, new_batch, 1) + +path.write_text(text) From 837d99724a9d97b5abda631828bb0fee19a400c0 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:36:04 +0500 Subject: [PATCH 23/38] chore: add one-time D1 transaction guard workflow --- .../apply-d1-transaction-guards-once.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/apply-d1-transaction-guards-once.yml diff --git a/.github/workflows/apply-d1-transaction-guards-once.yml b/.github/workflows/apply-d1-transaction-guards-once.yml new file mode 100644 index 0000000..2387697 --- /dev/null +++ b/.github/workflows/apply-d1-transaction-guards-once.yml @@ -0,0 +1,35 @@ +name: Apply D1 transaction guards once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/apply-d1-transaction-guards-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/apply-d1-transaction-guards.py + - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts + - run: git rm .github/workflows/apply-d1-transaction-guards-once.yml scripts/apply-d1-transaction-guards.py + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts + - run: pnpm check + - run: pnpm test:coverage + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/persistence/cloudflare/d1-runtime-persistence.ts migrations/0003_mutation_batch_guards.sql + git commit -m "fix: make durable lifecycle races transactional" + git push From 6c1911846bbf4ed4fa717bb9d6dff492da6a6642 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:36:47 +0500 Subject: [PATCH 24/38] fix: make mutation guard migration repeatable --- migrations/0003_mutation_batch_guards.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/0003_mutation_batch_guards.sql b/migrations/0003_mutation_batch_guards.sql index b10ea82..ab285d9 100644 --- a/migrations/0003_mutation_batch_guards.sql +++ b/migrations/0003_mutation_batch_guards.sql @@ -1,4 +1,4 @@ -CREATE TABLE mutation_batch_guards ( +CREATE TABLE IF NOT EXISTS mutation_batch_guards ( workspace_id TEXT NOT NULL, ok INTEGER NOT NULL, CONSTRAINT mutation_batch_guard_ok CHECK (ok = 1) From 4603519fd6c9d648c1fc5b4612f1f37a19d23537 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:37:01 +0500 Subject: [PATCH 25/38] chore: rerun D1 transaction guard workflow From b8753b415f573a3e2f5cf1438f3267500cc3cf41 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:37:31 +0500 Subject: [PATCH 26/38] chore: retrigger D1 transaction guard workflow --- .github/workflows/apply-d1-transaction-guards-once.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/apply-d1-transaction-guards-once.yml b/.github/workflows/apply-d1-transaction-guards-once.yml index 2387697..5e7e76e 100644 --- a/.github/workflows/apply-d1-transaction-guards-once.yml +++ b/.github/workflows/apply-d1-transaction-guards-once.yml @@ -1,5 +1,6 @@ name: Apply D1 transaction guards once +# Rerun after making the migration repeatable. on: push: branches: From fc0cf3e7d95e821fa25fade0d692709b49098600 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:38:31 +0500 Subject: [PATCH 27/38] chore: add final D1 transaction guard workflow --- .../finalize-d1-transaction-guards-once.yml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/finalize-d1-transaction-guards-once.yml diff --git a/.github/workflows/finalize-d1-transaction-guards-once.yml b/.github/workflows/finalize-d1-transaction-guards-once.yml new file mode 100644 index 0000000..4e478a1 --- /dev/null +++ b/.github/workflows/finalize-d1-transaction-guards-once.yml @@ -0,0 +1,36 @@ +name: Finalize D1 transaction guards once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/finalize-d1-transaction-guards-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/apply-d1-transaction-guards.py + - run: sed -i 's/ let seed = await openDispatcher/ const seed = await openDispatcher/g' test/application/durable-retry-cancellation.test.ts + - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/apply-d1-transaction-guards-once.yml .github/workflows/finalize-d1-transaction-guards-once.yml scripts/apply-d1-transaction-guards.py + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts + - run: pnpm check + - run: pnpm test:coverage + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/persistence/cloudflare/d1-runtime-persistence.ts migrations/0003_mutation_batch_guards.sql test/application/durable-retry-cancellation.test.ts + git commit -m "fix: make durable lifecycle races transactional" + git push From 2a74dd14f572319aa57e63249ab3de830b619af8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:39:15 +0000 Subject: [PATCH 28/38] fix: make durable lifecycle races transactional --- .../apply-d1-transaction-guards-once.yml | 36 -- .../finalize-d1-transaction-guards-once.yml | 36 -- scripts/apply-d1-transaction-guards.py | 550 ------------------ .../cloudflare/d1-runtime-persistence.ts | 116 ++-- .../durable-retry-cancellation.test.ts | 4 +- 5 files changed, 78 insertions(+), 664 deletions(-) delete mode 100644 .github/workflows/apply-d1-transaction-guards-once.yml delete mode 100644 .github/workflows/finalize-d1-transaction-guards-once.yml delete mode 100644 scripts/apply-d1-transaction-guards.py diff --git a/.github/workflows/apply-d1-transaction-guards-once.yml b/.github/workflows/apply-d1-transaction-guards-once.yml deleted file mode 100644 index 5e7e76e..0000000 --- a/.github/workflows/apply-d1-transaction-guards-once.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Apply D1 transaction guards once - -# Rerun after making the migration repeatable. -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/apply-d1-transaction-guards-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/apply-d1-transaction-guards.py - - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts - - run: git rm .github/workflows/apply-d1-transaction-guards-once.yml scripts/apply-d1-transaction-guards.py - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts - - run: pnpm check - - run: pnpm test:coverage - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/persistence/cloudflare/d1-runtime-persistence.ts migrations/0003_mutation_batch_guards.sql - git commit -m "fix: make durable lifecycle races transactional" - git push diff --git a/.github/workflows/finalize-d1-transaction-guards-once.yml b/.github/workflows/finalize-d1-transaction-guards-once.yml deleted file mode 100644 index 4e478a1..0000000 --- a/.github/workflows/finalize-d1-transaction-guards-once.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Finalize D1 transaction guards once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/finalize-d1-transaction-guards-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/apply-d1-transaction-guards.py - - run: sed -i 's/ let seed = await openDispatcher/ const seed = await openDispatcher/g' test/application/durable-retry-cancellation.test.ts - - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/apply-d1-transaction-guards-once.yml .github/workflows/finalize-d1-transaction-guards-once.yml scripts/apply-d1-transaction-guards.py - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts - - run: pnpm check - - run: pnpm test:coverage - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/persistence/cloudflare/d1-runtime-persistence.ts migrations/0003_mutation_batch_guards.sql test/application/durable-retry-cancellation.test.ts - git commit -m "fix: make durable lifecycle races transactional" - git push diff --git a/scripts/apply-d1-transaction-guards.py b/scripts/apply-d1-transaction-guards.py deleted file mode 100644 index cbcb591..0000000 --- a/scripts/apply-d1-transaction-guards.py +++ /dev/null @@ -1,550 +0,0 @@ -from pathlib import Path - -path = Path('src/persistence/cloudflare/d1-runtime-persistence.ts') -text = path.read_text() - - -def replace_region(start_marker: str, end_marker: str, replacement: str) -> None: - global text - start = text.index(start_marker) - end = text.index(end_marker, start) - text = text[:start] + replacement + '\n\n' + text[end:] - - -create_task = r''' async createTask(input: { - task: Task; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise> { - const { task } = input; - this.assertCanonical('Task', task); - this.assertRelatedAudit(task.workspaceId, input.auditEvent); - this.assertReceipt(task.workspaceId, input.receipt); - - return this.coordinator.runSerialized(task.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - await this.requireWorkspace(task.workspaceId); - const parentGoal = await this.getGoal(task.workspaceId, task.goalId); - if (!parentGoal) { - throw new PersistenceError('NOT_FOUND', `Goal ${task.goalId} was not found.`); - } - if (parentGoal.status !== 'active') { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Goal ${parentGoal.id} is terminal and cannot accept Task ${task.id}.`, - ); - } - await this.assertDependencies(task); - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `INSERT INTO tasks( - workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json - ) - SELECT ?, ?, ?, ?, ?, ?, ?, ? - WHERE EXISTS ( - SELECT 1 FROM goals - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - )`, - ) - .bind( - task.workspaceId, - task.id, - task.goalId, - task.revision, - task.status, - timestampMs(task.createdAt, 'Task.createdAt'), - timestampMs(task.updatedAt, 'Task.updatedAt'), - serializeJson(task, 'Task'), - task.workspaceId, - task.goalId, - parentGoal.revision, - ), - this.mutationChangesGuardStatement(task.workspaceId), - ...task.requiredCapabilities.map((capability) => - this.database - .prepare( - `INSERT INTO task_required_capabilities(workspace_id, task_id, capability) - VALUES (?, ?, ?)`, - ) - .bind(task.workspaceId, task.id, capability), - ), - ...task.dependencyTaskIds.map((dependencyTaskId) => - this.database - .prepare( - `INSERT INTO task_dependencies( - workspace_id, goal_id, task_id, dependency_task_id - ) VALUES (?, ?, ?, ?)`, - ) - .bind(task.workspaceId, task.goalId, task.id, dependencyTaskId), - ), - this.database - .prepare( - `INSERT INTO task_fencing_counters(workspace_id, task_id, last_fencing_token) - VALUES (?, ?, 0)`, - ) - .bind(task.workspaceId, task.id), - ]; - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); - await this.batch(statements, 'create Task'); - return { kind: 'committed', value: clone(task) }; - }); - }''' -replace_region(' async createTask(input: {', ' async claimTask(', create_task) - -retry_task = r''' async retryTask(input: { - task: Task; - expectedRevision: number; - now: string; - sessionCutoff: string; - receipt?: CommandReceiptInput; - auditEvent?: AuditEvent; - }): Promise> { - const { task } = input; - this.assertCanonical('Task', task); - this.assertRelatedAudit(task.workspaceId, input.auditEvent); - this.assertReceipt(task.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'retry Task now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); - - return this.coordinator.runSerialized(task.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const current = await this.getTask(task.workspaceId, task.id); - if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); - if (current.revision !== input.expectedRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, - ); - } - const goal = await this.getGoal(task.workspaceId, current.goalId); - if (!goal) { - throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); - } - if (goal.status !== 'active' || current.status !== 'failed') { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Task ${task.id} cannot be retried from current durable state.`, - ); - } - if (await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs)) { - throw new PersistenceError( - 'CONFLICT', - `Task ${task.id} still has active execution authority.`, - ); - } - - const expected: Task = { - ...clone(current), - status: 'ready', - revision: input.expectedRevision + 1, - updatedAt: input.now, - }; - delete expected.statusReason; - if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { - throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); - } - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed' - AND EXISTS ( - SELECT 1 FROM goals - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - )`, - ) - .bind( - task.revision, - task.status, - timestampMs(task.updatedAt, 'Task.updatedAt'), - serializeJson(task, 'Task'), - task.workspaceId, - task.id, - input.expectedRevision, - task.workspaceId, - current.goalId, - goal.revision, - ), - this.mutationChangesGuardStatement(task.workspaceId), - ]; - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); - await this.batch(statements, 'retry Task'); - return { kind: 'committed', value: clone(task) }; - }); - }''' -replace_region(' async retryTask(input: {', ' async cancelTask(', retry_task) - -cancel_task = r''' async cancelTask( - input: CancelTaskCommitInput, - ): Promise> { - this.assertCanonical('Task', input.task); - if (input.lease) this.assertCanonical('Lease', input.lease); - this.assertRelatedAudit(input.workspaceId, input.auditEvent); - this.assertReceipt(input.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'cancel Task now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); - - return this.coordinator.runSerialized(input.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const current = await this.getTask(input.workspaceId, input.task.id); - if (!current) { - throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); - } - if (current.revision !== input.expectedTaskRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, - ); - } - if (!isCancellableTaskStatus(current.status)) { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Task ${current.id} cannot be cancelled from ${current.status}.`, - ); - } - if (!input.task.statusReason) { - throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); - } - const expectedTask: Task = { - ...clone(current), - status: 'cancelled', - statusReason: clone(input.task.statusReason), - revision: input.expectedTaskRevision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); - } - - const effectiveLease = await this.getEffectiveActiveLease( - input.workspaceId, - current.id, - nowMs, - cutoffMs, - ); - if ((effectiveLease === undefined) !== (input.lease === undefined)) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); - } - if (effectiveLease && input.lease) { - const expectedLease: Lease = { - ...clone(effectiveLease), - status: 'revoked', - revision: effectiveLease.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.lease, 'Lease') !== - serializeJson(expectedLease, 'expected CancelTask Lease') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); - } - } - - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) - .bind( - input.task.revision, - input.task.status, - timestampMs(input.task.updatedAt, 'Task.updatedAt'), - serializeJson(input.task, 'Task'), - input.workspaceId, - input.task.id, - input.expectedTaskRevision, - current.status, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ]; - if (effectiveLease && input.lease) { - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - AND fencing_token = ?`, - ) - .bind( - input.lease.revision, - input.lease.status, - timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), - serializeJson(input.lease, 'Lease'), - input.workspaceId, - input.lease.id, - effectiveLease.revision, - effectiveLease.fencingToken, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - } - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); - await this.batch(statements, 'cancel Task'); - return { - kind: 'committed', - value: { - task: clone(input.task), - ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), - }, - }; - }); - }''' -replace_region(' async cancelTask(', ' async cancelGoal(', cancel_task) - -cancel_goal = r''' async cancelGoal( - input: CancelGoalCommitInput, - ): Promise> { - this.assertCanonical('Goal', input.goal); - for (const task of input.tasks) this.assertCanonical('Task', task); - for (const lease of input.leases) this.assertCanonical('Lease', lease); - this.assertRelatedAudit(input.workspaceId, input.auditEvent); - this.assertReceipt(input.workspaceId, input.receipt); - const nowMs = timestampMs(input.now, 'cancel Goal now'); - const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); - - return this.coordinator.runSerialized(input.workspaceId, async () => { - const replay = await this.resolveReceipt(input.receipt); - if (replay) return replay; - const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); - if (!currentGoal) { - throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); - } - if (currentGoal.revision !== input.expectedGoalRevision) { - throw new PersistenceError( - 'REVISION_MISMATCH', - `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, - ); - } - if (currentGoal.status !== 'active') { - throw new PersistenceError( - 'INVALID_STATE_TRANSITION', - `Goal ${currentGoal.id} is already terminal.`, - ); - } - const expectedGoal: Goal = { - ...clone(currentGoal), - status: 'cancelled', - revision: input.expectedGoalRevision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); - } - - const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); - const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); - const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); - if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); - } - const effectiveLeases: Lease[] = []; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id); - if (!output || !output.statusReason) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); - } - const expectedTask: Task = { - ...clone(currentTask), - status: 'cancelled', - statusReason: clone(output.statusReason), - revision: currentTask.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); - } - const effective = await this.getEffectiveActiveLease( - input.workspaceId, - currentTask.id, - nowMs, - cutoffMs, - ); - if (effective) effectiveLeases.push(effective); - } - - const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); - if ( - outputLeases.size !== input.leases.length || - input.leases.length !== effectiveLeases.length - ) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); - } - for (const effective of effectiveLeases) { - const output = outputLeases.get(effective.id); - if (!output) { - throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); - } - const expectedLease: Lease = { - ...clone(effective), - status: 'revoked', - revision: effective.revision + 1, - updatedAt: input.now, - }; - if ( - serializeJson(output, 'Lease') !== - serializeJson(expectedLease, 'expected CancelGoal Lease') - ) { - throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); - } - } - - const statements: D1PreparedStatementLike[] = []; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id)!; - statements.push( - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) - .bind( - output.revision, - output.status, - timestampMs(output.updatedAt, 'Task.updatedAt'), - serializeJson(output, 'Task'), - input.workspaceId, - output.id, - currentTask.revision, - currentTask.status, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - } - for (const effective of effectiveLeases) { - const output = outputLeases.get(effective.id)!; - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - AND fencing_token = ?`, - ) - .bind( - output.revision, - output.status, - timestampMs(output.updatedAt, 'Lease.updatedAt'), - serializeJson(output, 'Lease'), - input.workspaceId, - output.id, - effective.revision, - effective.fencingToken, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - } - statements.push( - this.database - .prepare( - `UPDATE goals - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' - AND NOT EXISTS ( - SELECT 1 FROM tasks - WHERE workspace_id = ? AND goal_id = ? - AND status IN ('pending', 'ready', 'running', 'blocked') - )`, - ) - .bind( - input.goal.revision, - input.goal.status, - timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), - serializeJson(input.goal, 'Goal'), - input.workspaceId, - input.goal.id, - input.expectedGoalRevision, - input.workspaceId, - input.goal.id, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - this.pushAuditStatement(statements, input.auditEvent); - this.pushReceiptStatement(statements, input.receipt); - statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); - await this.batch(statements, 'cancel Goal'); - return { - kind: 'committed', - value: { - goal: clone(input.goal), - tasks: clone(input.tasks), - leases: clone(input.leases), - }, - }; - }); - }''' -replace_region(' async cancelGoal(', ' private async commitTaskOutcome(', cancel_goal) - -helper_marker = ''' private async readRecord(sql: string, ...values: unknown[]): Promise {''' -helpers = r''' private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { - return this.database - .prepare( - `INSERT INTO mutation_batch_guards(workspace_id, ok) - VALUES (?, changes())`, - ) - .bind(workspaceId); - } - - private clearMutationBatchGuardsStatement(workspaceId: string): D1PreparedStatementLike { - return this.database - .prepare(`DELETE FROM mutation_batch_guards WHERE workspace_id = ?`) - .bind(workspaceId); - } - -''' -if text.count(helper_marker) != 1: - raise RuntimeError('readRecord marker mismatch') -text = text.replace(helper_marker, helpers + helper_marker, 1) - -old_batch = r''' private async batch( - statements: D1PreparedStatementLike[], - context: string, - ): Promise { - try { - return await this.database.batch(statements); - } catch (error) { - throw wrapDatabaseError(context, error); - } - }''' -new_batch = r''' private async batch( - statements: D1PreparedStatementLike[], - context: string, - ): Promise { - try { - return await this.database.batch(statements); - } catch (error) { - const message = errorMessage(error); - if (message.includes('mutation_batch_guard_ok')) { - const code = context === 'retry Task' ? 'REVISION_MISMATCH' : 'CONFLICT'; - throw new PersistenceError(code, `${context} lost its durable mutation race.`); - } - throw wrapDatabaseError(context, error); - } - }''' -if text.count(old_batch) != 1: - raise RuntimeError('batch method marker mismatch') -text = text.replace(old_batch, new_batch, 1) - -path.write_text(text) diff --git a/src/persistence/cloudflare/d1-runtime-persistence.ts b/src/persistence/cloudflare/d1-runtime-persistence.ts index b954008..e82c263 100644 --- a/src/persistence/cloudflare/d1-runtime-persistence.ts +++ b/src/persistence/cloudflare/d1-runtime-persistence.ts @@ -504,7 +504,12 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { .prepare( `INSERT INTO tasks( workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ? + WHERE EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, ) .bind( task.workspaceId, @@ -515,7 +520,11 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { timestampMs(task.createdAt, 'Task.createdAt'), timestampMs(task.updatedAt, 'Task.updatedAt'), serializeJson(task, 'Task'), + task.workspaceId, + task.goalId, + parentGoal.revision, ), + this.mutationChangesGuardStatement(task.workspaceId), ...task.requiredCapabilities.map((capability) => this.database .prepare( @@ -542,6 +551,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { ]; this.pushAuditStatement(statements, input.auditEvent); this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); await this.batch(statements, 'create Task'); return { kind: 'committed', value: clone(task) }; }); @@ -1098,7 +1108,11 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { .prepare( `UPDATE tasks SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed'`, + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed' + AND EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, ) .bind( task.revision, @@ -1108,14 +1122,16 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { task.workspaceId, task.id, input.expectedRevision, + task.workspaceId, + current.goalId, + goal.revision, ), + this.mutationChangesGuardStatement(task.workspaceId), ]; this.pushAuditStatement(statements, input.auditEvent); this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'retry Task'); - if (changes(results[0]!) !== 1) { - throw new PersistenceError('REVISION_MISMATCH', `Task ${task.id} lost its retry race.`); - } + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); + await this.batch(statements, 'retry Task'); return { kind: 'committed', value: clone(task) }; }); } @@ -1206,6 +1222,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { input.expectedTaskRevision, current.status, ), + this.mutationChangesGuardStatement(input.workspaceId), ]; if (effectiveLease && input.lease) { statements.push( @@ -1226,20 +1243,13 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { effectiveLease.revision, effectiveLease.fencingToken, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } this.pushAuditStatement(statements, input.auditEvent); this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'cancel Task'); - if (changes(results[0]!) !== 1) { - throw new PersistenceError('REVISION_MISMATCH', `Task ${current.id} lost its cancel race.`); - } - if (effectiveLease && changes(results[1]!) !== 1) { - throw new PersistenceError( - 'STALE_AUTHORITY', - `Task ${current.id} Lease lost its cancel race.`, - ); - } + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Task'); return { kind: 'committed', value: { @@ -1351,23 +1361,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { } } - const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE goals - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, - ) - .bind( - input.goal.revision, - input.goal.status, - timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), - serializeJson(input.goal, 'Goal'), - input.workspaceId, - input.goal.id, - input.expectedGoalRevision, - ), - ]; + const statements: D1PreparedStatementLike[] = []; for (const currentTask of cancellable) { const output = outputTasks.get(currentTask.id)!; statements.push( @@ -1387,6 +1381,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { currentTask.revision, currentTask.status, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } for (const effective of effectiveLeases) { @@ -1409,17 +1404,38 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { effective.revision, effective.fencingToken, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } + statements.push( + this.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND NOT EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND goal_id = ? + AND status IN ('pending', 'ready', 'running', 'blocked') + )`, + ) + .bind( + input.goal.revision, + input.goal.status, + timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), + serializeJson(input.goal, 'Goal'), + input.workspaceId, + input.goal.id, + input.expectedGoalRevision, + input.workspaceId, + input.goal.id, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); this.pushAuditStatement(statements, input.auditEvent); this.pushReceiptStatement(statements, input.receipt); - const results = await this.batch(statements, 'cancel Goal'); - const mutationCount = 1 + cancellable.length + effectiveLeases.length; - for (let index = 0; index < mutationCount; index += 1) { - if (changes(results[index]!) !== 1) { - throw new PersistenceError('CONFLICT', `Goal ${currentGoal.id} lost its cancel race.`); - } - } + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Goal'); return { kind: 'committed', value: { @@ -2552,6 +2568,21 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { ); } + private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + VALUES (?, changes())`, + ) + .bind(workspaceId); + } + + private clearMutationBatchGuardsStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare(`DELETE FROM mutation_batch_guards WHERE workspace_id = ?`) + .bind(workspaceId); + } + private async readRecord(sql: string, ...values: unknown[]): Promise { const row = await this.first(sql, ...values); return row ? parseJson(row.record_json, 'canonical record') : undefined; @@ -2593,6 +2624,11 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { try { return await this.database.batch(statements); } catch (error) { + const message = errorMessage(error); + if (message.includes('mutation_batch_guard_ok')) { + const code = context === 'retry Task' ? 'REVISION_MISMATCH' : 'CONFLICT'; + throw new PersistenceError(code, `${context} lost its durable mutation race.`); + } throw wrapDatabaseError(context, error); } } diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index 7092669..ac244e8 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -476,7 +476,7 @@ describe('durable retry and cancellation', () => { it('serializes stale CreateTask persistence behind CancelGoal through one Workspace authority', async () => { const path = databasePath(); const now = new Date('2026-08-30T21:00:00.000Z'); - let seed = await openDispatcher(path, 'cancel-race-seed', now); + const seed = await openDispatcher(path, 'cancel-race-seed', now); await seed.persistence.bootstrapWorkspace(workspace()); const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-race'); seed.database.close(); @@ -528,7 +528,7 @@ describe('durable retry and cancellation', () => { it('does not retain a success receipt when RetryTask loses its database CAS race', async () => { const path = databasePath(); const now = new Date('2026-08-30T21:30:00.000Z'); - let seed = await openDispatcher(path, 'retry-race-seed', now); + const seed = await openDispatcher(path, 'retry-race-seed', now); await seed.persistence.bootstrapWorkspace(workspace()); const seeded = await seedClaimedTask(seed.dispatcher, 'retry-race'); const failed = success( From 9893b6e9e5ca1529accabae9b9f6106058ebefc2 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:41:28 +0500 Subject: [PATCH 29/38] test: prepare cancellation claim race regressions --- scripts/add-cancel-claim-race-red.py | 162 +++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 scripts/add-cancel-claim-race-red.py diff --git a/scripts/add-cancel-claim-race-red.py b/scripts/add-cancel-claim-race-red.py new file mode 100644 index 0000000..b87ff61 --- /dev/null +++ b/scripts/add-cancel-claim-race-red.py @@ -0,0 +1,162 @@ +from pathlib import Path + +path = Path('test/application/durable-retry-cancellation.test.ts') +text = path.read_text() +new_tests = r''' + + it('prevents a paused recovery ClaimTask from minting authority after CancelTask commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:30:00.000Z'); + const seed = await openDispatcher(path, 'cancel-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'cancel-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'cancel-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'cancel-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Cancellation wins the recovery race.' }, + }), + ); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect(await claimApp.persistence.getCommandReceipt('ws-a', 'cancel-claim-race-claim-command')).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); + + it('prevents a paused recovery ClaimTask from minting authority after CancelGoal commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T23:00:00.000Z'); + const seed = await openDispatcher(path, 'goal-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'goal-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'goal-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'goal-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'goal-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'goal-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'goal-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Goal cancellation wins recovery race.' }, + }), + ); + expect(cancelled.goal.status).toBe('cancelled'); + expect(cancelled.tasks).toHaveLength(1); + expect(cancelled.leases).toHaveLength(0); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect(await claimApp.persistence.getCommandReceipt('ws-a', 'goal-claim-race-claim-command')).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); +''' +marker = '\n});\n' +if not text.endswith(marker): + raise RuntimeError('unexpected test suffix') +path.write_text(text[:-len(marker)] + new_tests + marker) From b3b0dd9dbda9d781caa09d95717b0a2f9105f66f Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:41:40 +0500 Subject: [PATCH 30/38] chore: add one-time cancellation claim race RED workflow --- .../add-cancel-claim-race-red-once.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/add-cancel-claim-race-red-once.yml diff --git a/.github/workflows/add-cancel-claim-race-red-once.yml b/.github/workflows/add-cancel-claim-race-red-once.yml new file mode 100644 index 0000000..c22422a --- /dev/null +++ b/.github/workflows/add-cancel-claim-race-red-once.yml @@ -0,0 +1,33 @@ +name: Add cancellation claim race RED once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/add-cancel-claim-race-red-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/add-cancel-claim-race-red.py + - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts + - run: git rm .github/workflows/add-cancel-claim-race-red-once.yml scripts/add-cancel-claim-race-red.py + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add test/application/durable-retry-cancellation.test.ts + git commit -m "test: cover cancellation recovery claim races" + git push + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts From 7730c0b1008d21225daf11964e832644184aa1f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:41:54 +0000 Subject: [PATCH 31/38] test: cover cancellation recovery claim races --- .../add-cancel-claim-race-red-once.yml | 33 ---- scripts/add-cancel-claim-race-red.py | 162 ------------------ .../durable-retry-cancellation.test.ts | 156 +++++++++++++++++ 3 files changed, 156 insertions(+), 195 deletions(-) delete mode 100644 .github/workflows/add-cancel-claim-race-red-once.yml delete mode 100644 scripts/add-cancel-claim-race-red.py diff --git a/.github/workflows/add-cancel-claim-race-red-once.yml b/.github/workflows/add-cancel-claim-race-red-once.yml deleted file mode 100644 index c22422a..0000000 --- a/.github/workflows/add-cancel-claim-race-red-once.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Add cancellation claim race RED once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/add-cancel-claim-race-red-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/add-cancel-claim-race-red.py - - run: pnpm exec prettier --write test/application/durable-retry-cancellation.test.ts - - run: git rm .github/workflows/add-cancel-claim-race-red-once.yml scripts/add-cancel-claim-race-red.py - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add test/application/durable-retry-cancellation.test.ts - git commit -m "test: cover cancellation recovery claim races" - git push - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts diff --git a/scripts/add-cancel-claim-race-red.py b/scripts/add-cancel-claim-race-red.py deleted file mode 100644 index b87ff61..0000000 --- a/scripts/add-cancel-claim-race-red.py +++ /dev/null @@ -1,162 +0,0 @@ -from pathlib import Path - -path = Path('test/application/durable-retry-cancellation.test.ts') -text = path.read_text() -new_tests = r''' - - it('prevents a paused recovery ClaimTask from minting authority after CancelTask commits first', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T22:30:00.000Z'); - const seed = await openDispatcher(path, 'cancel-claim-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-claim-race'); - success( - await seed.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'ReleaseLease', - commandId: 'cancel-claim-race-release', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - leaseId: seeded.claim.lease.id, - fencingToken: seeded.claim.lease.fencingToken, - expectedLeaseRevision: seeded.claim.lease.revision, - }), - ); - seed.database.close(); - - const claimApp = await openDispatcher(path, 'cancel-claim-race-claim', now); - const cancelApp = await openDispatcher(path, 'cancel-claim-race-cancel', now); - let claimArrivedResolve!: () => void; - let releaseClaim!: () => void; - const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); - const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); - claimApp.database.beforeNextBatch(async () => { - claimArrivedResolve(); - await claimRelease; - }); - - const claimPromise = claimApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'ClaimTask', - commandId: 'cancel-claim-race-claim-command', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - expectedTaskRevision: seeded.claim.task.revision, - }); - await claimArrived; - - const cancelled = success( - await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelTask', - commandId: 'cancel-claim-race-cancel-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - taskId: seeded.task.id, - expectedTaskRevision: seeded.claim.task.revision, - reason: { code: 'controller.cancelled', summary: 'Cancellation wins the recovery race.' }, - }), - ); - expect(cancelled.task.status).toBe('cancelled'); - expect(cancelled).not.toHaveProperty('lease'); - - releaseClaim(); - const staleClaim = await claimPromise; - expect(staleClaim).toHaveProperty('error'); - const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); - expect( - snapshot?.leases.some( - (lease) => lease.taskId === seeded.task.id && lease.status === 'active', - ), - ).toBe(false); - expect(await claimApp.persistence.getCommandReceipt('ws-a', 'cancel-claim-race-claim-command')).toBeUndefined(); - claimApp.database.close(); - cancelApp.database.close(); - }); - - it('prevents a paused recovery ClaimTask from minting authority after CancelGoal commits first', async () => { - const path = databasePath(); - const now = new Date('2026-08-30T23:00:00.000Z'); - const seed = await openDispatcher(path, 'goal-claim-race-seed', now); - await seed.persistence.bootstrapWorkspace(workspace()); - const seeded = await seedClaimedTask(seed.dispatcher, 'goal-claim-race'); - success( - await seed.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'ReleaseLease', - commandId: 'goal-claim-race-release', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - leaseId: seeded.claim.lease.id, - fencingToken: seeded.claim.lease.fencingToken, - expectedLeaseRevision: seeded.claim.lease.revision, - }), - ); - seed.database.close(); - - const claimApp = await openDispatcher(path, 'goal-claim-race-claim', now); - const cancelApp = await openDispatcher(path, 'goal-claim-race-cancel', now); - let claimArrivedResolve!: () => void; - let releaseClaim!: () => void; - const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); - const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); - claimApp.database.beforeNextBatch(async () => { - claimArrivedResolve(); - await claimRelease; - }); - - const claimPromise = claimApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'ClaimTask', - commandId: 'goal-claim-race-claim-command', - workspaceId: 'ws-a', - actor: seeded.agentActor, - taskId: seeded.task.id, - sessionId: seeded.session.id, - expectedTaskRevision: seeded.claim.task.revision, - }); - await claimArrived; - - const cancelled = success( - await cancelApp.dispatcher.dispatchCommand({ - protocolVersion: '0.1', - command: 'CancelGoal', - commandId: 'goal-claim-race-cancel-command', - workspaceId: 'ws-a', - actor: seeded.systemActor, - goalId: seeded.goal.id, - expectedGoalRevision: seeded.goal.revision, - reason: { code: 'controller.cancelled', summary: 'Goal cancellation wins recovery race.' }, - }), - ); - expect(cancelled.goal.status).toBe('cancelled'); - expect(cancelled.tasks).toHaveLength(1); - expect(cancelled.leases).toHaveLength(0); - - releaseClaim(); - const staleClaim = await claimPromise; - expect(staleClaim).toHaveProperty('error'); - const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); - expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); - expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); - expect( - snapshot?.leases.some( - (lease) => lease.taskId === seeded.task.id && lease.status === 'active', - ), - ).toBe(false); - expect(await claimApp.persistence.getCommandReceipt('ws-a', 'goal-claim-race-claim-command')).toBeUndefined(); - claimApp.database.close(); - cancelApp.database.close(); - }); -''' -marker = '\n});\n' -if not text.endswith(marker): - raise RuntimeError('unexpected test suffix') -path.write_text(text[:-len(marker)] + new_tests + marker) diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts index ac244e8..1fb5e5e 100644 --- a/test/application/durable-retry-cancellation.test.ts +++ b/test/application/durable-retry-cancellation.test.ts @@ -669,4 +669,160 @@ describe('durable retry and cancellation', () => { ).toBeUndefined(); app.database.close(); }); + + it('prevents a paused recovery ClaimTask from minting authority after CancelTask commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:30:00.000Z'); + const seed = await openDispatcher(path, 'cancel-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'cancel-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'cancel-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'cancel-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Cancellation wins the recovery race.' }, + }), + ); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect( + await claimApp.persistence.getCommandReceipt('ws-a', 'cancel-claim-race-claim-command'), + ).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); + + it('prevents a paused recovery ClaimTask from minting authority after CancelGoal commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T23:00:00.000Z'); + const seed = await openDispatcher(path, 'goal-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'goal-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'goal-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'goal-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'goal-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'goal-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'goal-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Goal cancellation wins recovery race.' }, + }), + ); + expect(cancelled.goal.status).toBe('cancelled'); + expect(cancelled.tasks).toHaveLength(1); + expect(cancelled.leases).toHaveLength(0); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect( + await claimApp.persistence.getCommandReceipt('ws-a', 'goal-claim-race-claim-command'), + ).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); }); From 9e82a7041a43504db49f1ab64e549dc86d7df242 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:42:35 +0500 Subject: [PATCH 32/38] fix: prepare cancellation claim linearization --- .../fix-cancellation-claim-linearization.py | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 scripts/fix-cancellation-claim-linearization.py diff --git a/scripts/fix-cancellation-claim-linearization.py b/scripts/fix-cancellation-claim-linearization.py new file mode 100644 index 0000000..8326e83 --- /dev/null +++ b/scripts/fix-cancellation-claim-linearization.py @@ -0,0 +1,251 @@ +from pathlib import Path + +path = Path('src/persistence/cloudflare/d1-runtime-persistence.ts') +text = path.read_text() + +old = r''' if (activeLease) { + const expiredLease: Lease = { + ...activeLease, + revision: activeLease.revision + 1, + updatedAt: input.now, + status: 'expired', + }; + this.assertCanonical('Lease', expiredLease); + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + expiredLease.revision, + expiredLease.status, + nowMs, + serializeJson(expiredLease, 'Lease'), + input.workspaceId, + activeLease.id, + activeLease.revision, + ), + ); + } + + statements.push( + this.database + .prepare( + `UPDATE task_fencing_counters + SET last_fencing_token = ? + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ?`, + ) + .bind(nextFence, input.workspaceId, input.taskId, counter), + ); +''' +new = r''' if (activeLease) { + const expiredLease: Lease = { + ...activeLease, + revision: activeLease.revision + 1, + updatedAt: input.now, + status: 'expired', + }; + this.assertCanonical('Lease', expiredLease); + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + expiredLease.revision, + expiredLease.status, + nowMs, + serializeJson(expiredLease, 'Lease'), + input.workspaceId, + activeLease.id, + activeLease.revision, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + + statements.push( + this.database + .prepare( + `UPDATE task_fencing_counters + SET last_fencing_token = ? + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + AND EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ? + )`, + ) + .bind( + nextFence, + input.workspaceId, + input.taskId, + counter, + input.workspaceId, + input.taskId, + task.revision, + task.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); +''' +if text.count(old) != 1: + raise RuntimeError('claim counter marker mismatch') +text = text.replace(old, new, 1) + +old = r''' statements.push( + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'ready'`, + ) + .bind( + nextTask.revision, + nextTask.status, + nowMs, + serializeJson(nextTask, 'Task'), + input.workspaceId, + input.taskId, + task.revision, + ), + ); + } + + statements.push(this.insertLeaseStatement(lease)); + this.pushAuditStatement(statements, input.auditEvent); + const value = { task: clone(nextTask), lease: clone(lease) }; + const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); + this.pushReceiptStatement(statements, finalReceipt); + await this.batch(statements, 'claim Task'); +''' +new = r''' statements.push( + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'ready'`, + ) + .bind( + nextTask.revision, + nextTask.status, + nowMs, + serializeJson(nextTask, 'Task'), + input.workspaceId, + input.taskId, + task.revision, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + + statements.push(this.insertLeaseStatement(lease)); + this.pushAuditStatement(statements, input.auditEvent); + const value = { task: clone(nextTask), lease: clone(lease) }; + const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); + this.pushReceiptStatement(statements, finalReceipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'claim Task'); +''' +if text.count(old) != 1: + raise RuntimeError('claim receipt marker mismatch') +text = text.replace(old, new, 1) + +old = r''' const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) +''' +new = r''' const fencingCounter = await this.getFencingCounter(input.workspaceId, current.id); + if (fencingCounter === undefined) { + throw new PersistenceError('INTEGRITY_ERROR', `Task ${current.id} has no fencing counter.`); + } + + const statements: D1PreparedStatementLike[] = [ + this.fencingCounterGuardStatement(input.workspaceId, current.id, fencingCounter), + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) +''' +# This marker appears only in cancelTask after latest transaction patch. +if text.count(old) != 1: + raise RuntimeError(f'cancelTask statements marker mismatch: {text.count(old)}') +text = text.replace(old, new, 1) + +old = r''' const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); +''' +new = r''' const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const fencingCounters = new Map(); + for (const currentTask of cancellable) { + const counter = await this.getFencingCounter(input.workspaceId, currentTask.id); + if (counter === undefined) { + throw new PersistenceError( + 'INTEGRITY_ERROR', + `Task ${currentTask.id} has no fencing counter.`, + ); + } + fencingCounters.set(currentTask.id, counter); + } + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); +''' +if text.count(old) != 1: + raise RuntimeError('cancelGoal counter snapshot marker mismatch') +text = text.replace(old, new, 1) + +old = r''' const statements: D1PreparedStatementLike[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.database +''' +new = r''' const statements: D1PreparedStatementLike[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.fencingCounterGuardStatement( + input.workspaceId, + currentTask.id, + fencingCounters.get(currentTask.id)!, + ), + this.database +''' +if text.count(old) != 1: + raise RuntimeError('cancelGoal statements marker mismatch') +text = text.replace(old, new, 1) + +marker = r''' private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { +''' +helper = r''' private fencingCounterGuardStatement( + workspaceId: string, + taskId: string, + expectedCounter: number, + ): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + SELECT ?, CASE WHEN EXISTS ( + SELECT 1 FROM task_fencing_counters + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + ) THEN 1 ELSE 0 END`, + ) + .bind(workspaceId, workspaceId, taskId, expectedCounter); + } + +''' +if text.count(marker) != 1: + raise RuntimeError('mutation guard helper marker mismatch') +text = text.replace(marker, helper + marker, 1) + +path.write_text(text) From 7eedfe7a171afabe20cb35040101b303429e88ab Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:42:47 +0500 Subject: [PATCH 33/38] chore: add one-time cancellation claim linearization workflow --- ...-cancellation-claim-linearization-once.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/fix-cancellation-claim-linearization-once.yml diff --git a/.github/workflows/fix-cancellation-claim-linearization-once.yml b/.github/workflows/fix-cancellation-claim-linearization-once.yml new file mode 100644 index 0000000..b2ff283 --- /dev/null +++ b/.github/workflows/fix-cancellation-claim-linearization-once.yml @@ -0,0 +1,35 @@ +name: Fix cancellation claim linearization once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/fix-cancellation-claim-linearization-once.yml + +permissions: + contents: write + +jobs: + apply: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/fix-cancellation-claim-linearization.py + - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts + - run: git rm .github/workflows/fix-cancellation-claim-linearization-once.yml scripts/fix-cancellation-claim-linearization.py + - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts + - run: pnpm check + - run: pnpm test:coverage + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/persistence/cloudflare/d1-runtime-persistence.ts + git commit -m "fix: serialize cancellation with recovery claims" + git push From 2165860c2ad8274d6a75f290c5a66b3750c32295 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:43:31 +0000 Subject: [PATCH 34/38] fix: serialize cancellation with recovery claims --- ...-cancellation-claim-linearization-once.yml | 35 --- .../fix-cancellation-claim-linearization.py | 251 ------------------ .../cloudflare/d1-runtime-persistence.ts | 59 +++- 3 files changed, 57 insertions(+), 288 deletions(-) delete mode 100644 .github/workflows/fix-cancellation-claim-linearization-once.yml delete mode 100644 scripts/fix-cancellation-claim-linearization.py diff --git a/.github/workflows/fix-cancellation-claim-linearization-once.yml b/.github/workflows/fix-cancellation-claim-linearization-once.yml deleted file mode 100644 index b2ff283..0000000 --- a/.github/workflows/fix-cancellation-claim-linearization-once.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Fix cancellation claim linearization once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/fix-cancellation-claim-linearization-once.yml - -permissions: - contents: write - -jobs: - apply: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/fix-cancellation-claim-linearization.py - - run: pnpm exec prettier --write src/persistence/cloudflare/d1-runtime-persistence.ts - - run: git rm .github/workflows/fix-cancellation-claim-linearization-once.yml scripts/fix-cancellation-claim-linearization.py - - run: pnpm vitest run test/application/durable-retry-cancellation.test.ts - - run: pnpm check - - run: pnpm test:coverage - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/persistence/cloudflare/d1-runtime-persistence.ts - git commit -m "fix: serialize cancellation with recovery claims" - git push diff --git a/scripts/fix-cancellation-claim-linearization.py b/scripts/fix-cancellation-claim-linearization.py deleted file mode 100644 index 8326e83..0000000 --- a/scripts/fix-cancellation-claim-linearization.py +++ /dev/null @@ -1,251 +0,0 @@ -from pathlib import Path - -path = Path('src/persistence/cloudflare/d1-runtime-persistence.ts') -text = path.read_text() - -old = r''' if (activeLease) { - const expiredLease: Lease = { - ...activeLease, - revision: activeLease.revision + 1, - updatedAt: input.now, - status: 'expired', - }; - this.assertCanonical('Lease', expiredLease); - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, - ) - .bind( - expiredLease.revision, - expiredLease.status, - nowMs, - serializeJson(expiredLease, 'Lease'), - input.workspaceId, - activeLease.id, - activeLease.revision, - ), - ); - } - - statements.push( - this.database - .prepare( - `UPDATE task_fencing_counters - SET last_fencing_token = ? - WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ?`, - ) - .bind(nextFence, input.workspaceId, input.taskId, counter), - ); -''' -new = r''' if (activeLease) { - const expiredLease: Lease = { - ...activeLease, - revision: activeLease.revision + 1, - updatedAt: input.now, - status: 'expired', - }; - this.assertCanonical('Lease', expiredLease); - statements.push( - this.database - .prepare( - `UPDATE leases - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, - ) - .bind( - expiredLease.revision, - expiredLease.status, - nowMs, - serializeJson(expiredLease, 'Lease'), - input.workspaceId, - activeLease.id, - activeLease.revision, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - } - - statements.push( - this.database - .prepare( - `UPDATE task_fencing_counters - SET last_fencing_token = ? - WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? - AND EXISTS ( - SELECT 1 FROM tasks - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ? - )`, - ) - .bind( - nextFence, - input.workspaceId, - input.taskId, - counter, - input.workspaceId, - input.taskId, - task.revision, - task.status, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); -''' -if text.count(old) != 1: - raise RuntimeError('claim counter marker mismatch') -text = text.replace(old, new, 1) - -old = r''' statements.push( - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'ready'`, - ) - .bind( - nextTask.revision, - nextTask.status, - nowMs, - serializeJson(nextTask, 'Task'), - input.workspaceId, - input.taskId, - task.revision, - ), - ); - } - - statements.push(this.insertLeaseStatement(lease)); - this.pushAuditStatement(statements, input.auditEvent); - const value = { task: clone(nextTask), lease: clone(lease) }; - const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); - this.pushReceiptStatement(statements, finalReceipt); - await this.batch(statements, 'claim Task'); -''' -new = r''' statements.push( - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'ready'`, - ) - .bind( - nextTask.revision, - nextTask.status, - nowMs, - serializeJson(nextTask, 'Task'), - input.workspaceId, - input.taskId, - task.revision, - ), - this.mutationChangesGuardStatement(input.workspaceId), - ); - } - - statements.push(this.insertLeaseStatement(lease)); - this.pushAuditStatement(statements, input.auditEvent); - const value = { task: clone(nextTask), lease: clone(lease) }; - const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); - this.pushReceiptStatement(statements, finalReceipt); - statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); - await this.batch(statements, 'claim Task'); -''' -if text.count(old) != 1: - raise RuntimeError('claim receipt marker mismatch') -text = text.replace(old, new, 1) - -old = r''' const statements: D1PreparedStatementLike[] = [ - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) -''' -new = r''' const fencingCounter = await this.getFencingCounter(input.workspaceId, current.id); - if (fencingCounter === undefined) { - throw new PersistenceError('INTEGRITY_ERROR', `Task ${current.id} has no fencing counter.`); - } - - const statements: D1PreparedStatementLike[] = [ - this.fencingCounterGuardStatement(input.workspaceId, current.id, fencingCounter), - this.database - .prepare( - `UPDATE tasks - SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? - WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, - ) -''' -# This marker appears only in cancelTask after latest transaction patch. -if text.count(old) != 1: - raise RuntimeError(f'cancelTask statements marker mismatch: {text.count(old)}') -text = text.replace(old, new, 1) - -old = r''' const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); - const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); - const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); -''' -new = r''' const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); - const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); - const fencingCounters = new Map(); - for (const currentTask of cancellable) { - const counter = await this.getFencingCounter(input.workspaceId, currentTask.id); - if (counter === undefined) { - throw new PersistenceError( - 'INTEGRITY_ERROR', - `Task ${currentTask.id} has no fencing counter.`, - ); - } - fencingCounters.set(currentTask.id, counter); - } - const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); -''' -if text.count(old) != 1: - raise RuntimeError('cancelGoal counter snapshot marker mismatch') -text = text.replace(old, new, 1) - -old = r''' const statements: D1PreparedStatementLike[] = []; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id)!; - statements.push( - this.database -''' -new = r''' const statements: D1PreparedStatementLike[] = []; - for (const currentTask of cancellable) { - const output = outputTasks.get(currentTask.id)!; - statements.push( - this.fencingCounterGuardStatement( - input.workspaceId, - currentTask.id, - fencingCounters.get(currentTask.id)!, - ), - this.database -''' -if text.count(old) != 1: - raise RuntimeError('cancelGoal statements marker mismatch') -text = text.replace(old, new, 1) - -marker = r''' private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { -''' -helper = r''' private fencingCounterGuardStatement( - workspaceId: string, - taskId: string, - expectedCounter: number, - ): D1PreparedStatementLike { - return this.database - .prepare( - `INSERT INTO mutation_batch_guards(workspace_id, ok) - SELECT ?, CASE WHEN EXISTS ( - SELECT 1 FROM task_fencing_counters - WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? - ) THEN 1 ELSE 0 END`, - ) - .bind(workspaceId, workspaceId, taskId, expectedCounter); - } - -''' -if text.count(marker) != 1: - raise RuntimeError('mutation guard helper marker mismatch') -text = text.replace(marker, helper + marker, 1) - -path.write_text(text) diff --git a/src/persistence/cloudflare/d1-runtime-persistence.ts b/src/persistence/cloudflare/d1-runtime-persistence.ts index e82c263..69e9c68 100644 --- a/src/persistence/cloudflare/d1-runtime-persistence.ts +++ b/src/persistence/cloudflare/d1-runtime-persistence.ts @@ -671,6 +671,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { activeLease.id, activeLease.revision, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } @@ -679,9 +680,23 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { .prepare( `UPDATE task_fencing_counters SET last_fencing_token = ? - WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ?`, + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + AND EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ? + )`, ) - .bind(nextFence, input.workspaceId, input.taskId, counter), + .bind( + nextFence, + input.workspaceId, + input.taskId, + counter, + input.workspaceId, + input.taskId, + task.revision, + task.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), ); let nextTask = task; @@ -709,6 +724,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { input.taskId, task.revision, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } @@ -717,6 +733,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { const value = { task: clone(nextTask), lease: clone(lease) }; const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); this.pushReceiptStatement(statements, finalReceipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); await this.batch(statements, 'claim Task'); return { kind: 'committed', value }; }); @@ -1205,7 +1222,13 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { } } + const fencingCounter = await this.getFencingCounter(input.workspaceId, current.id); + if (fencingCounter === undefined) { + throw new PersistenceError('INTEGRITY_ERROR', `Task ${current.id} has no fencing counter.`); + } + const statements: D1PreparedStatementLike[] = [ + this.fencingCounterGuardStatement(input.workspaceId, current.id, fencingCounter), this.database .prepare( `UPDATE tasks @@ -1304,6 +1327,17 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const fencingCounters = new Map(); + for (const currentTask of cancellable) { + const counter = await this.getFencingCounter(input.workspaceId, currentTask.id); + if (counter === undefined) { + throw new PersistenceError( + 'INTEGRITY_ERROR', + `Task ${currentTask.id} has no fencing counter.`, + ); + } + fencingCounters.set(currentTask.id, counter); + } const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); @@ -1365,6 +1399,11 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { for (const currentTask of cancellable) { const output = outputTasks.get(currentTask.id)!; statements.push( + this.fencingCounterGuardStatement( + input.workspaceId, + currentTask.id, + fencingCounters.get(currentTask.id)!, + ), this.database .prepare( `UPDATE tasks @@ -2568,6 +2607,22 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { ); } + private fencingCounterGuardStatement( + workspaceId: string, + taskId: string, + expectedCounter: number, + ): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + SELECT ?, CASE WHEN EXISTS ( + SELECT 1 FROM task_fencing_counters + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + ) THEN 1 ELSE 0 END`, + ) + .bind(workspaceId, workspaceId, taskId, expectedCounter); + } + private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { return this.database .prepare( From e15ae5aa3eb1b01e0ad1aab748f293afa7c5a3e3 Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:44:59 +0500 Subject: [PATCH 35/38] docs: prepare retry cancellation reconciliation --- ...concile-durable-retry-cancellation-docs.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 scripts/reconcile-durable-retry-cancellation-docs.py diff --git a/scripts/reconcile-durable-retry-cancellation-docs.py b/scripts/reconcile-durable-retry-cancellation-docs.py new file mode 100644 index 0000000..446c75a --- /dev/null +++ b/scripts/reconcile-durable-retry-cancellation-docs.py @@ -0,0 +1,74 @@ +from pathlib import Path + + +def replace_once(path: str, before: str, after: str) -> None: + file = Path(path) + text = file.read_text() + count = text.count(before) + if count != 1: + raise RuntimeError(f'{path}: expected one marker, found {count}') + file.write_text(text.replace(before, after, 1)) + + +replace_once( + 'docs/CURRENT_STATE.md', + '- The durable command loop is executable for `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`.', + '- The durable command loop is executable for the complete ADR-0005 v0.1 command surface: `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, `ResumeTask`, `RetryTask`, `CancelTask`, and `CancelGoal`.', +) +replace_once( + 'docs/CURRENT_STATE.md', + '- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority.', + '- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. `RetryTask`, `CancelTask`, and `CancelGoal` also commit through explicit durable mutations with controller authority, immutable receipts, Goal/Task/Lease revision checks, database-enforced Goal-versus-Task admission ordering, and fencing-counter guards that prevent recovery claims from surviving cancellation races.', +) +replace_once( + 'docs/CURRENT_STATE.md', + '- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds.', + '- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. Durable retry/cancellation review hardening culminated in run `33283962742`, which passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines. The regressions cover lost-response replay, `CancelTask` without an effective Lease, `CancelGoal` mid-batch rollback, stale Goal task admission, false-success receipt prevention after a lost CAS, and recovery-`ClaimTask` races against both Task and Goal cancellation.', +) +replace_once( + 'docs/CURRENT_STATE.md', + '- The durable dispatcher still leaves `RetryTask`, `CancelTask`, and `CancelGoal` explicitly unsupported until matching atomic persistence mutations are added.\n', + '', +) +replace_once( + 'docs/CURRENT_STATE.md', + '- Complete durable persistence mutations for the remaining v0.1 lifecycle/cancellation commands without introducing a second state machine or generic CRUD authority.', + '- Audit and harden any remaining conditional durable mutations against the same database-CAS/receipt and Goal-level concurrency invariants before treating the local persistence composition as complete.', +) + +replace_once( + 'docs/roadmap/V0_1.md', + 'The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. Remaining durable commands fail explicitly as unsupported until matching atomic persistence mutations exist.', + 'The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers the complete ADR-0005 v0.1 command surface, including `RetryTask`, `CancelTask`, and `CancelGoal`. Database-enforced mutation guards preserve receipt atomicity, Goal-versus-Task admission ordering, and cancellation-versus-recovery fencing under the SQLite D1-like reference harness.', +) +replace_once( + 'docs/roadmap/V0_1.md', + 'Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. The remaining local/runtime work is to extend atomic durable composition to `RetryTask`, `CancelTask`, and `CancelGoal` where required for dogfooding. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification.', + 'Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. Durable lifecycle/cancellation command parity is complete locally; remaining local work is bounded query parity plus persistence-concurrency hardening and final end-to-end reconciliation. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification.', +) +replace_once( + 'docs/roadmap/V0_1.md', + '**Current gap to that condition:** at least one real agent-host integration, deployed/reference-runtime hardening, and any remaining durable lifecycle mutations required by that integration are outstanding. The local durable HTTP composition now proves the control-plane persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path.', + '**Current gap to that condition:** at least one real agent-host integration, the three remaining durable read queries, persistence/deployed-reference-runtime hardening, and real Cloudflare verification are outstanding. The local durable HTTP composition now proves the command-side persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path.', +) + +replace_once( + 'CHANGELOG.md', + '- Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage.', + '- Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage.\n- Durable `RetryTask`, `CancelTask`, and `CancelGoal` composition with restart-safe receipts, controller authority, task-only cancellation when no effective Lease exists, atomic Goal/Task/Lease cancellation, and database-level concurrency guards against stale Goal admission and recovery-claim races.', +) +replace_once( + 'CHANGELOG.md', + '- Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease.', + '- Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease.\n- D1 mutation batches now use transaction-aborting guard rows for conditional mutation races. Task creation/retry revalidate an active parent Goal inside the database batch; Goal cancellation terminalizes the Goal only after its cancellable Tasks/Leases and only when no cancellable work remains; claim/cancellation share fencing-counter and Task-state predicates so neither can return success with execution authority beneath cancelled state.', +) +replace_once( + 'CHANGELOG.md', + '- Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry.', + '- Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry.\n- Durable retry/cancellation hardening run `33283962742` passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines after transaction-CAS, stale Goal admission, no-Lease cancellation, mid-batch rollback, false-receipt, and cancellation-versus-recovery claim regressions were made green.', +) +replace_once( + 'CHANGELOG.md', + '- The durable dispatcher currently supports `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. `RetryTask`, `CancelTask`, and `CancelGoal` remain canonical runtime behavior but are explicitly unsupported by the durable composition until matching atomic persistence paths are added.', + '- The durable dispatcher supports the complete ADR-0005 v0.1 command surface. The remaining explicit durable application gaps are read-side: `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView`.', +) From e2e5cbaa51993bb708ad9583752749b791ae3dfb Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:45:10 +0500 Subject: [PATCH 36/38] chore: add one-time retry cancellation docs workflow --- ...e-durable-retry-cancellation-docs-once.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml diff --git a/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml b/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml new file mode 100644 index 0000000..63e6723 --- /dev/null +++ b/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml @@ -0,0 +1,34 @@ +name: Reconcile durable retry cancellation docs once + +on: + push: + branches: + - feature/durable-retry-cancellation + paths: + - .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml + +permissions: + contents: write + +jobs: + reconcile: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version-file: package.json + package-manager-cache: false + - run: npm install --global "$(node -p "require('./package.json').packageManager")" + - run: pnpm install --frozen-lockfile + - run: python3 scripts/reconcile-durable-retry-cancellation-docs.py + - run: pnpm exec prettier --write docs/CURRENT_STATE.md docs/roadmap/V0_1.md CHANGELOG.md + - run: git rm .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml scripts/reconcile-durable-retry-cancellation-docs.py + - run: pnpm check + - run: pnpm test:coverage + - run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/CURRENT_STATE.md docs/roadmap/V0_1.md CHANGELOG.md + git commit -m "docs: reconcile durable retry cancellation" + git push From 681a5e6376f3f25f3d963052f00a818a7c132474 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:45:54 +0000 Subject: [PATCH 37/38] docs: reconcile durable retry cancellation --- ...e-durable-retry-cancellation-docs-once.yml | 34 --------- CHANGELOG.md | 5 +- docs/CURRENT_STATE.md | 9 +-- docs/roadmap/V0_1.md | 6 +- ...concile-durable-retry-cancellation-docs.py | 74 ------------------- 5 files changed, 11 insertions(+), 117 deletions(-) delete mode 100644 .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml delete mode 100644 scripts/reconcile-durable-retry-cancellation-docs.py diff --git a/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml b/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml deleted file mode 100644 index 63e6723..0000000 --- a/.github/workflows/reconcile-durable-retry-cancellation-docs-once.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Reconcile durable retry cancellation docs once - -on: - push: - branches: - - feature/durable-retry-cancellation - paths: - - .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml - -permissions: - contents: write - -jobs: - reconcile: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e - with: - node-version-file: package.json - package-manager-cache: false - - run: npm install --global "$(node -p "require('./package.json').packageManager")" - - run: pnpm install --frozen-lockfile - - run: python3 scripts/reconcile-durable-retry-cancellation-docs.py - - run: pnpm exec prettier --write docs/CURRENT_STATE.md docs/roadmap/V0_1.md CHANGELOG.md - - run: git rm .github/workflows/reconcile-durable-retry-cancellation-docs-once.yml scripts/reconcile-durable-retry-cancellation-docs.py - - run: pnpm check - - run: pnpm test:coverage - - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add docs/CURRENT_STATE.md docs/roadmap/V0_1.md CHANGELOG.md - git commit -m "docs: reconcile durable retry cancellation" - git push diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c94a2..dad5f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The project is in `0.x` development and does not yet have a public product relea - HTTP/application end-to-end regression covering `RegisterAgent -> StartSession -> CreateGoal -> CreateTask -> ClaimTask -> RecordCheckpoint -> RequestPermission(repository.write) -> RecordPermissionDecision(ALLOW) -> CompleteTask`, followed by HTTP verification of final Task, Goal, Lease, and checkpoint state. - Durable HTTP end-to-end regressions covering restart after claim, restart after `HUMAN_REQUIRED`, response-loss receipt replay after application replacement, competing independent application claims, and fencing advancement after Lease expiry/recovery against the SQLite D1-like persistence harness. - Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage. +- Durable `RetryTask`, `CancelTask`, and `CancelGoal` composition with restart-safe receipts, controller authority, task-only cancellation when no effective Lease exists, atomic Goal/Task/Lease cancellation, and database-level concurrency guards against stale Goal admission and recovery-claim races. ### Changed @@ -58,6 +59,7 @@ The project is in `0.x` development and does not yet have a public product relea - Admitted terminal semantic failures in the supported durable command loop now persist immutable `outcomeKind: error` command receipts and replay after restart instead of re-executing the command. - Durable Session/Lease liveness composition now supports `HeartbeatSession`, `EndSession`, `RenewLease`, and `ReleaseLease` with atomic mutation receipts, restart-safe replay, stable renewal fencing, and recoverable released/revoked execution authority. - Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease. +- D1 mutation batches now use transaction-aborting guard rows for conditional mutation races. Task creation/retry revalidate an active parent Goal inside the database batch; Goal cancellation terminalizes the Goal only after its cancellable Tasks/Leases and only when no cancellable work remains; claim/cancellation share fencing-counter and Task-state predicates so neither can return success with execution authority beneath cancelled state. ### Verification @@ -76,6 +78,7 @@ The project is in `0.x` development and does not yet have a public product relea - Permanent Quality run `33274903333` passed the restart-safe durable HTTP E2E tree with **29/29 test files and 123/123 tests**, plus `pnpm test:coverage`. Overall coverage reported 85.3% statements, 73.3% branches, 96.15% functions, and 86.95% lines. - Final correctness-review RED run `33275182068` failed exactly the two new terminal-error-receipt and recovery-discovery regressions while the previous 123 tests passed. Review-fix run `33275312677` then passed focused regressions, full `pnpm check`, and coverage with **30/30 test files and 125/125 tests**; overall coverage was 85.43% statements, 73.5% branches, 96.18% functions, and 87.07% lines. - Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry. +- Durable retry/cancellation hardening run `33283962742` passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines after transaction-CAS, stale Goal admission, no-Lease cancellation, mid-batch rollback, false-receipt, and cancellation-versus-recovery claim regressions were made green. - Frozen installation resolves `@mindrail/contracts 0.0.0 <- packages/contracts`, confirming the root runtime uses the workspace contract package. - No new third-party runtime dependency was introduced for schema admission, protocol admission, bootstrap, transport, permission, persistence composition, rehydration, or durable query semantics. @@ -83,7 +86,7 @@ The project is in `0.x` development and does not yet have a public product relea - `Quality` is not yet enforced as a required `main` merge gate; repository protection remains tracked separately in issue #3. - The durable application composition is verified against the local SQLite D1-like test harness, not a deployed Cloudflare Worker/Durable Object/D1 environment. -- The durable dispatcher currently supports `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. `RetryTask`, `CancelTask`, and `CancelGoal` remain canonical runtime behavior but are explicitly unsupported by the durable composition until matching atomic persistence paths are added. +- The durable dispatcher supports the complete ADR-0005 v0.1 command surface. The remaining explicit durable application gaps are read-side: `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView`. - Durable application queries `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain explicitly unsupported. - MindRail does not yet expose a verified deployed Cloudflare control-plane service, GitHub adapter, real Codex/ChatGPT integration, or unattended continuation of a real external agent across host/platform termination. - The deterministic v0.1 permission policy is intentionally small and is not an IAM system, credential manager, arbitrary policy DSL, or model-based authority mechanism. diff --git a/docs/CURRENT_STATE.md b/docs/CURRENT_STATE.md index 75dba52..dacc064 100644 --- a/docs/CURRENT_STATE.md +++ b/docs/CURRENT_STATE.md @@ -27,9 +27,9 @@ The following facts are supported by repository state and executed GitHub Action - `ClaimTask` uses a deferred receipt snapshot so the returned/stored result is built only after the persistence layer allocates the authoritative fencing token. Executed regression coverage proves a replay returns the persisted Lease/fence rather than a speculative runtime token. - `InMemoryControlPlane.rehydrate()` restores canonical Workspace/Goal/Task/Agent/Session/Lease/checkpoint/permission state plus durable per-Task fencing counters from a persistence snapshot. Rehydration validates record relationships, reconstructs effective Lease authority from canonical records and authoritative time, and fails closed on inconsistent state. - `createDurableApplicationDispatcher(...)` composes the canonical runtime semantics with `DurableRuntimePersistence`. Each supported command loads authoritative durable state, rehydrates an ephemeral runtime, executes the existing semantics, and commits through explicit persistence methods. The dispatcher does not retain an in-memory fallback between requests. -- The durable command loop is executable for `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. +- The durable command loop is executable for the complete ADR-0005 v0.1 command surface: `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, `ResumeTask`, `RetryTask`, `CancelTask`, and `CancelGoal`. - Durable Session/Lease liveness preserves the runtime authority model: heartbeat advances only Session liveness/revision, Lease renewal keeps the fencing token stable, release leaves the Task running/recoverable, and ending a Session revokes its still-effective active Leases without completing the Task. -- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. +- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. `RetryTask`, `CancelTask`, and `CancelGoal` also commit through explicit durable mutations with controller authority, immutable receipts, Goal/Task/Lease revision checks, database-enforced Goal-versus-Task admission ordering, and fencing-counter guards that prevent recovery claims from surviving cancellation races. - Durable command replay first reads the persisted `(workspaceId, commandId)` receipt. Exact retries survive application/database-handle replacement, return `replayed: true`, preserve the immutable stored result/error snapshot, and reflect the current correlation id. Semantic command-id drift fails with `IDEMPOTENCY_CONFLICT`. - Admitted terminal semantic failures on the supported durable command loop are also persisted as immutable error receipts, so an exact retry after restart replays the original terminal error instead of silently re-executing the command. - Explicit durable read ports and application queries are implemented for `GetWorkspace`, `GetGoal`, `GetTask`, `GetLease`, `GetAgent`, `GetSession`, `GetPermissionRequest`, `ListTaskCheckpoints`, `ListPendingHumanPermissions`, `ListPermissionDecisions`, and advisory `ListClaimableTasks`. List queries use bounded deterministic cursor paging. `ListClaimableTasks` includes capability-compatible `ready` Tasks plus `running` Tasks whose prior Lease/Session authority is no longer effective at the authoritative server time; work acquisition still revalidates authority atomically at `ClaimTask`. @@ -40,14 +40,13 @@ The following facts are supported by repository state and executed GitHub Action - Permanent `Quality` run `33274903333` on the durable HTTP E2E tree passed formatting, lint, strict TypeScript, generated-contract drift checks, **29/29 test files and 123/123 tests**, and coverage. Reported overall coverage was 85.3% statements, 73.3% branches, 96.15% functions, and 86.95% lines. - Final correctness review RED run `33275182068` demonstrated both remaining defects: the two new regressions failed because terminal semantic errors had no durable receipt and recovery work discovery omitted a `running` Task after its effective Lease expired, while the previous **123 tests passed**. Review-fix run `33275312677` then passed focused regressions, full `pnpm check`, and coverage with **30/30 test files and 125/125 tests**. Overall coverage was 85.43% statements, 73.5% branches, 96.18% functions, and 87.07% lines. - Durable Session/Lease liveness RED Quality run `33276800987` passed formatting/lint/typecheck/contracts and the previous **125 tests**, while all four new HTTP E2E regressions failed exactly because `HeartbeatSession`, `EndSession`, `RenewLease`, and `ReleaseLease` returned `UNSUPPORTED_OPERATION`. GREEN run `33276973691` then passed the focused 4/4 liveness E2E tests, full `pnpm check` with **31/31 test files and 129/129 tests**, and coverage at 85.19% statements, 73.88% branches, 96.3% functions, and 86.83% lines. -- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. +- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. Durable retry/cancellation review hardening culminated in run `33283962742`, which passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines. The regressions cover lost-response replay, `CancelTask` without an effective Lease, `CancelGoal` mid-batch rollback, stale Goal task admission, false-success receipt prevention after a lost CAS, and recovery-`ClaimTask` races against both Task and Goal cancellation. - Runtime Surface PR #20 merged with post-merge Quality PASS. Persistence PR #24 merged with post-merge Quality #248 PASS. HTTP/MCP Transport PR #25 merged at `e142c1399aed5de3d8df53ad876499583728a6b4`; permanent Quality #249 and post-merge Quality #250 both passed full quality and coverage gates. - Permanent `Quality` CI remains least-privilege and uses pinned GitHub-owned action commits. ## Implemented but not yet fully deployed / externally integrated - The durable application composition is verified locally against the SQLite D1-like harness used by persistence tests. This is executable restart/concurrency evidence for the application/persistence contract, but it is **not** evidence of a deployed Cloudflare Worker, Durable Object, or production D1 environment. -- The durable dispatcher still leaves `RetryTask`, `CancelTask`, and `CancelGoal` explicitly unsupported until matching atomic persistence mutations are added. - Durable queries `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain explicitly unsupported rather than inferred from retained application state. - No deployed Cloudflare Worker/Durable Object service is claimed. Deployment configuration, environment provisioning, deployed-runtime restart verification, and real Cloudflare concurrency verification remain outstanding. - The v0.1 permission policy is intentionally small, explicit, hard-coded, and versioned. It is not a policy DSL, IAM system, credential manager, model judge, or arbitrary-code policy runtime. @@ -55,7 +54,7 @@ The following facts are supported by repository state and executed GitHub Action ## Next implementation slices -- Complete durable persistence mutations for the remaining v0.1 lifecycle/cancellation commands without introducing a second state machine or generic CRUD authority. +- Audit and harden any remaining conditional durable mutations against the same database-CAS/receipt and Goal-level concurrency invariants before treating the local persistence composition as complete. - Add the remaining bounded durable queries only where required by agent/human workflows. - Add GitHub integration while keeping GitHub as an adapter/projection rather than canonical state authority. - Add minimal real Codex, ChatGPT-compatible, generic MCP, and generic HTTP agent bootstrap/worker paths on top of the stable protocol/application boundary. diff --git a/docs/roadmap/V0_1.md b/docs/roadmap/V0_1.md index dc08a50..b90f5f9 100644 --- a/docs/roadmap/V0_1.md +++ b/docs/roadmap/V0_1.md @@ -41,7 +41,7 @@ The slice includes strict schema validation, representative positive/negative fi ADR-0005 defines the transport-neutral command/query semantics, idempotency scope, fencing/revision authority, recovery behavior, error model, and HTTP/MCP mapping principles. -The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. Remaining durable commands fail explicitly as unsupported until matching atomic persistence mutations exist. +The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers the complete ADR-0005 v0.1 command surface, including `RetryTask`, `CancelTask`, and `CancelGoal`. Database-enforced mutation guards preserve receipt atomicity, Goal-versus-Task admission ordering, and cancellation-versus-recovery fencing under the SQLite D1-like reference harness. Durable read/query support now includes single-resource execution/permission reads, checkpoint and permission-decision history, the pending-human queue, and advisory claimable work. `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain pending. @@ -66,7 +66,7 @@ Implemented behavior includes: - durable queries for `GetWorkspace`, `GetGoal`, `GetTask`, `GetLease`, `GetAgent`, `GetSession`, `GetPermissionRequest`, `ListTaskCheckpoints`, `ListPendingHumanPermissions`, `ListPermissionDecisions`, and advisory `ListClaimableTasks`; - HTTP E2E evidence for restart-after-claim, restart-after-`HUMAN_REQUIRED`, response-loss replay, competing claims, and monotonic fencing recovery against the SQLite D1-like test harness. -Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. The remaining local/runtime work is to extend atomic durable composition to `RetryTask`, `CancelTask`, and `CancelGoal` where required for dogfooding. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification. +Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. Durable lifecycle/cancellation command parity is complete locally; remaining local work is bounded query parity plus persistence-concurrency hardening and final end-to-end reconciliation. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification. ## Slice 4 — GitHub integration @@ -111,4 +111,4 @@ Add optional human-facing projections such as Google Sheets or a lightweight UI A user can give a goal to a supported real agent, the agent can obtain a **durable** task/context/policy assignment from MindRail, report checkpoints and evidence, continue to the next action without asking the user by default, and escalate only when a policy/decision boundary requires human input. -**Current gap to that condition:** at least one real agent-host integration, deployed/reference-runtime hardening, and any remaining durable lifecycle mutations required by that integration are outstanding. The local durable HTTP composition now proves the control-plane persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path. +**Current gap to that condition:** at least one real agent-host integration, the three remaining durable read queries, persistence/deployed-reference-runtime hardening, and real Cloudflare verification are outstanding. The local durable HTTP composition now proves the command-side persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path. diff --git a/scripts/reconcile-durable-retry-cancellation-docs.py b/scripts/reconcile-durable-retry-cancellation-docs.py deleted file mode 100644 index 446c75a..0000000 --- a/scripts/reconcile-durable-retry-cancellation-docs.py +++ /dev/null @@ -1,74 +0,0 @@ -from pathlib import Path - - -def replace_once(path: str, before: str, after: str) -> None: - file = Path(path) - text = file.read_text() - count = text.count(before) - if count != 1: - raise RuntimeError(f'{path}: expected one marker, found {count}') - file.write_text(text.replace(before, after, 1)) - - -replace_once( - 'docs/CURRENT_STATE.md', - '- The durable command loop is executable for `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`.', - '- The durable command loop is executable for the complete ADR-0005 v0.1 command surface: `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, `ResumeTask`, `RetryTask`, `CancelTask`, and `CancelGoal`.', -) -replace_once( - 'docs/CURRENT_STATE.md', - '- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority.', - '- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. `RetryTask`, `CancelTask`, and `CancelGoal` also commit through explicit durable mutations with controller authority, immutable receipts, Goal/Task/Lease revision checks, database-enforced Goal-versus-Task admission ordering, and fencing-counter guards that prevent recovery claims from surviving cancellation races.', -) -replace_once( - 'docs/CURRENT_STATE.md', - '- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds.', - '- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. Durable retry/cancellation review hardening culminated in run `33283962742`, which passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines. The regressions cover lost-response replay, `CancelTask` without an effective Lease, `CancelGoal` mid-batch rollback, stale Goal task admission, false-success receipt prevention after a lost CAS, and recovery-`ClaimTask` races against both Task and Goal cancellation.', -) -replace_once( - 'docs/CURRENT_STATE.md', - '- The durable dispatcher still leaves `RetryTask`, `CancelTask`, and `CancelGoal` explicitly unsupported until matching atomic persistence mutations are added.\n', - '', -) -replace_once( - 'docs/CURRENT_STATE.md', - '- Complete durable persistence mutations for the remaining v0.1 lifecycle/cancellation commands without introducing a second state machine or generic CRUD authority.', - '- Audit and harden any remaining conditional durable mutations against the same database-CAS/receipt and Goal-level concurrency invariants before treating the local persistence composition as complete.', -) - -replace_once( - 'docs/roadmap/V0_1.md', - 'The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. Remaining durable commands fail explicitly as unsupported until matching atomic persistence mutations exist.', - 'The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers the complete ADR-0005 v0.1 command surface, including `RetryTask`, `CancelTask`, and `CancelGoal`. Database-enforced mutation guards preserve receipt atomicity, Goal-versus-Task admission ordering, and cancellation-versus-recovery fencing under the SQLite D1-like reference harness.', -) -replace_once( - 'docs/roadmap/V0_1.md', - 'Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. The remaining local/runtime work is to extend atomic durable composition to `RetryTask`, `CancelTask`, and `CancelGoal` where required for dogfooding. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification.', - 'Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. Durable lifecycle/cancellation command parity is complete locally; remaining local work is bounded query parity plus persistence-concurrency hardening and final end-to-end reconciliation. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification.', -) -replace_once( - 'docs/roadmap/V0_1.md', - '**Current gap to that condition:** at least one real agent-host integration, deployed/reference-runtime hardening, and any remaining durable lifecycle mutations required by that integration are outstanding. The local durable HTTP composition now proves the control-plane persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path.', - '**Current gap to that condition:** at least one real agent-host integration, the three remaining durable read queries, persistence/deployed-reference-runtime hardening, and real Cloudflare verification are outstanding. The local durable HTTP composition now proves the command-side persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path.', -) - -replace_once( - 'CHANGELOG.md', - '- Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage.', - '- Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage.\n- Durable `RetryTask`, `CancelTask`, and `CancelGoal` composition with restart-safe receipts, controller authority, task-only cancellation when no effective Lease exists, atomic Goal/Task/Lease cancellation, and database-level concurrency guards against stale Goal admission and recovery-claim races.', -) -replace_once( - 'CHANGELOG.md', - '- Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease.', - '- Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease.\n- D1 mutation batches now use transaction-aborting guard rows for conditional mutation races. Task creation/retry revalidate an active parent Goal inside the database batch; Goal cancellation terminalizes the Goal only after its cancellable Tasks/Leases and only when no cancellable work remains; claim/cancellation share fencing-counter and Task-state predicates so neither can return success with execution authority beneath cancelled state.', -) -replace_once( - 'CHANGELOG.md', - '- Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry.', - '- Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry.\n- Durable retry/cancellation hardening run `33283962742` passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines after transaction-CAS, stale Goal admission, no-Lease cancellation, mid-batch rollback, false-receipt, and cancellation-versus-recovery claim regressions were made green.', -) -replace_once( - 'CHANGELOG.md', - '- The durable dispatcher currently supports `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. `RetryTask`, `CancelTask`, and `CancelGoal` remain canonical runtime behavior but are explicitly unsupported by the durable composition until matching atomic persistence paths are added.', - '- The durable dispatcher supports the complete ADR-0005 v0.1 command surface. The remaining explicit durable application gaps are read-side: `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView`.', -) From a8a022e9a24b1a176b5313125cb18d6a2787271c Mon Sep 17 00:00:00 2001 From: tim8es <108188608+tim8es@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:57:05 +0500 Subject: [PATCH 38/38] chore: trigger final quality gate