diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c94a28..dad5f6e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The project is in `0.x` development and does not yet have a public product relea - HTTP/application end-to-end regression covering `RegisterAgent -> StartSession -> CreateGoal -> CreateTask -> ClaimTask -> RecordCheckpoint -> RequestPermission(repository.write) -> RecordPermissionDecision(ALLOW) -> CompleteTask`, followed by HTTP verification of final Task, Goal, Lease, and checkpoint state. - Durable HTTP end-to-end regressions covering restart after claim, restart after `HUMAN_REQUIRED`, response-loss receipt replay after application replacement, competing independent application claims, and fencing advancement after Lease expiry/recovery against the SQLite D1-like persistence harness. - Durable `FailTask`, `BlockTask`, and `ResumeTask` composition with atomic Task/Lease/Checkpoint/receipt persistence where applicable, restart-safe immutable replay, persisted blocked/failed evidence, controller-authorized resume, and D1-like fault-injection rollback coverage. +- Durable `RetryTask`, `CancelTask`, and `CancelGoal` composition with restart-safe receipts, controller authority, task-only cancellation when no effective Lease exists, atomic Goal/Task/Lease cancellation, and database-level concurrency guards against stale Goal admission and recovery-claim races. ### Changed @@ -58,6 +59,7 @@ The project is in `0.x` development and does not yet have a public product relea - Admitted terminal semantic failures in the supported durable command loop now persist immutable `outcomeKind: error` command receipts and replay after restart instead of re-executing the command. - Durable Session/Lease liveness composition now supports `HeartbeatSession`, `EndSession`, `RenewLease`, and `ReleaseLease` with atomic mutation receipts, restart-safe replay, stable renewal fencing, and recoverable released/revoked execution authority. - Durable task-outcome persistence now revalidates execution/revision authority at commit time and stores failure/block transitions and their released Lease plus Checkpoint in one batch; `ResumeTask` recomputes dependency readiness and removes the blocking reason without granting a Lease. +- D1 mutation batches now use transaction-aborting guard rows for conditional mutation races. Task creation/retry revalidate an active parent Goal inside the database batch; Goal cancellation terminalizes the Goal only after its cancellable Tasks/Leases and only when no cancellable work remains; claim/cancellation share fencing-counter and Task-state predicates so neither can return success with execution authority beneath cancelled state. ### Verification @@ -76,6 +78,7 @@ The project is in `0.x` development and does not yet have a public product relea - Permanent Quality run `33274903333` passed the restart-safe durable HTTP E2E tree with **29/29 test files and 123/123 tests**, plus `pnpm test:coverage`. Overall coverage reported 85.3% statements, 73.3% branches, 96.15% functions, and 86.95% lines. - Final correctness-review RED run `33275182068` failed exactly the two new terminal-error-receipt and recovery-discovery regressions while the previous 123 tests passed. Review-fix run `33275312677` then passed focused regressions, full `pnpm check`, and coverage with **30/30 test files and 125/125 tests**; overall coverage was 85.43% statements, 73.5% branches, 96.18% functions, and 87.07% lines. - Durable task-outcome RED run `33278105807` preserved the previous 130 passing tests while the new FailTask/BlockTask regressions failed at the expected unsupported boundary. Review/atomicity run `33278493339` passed **32/32 test files and 133/133 tests**, plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines, including a forced mid-batch rollback followed by successful exact retry. +- Durable retry/cancellation hardening run `33283962742` passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines after transaction-CAS, stale Goal admission, no-Lease cancellation, mid-batch rollback, false-receipt, and cancellation-versus-recovery claim regressions were made green. - Frozen installation resolves `@mindrail/contracts 0.0.0 <- packages/contracts`, confirming the root runtime uses the workspace contract package. - No new third-party runtime dependency was introduced for schema admission, protocol admission, bootstrap, transport, permission, persistence composition, rehydration, or durable query semantics. @@ -83,7 +86,7 @@ The project is in `0.x` development and does not yet have a public product relea - `Quality` is not yet enforced as a required `main` merge gate; repository protection remains tracked separately in issue #3. - The durable application composition is verified against the local SQLite D1-like test harness, not a deployed Cloudflare Worker/Durable Object/D1 environment. -- The durable dispatcher currently supports `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. `RetryTask`, `CancelTask`, and `CancelGoal` remain canonical runtime behavior but are explicitly unsupported by the durable composition until matching atomic persistence paths are added. +- The durable dispatcher supports the complete ADR-0005 v0.1 command surface. The remaining explicit durable application gaps are read-side: `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView`. - Durable application queries `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain explicitly unsupported. - MindRail does not yet expose a verified deployed Cloudflare control-plane service, GitHub adapter, real Codex/ChatGPT integration, or unattended continuation of a real external agent across host/platform termination. - The deterministic v0.1 permission policy is intentionally small and is not an IAM system, credential manager, arbitrary policy DSL, or model-based authority mechanism. diff --git a/docs/CURRENT_STATE.md b/docs/CURRENT_STATE.md index 75dba52c..dacc0641 100644 --- a/docs/CURRENT_STATE.md +++ b/docs/CURRENT_STATE.md @@ -27,9 +27,9 @@ The following facts are supported by repository state and executed GitHub Action - `ClaimTask` uses a deferred receipt snapshot so the returned/stored result is built only after the persistence layer allocates the authoritative fencing token. Executed regression coverage proves a replay returns the persisted Lease/fence rather than a speculative runtime token. - `InMemoryControlPlane.rehydrate()` restores canonical Workspace/Goal/Task/Agent/Session/Lease/checkpoint/permission state plus durable per-Task fencing counters from a persistence snapshot. Rehydration validates record relationships, reconstructs effective Lease authority from canonical records and authoritative time, and fails closed on inconsistent state. - `createDurableApplicationDispatcher(...)` composes the canonical runtime semantics with `DurableRuntimePersistence`. Each supported command loads authoritative durable state, rehydrates an ephemeral runtime, executes the existing semantics, and commits through explicit persistence methods. The dispatcher does not retain an in-memory fallback between requests. -- The durable command loop is executable for `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. +- The durable command loop is executable for the complete ADR-0005 v0.1 command surface: `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, `ResumeTask`, `RetryTask`, `CancelTask`, and `CancelGoal`. - Durable Session/Lease liveness preserves the runtime authority model: heartbeat advances only Session liveness/revision, Lease renewal keeps the fencing token stable, release leaves the Task running/recoverable, and ending a Session revokes its still-effective active Leases without completing the Task. -- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. +- Durable task outcomes now preserve the same runtime semantics through persistence: `FailTask` and `BlockTask` atomically commit the Task transition, released Lease, terminal/blocked Checkpoint, and immutable command receipt; `ResumeTask` atomically commits the controller-authorized blocked-to-ready/pending Task transition and receipt without minting execution authority. `RetryTask`, `CancelTask`, and `CancelGoal` also commit through explicit durable mutations with controller authority, immutable receipts, Goal/Task/Lease revision checks, database-enforced Goal-versus-Task admission ordering, and fencing-counter guards that prevent recovery claims from surviving cancellation races. - Durable command replay first reads the persisted `(workspaceId, commandId)` receipt. Exact retries survive application/database-handle replacement, return `replayed: true`, preserve the immutable stored result/error snapshot, and reflect the current correlation id. Semantic command-id drift fails with `IDEMPOTENCY_CONFLICT`. - Admitted terminal semantic failures on the supported durable command loop are also persisted as immutable error receipts, so an exact retry after restart replays the original terminal error instead of silently re-executing the command. - Explicit durable read ports and application queries are implemented for `GetWorkspace`, `GetGoal`, `GetTask`, `GetLease`, `GetAgent`, `GetSession`, `GetPermissionRequest`, `ListTaskCheckpoints`, `ListPendingHumanPermissions`, `ListPermissionDecisions`, and advisory `ListClaimableTasks`. List queries use bounded deterministic cursor paging. `ListClaimableTasks` includes capability-compatible `ready` Tasks plus `running` Tasks whose prior Lease/Session authority is no longer effective at the authoritative server time; work acquisition still revalidates authority atomically at `ClaimTask`. @@ -40,14 +40,13 @@ The following facts are supported by repository state and executed GitHub Action - Permanent `Quality` run `33274903333` on the durable HTTP E2E tree passed formatting, lint, strict TypeScript, generated-contract drift checks, **29/29 test files and 123/123 tests**, and coverage. Reported overall coverage was 85.3% statements, 73.3% branches, 96.15% functions, and 86.95% lines. - Final correctness review RED run `33275182068` demonstrated both remaining defects: the two new regressions failed because terminal semantic errors had no durable receipt and recovery work discovery omitted a `running` Task after its effective Lease expired, while the previous **123 tests passed**. Review-fix run `33275312677` then passed focused regressions, full `pnpm check`, and coverage with **30/30 test files and 125/125 tests**. Overall coverage was 85.43% statements, 73.5% branches, 96.18% functions, and 87.07% lines. - Durable Session/Lease liveness RED Quality run `33276800987` passed formatting/lint/typecheck/contracts and the previous **125 tests**, while all four new HTTP E2E regressions failed exactly because `HeartbeatSession`, `EndSession`, `RenewLease`, and `ReleaseLease` returned `UNSUPPORTED_OPERATION`. GREEN run `33276973691` then passed the focused 4/4 liveness E2E tests, full `pnpm check` with **31/31 test files and 129/129 tests**, and coverage at 85.19% statements, 73.88% branches, 96.3% functions, and 86.83% lines. -- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. +- Durable task-outcome RED Quality run `33278105807` kept the previous **130 tests green** while both new outcome regressions failed on the expected `UNSUPPORTED_OPERATION` boundary. GREEN run `33278271723` passed full `pnpm check` and coverage after self-cleaning the one-time implementation harness. Review-hardening run `33278367867` then proved persisted FailTask Lease state, lost-response BlockTask replay, durable blocked Checkpoint storage, and complete persisted ResumeTask state. Atomicity run `33278493339` added D1-like mid-batch fault injection and passed **32/32 test files and 133/133 tests** plus coverage at 84.99% statements, 74.15% branches, 96.36% functions, and 86.73% lines; the injected failure rolls back Checkpoint, Task, Lease, and receipt together before an exact retry succeeds. Durable retry/cancellation review hardening culminated in run `33283962742`, which passed **33/33 test files and 143/143 tests**, full `pnpm check`, and coverage at 85.0% statements, 74.3% branches, 96.51% functions, and 86.71% lines. The regressions cover lost-response replay, `CancelTask` without an effective Lease, `CancelGoal` mid-batch rollback, stale Goal task admission, false-success receipt prevention after a lost CAS, and recovery-`ClaimTask` races against both Task and Goal cancellation. - Runtime Surface PR #20 merged with post-merge Quality PASS. Persistence PR #24 merged with post-merge Quality #248 PASS. HTTP/MCP Transport PR #25 merged at `e142c1399aed5de3d8df53ad876499583728a6b4`; permanent Quality #249 and post-merge Quality #250 both passed full quality and coverage gates. - Permanent `Quality` CI remains least-privilege and uses pinned GitHub-owned action commits. ## Implemented but not yet fully deployed / externally integrated - The durable application composition is verified locally against the SQLite D1-like harness used by persistence tests. This is executable restart/concurrency evidence for the application/persistence contract, but it is **not** evidence of a deployed Cloudflare Worker, Durable Object, or production D1 environment. -- The durable dispatcher still leaves `RetryTask`, `CancelTask`, and `CancelGoal` explicitly unsupported until matching atomic persistence mutations are added. - Durable queries `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain explicitly unsupported rather than inferred from retained application state. - No deployed Cloudflare Worker/Durable Object service is claimed. Deployment configuration, environment provisioning, deployed-runtime restart verification, and real Cloudflare concurrency verification remain outstanding. - The v0.1 permission policy is intentionally small, explicit, hard-coded, and versioned. It is not a policy DSL, IAM system, credential manager, model judge, or arbitrary-code policy runtime. @@ -55,7 +54,7 @@ The following facts are supported by repository state and executed GitHub Action ## Next implementation slices -- Complete durable persistence mutations for the remaining v0.1 lifecycle/cancellation commands without introducing a second state machine or generic CRUD authority. +- Audit and harden any remaining conditional durable mutations against the same database-CAS/receipt and Goal-level concurrency invariants before treating the local persistence composition as complete. - Add the remaining bounded durable queries only where required by agent/human workflows. - Add GitHub integration while keeping GitHub as an adapter/projection rather than canonical state authority. - Add minimal real Codex, ChatGPT-compatible, generic MCP, and generic HTTP agent bootstrap/worker paths on top of the stable protocol/application boundary. diff --git a/docs/roadmap/V0_1.md b/docs/roadmap/V0_1.md index dc08a50a..b90f5f9e 100644 --- a/docs/roadmap/V0_1.md +++ b/docs/roadmap/V0_1.md @@ -41,7 +41,7 @@ The slice includes strict schema validation, representative positive/negative fi ADR-0005 defines the transport-neutral command/query semantics, idempotency scope, fencing/revision authority, recovery behavior, error model, and HTTP/MCP mapping principles. -The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers `RegisterAgent`, `StartSession`, `HeartbeatSession`, `EndSession`, `CreateGoal`, `CreateTask`, `ClaimTask`, `RenewLease`, `ReleaseLease`, `RecordCheckpoint`, `RequestPermission`, `RecordPermissionDecision`, `CompleteTask`, `FailTask`, `BlockTask`, and `ResumeTask`. Remaining durable commands fail explicitly as unsupported until matching atomic persistence mutations exist. +The canonical runtime/application command surface implements all ADR-0005 v0.1 commands, including Agent registration and Session bootstrap, lifecycle/recovery, permission request/decision, cancellation, and idempotent replay. Durable composition now covers the complete ADR-0005 v0.1 command surface, including `RetryTask`, `CancelTask`, and `CancelGoal`. Database-enforced mutation guards preserve receipt atomicity, Goal-versus-Task admission ordering, and cancellation-versus-recovery fencing under the SQLite D1-like reference harness. Durable read/query support now includes single-resource execution/permission reads, checkpoint and permission-decision history, the pending-human queue, and advisory claimable work. `ListGoals`, `ListGoalTasks`, and `GetTaskExecutionView` remain pending. @@ -66,7 +66,7 @@ Implemented behavior includes: - durable queries for `GetWorkspace`, `GetGoal`, `GetTask`, `GetLease`, `GetAgent`, `GetSession`, `GetPermissionRequest`, `ListTaskCheckpoints`, `ListPendingHumanPermissions`, `ListPermissionDecisions`, and advisory `ListClaimableTasks`; - HTTP E2E evidence for restart-after-claim, restart-after-`HUMAN_REQUIRED`, response-loss replay, competing claims, and monotonic fencing recovery against the SQLite D1-like test harness. -Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. The remaining local/runtime work is to extend atomic durable composition to `RetryTask`, `CancelTask`, and `CancelGoal` where required for dogfooding. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification. +Durable Session/Lease liveness is now covered through HTTP restart E2E: heartbeat does not extend Lease authority, renew preserves the fence, release permits higher-fence recovery, and EndSession durably revokes active Lease authority before recovery. Durable lifecycle/cancellation command parity is complete locally; remaining local work is bounded query parity plus persistence-concurrency hardening and final end-to-end reconciliation. The executed SQLite/D1-like tests do not substitute for deployed Cloudflare verification. ## Slice 4 — GitHub integration @@ -111,4 +111,4 @@ Add optional human-facing projections such as Google Sheets or a lightweight UI A user can give a goal to a supported real agent, the agent can obtain a **durable** task/context/policy assignment from MindRail, report checkpoints and evidence, continue to the next action without asking the user by default, and escalate only when a policy/decision boundary requires human input. -**Current gap to that condition:** at least one real agent-host integration, deployed/reference-runtime hardening, and any remaining durable lifecycle mutations required by that integration are outstanding. The local durable HTTP composition now proves the control-plane persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path. +**Current gap to that condition:** at least one real agent-host integration, the three remaining durable read queries, persistence/deployed-reference-runtime hardening, and real Cloudflare verification are outstanding. The local durable HTTP composition now proves the command-side persistence/restart contract, but it is not yet a real-agent or deployed-Cloudflare product path. diff --git a/migrations/0003_mutation_batch_guards.sql b/migrations/0003_mutation_batch_guards.sql new file mode 100644 index 00000000..ab285d9d --- /dev/null +++ b/migrations/0003_mutation_batch_guards.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS mutation_batch_guards ( + workspace_id TEXT NOT NULL, + ok INTEGER NOT NULL, + CONSTRAINT mutation_batch_guard_ok CHECK (ok = 1) +); diff --git a/src/application/durable-dispatcher.ts b/src/application/durable-dispatcher.ts index 345d0d70..0223d567 100644 --- a/src/application/durable-dispatcher.ts +++ b/src/application/durable-dispatcher.ts @@ -22,6 +22,8 @@ import type { CanonicalDomainValidator } from '../runtime/domain-validation.ts'; import { InMemoryControlPlane, type BlockTaskResult, + type CancelGoalResult, + type CancelTaskResult, type ClaimTaskResult, type CompleteTaskResult, type EndSessionResult, @@ -258,7 +260,10 @@ function isFirstDurableLoopCommand(command: ProtocolCommand): boolean { command.command === 'CompleteTask' || command.command === 'FailTask' || command.command === 'BlockTask' || - command.command === 'ResumeTask' + command.command === 'ResumeTask' || + command.command === 'RetryTask' || + command.command === 'CancelTask' || + command.command === 'CancelGoal' ); } @@ -543,6 +548,68 @@ async function commitDurableSuccess( }), ); } + case 'RetryTask': { + const result = semanticResponse.result as Task; + const now = result.updatedAt; + return resolveMutationResult( + command, + await options.persistence.retryTask({ + task: result, + expectedRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelTask': { + const result = semanticResponse.result as CancelTaskResult; + const now = result.task.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelTask({ + workspaceId: command.workspaceId, + task: result.task, + ...(result.lease === undefined ? {} : { lease: result.lease }), + expectedTaskRevision: command.expectedTaskRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } + case 'CancelGoal': { + const result = semanticResponse.result as CancelGoalResult; + const now = result.goal.updatedAt; + return resolveMutationResult( + command, + await options.persistence.cancelGoal({ + workspaceId: command.workspaceId, + goal: result.goal, + tasks: result.tasks, + leases: result.leases, + expectedGoalRevision: command.expectedGoalRevision, + now, + sessionCutoff: new Date(Date.parse(now) - options.sessionTimeoutMs).toISOString(), + receipt: receiptFor( + command, + fingerprint, + successResponse(command, result), + options.now(), + ), + }), + ); + } default: return commandFailure(command, 'UNSUPPORTED_OPERATION', 'Command is not durable yet.'); } diff --git a/src/persistence/cloudflare/d1-runtime-persistence.ts b/src/persistence/cloudflare/d1-runtime-persistence.ts index ff0cc237..69e9c681 100644 --- a/src/persistence/cloudflare/d1-runtime-persistence.ts +++ b/src/persistence/cloudflare/d1-runtime-persistence.ts @@ -13,6 +13,10 @@ import type { import { PersistenceError, + type CancelGoalCommitInput, + type CancelGoalCommitValue, + type CancelTaskCommitInput, + type CancelTaskCommitValue, type ClaimTaskCommitInput, type ClaimTaskCommitValue, type CommandReceiptInput, @@ -500,7 +504,12 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { .prepare( `INSERT INTO tasks( workspace_id, id, goal_id, revision, status, created_at_ms, updated_at_ms, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ? + WHERE EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, ) .bind( task.workspaceId, @@ -511,7 +520,11 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { timestampMs(task.createdAt, 'Task.createdAt'), timestampMs(task.updatedAt, 'Task.updatedAt'), serializeJson(task, 'Task'), + task.workspaceId, + task.goalId, + parentGoal.revision, ), + this.mutationChangesGuardStatement(task.workspaceId), ...task.requiredCapabilities.map((capability) => this.database .prepare( @@ -538,6 +551,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { ]; this.pushAuditStatement(statements, input.auditEvent); this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); await this.batch(statements, 'create Task'); return { kind: 'committed', value: clone(task) }; }); @@ -657,6 +671,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { activeLease.id, activeLease.revision, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } @@ -665,9 +680,23 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { .prepare( `UPDATE task_fencing_counters SET last_fencing_token = ? - WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ?`, + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + AND EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ? + )`, ) - .bind(nextFence, input.workspaceId, input.taskId, counter), + .bind( + nextFence, + input.workspaceId, + input.taskId, + counter, + input.workspaceId, + input.taskId, + task.revision, + task.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), ); let nextTask = task; @@ -695,6 +724,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { input.taskId, task.revision, ), + this.mutationChangesGuardStatement(input.workspaceId), ); } @@ -703,6 +733,7 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { const value = { task: clone(nextTask), lease: clone(lease) }; const finalReceipt = this.materializeReceipt(input.receipt, input.deferredReceipt, value); this.pushReceiptStatement(statements, finalReceipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); await this.batch(statements, 'claim Task'); return { kind: 'committed', value }; }); @@ -1035,6 +1066,426 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { }); } + async retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise> { + const { task } = input; + this.assertCanonical('Task', task); + this.assertRelatedAudit(task.workspaceId, input.auditEvent); + this.assertReceipt(task.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'retry Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'retry Task session cutoff'); + + return this.coordinator.runSerialized(task.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(task.workspaceId, task.id); + if (!current) throw new PersistenceError('NOT_FOUND', `Task ${task.id} was not found.`); + if (current.revision !== input.expectedRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${task.id} revision ${current.revision} does not match ${input.expectedRevision}.`, + ); + } + const goal = await this.getGoal(task.workspaceId, current.goalId); + if (!goal) { + throw new PersistenceError('INTEGRITY_ERROR', `Goal ${current.goalId} was not found.`); + } + if (goal.status !== 'active' || current.status !== 'failed') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${task.id} cannot be retried from current durable state.`, + ); + } + if (await this.getEffectiveActiveLease(task.workspaceId, task.id, nowMs, cutoffMs)) { + throw new PersistenceError( + 'CONFLICT', + `Task ${task.id} still has active execution authority.`, + ); + } + + const expected: Task = { + ...clone(current), + status: 'ready', + revision: input.expectedRevision + 1, + updatedAt: input.now, + }; + delete expected.statusReason; + if (serializeJson(task, 'Task') !== serializeJson(expected, 'expected RetryTask')) { + throw new PersistenceError('INVALID_RECORD', 'RetryTask replacement is invalid.'); + } + + const statements: D1PreparedStatementLike[] = [ + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'failed' + AND EXISTS ( + SELECT 1 FROM goals + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + )`, + ) + .bind( + task.revision, + task.status, + timestampMs(task.updatedAt, 'Task.updatedAt'), + serializeJson(task, 'Task'), + task.workspaceId, + task.id, + input.expectedRevision, + task.workspaceId, + current.goalId, + goal.revision, + ), + this.mutationChangesGuardStatement(task.workspaceId), + ]; + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(task.workspaceId)); + await this.batch(statements, 'retry Task'); + return { kind: 'committed', value: clone(task) }; + }); + } + + async cancelTask( + input: CancelTaskCommitInput, + ): Promise> { + this.assertCanonical('Task', input.task); + if (input.lease) this.assertCanonical('Lease', input.lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Task now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Task session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const current = await this.getTask(input.workspaceId, input.task.id); + if (!current) { + throw new PersistenceError('NOT_FOUND', `Task ${input.task.id} was not found.`); + } + if (current.revision !== input.expectedTaskRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Task ${current.id} revision ${current.revision} does not match ${input.expectedTaskRevision}.`, + ); + } + if (!isCancellableTaskStatus(current.status)) { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Task ${current.id} cannot be cancelled from ${current.status}.`, + ); + } + if (!input.task.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'Cancelled Task must include statusReason.'); + } + const expectedTask: Task = { + ...clone(current), + status: 'cancelled', + statusReason: clone(input.task.statusReason), + revision: input.expectedTaskRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.task, 'Task') !== serializeJson(expectedTask, 'expected CancelTask') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask replacement is invalid.'); + } + + const effectiveLease = await this.getEffectiveActiveLease( + input.workspaceId, + current.id, + nowMs, + cutoffMs, + ); + if ((effectiveLease === undefined) !== (input.lease === undefined)) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelTask Lease view is stale.'); + } + if (effectiveLease && input.lease) { + const expectedLease: Lease = { + ...clone(effectiveLease), + status: 'revoked', + revision: effectiveLease.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.lease, 'Lease') !== + serializeJson(expectedLease, 'expected CancelTask Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelTask Lease replacement is invalid.'); + } + } + + const fencingCounter = await this.getFencingCounter(input.workspaceId, current.id); + if (fencingCounter === undefined) { + throw new PersistenceError('INTEGRITY_ERROR', `Task ${current.id} has no fencing counter.`); + } + + const statements: D1PreparedStatementLike[] = [ + this.fencingCounterGuardStatement(input.workspaceId, current.id, fencingCounter), + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + input.task.revision, + input.task.status, + timestampMs(input.task.updatedAt, 'Task.updatedAt'), + serializeJson(input.task, 'Task'), + input.workspaceId, + input.task.id, + input.expectedTaskRevision, + current.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ]; + if (effectiveLease && input.lease) { + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + input.lease.revision, + input.lease.status, + timestampMs(input.lease.updatedAt, 'Lease.updatedAt'), + serializeJson(input.lease, 'Lease'), + input.workspaceId, + input.lease.id, + effectiveLease.revision, + effectiveLease.fencingToken, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Task'); + return { + kind: 'committed', + value: { + task: clone(input.task), + ...(input.lease === undefined ? {} : { lease: clone(input.lease) }), + }, + }; + }); + } + + async cancelGoal( + input: CancelGoalCommitInput, + ): Promise> { + this.assertCanonical('Goal', input.goal); + for (const task of input.tasks) this.assertCanonical('Task', task); + for (const lease of input.leases) this.assertCanonical('Lease', lease); + this.assertRelatedAudit(input.workspaceId, input.auditEvent); + this.assertReceipt(input.workspaceId, input.receipt); + const nowMs = timestampMs(input.now, 'cancel Goal now'); + const cutoffMs = timestampMs(input.sessionCutoff, 'cancel Goal session cutoff'); + + return this.coordinator.runSerialized(input.workspaceId, async () => { + const replay = await this.resolveReceipt(input.receipt); + if (replay) return replay; + const currentGoal = await this.getGoal(input.workspaceId, input.goal.id); + if (!currentGoal) { + throw new PersistenceError('NOT_FOUND', `Goal ${input.goal.id} was not found.`); + } + if (currentGoal.revision !== input.expectedGoalRevision) { + throw new PersistenceError( + 'REVISION_MISMATCH', + `Goal ${currentGoal.id} revision ${currentGoal.revision} does not match ${input.expectedGoalRevision}.`, + ); + } + if (currentGoal.status !== 'active') { + throw new PersistenceError( + 'INVALID_STATE_TRANSITION', + `Goal ${currentGoal.id} is already terminal.`, + ); + } + const expectedGoal: Goal = { + ...clone(currentGoal), + status: 'cancelled', + revision: input.expectedGoalRevision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(input.goal, 'Goal') !== serializeJson(expectedGoal, 'expected CancelGoal') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal replacement is invalid.'); + } + + const currentTasks = await this.listGoalTasks(input.workspaceId, currentGoal.id); + const cancellable = currentTasks.filter((task) => isCancellableTaskStatus(task.status)); + const fencingCounters = new Map(); + for (const currentTask of cancellable) { + const counter = await this.getFencingCounter(input.workspaceId, currentTask.id); + if (counter === undefined) { + throw new PersistenceError( + 'INTEGRITY_ERROR', + `Task ${currentTask.id} has no fencing counter.`, + ); + } + fencingCounters.set(currentTask.id, counter); + } + const outputTasks = new Map(input.tasks.map((task) => [task.id, task])); + if (outputTasks.size !== input.tasks.length || input.tasks.length !== cancellable.length) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task set is invalid.'); + } + const effectiveLeases: Lease[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id); + if (!output || !output.statusReason) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is missing.'); + } + const expectedTask: Task = { + ...clone(currentTask), + status: 'cancelled', + statusReason: clone(output.statusReason), + revision: currentTask.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Task') !== serializeJson(expectedTask, 'expected CancelGoal Task') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Task replacement is invalid.'); + } + const effective = await this.getEffectiveActiveLease( + input.workspaceId, + currentTask.id, + nowMs, + cutoffMs, + ); + if (effective) effectiveLeases.push(effective); + } + + const outputLeases = new Map(input.leases.map((lease) => [lease.id, lease])); + if ( + outputLeases.size !== input.leases.length || + input.leases.length !== effectiveLeases.length + ) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal Lease set is stale.'); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id); + if (!output) { + throw new PersistenceError('STALE_AUTHORITY', 'CancelGoal effective Lease is missing.'); + } + const expectedLease: Lease = { + ...clone(effective), + status: 'revoked', + revision: effective.revision + 1, + updatedAt: input.now, + }; + if ( + serializeJson(output, 'Lease') !== + serializeJson(expectedLease, 'expected CancelGoal Lease') + ) { + throw new PersistenceError('INVALID_RECORD', 'CancelGoal Lease replacement is invalid.'); + } + } + + const statements: D1PreparedStatementLike[] = []; + for (const currentTask of cancellable) { + const output = outputTasks.get(currentTask.id)!; + statements.push( + this.fencingCounterGuardStatement( + input.workspaceId, + currentTask.id, + fencingCounters.get(currentTask.id)!, + ), + this.database + .prepare( + `UPDATE tasks + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Task.updatedAt'), + serializeJson(output, 'Task'), + input.workspaceId, + output.id, + currentTask.revision, + currentTask.status, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + for (const effective of effectiveLeases) { + const output = outputLeases.get(effective.id)!; + statements.push( + this.database + .prepare( + `UPDATE leases + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND fencing_token = ?`, + ) + .bind( + output.revision, + output.status, + timestampMs(output.updatedAt, 'Lease.updatedAt'), + serializeJson(output, 'Lease'), + input.workspaceId, + output.id, + effective.revision, + effective.fencingToken, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + } + statements.push( + this.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active' + AND NOT EXISTS ( + SELECT 1 FROM tasks + WHERE workspace_id = ? AND goal_id = ? + AND status IN ('pending', 'ready', 'running', 'blocked') + )`, + ) + .bind( + input.goal.revision, + input.goal.status, + timestampMs(input.goal.updatedAt, 'Goal.updatedAt'), + serializeJson(input.goal, 'Goal'), + input.workspaceId, + input.goal.id, + input.expectedGoalRevision, + input.workspaceId, + input.goal.id, + ), + this.mutationChangesGuardStatement(input.workspaceId), + ); + this.pushAuditStatement(statements, input.auditEvent); + this.pushReceiptStatement(statements, input.receipt); + statements.push(this.clearMutationBatchGuardsStatement(input.workspaceId)); + await this.batch(statements, 'cancel Goal'); + return { + kind: 'committed', + value: { + goal: clone(input.goal), + tasks: clone(input.tasks), + leases: clone(input.leases), + }, + }; + }); + } + private async commitTaskOutcome( input: TaskOutcomeCommitInput, expectedTaskStatus: 'failed' | 'blocked', @@ -2105,6 +2556,25 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { }); } + private async getEffectiveActiveLease( + workspaceId: string, + taskId: string, + nowMs: number, + sessionCutoffMs: number, + ): Promise { + const lease = await this.getActiveLease(workspaceId, taskId); + if (!lease || timestampMs(lease.expiresAt, 'Lease.expiresAt') <= nowMs) return undefined; + const session = await this.getSession(workspaceId, lease.sessionId); + if ( + !session || + session.status !== 'active' || + timestampMs(session.lastSeenAt, 'Session.lastSeenAt') <= sessionCutoffMs + ) { + return undefined; + } + return lease; + } + private async getActiveLease(workspaceId: string, taskId: string): Promise { return this.readRecord( `SELECT record_json FROM leases @@ -2137,6 +2607,37 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { ); } + private fencingCounterGuardStatement( + workspaceId: string, + taskId: string, + expectedCounter: number, + ): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + SELECT ?, CASE WHEN EXISTS ( + SELECT 1 FROM task_fencing_counters + WHERE workspace_id = ? AND task_id = ? AND last_fencing_token = ? + ) THEN 1 ELSE 0 END`, + ) + .bind(workspaceId, workspaceId, taskId, expectedCounter); + } + + private mutationChangesGuardStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare( + `INSERT INTO mutation_batch_guards(workspace_id, ok) + VALUES (?, changes())`, + ) + .bind(workspaceId); + } + + private clearMutationBatchGuardsStatement(workspaceId: string): D1PreparedStatementLike { + return this.database + .prepare(`DELETE FROM mutation_batch_guards WHERE workspace_id = ?`) + .bind(workspaceId); + } + private async readRecord(sql: string, ...values: unknown[]): Promise { const row = await this.first(sql, ...values); return row ? parseJson(row.record_json, 'canonical record') : undefined; @@ -2178,11 +2679,20 @@ export class D1RuntimePersistence implements DurableRuntimePersistence { try { return await this.database.batch(statements); } catch (error) { + const message = errorMessage(error); + if (message.includes('mutation_batch_guard_ok')) { + const code = context === 'retry Task' ? 'REVISION_MISMATCH' : 'CONFLICT'; + throw new PersistenceError(code, `${context} lost its durable mutation race.`); + } throw wrapDatabaseError(context, error); } } } +function isCancellableTaskStatus(status: Task['status']): boolean { + return status === 'pending' || status === 'ready' || status === 'running' || status === 'blocked'; +} + function serializeJson(value: unknown, label: string, maxBytes?: number): string { let json: string | undefined; try { diff --git a/src/persistence/ports.ts b/src/persistence/ports.ts index 09b7dc66..7754ff65 100644 --- a/src/persistence/ports.ts +++ b/src/persistence/ports.ts @@ -162,6 +162,40 @@ export interface TaskOutcomeCommitValue { checkpoint: Checkpoint; } +export interface CancelTaskCommitInput { + workspaceId: string; + task: Task; + lease?: Lease; + expectedTaskRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelTaskCommitValue { + task: Task; + lease?: Lease; +} + +export interface CancelGoalCommitInput { + workspaceId: string; + goal: Goal; + tasks: Task[]; + leases: Lease[]; + expectedGoalRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; +} + +export interface CancelGoalCommitValue { + goal: Goal; + tasks: Task[]; + leases: Lease[]; +} + export interface DurableRuntimePersistence { bootstrapWorkspace(workspace: Workspace): Promise; createAgent(input: { @@ -234,6 +268,16 @@ export interface DurableRuntimePersistence { receipt?: CommandReceiptInput; auditEvent?: AuditEvent; }): Promise>; + retryTask(input: { + task: Task; + expectedRevision: number; + now: string; + sessionCutoff: string; + receipt?: CommandReceiptInput; + auditEvent?: AuditEvent; + }): Promise>; + cancelTask(input: CancelTaskCommitInput): Promise>; + cancelGoal(input: CancelGoalCommitInput): Promise>; appendAuditEvent(input: { auditEvent: AuditEvent }): Promise; appendPermissionRequestWithInitialDecision(input: { request: PermissionRequest; diff --git a/test/application/durable-retry-cancellation.test.ts b/test/application/durable-retry-cancellation.test.ts new file mode 100644 index 00000000..1fb5e5e6 --- /dev/null +++ b/test/application/durable-retry-cancellation.test.ts @@ -0,0 +1,828 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { Agent, Goal, Lease, Session, Task } from '@mindrail/contracts'; +import { describe, expect, it } from 'vitest'; + +import { createDurableApplicationDispatcher } from '../../src/application/durable-dispatcher.ts'; +import { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts'; +import type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts'; +import type { ApplicationDispatcher } from '../../src/application/ports.ts'; +import type { + CancelGoalResult, + CancelTaskResult, + FailTaskResult, +} from '../../src/runtime/in-memory-control-plane.ts'; +import { workspace } from '../persistence/fixtures.ts'; +import { openPersistence } from '../persistence/setup.ts'; +import { canonicalDomainValidator } from '../runtime/canonical-domain-validator.ts'; + +type CommandResponse = Awaited>; + +function databasePath(): string { + return join(mkdtempSync(join(tmpdir(), 'mindrail-durable-retry-cancel-')), 'runtime.sqlite'); +} + +async function openDispatcher( + path: string, + prefix: string, + now: Date, + coordinator?: WorkspaceMutationCoordinator, +) { + const opened = await openPersistence(path, coordinator); + let sequence = 0; + return { + ...opened, + dispatcher: createDurableApplicationDispatcher({ + persistence: opened.persistence, + now: () => new Date(now), + idFactory: (kind) => `${prefix}-${kind}-${++sequence}`, + leaseDurationMs: 120_000, + sessionTimeoutMs: 300_000, + validateCanonicalDomainRecord: canonicalDomainValidator, + }), + }; +} + +class OrderedTwoPartyCoordinator implements WorkspaceMutationCoordinator { + readonly firstArrived: Promise; + private firstArrivedResolve!: () => void; + private secondArrived: Promise; + private secondArrivedResolve!: () => void; + private firstQueued: Promise; + private firstQueuedResolve!: () => void; + private arrivals = 0; + + constructor(private readonly inner: WorkspaceMutationCoordinator) { + this.firstArrived = new Promise((resolve) => { + this.firstArrivedResolve = resolve; + }); + this.secondArrived = new Promise((resolve) => { + this.secondArrivedResolve = resolve; + }); + this.firstQueued = new Promise((resolve) => { + this.firstQueuedResolve = resolve; + }); + } + + runSerialized(workspaceId: string, operation: () => Promise): Promise { + this.arrivals += 1; + const position = this.arrivals; + if (position === 1) { + this.firstArrivedResolve(); + return this.queueFirst(workspaceId, operation); + } + if (position === 2) { + this.secondArrivedResolve(); + return this.queueSecond(workspaceId, operation); + } + return this.inner.runSerialized(workspaceId, operation); + } + + private async queueFirst(workspaceId: string, operation: () => Promise): Promise { + await this.secondArrived; + const result = this.inner.runSerialized(workspaceId, operation); + this.firstQueuedResolve(); + return result; + } + + private async queueSecond(workspaceId: string, operation: () => Promise): Promise { + await this.firstQueued; + return this.inner.runSerialized(workspaceId, operation); + } +} + +function success(response: CommandResponse): T { + expect(response).not.toHaveProperty('error'); + if ('error' in response) throw new Error(`Expected success, got ${response.error.code}.`); + return response.result as T; +} + +async function seedClaimedTask(dispatcher: ApplicationDispatcher, prefix: string) { + const systemActor = { type: 'system' as const, id: 'system-1' }; + const agent = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RegisterAgent', + commandId: `${prefix}-register`, + workspaceId: 'ws-a', + actor: systemActor, + displayName: 'Cancellation worker', + capabilities: ['repo.write'], + }), + ); + const agentActor = { type: 'agent' as const, id: agent.id }; + const session = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'StartSession', + commandId: `${prefix}-session`, + workspaceId: 'ws-a', + actor: systemActor, + agentId: agent.id, + }), + ); + const goal = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateGoal', + commandId: `${prefix}-goal`, + workspaceId: 'ws-a', + actor: systemActor, + title: 'Retry cancellation goal', + objective: 'Exercise durable controller transitions.', + successCriteria: ['Controller transitions survive restart.'], + }), + ); + const task = success( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: `${prefix}-task`, + workspaceId: 'ws-a', + actor: systemActor, + goalId: goal.id, + title: 'Retry cancellation task', + objective: 'Exercise retry and cancellation.', + acceptanceCriteria: ['State is durable.'], + requiredCapabilities: ['repo.write'], + dependencyTaskIds: [], + }), + ); + const claim = success<{ task: Task; lease: Lease }>( + await dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: `${prefix}-claim`, + workspaceId: 'ws-a', + actor: agentActor, + taskId: task.id, + sessionId: session.id, + expectedTaskRevision: task.revision, + }), + ); + return { systemActor, agentActor, agent, session, goal, task, claim }; +} + +describe('durable retry and cancellation', () => { + it('retries a durably failed Task through controller authority and replays after restart', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T18:00:00.000Z'); + let app = await openDispatcher(path, 'retry-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'retry'); + const failed = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'FailTask', + commandId: 'retry-fail', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'execution.failed', summary: 'Retryable deterministic failure.' }, + summary: 'Retry this work.', + evidence: [], + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'retry-after', now); + const retryResponse = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-command', + correlationId: 'retry-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + const retried = success(retryResponse); + expect(retried).toMatchObject({ status: 'ready', revision: failed.task.revision + 1 }); + expect(retried).not.toHaveProperty('statusReason'); + app.database.close(); + + app = await openDispatcher(path, 'retry-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-command', + correlationId: 'retry-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'retry-replay' }); + expect(success(replay)).toEqual(retried); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(retried); + app.database.close(); + }); + + it('cancels a running Task with its active Lease in one durable controller mutation', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T19:00:00.000Z'); + let app = await openDispatcher(path, 'cancel-task-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-task'); + app.database.close(); + + app = await openDispatcher(path, 'cancel-task-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-task-command', + correlationId: 'cancel-task-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Work is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.task).toMatchObject({ + status: 'cancelled', + revision: seeded.claim.task.revision + 1, + }); + expect(cancelled.lease).toMatchObject({ + status: 'revoked', + revision: seeded.claim.lease.revision + 1, + fencingToken: seeded.claim.lease.fencingToken, + }); + app.database.close(); + + app = await openDispatcher(path, 'cancel-task-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-task-command', + correlationId: 'cancel-task-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Work is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-task-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual(cancelled.lease); + app.database.close(); + }); + + it('cancels a Goal, its cancellable Tasks, active Lease, and receipt atomically', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T20:00:00.000Z'); + let app = await openDispatcher(path, 'cancel-goal-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal'); + const secondTask = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-goal-second-task', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Second cancellable task', + objective: 'Remain unclaimed until cancellation.', + acceptanceCriteria: ['Cancellation is durable.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-command', + correlationId: 'cancel-goal-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'The goal is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.goal).toMatchObject({ + status: 'cancelled', + revision: seeded.goal.revision + 1, + }); + expect(cancelled.tasks).toHaveLength(2); + expect(cancelled.tasks.every((task) => task.status === 'cancelled')).toBe(true); + expect(cancelled.leases).toEqual([ + expect.objectContaining({ + id: seeded.claim.lease.id, + status: 'revoked', + fencingToken: seeded.claim.lease.fencingToken, + }), + ]); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-command', + correlationId: 'cancel-goal-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'The goal is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-goal-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(cancelled.goal); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual( + cancelled.tasks.find((task) => task.id === seeded.task.id), + ); + expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual( + cancelled.tasks.find((task) => task.id === secondTask.id), + ); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual( + cancelled.leases[0], + ); + app.database.close(); + }); + + it('cancels a recoverable running Task without requiring an effective Lease', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T19:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-no-lease-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-no-lease'); + success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-no-lease-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-after', now); + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-first', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + const cancelled = success(response); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + app.database.close(); + + app = await openDispatcher(path, 'cancel-no-lease-replay', now); + const replay = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-no-lease-command', + correlationId: 'cancel-no-lease-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Recoverable work is no longer required.' }, + }); + expect(replay).toMatchObject({ replayed: true, correlationId: 'cancel-no-lease-replay' }); + expect(success(replay)).toEqual(cancelled); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(cancelled.task); + app.database.close(); + }); + + it('rolls back Goal, Tasks, Lease, and receipt when CancelGoal batch fails mid-transaction', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T20:30:00.000Z'); + let app = await openDispatcher(path, 'cancel-goal-atomic-before', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'cancel-goal-atomic'); + const secondTask = success( + await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-goal-atomic-second-task', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Second atomic cancellation task', + objective: 'Remain ready before injected cancellation failure.', + acceptanceCriteria: ['Rollback preserves this Task.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }), + ); + + app.database.failNextBatchAfterStatements(2); + const failed = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + expect(failed).toHaveProperty('error'); + expect(await app.persistence.getGoal('ws-a', seeded.goal.id)).toEqual(seeded.goal); + expect(await app.persistence.getTask('ws-a', seeded.task.id)).toEqual(seeded.claim.task); + expect(await app.persistence.getTask('ws-a', secondTask.id)).toEqual(secondTask); + expect(await app.persistence.getLease('ws-a', seeded.claim.lease.id)).toEqual( + seeded.claim.lease, + ); + expect( + await app.persistence.getCommandReceipt('ws-a', 'cancel-goal-atomic-command'), + ).toBeUndefined(); + app.database.close(); + + app = await openDispatcher(path, 'cancel-goal-atomic-retry', now); + const retry = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-goal-atomic-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Injected atomic cancellation failure.' }, + }); + const committed = success(retry); + expect(committed.goal.status).toBe('cancelled'); + expect(committed.tasks).toHaveLength(2); + expect(committed.leases).toHaveLength(1); + app.database.close(); + }); + + it('serializes stale CreateTask persistence behind CancelGoal through one Workspace authority', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:00:00.000Z'); + const seed = await openDispatcher(path, 'cancel-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-race'); + seed.database.close(); + + const sharedAuthority = new WorkspaceDurableObjectCoordinator('ws-a'); + const rendezvous = new OrderedTwoPartyCoordinator(sharedAuthority); + const cancelApp = await openDispatcher(path, 'cancel-race-controller', now, rendezvous); + const createApp = await openDispatcher(path, 'cancel-race-creator', now, rendezvous); + + const cancelPromise = cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'cancel-race-goal-shared', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Cancel before stale Task commit.' }, + }); + await rendezvous.firstArrived; + + const createPromise = createApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'cancel-race-create-shared', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Stale observed Task', + objective: 'Must not survive beneath a cancelled Goal.', + acceptanceCriteria: ['Persistence rejects stale creation.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + + const [cancelledResponse, createResponse] = await Promise.all([cancelPromise, createPromise]); + expect(success(cancelledResponse).goal.status).toBe('cancelled'); + expect('error' in createResponse && createResponse.error.code).toBe('INVALID_STATE_TRANSITION'); + + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + const goalTasks = snapshot?.tasks.filter((task) => task.goalId === seeded.goal.id) ?? []; + expect(goalTasks).toHaveLength(1); + expect(goalTasks.every((task) => task.status === 'cancelled')).toBe(true); + cancelApp.database.close(); + createApp.database.close(); + }); + + it('does not retain a success receipt when RetryTask loses its database CAS race', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T21:30:00.000Z'); + const seed = await openDispatcher(path, 'retry-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'retry-race'); + const failed = success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'FailTask', + commandId: 'retry-race-fail', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'execution.failed', summary: 'Prepare a retry race.' }, + summary: 'Race retries.', + evidence: [], + }), + ); + seed.database.close(); + + const first = await openDispatcher(path, 'retry-race-first', now); + const second = await openDispatcher(path, 'retry-race-second', now); + let firstArrivedResolve!: () => void; + let secondArrivedResolve!: () => void; + let releaseFirst!: () => void; + let releaseSecond!: () => void; + const firstArrived = new Promise((resolve) => (firstArrivedResolve = resolve)); + const secondArrived = new Promise((resolve) => (secondArrivedResolve = resolve)); + const firstRelease = new Promise((resolve) => (releaseFirst = resolve)); + const secondRelease = new Promise((resolve) => (releaseSecond = resolve)); + first.database.beforeNextBatch(async () => { + firstArrivedResolve(); + await firstRelease; + }); + second.database.beforeNextBatch(async () => { + secondArrivedResolve(); + await secondRelease; + }); + + const firstPromise = first.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-first-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await firstArrived; + const secondPromise = second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + await secondArrived; + + releaseFirst(); + success(await firstPromise); + releaseSecond(); + const lost = await secondPromise; + expect('error' in lost && lost.error.code).toBe('REVISION_MISMATCH'); + expect( + await second.persistence.getCommandReceipt('ws-a', 'retry-race-second-command'), + ).toBeUndefined(); + + const replayAttempt = await second.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'RetryTask', + commandId: 'retry-race-second-command', + correlationId: 'retry-race-replay', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: failed.task.revision, + }); + expect('error' in replayAttempt && replayAttempt.error.code).toBe('REVISION_MISMATCH'); + expect(replayAttempt).not.toHaveProperty('replayed', true); + first.database.close(); + second.database.close(); + }); + + it('rejects CreateTask when its Goal becomes terminal after precheck but before the durable batch', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:00:00.000Z'); + const app = await openDispatcher(path, 'create-goal-race', now); + await app.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(app.dispatcher, 'create-goal-race'); + const cancelledGoal: Goal = { + ...seeded.goal, + revision: seeded.goal.revision + 1, + status: 'cancelled', + updatedAt: now.toISOString(), + }; + app.database.beforeNextBatch(async () => { + await app.database + .prepare( + `UPDATE goals + SET revision = ?, status = ?, updated_at_ms = ?, record_json = ? + WHERE workspace_id = ? AND id = ? AND revision = ? AND status = 'active'`, + ) + .bind( + cancelledGoal.revision, + cancelledGoal.status, + now.getTime(), + JSON.stringify(cancelledGoal), + cancelledGoal.workspaceId, + cancelledGoal.id, + seeded.goal.revision, + ) + .run(); + }); + + const response = await app.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CreateTask', + commandId: 'create-goal-race-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + title: 'Stale Goal Task', + objective: 'Must not commit beneath a terminal Goal.', + acceptanceCriteria: ['Database predicate rejects stale admission.'], + requiredCapabilities: [], + dependencyTaskIds: [], + }); + expect(response).toHaveProperty('error'); + + const snapshot = await app.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)).toEqual(cancelledGoal); + expect(snapshot?.tasks.some((task) => task.title === 'Stale Goal Task')).toBe(false); + expect( + await app.persistence.getCommandReceipt('ws-a', 'create-goal-race-command'), + ).toBeUndefined(); + app.database.close(); + }); + + it('prevents a paused recovery ClaimTask from minting authority after CancelTask commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T22:30:00.000Z'); + const seed = await openDispatcher(path, 'cancel-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'cancel-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'cancel-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'cancel-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'cancel-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'cancel-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelTask', + commandId: 'cancel-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + taskId: seeded.task.id, + expectedTaskRevision: seeded.claim.task.revision, + reason: { code: 'controller.cancelled', summary: 'Cancellation wins the recovery race.' }, + }), + ); + expect(cancelled.task.status).toBe('cancelled'); + expect(cancelled).not.toHaveProperty('lease'); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect( + await claimApp.persistence.getCommandReceipt('ws-a', 'cancel-claim-race-claim-command'), + ).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); + + it('prevents a paused recovery ClaimTask from minting authority after CancelGoal commits first', async () => { + const path = databasePath(); + const now = new Date('2026-08-30T23:00:00.000Z'); + const seed = await openDispatcher(path, 'goal-claim-race-seed', now); + await seed.persistence.bootstrapWorkspace(workspace()); + const seeded = await seedClaimedTask(seed.dispatcher, 'goal-claim-race'); + success( + await seed.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ReleaseLease', + commandId: 'goal-claim-race-release', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + leaseId: seeded.claim.lease.id, + fencingToken: seeded.claim.lease.fencingToken, + expectedLeaseRevision: seeded.claim.lease.revision, + }), + ); + seed.database.close(); + + const claimApp = await openDispatcher(path, 'goal-claim-race-claim', now); + const cancelApp = await openDispatcher(path, 'goal-claim-race-cancel', now); + let claimArrivedResolve!: () => void; + let releaseClaim!: () => void; + const claimArrived = new Promise((resolve) => (claimArrivedResolve = resolve)); + const claimRelease = new Promise((resolve) => (releaseClaim = resolve)); + claimApp.database.beforeNextBatch(async () => { + claimArrivedResolve(); + await claimRelease; + }); + + const claimPromise = claimApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'ClaimTask', + commandId: 'goal-claim-race-claim-command', + workspaceId: 'ws-a', + actor: seeded.agentActor, + taskId: seeded.task.id, + sessionId: seeded.session.id, + expectedTaskRevision: seeded.claim.task.revision, + }); + await claimArrived; + + const cancelled = success( + await cancelApp.dispatcher.dispatchCommand({ + protocolVersion: '0.1', + command: 'CancelGoal', + commandId: 'goal-claim-race-cancel-command', + workspaceId: 'ws-a', + actor: seeded.systemActor, + goalId: seeded.goal.id, + expectedGoalRevision: seeded.goal.revision, + reason: { code: 'controller.cancelled', summary: 'Goal cancellation wins recovery race.' }, + }), + ); + expect(cancelled.goal.status).toBe('cancelled'); + expect(cancelled.tasks).toHaveLength(1); + expect(cancelled.leases).toHaveLength(0); + + releaseClaim(); + const staleClaim = await claimPromise; + expect(staleClaim).toHaveProperty('error'); + const snapshot = await cancelApp.persistence.loadWorkspaceState('ws-a'); + expect(snapshot?.goals.find((goal) => goal.id === seeded.goal.id)?.status).toBe('cancelled'); + expect(snapshot?.tasks.find((task) => task.id === seeded.task.id)?.status).toBe('cancelled'); + expect( + snapshot?.leases.some( + (lease) => lease.taskId === seeded.task.id && lease.status === 'active', + ), + ).toBe(false); + expect( + await claimApp.persistence.getCommandReceipt('ws-a', 'goal-claim-race-claim-command'), + ).toBeUndefined(); + claimApp.database.close(); + cancelApp.database.close(); + }); +}); diff --git a/test/persistence/d1-sqlite-harness.ts b/test/persistence/d1-sqlite-harness.ts index 899ec702..c80d73e9 100644 --- a/test/persistence/d1-sqlite-harness.ts +++ b/test/persistence/d1-sqlite-harness.ts @@ -9,6 +9,7 @@ import type { export class SqliteD1Database implements D1DatabaseLike { private readonly database: DatabaseSync; private failNextBatchAfterStatementCount: number | undefined; + private beforeNextBatchHook: (() => Promise) | undefined; constructor(path: string) { this.database = new DatabaseSync(path); @@ -20,6 +21,9 @@ export class SqliteD1Database implements D1DatabaseLike { } async batch(statements: D1PreparedStatementLike[]): Promise { + const beforeBatch = this.beforeNextBatchHook; + this.beforeNextBatchHook = undefined; + if (beforeBatch) await beforeBatch(); this.database.exec('BEGIN IMMEDIATE;'); try { const results: D1ResultLike[] = []; @@ -52,6 +56,10 @@ export class SqliteD1Database implements D1DatabaseLike { this.failNextBatchAfterStatementCount = statementCount; } + beforeNextBatch(hook: () => Promise): void { + this.beforeNextBatchHook = hook; + } + async exec(sql: string): Promise { this.database.exec(sql); } diff --git a/test/persistence/setup.ts b/test/persistence/setup.ts index bdd46757..eca52d84 100644 --- a/test/persistence/setup.ts +++ b/test/persistence/setup.ts @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { D1RuntimePersistence } from '../../src/persistence/cloudflare/d1-runtime-persistence.ts'; +import type { WorkspaceMutationCoordinator } from '../../src/persistence/ports.ts'; import { WorkspaceDurableObjectCoordinator } from '../../src/persistence/cloudflare/workspace-durable-object-coordinator.ts'; import { persistenceCanonicalValidator } from './canonical-domain-validator.ts'; import { SqliteD1Database } from './d1-sqlite-harness.ts'; @@ -14,7 +15,10 @@ const migrations = readdirSync(migrationDirectory) .sort() .map((name) => readFileSync(join(migrationDirectory, name), 'utf8')); -export async function openPersistence(path: string): Promise<{ +export async function openPersistence( + path: string, + coordinator: WorkspaceMutationCoordinator = new WorkspaceDurableObjectCoordinator(), +): Promise<{ database: SqliteD1Database; persistence: D1RuntimePersistence; }> { @@ -24,7 +28,7 @@ export async function openPersistence(path: string): Promise<{ } const persistence = new D1RuntimePersistence({ database, - coordinator: new WorkspaceDurableObjectCoordinator(), + coordinator, validateCanonicalDomainRecord: persistenceCanonicalValidator, }); return { database, persistence };