From bc988edb149acfdf0f5e94f334a91d7fb784504f Mon Sep 17 00:00:00 2001 From: Quinn Rafferty Date: Sat, 25 Jul 2026 14:16:52 +0000 Subject: [PATCH] feat(worker): emit app update cleanup terminal receipt Signed-off-by: Quinn Rafferty --- .../sql/0054_app_update_cleanup_terminal.sql | 58 ++ worker/src/index.ts | 6 + worker/src/openapi/releases.ts | 33 + worker/src/routes/app_update_cleanup.ts | 929 ++++++++++++++++++ worker/src/routes/auth.ts | 19 + worker/src/routes/operations.ts | 15 +- worker/src/routes/public_v2.ts | 278 ++++-- worker/src/routes/webhooks.ts | 7 +- .../test/app_update_cleanup_migration.test.ts | 149 +++ worker/test/routes.test.ts | 533 ++++++++++ worker/test/webhook_reaper.test.ts | 1 + 11 files changed, 1915 insertions(+), 113 deletions(-) create mode 100644 migrations/sql/0054_app_update_cleanup_terminal.sql create mode 100644 worker/src/routes/app_update_cleanup.ts create mode 100644 worker/test/app_update_cleanup_migration.test.ts diff --git a/migrations/sql/0054_app_update_cleanup_terminal.sql b/migrations/sql/0054_app_update_cleanup_terminal.sql new file mode 100644 index 0000000..cc71dab --- /dev/null +++ b/migrations/sql/0054_app_update_cleanup_terminal.sql @@ -0,0 +1,58 @@ +-- Immutable Hands producer receipt for Stamp's Android App Update cleanup +-- terminal. The operation, server-side inactive readback, exact event bytes, +-- subscriber, and signing generation are frozen in one D1 batch. + +CREATE TABLE app_update_cleanup_terminal_receipts ( + operation_id TEXT PRIMARY KEY + REFERENCES operation_logs(id) ON DELETE RESTRICT, + receipt_digest TEXT NOT NULL UNIQUE + CHECK (substr(receipt_digest, 1, 7) = 'sha256:' AND length(receipt_digest) = 71 + AND substr(receipt_digest, 8) NOT GLOB '*[^0-9a-f]*'), + run_case_id TEXT NOT NULL UNIQUE, + attempt INTEGER NOT NULL CHECK (attempt > 0), + artifact_bundle_digest TEXT NOT NULL + CHECK (length(artifact_bundle_digest) = 64 + AND artifact_bundle_digest NOT GLOB '*[^0-9a-f]*'), + app_id TEXT NOT NULL, + release_id TEXT NOT NULL UNIQUE, + release_revision INTEGER NOT NULL CHECK (release_revision >= 0), + build_id TEXT NOT NULL, + app_slug TEXT NOT NULL, + channel_slug TEXT NOT NULL, + target_artifact_sha256 TEXT NOT NULL + CHECK (length(target_artifact_sha256) = 64 + AND target_artifact_sha256 NOT GLOB '*[^0-9a-f]*'), + target_version_code INTEGER NOT NULL CHECK (target_version_code > 0), + target_installation_digest TEXT NOT NULL + CHECK (substr(target_installation_digest, 1, 7) = 'sha256:' AND length(target_installation_digest) = 71 + AND substr(target_installation_digest, 8) NOT GLOB '*[^0-9a-f]*'), + cancel_readback TEXT NOT NULL CHECK (cancel_readback = 'inactive'), + scope_deactivated INTEGER NOT NULL CHECK (scope_deactivated = 1), + scope_readback_json TEXT NOT NULL CHECK (json_valid(scope_readback_json)), + delivery_bindings_json TEXT NOT NULL CHECK (json_valid(delivery_bindings_json)), + canonical_request_json TEXT NOT NULL CHECK (json_valid(canonical_request_json)), + event_payload_json TEXT NOT NULL CHECK (json_valid(event_payload_json)), + canonical_receipt_json TEXT NOT NULL CHECK (json_valid(canonical_receipt_json)), + created_at INTEGER NOT NULL +); + +-- Keep trigger bodies on one physical line for Cloudflare's remote D1 parser. +CREATE TRIGGER app_update_cleanup_terminal_receipts_no_update BEFORE UPDATE ON app_update_cleanup_terminal_receipts BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal receipts are immutable'); END; + +CREATE TRIGGER app_update_cleanup_terminal_receipts_no_delete BEFORE DELETE ON app_update_cleanup_terminal_receipts BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal receipts are immutable'); END; + +-- A delivery references the immutable producer receipt instead of overloading +-- feedback event foreign keys. One subscriber can receive a receipt once. +ALTER TABLE webhook_deliveries + ADD COLUMN app_update_terminal_receipt_id TEXT + REFERENCES app_update_cleanup_terminal_receipts(operation_id) ON DELETE RESTRICT; + +CREATE UNIQUE INDEX idx_webhook_deliveries_app_update_terminal + ON webhook_deliveries(webhook_id, app_update_terminal_receipt_id) + WHERE app_update_terminal_receipt_id IS NOT NULL; + +-- Generic operation mutation endpoints must never rewrite or delete the +-- durable intent/receipt identity, even if a future handler forgets the guard. +CREATE TRIGGER app_update_cleanup_terminal_operation_no_update BEFORE UPDATE ON operation_logs WHEN OLD.kind = 'app-update-cleanup-terminal' BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal operations are immutable'); END; + +CREATE TRIGGER app_update_cleanup_terminal_operation_no_delete BEFORE DELETE ON operation_logs WHEN OLD.kind = 'app-update-cleanup-terminal' BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal operations are immutable'); END; diff --git a/worker/src/index.ts b/worker/src/index.ts index a43db25..740a343 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -177,6 +177,7 @@ import { handleUpsertReleaseCheck, handleListReleaseChecks, } from "./routes/releases"; +import { handleEmitAppUpdateCleanupTerminal } from "./routes/app_update_cleanup"; import { handleListChannels, handleCreateChannel, handleUpdateChannel, handleDeleteChannel } from "./routes/channels"; import { handleAddDeviceGroupMember, @@ -738,6 +739,11 @@ admin.patch("/api/apps/:appId/releases/:releaseId", requireAppRole("publisher"), admin.post("/api/apps/:appId/releases/:releaseId/publish", requireAppRole("publisher"), handlePublishRelease); admin.delete("/api/apps/:appId/releases/:releaseId", requireAppRole("publisher"), handleDeleteRelease); admin.post("/api/apps/:appId/releases/:releaseId/rollback", requireAppRole("publisher"), handleRollbackRelease); +admin.post( + "/api/apps/:appId/releases/:releaseId/app-update-cleanup-terminal", + requireAppRole("publisher"), + handleEmitAppUpdateCleanupTerminal, +); admin.post("/api/apps/:appId/releases/:releaseId/bump-rollout", requireAppRole("publisher"), handleBumpRollout); admin.post("/api/apps/:appId/releases/:releaseId/force-update", requireAppRole("publisher"), handleForceUpdate); admin.get("/api/apps/:appId/releases/:releaseId/checks", requireAppRole("viewer"), handleListReleaseChecks); diff --git a/worker/src/openapi/releases.ts b/worker/src/openapi/releases.ts index 91a2937..8dc7e17 100644 --- a/worker/src/openapi/releases.ts +++ b/worker/src/openapi/releases.ts @@ -66,6 +66,17 @@ const PublishReleaseInput = z .catchall(z.unknown()) .openapi("PublishReleaseInput"); +const AppUpdateCleanupTerminalInput = z.object({ + operation_id: z.string().min(1).max(128), + run_case_id: z.string().min(1).max(128), + attempt: z.number().int().positive(), + artifact_bundle_digest: z.string().regex(/^[0-9a-f]{64}$/), + expected_release_revision: z.number().int().nonnegative(), + target_device_id: z.string().min(1).max(200).openapi({ + description: "Exact installation id used for resolver readback. Hands stores only its SHA-256 digest.", + }), +}).openapi("AppUpdateCleanupTerminalInput"); + const ReleaseShare = z .object({ id: z.string(), @@ -297,6 +308,28 @@ export function registerReleaseRoutes(registry: OpenApiRegistry) { }); } + register(registry, { + method: "post", + path: "/api/apps/{appId}/releases/{releaseId}/app-update-cleanup-terminal", + tags: ["Releases"], + summary: "Freeze one signed Android App Update cleanup terminal receipt", + description: + "Live-facing. Reads an already-cancelled current release generation through the Hands resolver, then atomically freezes one immutable operation, receipt, and signed delivery set. It never cancels or mutates the release.", + security: auth, + request: { + params: AppReleaseParams, + body: { content: json(AppUpdateCleanupTerminalInput), required: true }, + }, + responses: { + 200: success("Exact response-loss replay of the frozen receipt.", GenericObject), + 201: success("Created the immutable receipt and delivery set.", GenericObject), + 400: error("Input is not the exact closed contract."), + 403: error("Current principal cannot emit this terminal receipt."), + 404: error("Release was not found."), + 409: error("Release generation, target scope, artifact, subscriber, or immutable binding conflict."), + }, + }); + register(registry, { method: "get", path: "/api/apps/{appId}/shares", diff --git a/worker/src/routes/app_update_cleanup.ts b/worker/src/routes/app_update_cleanup.ts new file mode 100644 index 0000000..abcac1f --- /dev/null +++ b/worker/src/routes/app_update_cleanup.ts @@ -0,0 +1,929 @@ +import { currentActor } from "../middleware/auth"; +import type { AdminContext } from "../lib/permissions"; +import { + resolveActiveReleaseForClient, + rolloutIncludes, +} from "./public_v2"; + +const EVENT = "app_update:cleanup_terminal"; +const OPERATION_KIND = "app-update-cleanup-terminal"; +const SAFE_REF = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/; +const SHA256_HEX = /^[0-9a-f]{64}$/; + +interface CleanupTerminalInput { + operation_id: string; + run_case_id: string; + attempt: number; + artifact_bundle_digest: string; + expected_release_revision: number; + target_device_id: string; +} + +interface ReleaseAuthority { + release_id: string; + release_revision: number; + release_status: string; + rollout_cohort_count: number | null; + build_id: string; + product_type: string; + release_type: string; + version_code: number; + app_id: string; + app_slug: string; + app_platform: string; + org_id: string; + channel_id: string; + channel_slug: string; +} + +interface ReleaseScope { + scope_type: string; + scope_value: string; +} + +interface TargetAsset { + id: string; + platform: string; + file_hash: string; +} + +interface ResolverWinnerAsset { + id: string; + file_hash: string; +} + +interface CleanupTerminalReceiptRow { + operation_id: string; + receipt_digest: string; + run_case_id: string; + attempt: number; + artifact_bundle_digest: string; + app_id: string; + release_id: string; + release_revision: number; + build_id: string; + app_slug: string; + channel_slug: string; + target_artifact_sha256: string; + target_version_code: number; + target_installation_digest: string; + cancel_readback: "inactive"; + scope_deactivated: number; + scope_readback_json: string; + delivery_bindings_json: string; + canonical_request_json: string; + event_payload_json: string; + canonical_receipt_json: string; + created_at: number; +} + +function isExactInput(value: unknown): value is CleanupTerminalInput { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const input = value as Record; + const expected = [ + "artifact_bundle_digest", + "attempt", + "expected_release_revision", + "operation_id", + "run_case_id", + "target_device_id", + ]; + if (Object.keys(input).sort().join(",") !== expected.join(",")) return false; + return typeof input.operation_id === "string" && + input.operation_id.length <= 128 && SAFE_REF.test(input.operation_id) && + typeof input.run_case_id === "string" && input.run_case_id.length <= 128 && + SAFE_REF.test(input.run_case_id) && + Number.isInteger(input.attempt) && Number(input.attempt) > 0 && + typeof input.artifact_bundle_digest === "string" && + SHA256_HEX.test(input.artifact_bundle_digest) && + Number.isInteger(input.expected_release_revision) && + Number(input.expected_release_revision) >= 0 && + typeof input.target_device_id === "string" && + input.target_device_id === input.target_device_id.trim() && + input.target_device_id.length >= 1 && input.target_device_id.length <= 200; +} + +async function sha256(value: string): Promise { + const bytes = new TextEncoder().encode(value); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function subscriptionPredicate(alias: string, orgParam: string, appParam: string): string { + const safeEvents = `CASE WHEN json_valid(${alias}.events_json) THEN ${alias}.events_json ELSE 'null' END`; + return `${alias}.org_id = ${orgParam} + AND ${alias}.enabled = 1 + AND ${alias}.archived_at IS NULL + AND (${alias}.app_id IS NULL OR ${alias}.app_id = ${appParam}) + AND ( + json_array_length(${safeEvents}) = 0 + OR EXISTS ( + SELECT 1 FROM json_each(${safeEvents}) event + WHERE event.value IN ('*', '${EVENT}') + ) + )`; +} + +function receiptResponse(row: CleanupTerminalReceiptRow, replay: boolean) { + return { + replay, + operation: { + id: row.operation_id, + kind: OPERATION_KIND, + status: "success", + }, + event: JSON.parse(row.event_payload_json) as Record, + readback: JSON.parse(row.scope_readback_json) as Record, + receipt: JSON.parse(row.canonical_receipt_json) as Record, + deliveries_frozen: true, + }; +} + +async function findReceiptCollision( + db: D1Database, + operationId: string, + runCaseId: string, + releaseId: string, +): Promise { + return await db.prepare( + `SELECT * FROM app_update_cleanup_terminal_receipts + WHERE operation_id = ?1 OR run_case_id = ?2 OR release_id = ?3 + ORDER BY CASE WHEN operation_id = ?1 THEN 0 WHEN run_case_id = ?2 THEN 1 ELSE 2 END + LIMIT 1`, + ).bind(operationId, runCaseId, releaseId).first(); +} + +function conflictResponse(c: AdminContext, code: string, error: string) { + return c.json({ error, code }, 409); +} + +/** + * Persist the one immutable Hands terminal producer receipt for an Android App + * Update cleanup. This endpoint never cancels or otherwise mutates a release; + * it only proves an already-cancelled current generation is unreachable to the + * exact target installation and freezes the resulting webhook event. + */ +export async function handleEmitAppUpdateCleanupTerminal(c: AdminContext) { + const appId = c.req.param("appId") ?? ""; + const releaseId = c.req.param("releaseId") ?? ""; + const raw = await c.req.json().catch(() => null); + if (!isExactInput(raw)) { + return c.json({ + error: "exact operation/run/attempt/bundle/revision/target-device input required", + code: "INVALID_APP_UPDATE_CLEANUP_TERMINAL_INPUT", + }, 400); + } + const input = raw; + const targetInstallationDigest = `sha256:${await sha256(input.target_device_id)}`; + const canonicalRequest = JSON.stringify({ + operation_id: input.operation_id, + run_case_id: input.run_case_id, + attempt: input.attempt, + artifact_bundle_digest: input.artifact_bundle_digest, + app_id: appId, + release_id: releaseId, + expected_release_revision: input.expected_release_revision, + target_installation_digest: targetInstallationDigest, + }); + // Response-loss replay is resolved before any mutable release/subscriber + // read. A later key rotation, archive, or release transition must not make + // the caller regenerate or silently rebind the frozen receipt. + const earlyExisting = await findReceiptCollision( + c.env.DB, + input.operation_id, + input.run_case_id, + releaseId, + ); + if (earlyExisting) { + if (earlyExisting.operation_id === input.operation_id && + earlyExisting.canonical_request_json === canonicalRequest) { + return c.json(receiptResponse(earlyExisting, true)); + } + return conflictResponse( + c, + "APP_UPDATE_TERMINAL_BINDING_CONFLICT", + "operation, run case, or release is already bound to another terminal receipt", + ); + } + + const authority = await c.env.DB.prepare( + `SELECT r.id AS release_id, r.revision AS release_revision, + r.status AS release_status, r.rollout_cohort_count, + r.build_id, r.product_type, r.release_type, + b.version_code, a.id AS app_id, a.slug AS app_slug, + a.platform AS app_platform, a.org_id, + ch.id AS channel_id, ch.slug AS channel_slug + FROM releases r + JOIN builds b ON b.id = r.build_id + JOIN apps a ON a.id = r.app_id + JOIN channels ch ON ch.id = r.channel_id AND ch.app_id = r.app_id + WHERE r.app_id = ?1 AND r.id = ?2`, + ).bind(appId, releaseId).first(); + if (!authority) return c.json({ error: "release not found" }, 404); + if (authority.release_revision !== input.expected_release_revision) { + return c.json({ + error: "release changed before cleanup terminal readback", + code: "RELEASE_REVISION_CONFLICT", + expected_revision: input.expected_release_revision, + current_revision: authority.release_revision, + }, 409); + } + if (authority.release_status !== "cancelled") { + return conflictResponse( + c, + "APP_UPDATE_RELEASE_NOT_INACTIVE", + "cleanup terminal requires a currently cancelled release", + ); + } + if (authority.app_platform !== "android" || authority.product_type !== "android-apk") { + return conflictResponse( + c, + "APP_UPDATE_TARGET_NOT_ANDROID", + "cleanup terminal requires an Android APK release", + ); + } + if (authority.app_slug.length > 64 || !SAFE_REF.test(authority.app_slug) || + authority.channel_slug.length > 64 || !SAFE_REF.test(authority.channel_slug)) { + return conflictResponse( + c, + "APP_UPDATE_TARGET_REFERENCE_INVALID", + "app or channel slug cannot be represented in the exact terminal contract", + ); + } + const orgId = c.get("org_id"); + if (!orgId || authority.org_id !== orgId) { + return c.json({ error: "release organization mismatch" }, 403); + } + + const { results: scopes } = await c.env.DB.prepare( + `SELECT scope_type, scope_value FROM release_scopes + WHERE release_id = ?1 ORDER BY scope_type, scope_value`, + ).bind(releaseId).all(); + if (scopes.length === 0) { + return conflictResponse( + c, + "APP_UPDATE_RELEASE_SCOPE_MISSING", + "release has no frozen scope identity", + ); + } + + const { results: assets } = await c.env.DB.prepare( + `SELECT id, platform, file_hash FROM build_assets + WHERE build_id = ?1 AND artifact_kind = 'installable' AND filetype = 'apk' + ORDER BY id`, + ).bind(authority.build_id).all(); + if (assets.length !== 1) { + return conflictResponse( + c, + assets.length === 0 ? "APP_UPDATE_TARGET_ARTIFACT_MISSING" : "APP_UPDATE_TARGET_ARTIFACT_AMBIGUOUS", + "cleanup terminal requires exactly one installable APK target", + ); + } + const asset = assets[0]!; + if (!SHA256_HEX.test(asset.file_hash) || authority.version_code < 1) { + return conflictResponse( + c, + "APP_UPDATE_TARGET_IDENTITY_INVALID", + "target APK digest or version code is invalid", + ); + } + + const { results: memberships } = await c.env.DB.prepare( + `SELECT m.group_id FROM device_group_members m + JOIN device_groups g ON g.id = m.group_id + WHERE g.app_id = ?1 AND m.device_id = ?2 + ORDER BY m.group_id`, + ).bind(appId, input.target_device_id).all<{ group_id: string }>(); + const targetGroups = new Set(memberships.map((membership) => membership.group_id)); + const scopeMatchedBeforeCancellation = scopes.some((scope) => { + if (scope.scope_type === "device_group") return targetGroups.has(scope.scope_value); + if (scope.scope_type === "full" && scope.scope_value === "all") { + return rolloutIncludes(releaseId, authority.rollout_cohort_count, input.target_device_id); + } + if (scope.scope_type === "platform") { + return scope.scope_value.split(",").map((value) => value.trim()).includes(asset.platform); + } + return false; + }); + if (!scopeMatchedBeforeCancellation) { + return conflictResponse( + c, + "APP_UPDATE_TARGET_OUTSIDE_RELEASE_SCOPE", + "target installation is not bound to the preserved release scope", + ); + } + + // Run the exact same candidate/scope/rollout/winner resolver as the public + // update-check path. A cancelled row by itself proves nothing: historical + // duplicate releases may still resolve the same B version/artifact. + const activeResolution = await resolveActiveReleaseForClient(c.env.DB, { + appId, + channelId: authority.channel_id, + productType: authority.product_type, + deviceId: input.target_device_id, + cohort: null, + clientPlatform: asset.platform, + clientIp: null, + }); + const resolverWinnerCandidate = activeResolution.winner + ? activeResolution.candidates.find( + (candidate) => candidate.id === activeResolution.winner?.release_id, + ) ?? null + : null; + let resolverWinnerVersionCode: number | null = null; + let resolverWinnerAssets: ResolverWinnerAsset[] = []; + if (resolverWinnerCandidate) { + const winnerBuild = await c.env.DB.prepare( + "SELECT version_code FROM builds WHERE id = ?1", + ).bind(resolverWinnerCandidate.build_id).first<{ version_code: number }>(); + resolverWinnerVersionCode = winnerBuild?.version_code ?? null; + const { results: winnerAssets } = await c.env.DB.prepare( + `SELECT id, file_hash FROM build_assets + WHERE build_id = ?1 AND artifact_kind = 'installable' AND filetype = 'apk' + ORDER BY id`, + ).bind(resolverWinnerCandidate.build_id).all(); + resolverWinnerAssets = winnerAssets; + } + const targetBStillReachable = resolverWinnerVersionCode === authority.version_code && + resolverWinnerAssets.some((winnerAsset) => winnerAsset.file_hash === asset.file_hash); + if (targetBStillReachable) { + return conflictResponse( + c, + "APP_UPDATE_TARGET_STILL_REACHABLE", + "canonical resolver still reaches the target release", + ); + } + + const { results: subscribers } = await c.env.DB.prepare( + `SELECT id, secret, signature_key_version FROM webhooks w + WHERE ${subscriptionPredicate("w", "?1", "?2")} + ORDER BY id`, + ).bind(orgId, appId).all<{ + id: string; + secret: string; + signature_key_version: string; + }>(); + if (subscribers.length === 0) { + return conflictResponse( + c, + "APP_UPDATE_TERMINAL_SUBSCRIBER_MISSING", + "no active webhook subscribes to the cleanup terminal event", + ); + } + + const scopeIdentityJson = JSON.stringify(scopes); + const scopeIdentityDigest = `sha256:${await sha256(scopeIdentityJson)}`; + const deliveryBindings = subscribers.map((subscriber) => ({ + subscriber_id: subscriber.id, + signature_key_version: subscriber.signature_key_version, + })); + const deliveryBindingsJson = JSON.stringify(deliveryBindings); + const deliveryBindingsDigest = `sha256:${await sha256(deliveryBindingsJson)}`; + const resolverSnapshotJson = JSON.stringify({ + candidates: [...activeResolution.candidates].sort((left, right) => + left.id.localeCompare(right.id)), + scopes: [...activeResolution.scopes].sort((left, right) => + left.release_id.localeCompare(right.release_id) || + left.scope_type.localeCompare(right.scope_type) || + left.scope_value.localeCompare(right.scope_value)), + device_group_ids: activeResolution.deviceGroupIds, + matched: activeResolution.matched, + winner: activeResolution.winner, + winner_version_code: resolverWinnerVersionCode, + winner_assets: resolverWinnerAssets, + }); + const resolverSnapshotDigest = `sha256:${await sha256(resolverSnapshotJson)}`; + const operationCollision = await c.env.DB.prepare( + "SELECT id FROM operation_logs WHERE id = ?1", + ).bind(input.operation_id).first<{ id: string }>(); + if (operationCollision) { + return conflictResponse( + c, + "APP_UPDATE_TERMINAL_OPERATION_CONFLICT", + "operation id is already used by another operation", + ); + } + + const readback = { + source: "hands_current_release_resolver", + release_id: releaseId, + release_revision: authority.release_revision, + release_status: "cancelled", + target_installation_digest: targetInstallationDigest, + target_artifact_sha256: asset.file_hash, + target_version_code: authority.version_code, + preserved_scope_identity_digest: scopeIdentityDigest, + preserved_scope_count: scopes.length, + target_matched_preserved_scope: true, + resolver_snapshot_digest: resolverSnapshotDigest, + resolver_candidate_count: activeResolution.candidates.length, + resolver_winner_release_id: activeResolution.winner?.release_id ?? null, + resolver_winner_version_code: resolverWinnerVersionCode, + target_release_reachable: false, + scope_inactive: true, + }; + const scopeReadbackJson = JSON.stringify(readback); + const receiptMaterial = { + operation_id: input.operation_id, + run_case_id: input.run_case_id, + attempt: input.attempt, + artifact_bundle_digest: input.artifact_bundle_digest, + app_id: appId, + release_id: releaseId, + release_revision: authority.release_revision, + build_id: authority.build_id, + app_slug: authority.app_slug, + channel: authority.channel_slug, + target_artifact_sha256: asset.file_hash, + target_version_code: authority.version_code, + target_installation_digest: targetInstallationDigest, + scope_readback_digest: `sha256:${await sha256(scopeReadbackJson)}`, + delivery_bindings_digest: deliveryBindingsDigest, + delivery_count: deliveryBindings.length, + cancel_readback: "inactive", + scope_deactivated: true, + }; + const receiptDigest = `sha256:${await sha256(JSON.stringify(receiptMaterial))}`; + const eventPayload = { + operation_id: input.operation_id, + receipt_digest: receiptDigest, + run_case_id: input.run_case_id, + attempt: input.attempt, + artifact_bundle_digest: input.artifact_bundle_digest, + app_slug: authority.app_slug, + channel: authority.channel_slug, + target_artifact_sha256: asset.file_hash, + target_version_code: authority.version_code, + cancel_readback: "inactive", + scope_deactivated: true, + }; + const eventPayloadJson = JSON.stringify(eventPayload); + const canonicalReceiptJson = JSON.stringify({ ...receiptMaterial, receipt_digest: receiptDigest }); + const now = Date.now(); + const envelopeJson = JSON.stringify({ + event: EVENT, + delivered_at: now, + org_id: orgId, + app_id: appId, + payload: eventPayload, + }); + const operationInput = JSON.stringify({ + run_case_id: input.run_case_id, + attempt: input.attempt, + artifact_bundle_digest: input.artifact_bundle_digest, + release_id: releaseId, + expected_release_revision: input.expected_release_revision, + target_installation_digest: targetInstallationDigest, + }); + const operationOutput = JSON.stringify({ + receipt_digest: receiptDigest, + event: EVENT, + release_id: releaseId, + release_revision: authority.release_revision, + scope_readback_digest: receiptMaterial.scope_readback_digest, + }); + const actor = currentActor(c); + + // Build one shared current-generation predicate. Every statement in the D1 + // transaction is gated on the immutable release/build/asset coordinate, + // preserved exact scope set, target membership, and current subscriber. + const binds: Array = []; + const bind = (value: string | number | null) => { + binds.push(value); + return `?${binds.length}`; + }; + const releaseIdParam = bind(releaseId); + const appIdParam = bind(appId); + const revisionParam = bind(authority.release_revision); + const buildIdParam = bind(authority.build_id); + const channelIdParam = bind(authority.channel_id); + const channelSlugParam = bind(authority.channel_slug); + const appSlugParam = bind(authority.app_slug); + const authorityOrgParam = bind(authority.org_id); + const productTypeParam = bind(authority.product_type); + const releaseTypeParam = bind(authority.release_type); + const versionCodeParam = bind(authority.version_code); + const rolloutPredicate = authority.rollout_cohort_count === null + ? "r.rollout_cohort_count IS NULL" + : `r.rollout_cohort_count = ${bind(authority.rollout_cohort_count)}`; + const assetIdParam = bind(asset.id); + const assetHashParam = bind(asset.file_hash); + const assetPlatformParam = bind(asset.platform); + const scopeCountParam = bind(scopes.length); + const scopePredicates = scopes.map((scope) => { + const typeParam = bind(scope.scope_type); + const valueParam = bind(scope.scope_value); + return `EXISTS ( + SELECT 1 FROM release_scopes exact_scope + WHERE exact_scope.release_id = r.id + AND exact_scope.scope_type = ${typeParam} + AND exact_scope.scope_value = ${valueParam} + )`; + }); + const orgParam = bind(orgId); + const subscriberAppParam = bind(appId); + const subscriberCountParam = bind(subscribers.length); + const subscriberPredicates = subscribers.map((subscriber) => { + const idParam = bind(subscriber.id); + const secretParam = bind(subscriber.secret); + const keyVersionParam = bind(subscriber.signature_key_version); + return `EXISTS ( + SELECT 1 FROM webhooks exact_subscriber + WHERE exact_subscriber.id = ${idParam} + AND exact_subscriber.secret = ${secretParam} + AND exact_subscriber.signature_key_version = ${keyVersionParam} + AND ${subscriptionPredicate("exact_subscriber", orgParam, subscriberAppParam)} + )`; + }); + const targetScopePredicates: string[] = []; + for (const scope of scopes) { + if (scope.scope_type === "device_group" && targetGroups.has(scope.scope_value)) { + const groupParam = bind(scope.scope_value); + const deviceParam = bind(input.target_device_id); + targetScopePredicates.push(`EXISTS ( + SELECT 1 FROM release_scopes target_scope + JOIN device_group_members target_member ON target_member.group_id = target_scope.scope_value + WHERE target_scope.release_id = r.id + AND target_scope.scope_type = 'device_group' + AND target_scope.scope_value = ${groupParam} + AND target_member.device_id = ${deviceParam} + )`); + } else if (scope.scope_type === "full" && scope.scope_value === "all" && + rolloutIncludes(releaseId, authority.rollout_cohort_count, input.target_device_id)) { + targetScopePredicates.push(`EXISTS ( + SELECT 1 FROM release_scopes target_scope + WHERE target_scope.release_id = r.id + AND target_scope.scope_type = 'full' AND target_scope.scope_value = 'all' + )`); + } else if (scope.scope_type === "platform" && + scope.scope_value.split(",").map((value) => value.trim()).includes(asset.platform)) { + const platformScopeParam = bind(scope.scope_value); + targetScopePredicates.push(`EXISTS ( + SELECT 1 FROM release_scopes target_scope + WHERE target_scope.release_id = r.id + AND target_scope.scope_type = 'platform' + AND target_scope.scope_value = ${platformScopeParam} + )`); + } + } + + // Freeze every mutable input to the shared resolver, then repeat its + // priority/activation/id winner ordering inside the commit transaction. + // rollout eligibility is precomputed by the shared JS resolver; exact + // candidate rollout values are CAS-checked below, so that set cannot drift. + const resolverCandidatePredicates = activeResolution.candidates.map((candidate) => { + const candidateIdParam = bind(candidate.id); + const candidateBuildParam = bind(candidate.build_id); + const candidateActivatedParam = bind(candidate.activated_at); + const candidateProductParam = bind(candidate.product_type); + const candidateRolloutPredicate = candidate.rollout_cohort_count === null + ? "resolver_candidate.rollout_cohort_count IS NULL" + : `resolver_candidate.rollout_cohort_count = ${bind(candidate.rollout_cohort_count)}`; + return `EXISTS ( + SELECT 1 FROM releases resolver_candidate + JOIN builds resolver_build ON resolver_build.id = resolver_candidate.build_id + WHERE resolver_candidate.id = ${candidateIdParam} + AND resolver_candidate.app_id = ${appIdParam} + AND resolver_candidate.channel_id = ${channelIdParam} + AND resolver_candidate.product_type = ${candidateProductParam} + AND resolver_candidate.status = 'active' + AND resolver_candidate.build_id = ${candidateBuildParam} + AND COALESCE(resolver_candidate.activated_at, resolver_candidate.created_at) = ${candidateActivatedParam} + AND ${candidateRolloutPredicate} + AND resolver_build.product_type != 'ios-simulator-qa' + AND resolver_build.release_type != 'qa' + )`; + }); + const resolverCandidateCountParam = bind(activeResolution.candidates.length); + const resolverCandidateCountPredicate = `(SELECT COUNT(*) + FROM releases resolver_counted + JOIN builds resolver_counted_build ON resolver_counted_build.id = resolver_counted.build_id + WHERE resolver_counted.app_id = ${appIdParam} + AND resolver_counted.channel_id = ${channelIdParam} + AND resolver_counted.product_type = ${productTypeParam} + AND resolver_counted.status = 'active' + AND resolver_counted_build.product_type != 'ios-simulator-qa' + AND resolver_counted_build.release_type != 'qa') = ${resolverCandidateCountParam}`; + + const resolverScopePredicates = activeResolution.scopes.map((scope) => { + const releaseParam = bind(scope.release_id); + const typeParam = bind(scope.scope_type); + const valueParam = bind(scope.scope_value); + return `EXISTS ( + SELECT 1 FROM release_scopes resolver_scope + WHERE resolver_scope.release_id = ${releaseParam} + AND resolver_scope.scope_type = ${typeParam} + AND resolver_scope.scope_value = ${valueParam} + )`; + }); + const resolverScopeCountParam = bind(activeResolution.scopes.length); + const resolverCandidateIds = activeResolution.candidates.map((candidate) => bind(candidate.id)); + const resolverScopeCountPredicate = resolverCandidateIds.length > 0 + ? `(SELECT COUNT(*) FROM release_scopes resolver_scope_counted + WHERE resolver_scope_counted.release_id IN (${resolverCandidateIds.join(",")})) = ${resolverScopeCountParam}` + : `${resolverScopeCountParam} = 0`; + + const resolverGroupPredicates = activeResolution.deviceGroupIds.map((groupId) => { + const groupParam = bind(groupId); + const deviceParam = bind(input.target_device_id); + return `EXISTS ( + SELECT 1 FROM device_group_members resolver_member + JOIN device_groups resolver_group ON resolver_group.id = resolver_member.group_id + WHERE resolver_group.app_id = ${appIdParam} + AND resolver_member.group_id = ${groupParam} + AND resolver_member.device_id = ${deviceParam} + )`; + }); + const resolverGroupCountParam = bind(activeResolution.deviceGroupIds.length); + const resolverGroupDeviceParam = bind(input.target_device_id); + const resolverGroupCountPredicate = `(SELECT COUNT(*) + FROM device_group_members resolver_member_counted + JOIN device_groups resolver_group_counted + ON resolver_group_counted.id = resolver_member_counted.group_id + WHERE resolver_group_counted.app_id = ${appIdParam} + AND resolver_member_counted.device_id = ${resolverGroupDeviceParam}) = ${resolverGroupCountParam}`; + + const resolverMatchSelects = activeResolution.matched.map((match) => { + const releaseParam = bind(match.release_id); + const typeParam = bind(match.scope_type); + const valueParam = bind(match.scope_value); + const priorityParam = bind(match.priority); + const activatedParam = bind(match.release_activated_at); + return `SELECT ${releaseParam} AS release_id, + ${priorityParam} AS priority, + ${activatedParam} AS activated_at + WHERE EXISTS ( + SELECT 1 FROM releases resolver_match_release + JOIN release_scopes resolver_match_scope + ON resolver_match_scope.release_id = resolver_match_release.id + WHERE resolver_match_release.id = ${releaseParam} + AND resolver_match_release.status = 'active' + AND resolver_match_scope.scope_type = ${typeParam} + AND resolver_match_scope.scope_value = ${valueParam} + )`; + }); + const resolverWinnerSql = resolverMatchSelects.length > 0 + ? `(SELECT resolver_match.release_id + FROM (${resolverMatchSelects.join(" UNION ALL ")}) resolver_match + ORDER BY resolver_match.priority DESC, + resolver_match.activated_at DESC, + resolver_match.release_id ASC + LIMIT 1)` + : "NULL"; + const resolverWinnerPredicate = activeResolution.winner + ? `${resolverWinnerSql} = ${bind(activeResolution.winner.release_id)}` + : `${resolverWinnerSql} IS NULL`; + const resolverWinnerBuildPredicate = resolverWinnerCandidate && + resolverWinnerVersionCode !== null + ? `EXISTS ( + SELECT 1 FROM builds resolver_winner_exact_build + WHERE resolver_winner_exact_build.id = ${bind(resolverWinnerCandidate.build_id)} + AND resolver_winner_exact_build.version_code = ${bind(resolverWinnerVersionCode)} + )` + : activeResolution.winner ? "0" : "1"; + const resolverWinnerAssetPredicates = resolverWinnerCandidate + ? resolverWinnerAssets.map((winnerAsset) => { + const idParam = bind(winnerAsset.id); + const hashParam = bind(winnerAsset.file_hash); + const buildParam = bind(resolverWinnerCandidate.build_id); + return `EXISTS ( + SELECT 1 FROM build_assets resolver_winner_exact_asset + WHERE resolver_winner_exact_asset.id = ${idParam} + AND resolver_winner_exact_asset.build_id = ${buildParam} + AND resolver_winner_exact_asset.artifact_kind = 'installable' + AND resolver_winner_exact_asset.filetype = 'apk' + AND resolver_winner_exact_asset.file_hash = ${hashParam} + )`; + }) + : []; + const resolverWinnerAssetCountPredicate = resolverWinnerCandidate + ? `(SELECT COUNT(*) FROM build_assets resolver_winner_counted_asset + WHERE resolver_winner_counted_asset.build_id = ${bind(resolverWinnerCandidate.build_id)} + AND resolver_winner_counted_asset.artifact_kind = 'installable' + AND resolver_winner_counted_asset.filetype = 'apk') = ${bind(resolverWinnerAssets.length)}` + : "1"; + const resolverTargetVersionParam = bind(authority.version_code); + const resolverTargetHashParam = bind(asset.file_hash); + const resolverTargetBUnreachablePredicate = `NOT EXISTS ( + SELECT 1 FROM releases resolver_winner_release + JOIN builds resolver_winner_build + ON resolver_winner_build.id = resolver_winner_release.build_id + JOIN build_assets resolver_winner_asset + ON resolver_winner_asset.build_id = resolver_winner_build.id + WHERE resolver_winner_release.id = ${resolverWinnerSql} + AND resolver_winner_build.version_code = ${resolverTargetVersionParam} + AND resolver_winner_asset.artifact_kind = 'installable' + AND resolver_winner_asset.filetype = 'apk' + AND resolver_winner_asset.file_hash = ${resolverTargetHashParam} + )`; + const resolverCasPredicate = [ + resolverCandidateCountPredicate, + ...resolverCandidatePredicates, + resolverScopeCountPredicate, + ...resolverScopePredicates, + resolverGroupCountPredicate, + ...resolverGroupPredicates, + resolverWinnerPredicate, + resolverWinnerBuildPredicate, + resolverWinnerAssetCountPredicate, + ...resolverWinnerAssetPredicates, + resolverTargetBUnreachablePredicate, + ].join(" AND "); + const operationIdParam = bind(input.operation_id); + const eligibility = `r.id = ${releaseIdParam} + AND r.app_id = ${appIdParam} + AND r.revision = ${revisionParam} + AND r.status = 'cancelled' + AND r.build_id = ${buildIdParam} + AND r.channel_id = ${channelIdParam} + AND r.product_type = ${productTypeParam} + AND r.release_type = ${releaseTypeParam} + AND ${rolloutPredicate} + AND b.id = r.build_id + AND b.product_type = ${productTypeParam} + AND b.release_type = ${releaseTypeParam} + AND b.version_code = ${versionCodeParam} + AND a.id = r.app_id AND a.platform = 'android' + AND a.slug = ${appSlugParam} AND a.org_id = ${authorityOrgParam} + AND ch.id = r.channel_id AND ch.app_id = r.app_id AND ch.slug = ${channelSlugParam} + AND ba.id = ${assetIdParam} AND ba.build_id = b.id + AND ba.file_hash = ${assetHashParam} + AND ba.platform = ${assetPlatformParam} + AND ba.artifact_kind = 'installable' AND ba.filetype = 'apk' + AND (SELECT COUNT(*) FROM build_assets exact_asset + WHERE exact_asset.build_id = b.id + AND exact_asset.artifact_kind = 'installable' + AND exact_asset.filetype = 'apk') = 1 + AND (SELECT COUNT(*) FROM release_scopes counted WHERE counted.release_id = r.id) = ${scopeCountParam} + AND ${scopePredicates.join(" AND ")} + AND (${targetScopePredicates.join(" OR ")}) + AND ${resolverCasPredicate} + AND EXISTS ( + SELECT 1 FROM webhooks w + WHERE ${subscriptionPredicate("w", orgParam, subscriberAppParam)} + ) + AND (SELECT COUNT(*) FROM webhooks counted_subscriber + WHERE ${subscriptionPredicate("counted_subscriber", orgParam, subscriberAppParam)}) = ${subscriberCountParam} + AND ${subscriberPredicates.join(" AND ")} + AND NOT EXISTS ( + SELECT 1 FROM app_update_cleanup_terminal_receipts receipt + WHERE receipt.operation_id = ${operationIdParam} + OR receipt.run_case_id = ${bind(input.run_case_id)} + OR receipt.release_id = r.id + ) + AND NOT EXISTS (SELECT 1 FROM operation_logs existing_op WHERE existing_op.id = ${operationIdParam})`; + + const operationStatement = c.env.DB.prepare( + `INSERT INTO operation_logs + (id, app_id, kind, status, parent_op_id, step_number, actor, + input, output, error, progress, retry_count, created_at, updated_at, completed_at) + SELECT ?${binds.length + 1}, ?${binds.length + 2}, '${OPERATION_KIND}', 'success', + NULL, NULL, ?${binds.length + 3}, ?${binds.length + 4}, ?${binds.length + 5}, + NULL, 1, 0, ?${binds.length + 6}, ?${binds.length + 6}, ?${binds.length + 6} + FROM releases r + JOIN builds b ON b.id = r.build_id + JOIN apps a ON a.id = r.app_id + JOIN channels ch ON ch.id = r.channel_id + JOIN build_assets ba ON ba.build_id = b.id + WHERE ${eligibility}`, + ).bind( + ...binds, + input.operation_id, + appId, + actor, + operationInput, + operationOutput, + now, + ); + + const receiptStatement = c.env.DB.prepare( + `INSERT INTO app_update_cleanup_terminal_receipts + (operation_id, receipt_digest, run_case_id, attempt, artifact_bundle_digest, + app_id, release_id, release_revision, build_id, app_slug, channel_slug, + target_artifact_sha256, target_version_code, target_installation_digest, + cancel_readback, scope_deactivated, scope_readback_json, delivery_bindings_json, + canonical_request_json, event_payload_json, canonical_receipt_json, created_at) + SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, + ?12, ?13, ?14, 'inactive', 1, ?15, ?16, ?17, ?18, ?19, ?20 + WHERE EXISTS ( + SELECT 1 FROM operation_logs + WHERE id = ?1 AND kind = '${OPERATION_KIND}' AND status = 'success' + )`, + ).bind( + input.operation_id, + receiptDigest, + input.run_case_id, + input.attempt, + input.artifact_bundle_digest, + appId, + releaseId, + authority.release_revision, + authority.build_id, + authority.app_slug, + authority.channel_slug, + asset.file_hash, + authority.version_code, + targetInstallationDigest, + scopeReadbackJson, + deliveryBindingsJson, + canonicalRequest, + eventPayloadJson, + canonicalReceiptJson, + now, + ); + + const deliveryStatement = c.env.DB.prepare( + `INSERT INTO webhook_deliveries + (id, webhook_id, event_type, payload_json, signing_secret, + signature_key_version, reporter_delivery, app_update_terminal_receipt_id, + status, attempts, max_attempts, last_attempt_at, next_attempt_at, + created_at, updated_at) + SELECT 'app-update:' || ?1 || ':' || w.id, w.id, '${EVENT}', ?2, w.secret, + w.signature_key_version, 0, ?1, + 'pending', 0, 3, NULL, ?3, ?3, ?3 + FROM webhooks w + JOIN app_update_cleanup_terminal_receipts receipt ON receipt.operation_id = ?1 + WHERE ${subscriptionPredicate("w", "?4", "?5")}`, + ).bind(input.operation_id, envelopeJson, now, orgId, appId); + + const auditStatement = c.env.DB.prepare( + `INSERT INTO audit_logs (id, app_id, action, actor, payload, created_at) + SELECT ?1, ?2, 'release.app_update_cleanup_terminal', ?3, ?4, ?5 + WHERE EXISTS ( + SELECT 1 FROM app_update_cleanup_terminal_receipts WHERE operation_id = ?6 + )`, + ).bind( + crypto.randomUUID(), + appId, + actor, + JSON.stringify({ + operation_id: input.operation_id, + receipt_digest: receiptDigest, + release_id: releaseId, + release_revision: authority.release_revision, + run_case_id: input.run_case_id, + attempt: input.attempt, + artifact_bundle_digest: input.artifact_bundle_digest, + target_installation_digest: targetInstallationDigest, + }), + now, + input.operation_id, + ); + + try { + const results = await c.env.DB.batch([ + operationStatement, + receiptStatement, + deliveryStatement, + auditStatement, + ]); + const operationChanges = Number(results[0]?.meta?.changes ?? 0); + const receiptChanges = Number(results[1]?.meta?.changes ?? 0); + const deliveryChanges = Number(results[2]?.meta?.changes ?? 0); + const auditChanges = Number(results[3]?.meta?.changes ?? 0); + if (operationChanges !== 1 || receiptChanges !== 1 || deliveryChanges < 1 || auditChanges !== 1) { + const collision = await findReceiptCollision( + c.env.DB, + input.operation_id, + input.run_case_id, + releaseId, + ); + if (collision?.operation_id === input.operation_id && + collision.canonical_request_json === canonicalRequest) { + return c.json(receiptResponse(collision, true)); + } + return conflictResponse( + c, + "APP_UPDATE_TERMINAL_PRECONDITION_CHANGED", + "release, target scope, subscriber, or operation binding changed before commit", + ); + } + } catch { + const collision = await findReceiptCollision( + c.env.DB, + input.operation_id, + input.run_case_id, + releaseId, + ); + if (collision?.operation_id === input.operation_id && + collision.canonical_request_json === canonicalRequest) { + return c.json(receiptResponse(collision, true)); + } + return conflictResponse( + c, + "APP_UPDATE_TERMINAL_COMMIT_CONFLICT", + "cleanup terminal receipt could not be committed atomically", + ); + } + + const created = await findReceiptCollision( + c.env.DB, + input.operation_id, + input.run_case_id, + releaseId, + ); + if (!created || created.operation_id !== input.operation_id) { + return c.json({ error: "terminal receipt readback missing" }, 500); + } + return c.json(receiptResponse(created, false), 201); +} diff --git a/worker/src/routes/auth.ts b/worker/src/routes/auth.ts index 0b2c639..5680c62 100644 --- a/worker/src/routes/auth.ts +++ b/worker/src/routes/auth.ts @@ -1096,6 +1096,25 @@ export async function handleAgentManifest(c: Context<{ Bindings: Env }>) { expected_revision: { type: "number", in: "query", required: false, description: "Revision from fresh release detail." }, }, }, + { + name: "emit-app-update-cleanup-terminal", + description: + "Live-facing: after an Android release has already been cancelled, prove the exact target installation cannot resolve that current release generation, then atomically freeze one immutable cleanup terminal operation/receipt and its signed webhook deliveries. Exact response-loss replay reuses the same operation, receipt, payload bytes, subscribers, and signing generations. This action never cancels or mutates a release. Requires app publisher and explicit human authorization.", + endpoint: { + method: "POST", + path: "/api/apps/{app_id}/releases/{release_id}/app-update-cleanup-terminal", + }, + parameters: { + app_id: { type: "string", in: "path", required: true, description: "Hands Android app UUID." }, + release_id: { type: "string", in: "path", required: true, description: "Exact already-cancelled Hands release UUID." }, + operation_id: { type: "string", in: "body", required: true, description: "Caller-frozen idempotency identity. Replays must reuse it exactly." }, + run_case_id: { type: "string", in: "body", required: true, description: "Frozen Stamp run-case identity." }, + attempt: { type: "number", in: "body", required: true, description: "Positive frozen Stamp attempt." }, + artifact_bundle_digest: { type: "string", in: "body", required: true, description: "Lowercase 64-hex frozen Stamp artifact-bundle digest." }, + expected_release_revision: { type: "number", in: "body", required: true, description: "Current cancelled release revision used for the server-side inactive readback." }, + target_device_id: { type: "string", in: "body", required: true, description: "Exact Hands installation device id used only for resolver readback; stored only as SHA-256." }, + }, + }, { name: "restore-release", description: "Restore the same release row. A never-published cancelled draft returns to draft and must pass normal publish gates; a previously active release returns to active. Requires app publisher.", diff --git a/worker/src/routes/operations.ts b/worker/src/routes/operations.ts index ffddfc3..2b71d0d 100644 --- a/worker/src/routes/operations.ts +++ b/worker/src/routes/operations.ts @@ -27,6 +27,7 @@ export interface OperationLog { | "testflight-upload" | "testflight-publish" | "testflight-expire" + | "app-update-cleanup-terminal" | "delta-generate"; status: "pending" | "in_progress" | "success" | "failed" | "cancelled"; parent_op_id: string | null; @@ -185,10 +186,12 @@ export async function handleRetryOperation(c: Context<{ Bindings: Env }>) { const id = c.req.param("opId") ?? ""; const existing = await getOperation(c.env.DB, id); if (!existing) return c.json({ error: "not found" }, 404); - if (existing.kind === "testflight-expire") { + if (existing.kind === "testflight-expire" || existing.kind === "app-update-cleanup-terminal") { return c.json( { - error: "retry cannot rewrite a durable TestFlight expire receipt; invoke the exact expire action again", + error: existing.kind === "testflight-expire" + ? "retry cannot rewrite a durable TestFlight expire receipt; invoke the exact expire action again" + : "retry cannot rewrite a durable App Update cleanup terminal receipt; replay the exact terminal action", operation: existing, }, 400, @@ -241,9 +244,13 @@ export async function handleDeleteOperation(c: Context<{ Bindings: Env }>) { const id = c.req.param("opId") ?? ""; const existing = await getOperation(c.env.DB, id); if (!existing) return c.json({ error: "not found" }, 404); - if (existing.kind === "testflight-expire") { + if (existing.kind === "testflight-expire" || existing.kind === "app-update-cleanup-terminal") { return c.json( - { error: "durable TestFlight expire receipts cannot be deleted" }, + { + error: existing.kind === "testflight-expire" + ? "durable TestFlight expire receipts cannot be deleted" + : "durable App Update cleanup terminal receipts cannot be deleted", + }, 409, ); } diff --git a/worker/src/routes/public_v2.ts b/worker/src/routes/public_v2.ts index 9399610..b47f82f 100644 --- a/worker/src/routes/public_v2.ts +++ b/worker/src/routes/public_v2.ts @@ -24,7 +24,7 @@ import { presignR2DownloadUrl } from "../lib/r2_presign"; import { parseReleaseNotes, resolveReleaseNote, type ReleaseNotes } from "../lib/release_notes"; import { isFeatureEnabled } from "../lib/feature_flags"; -interface ScopedResolution { +export interface ScopedResolution { release_id: string; scope_type: "full" | "platform" | "user_cohort" | "ip_range" | "device_group"; scope_value: string; @@ -32,6 +32,41 @@ interface ScopedResolution { release_activated_at: number; } +export interface ActiveReleaseCandidate { + id: string; + build_id: string; + created_at: number; + activated_at: number; + product_type: string; + rollout_cohort_count: number | null; + changelog: string | null; +} + +export interface ActiveReleaseResolution { + candidates: ActiveReleaseCandidate[]; + scopes: ActiveReleaseScope[]; + deviceGroupIds: string[]; + rolloutEligibleReleaseIds: string[]; + matched: ScopedResolution[]; + winner: ScopedResolution | null; +} + +export interface ActiveReleaseScope { + release_id: string; + scope_type: string; + scope_value: string; +} + +export interface ActiveReleaseClientFacts { + appId: string; + channelId: string; + productType: string | null; + deviceId: string | null; + cohort: string | null; + clientPlatform: string | null; + clientIp: string | null; +} + type PublicAssetResponse = { platform: string; arch: string | null; @@ -76,6 +111,123 @@ const PRIORITY = { full: 1, } as const; +/** + * Canonical active-release candidate + scope + rollout resolver. Public + * update checks and server-side terminal readbacks share this function so a + * producer cannot substitute a lifecycle-status shortcut for actual client + * resolution. + */ +export async function resolveActiveReleaseForClient( + db: D1Database, + facts: ActiveReleaseClientFacts, +): Promise { + const candidateSql = facts.productType + ? `SELECT r.id, r.build_id, r.created_at, + COALESCE(r.activated_at, r.created_at) AS activated_at, + r.product_type, r.rollout_cohort_count, r.changelog + FROM releases r + JOIN builds b ON b.id = r.build_id + WHERE r.app_id = ?1 AND r.channel_id = ?2 AND r.product_type = ?3 + AND r.status = 'active' + AND b.product_type != 'ios-simulator-qa' AND b.release_type != 'qa' + ORDER BY COALESCE(r.activated_at, r.created_at) DESC` + : `SELECT r.id, r.build_id, r.created_at, + COALESCE(r.activated_at, r.created_at) AS activated_at, + r.product_type, r.rollout_cohort_count, r.changelog + FROM releases r + JOIN builds b ON b.id = r.build_id + WHERE r.app_id = ?1 AND r.channel_id = ?2 + AND r.status = 'active' + AND b.product_type != 'ios-simulator-qa' AND b.release_type != 'qa' + ORDER BY COALESCE(r.activated_at, r.created_at) DESC`; + const statement = db.prepare(candidateSql); + const { results: candidates } = await (facts.productType + ? statement.bind(facts.appId, facts.channelId, facts.productType) + : statement.bind(facts.appId, facts.channelId) + ).all(); + + // Membership is an input to resolution, not a property of the candidate + // set. Preserve it even when there are currently no active candidates so a + // caller that repeats this resolution in a commit CAS can detect membership + // drift rather than snapshotting a misleading empty set. + const deviceGroupIds = new Set(); + if (facts.deviceId) { + const { results: memberships } = await db.prepare( + `SELECT m.group_id + FROM device_group_members m + JOIN device_groups g ON g.id = m.group_id + WHERE g.app_id = ?1 AND m.device_id = ?2`, + ).bind(facts.appId, facts.deviceId).all<{ group_id: string }>(); + for (const membership of memberships) deviceGroupIds.add(membership.group_id); + } + if (candidates.length === 0) { + return { + candidates, + scopes: [], + deviceGroupIds: [...deviceGroupIds].sort(), + rolloutEligibleReleaseIds: [], + matched: [], + winner: null, + }; + } + + const candidateIds = candidates.map((candidate) => candidate.id); + const placeholders = candidateIds.map(() => "?").join(","); + const { results: scopes } = await db.prepare( + `SELECT release_id, scope_type, scope_value + FROM release_scopes + WHERE release_id IN (${placeholders})`, + ).bind(...candidateIds).all(); + + const matched: ScopedResolution[] = []; + const rolloutEligibleReleaseIds = new Set( + candidates + .filter((release) => rolloutIncludes( + release.id, + release.rollout_cohort_count, + facts.deviceId, + )) + .map((release) => release.id), + ); + for (const release of candidates) { + for (const scope of scopes) { + if (scope.release_id !== release.id) continue; + if (!matchesScope( + scope.scope_type, + scope.scope_value, + facts.cohort, + facts.clientPlatform, + facts.clientIp, + deviceGroupIds, + )) continue; + if (scope.scope_type !== "device_group" && + !rolloutEligibleReleaseIds.has(release.id)) continue; + matched.push({ + release_id: release.id, + scope_type: scope.scope_type as ScopedResolution["scope_type"], + scope_value: scope.scope_value, + priority: PRIORITY[scope.scope_type as keyof typeof PRIORITY] ?? 0, + release_activated_at: release.activated_at, + }); + } + } + matched.sort((left, right) => { + if (left.priority !== right.priority) return right.priority - left.priority; + if (left.release_activated_at !== right.release_activated_at) { + return right.release_activated_at - left.release_activated_at; + } + return left.release_id < right.release_id ? -1 : 1; + }); + return { + candidates, + scopes, + deviceGroupIds: [...deviceGroupIds].sort(), + rolloutEligibleReleaseIds: [...rolloutEligibleReleaseIds].sort(), + matched, + winner: matched[0] ?? null, + }; +} + export async function handlePublicV2Latest(c: Context<{ Bindings: Env }>) { const slug = c.req.param("slug"); const channel = c.req.query("channel") ?? "main"; @@ -111,42 +263,17 @@ export async function handlePublicV2Latest(c: Context<{ Bindings: Env }>) { ); } - // Candidates: active releases on (channel, [product_type]). No time window: - // an active release must stay resolvable no matter how old it is. - const candidateSql = productType - ? `SELECT r.id, r.build_id, r.created_at, - COALESCE(r.activated_at, r.created_at) AS activated_at, - r.product_type, r.rollout_cohort_count, r.changelog - FROM releases r - JOIN builds b ON b.id = r.build_id - WHERE r.app_id = ?1 AND r.channel_id = ?2 AND r.product_type = ?3 - AND r.status = 'active' - AND b.product_type != 'ios-simulator-qa' AND b.release_type != 'qa' - ORDER BY COALESCE(r.activated_at, r.created_at) DESC` - : `SELECT r.id, r.build_id, r.created_at, - COALESCE(r.activated_at, r.created_at) AS activated_at, - r.product_type, r.rollout_cohort_count, r.changelog - FROM releases r - JOIN builds b ON b.id = r.build_id - WHERE r.app_id = ?1 AND r.channel_id = ?2 - AND r.status = 'active' - AND b.product_type != 'ios-simulator-qa' AND b.release_type != 'qa' - ORDER BY COALESCE(r.activated_at, r.created_at) DESC`; - const candidateStmt = c.env.DB.prepare(candidateSql); - const allCandidates = await (productType - ? candidateStmt.bind(app.id, channelRow.id, productType) - : candidateStmt.bind(app.id, channelRow.id) - ).all<{ - id: string; - build_id: string; - created_at: number; - activated_at: number; - product_type: string; - rollout_cohort_count: number | null; - changelog: string | null; - }>(); - - if (allCandidates.results.length === 0) { + const resolution = await resolveActiveReleaseForClient(c.env.DB, { + appId: app.id, + channelId: channelRow.id, + productType: productType ?? null, + deviceId, + cohort, + clientPlatform, + clientIp, + }); + const allCandidates = { results: resolution.candidates }; + if (resolution.candidates.length === 0) { return c.json( { error: `no active release for this client on channel '${channel}'`, @@ -157,67 +284,7 @@ export async function handlePublicV2Latest(c: Context<{ Bindings: Env }>) { ); } - // Pull all scopes for those releases in one query. - const candidateIds = allCandidates.results.map((r) => r.id); - if (candidateIds.length === 0) { - return c.json({ error: "no candidates" }, 404); - } - const placeholders = candidateIds.map(() => "?").join(","); - const { results: scopes } = await c.env.DB.prepare( - `SELECT release_id, scope_type, scope_value - FROM release_scopes - WHERE release_id IN (${placeholders})`, - ) - .bind(...candidateIds) - .all<{ release_id: string; scope_type: string; scope_value: string }>(); - - const deviceGroupIds = new Set(); - if (deviceId) { - const { results: memberships } = await c.env.DB.prepare( - `SELECT m.group_id - FROM device_group_members m - JOIN device_groups g ON g.id = m.group_id - WHERE g.app_id = ?1 AND m.device_id = ?2`, - ).bind(app.id, deviceId).all<{ group_id: string }>(); - for (const membership of memberships) deviceGroupIds.add(membership.group_id); - } - - // Build match list (release, scope, priority). - const matched: ScopedResolution[] = []; - const rolloutEligibleReleaseIds = new Set( - allCandidates.results - .filter((release) => rolloutIncludes(release.id, release.rollout_cohort_count, deviceId)) - .map((release) => release.id), - ); - for (const release of allCandidates.results) { - for (const s of scopes) { - if (s.release_id !== release.id) continue; - const ok = matchesScope( - s.scope_type, - s.scope_value, - cohort, - clientPlatform, - clientIp, - deviceGroupIds, - ); - if (!ok) continue; - // Device groups are mandatory overrides on a staged release: a listed QA - // or customer device always receives the release, while its full:all - // scope remains percentage-gated for everybody else. - if (s.scope_type !== "device_group" && !rolloutEligibleReleaseIds.has(release.id)) continue; - const prio = - PRIORITY[s.scope_type as keyof typeof PRIORITY] ?? 0; - matched.push({ - release_id: release.id, - scope_type: s.scope_type as ScopedResolution["scope_type"], - scope_value: s.scope_value, - priority: prio, - release_activated_at: release.activated_at, - }); - } - } - - if (matched.length === 0) { + if (!resolution.winner) { return c.json( { error: "no active release matches this client", @@ -229,16 +296,11 @@ export async function handlePublicV2Latest(c: Context<{ Bindings: Env }>) { ); } - // Sort by priority DESC, latest activation DESC, release_id ASC. - matched.sort((a, b) => { - if (a.priority !== b.priority) return b.priority - a.priority; - if (a.release_activated_at !== b.release_activated_at) { - return b.release_activated_at - a.release_activated_at; - } - return a.release_id < b.release_id ? -1 : 1; - }); - - const winner = matched[0]!; + const winner = resolution.winner; + const candidateIds = resolution.candidates.map((candidate) => candidate.id); + const rolloutEligibleReleaseIds = new Set( + resolution.rolloutEligibleReleaseIds, + ); // Build the response: build + assets + scoped block + fallback release. const build = await c.env.DB.prepare( diff --git a/worker/src/routes/webhooks.ts b/worker/src/routes/webhooks.ts index d63ca96..593822f 100644 --- a/worker/src/routes/webhooks.ts +++ b/worker/src/routes/webhooks.ts @@ -35,6 +35,7 @@ type WebhookEventType = | "release:superseded" | "release:rolled_back" | "release:cancelled" + | "app_update:cleanup_terminal" | "build:succeeded" | "build:failed"; @@ -340,7 +341,11 @@ export async function reapWebhookDeliveries( // occupying the oldest slots forever. const { results: due } = await env.DB.prepare( `SELECT d.id, d.webhook_id, - COALESCE(d.event_id, d.feedback_submission_event_id) AS event_id, + COALESCE( + d.event_id, + d.feedback_submission_event_id, + d.app_update_terminal_receipt_id + ) AS event_id, d.attempts, d.max_attempts, d.payload_json, d.signing_secret, w.id AS resolved_webhook_id, w.url AS webhook_url, w.secret AS webhook_secret, w.enabled AS webhook_enabled, diff --git a/worker/test/app_update_cleanup_migration.test.ts b/worker/test/app_update_cleanup_migration.test.ts new file mode 100644 index 0000000..8e8f8ee --- /dev/null +++ b/worker/test/app_update_cleanup_migration.test.ts @@ -0,0 +1,149 @@ +import { readFileSync } from "node:fs"; +import Database from "better-sqlite3"; +import { describe, expect, it } from "vitest"; + +const migrationPath = new URL( + "../../migrations/sql/0054_app_update_cleanup_terminal.sql", + import.meta.url, +); + +function baseDb() { + const db = new Database(":memory:"); + db.pragma("foreign_keys = ON"); + db.exec(` + CREATE TABLE operation_logs ( + id TEXT PRIMARY KEY, + app_id TEXT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + parent_op_id TEXT, + step_number INTEGER, + actor TEXT NOT NULL DEFAULT 'admin', + input TEXT NOT NULL DEFAULT '{}', + output TEXT NOT NULL DEFAULT '{}', + error TEXT, + progress REAL NOT NULL DEFAULT 0, + retry_count INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ); + CREATE TABLE webhooks (id TEXT PRIMARY KEY); + CREATE TABLE webhook_deliveries ( + id TEXT PRIMARY KEY, + webhook_id TEXT NOT NULL REFERENCES webhooks(id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + payload_json TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + last_attempt_at INTEGER, + next_attempt_at INTEGER, + last_response_status INTEGER, + last_response_body TEXT, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ); + `); + return db; +} + +function seedOperationAndReceipt(db: Database.Database) { + const hex = "a".repeat(64); + db.prepare(` + INSERT INTO operation_logs + (id, app_id, kind, status, actor, input, output, progress, + retry_count, created_at, updated_at, completed_at) + VALUES ('operation-1', 'app-1', 'app-update-cleanup-terminal', 'success', + 'tester', '{}', '{}', 1, 0, 1, 1, 1) + `).run(); + db.prepare(` + INSERT INTO app_update_cleanup_terminal_receipts + (operation_id, receipt_digest, run_case_id, attempt, + artifact_bundle_digest, app_id, release_id, release_revision, + build_id, app_slug, channel_slug, target_artifact_sha256, + target_version_code, target_installation_digest, cancel_readback, + scope_deactivated, scope_readback_json, delivery_bindings_json, + canonical_request_json, event_payload_json, canonical_receipt_json, + created_at) + VALUES (?, ?, 'run-1', 1, ?, 'app-1', 'release-1', 2, 'build-1', + 'app-one', 'main', ?, 10, ?, 'inactive', 1, + '{}', '[]', '{}', '{}', '{}', 1) + `).run("operation-1", `sha256:${hex}`, hex, hex, `sha256:${hex}`); +} + +describe("0054 App Update cleanup terminal migration", () => { + it("applies cleanly and adds the delivery receipt reference", () => { + const sql = readFileSync(migrationPath, "utf8"); + const db = baseDb(); + expect(() => db.exec(sql)).not.toThrow(); + expect(db.prepare( + "SELECT name FROM pragma_table_info('webhook_deliveries') WHERE name='app_update_terminal_receipt_id'", + ).get()).toEqual({ name: "app_update_terminal_receipt_id" }); + db.close(); + }); + + it("enforces immutable operation/receipt identity and one delivery per subscriber", () => { + const db = baseDb(); + db.exec(readFileSync(migrationPath, "utf8")); + seedOperationAndReceipt(db); + db.prepare("INSERT INTO webhooks (id) VALUES ('hook-1')").run(); + db.prepare(` + INSERT INTO webhook_deliveries + (id, webhook_id, event_type, payload_json, + app_update_terminal_receipt_id, created_at, updated_at) + VALUES ('delivery-1', 'hook-1', 'app_update:cleanup_terminal', '{}', + 'operation-1', 1, 1) + `).run(); + + expect(() => db.prepare( + "UPDATE operation_logs SET status='failed' WHERE id='operation-1'", + ).run()).toThrow(/immutable/i); + expect(() => db.prepare( + "DELETE FROM operation_logs WHERE id='operation-1'", + ).run()).toThrow(/immutable/i); + expect(() => db.prepare( + "UPDATE app_update_cleanup_terminal_receipts SET attempt=2 WHERE operation_id='operation-1'", + ).run()).toThrow(/immutable/i); + expect(() => db.prepare(` + INSERT INTO webhook_deliveries + (id, webhook_id, event_type, payload_json, + app_update_terminal_receipt_id, created_at, updated_at) + VALUES ('delivery-2', 'hook-1', 'app_update:cleanup_terminal', '{}', + 'operation-1', 1, 1) + `).run()).toThrow(/UNIQUE/i); + db.close(); + }); + + it("rejects malformed digests at the storage boundary", () => { + const db = baseDb(); + db.exec(readFileSync(migrationPath, "utf8")); + db.prepare(` + INSERT INTO operation_logs + (id, kind, status, actor, input, output, created_at, updated_at) + VALUES ('operation-bad', 'app-update-cleanup-terminal', 'success', + 'tester', '{}', '{}', 1, 1) + `).run(); + expect(() => db.prepare(` + INSERT INTO app_update_cleanup_terminal_receipts + (operation_id, receipt_digest, run_case_id, attempt, + artifact_bundle_digest, app_id, release_id, release_revision, + build_id, app_slug, channel_slug, target_artifact_sha256, + target_version_code, target_installation_digest, cancel_readback, + scope_deactivated, scope_readback_json, delivery_bindings_json, + canonical_request_json, event_payload_json, canonical_receipt_json, + created_at) + VALUES ('operation-bad', ?, 'run-bad', 1, ?, 'app', 'release', 1, + 'build', 'app', 'main', ?, 1, ?, 'inactive', 1, + '{}', '[]', '{}', '{}', '{}', 1) + `).run( + `sha256:${"g".repeat(64)}`, + "b".repeat(64), + "c".repeat(64), + `sha256:${"d".repeat(64)}`, + )).toThrow(/CHECK/i); + db.close(); + }); +}); diff --git a/worker/test/routes.test.ts b/worker/test/routes.test.ts index e7d5cf2..d88fd3a 100644 --- a/worker/test/routes.test.ts +++ b/worker/test/routes.test.ts @@ -49,6 +49,7 @@ import { handleGetIosSimulatorArtifact, handleListIosSimulatorArtifacts, } from "../src/routes/qa_artifacts"; +import { handleEmitAppUpdateCleanupTerminal } from "../src/routes/app_update_cleanup"; // ---------- Test harness ---------- @@ -102,6 +103,7 @@ describe("quiver OpenAPI document", () => { "/api/apps/{appId}/qa-artifacts/ios-simulator/{assetId}/complete", "/api/apps/{appId}/qa-artifacts/ios-simulator/{assetId}/download", "/api/apps/{appId}/releases/{releaseId}/publish", + "/api/apps/{appId}/releases/{releaseId}/app-update-cleanup-terminal", "/api/apps/{appId}/feedback/{ticketId}/comments", "/api/app-permissions", "/api/apps/{appId}/client-key", @@ -613,6 +615,44 @@ function makeMockDb() { updated_at INTEGER NOT NULL, archived_at INTEGER ); + CREATE TABLE app_update_cleanup_terminal_receipts ( + operation_id TEXT PRIMARY KEY REFERENCES operation_logs(id) ON DELETE RESTRICT, + receipt_digest TEXT NOT NULL UNIQUE, + run_case_id TEXT NOT NULL UNIQUE, + attempt INTEGER NOT NULL CHECK (attempt > 0), + artifact_bundle_digest TEXT NOT NULL, + app_id TEXT NOT NULL, + release_id TEXT NOT NULL UNIQUE, + release_revision INTEGER NOT NULL, + build_id TEXT NOT NULL, + app_slug TEXT NOT NULL, + channel_slug TEXT NOT NULL, + target_artifact_sha256 TEXT NOT NULL, + target_version_code INTEGER NOT NULL, + target_installation_digest TEXT NOT NULL, + cancel_readback TEXT NOT NULL CHECK (cancel_readback = 'inactive'), + scope_deactivated INTEGER NOT NULL CHECK (scope_deactivated = 1), + scope_readback_json TEXT NOT NULL, + delivery_bindings_json TEXT NOT NULL, + canonical_request_json TEXT NOT NULL, + event_payload_json TEXT NOT NULL, + canonical_receipt_json TEXT NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE TRIGGER app_update_cleanup_terminal_receipts_no_update + BEFORE UPDATE ON app_update_cleanup_terminal_receipts + BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal receipts are immutable'); END; + CREATE TRIGGER app_update_cleanup_terminal_receipts_no_delete + BEFORE DELETE ON app_update_cleanup_terminal_receipts + BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal receipts are immutable'); END; + CREATE TRIGGER app_update_cleanup_terminal_operation_no_update + BEFORE UPDATE ON operation_logs + WHEN OLD.kind = 'app-update-cleanup-terminal' + BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal operations are immutable'); END; + CREATE TRIGGER app_update_cleanup_terminal_operation_no_delete + BEFORE DELETE ON operation_logs + WHEN OLD.kind = 'app-update-cleanup-terminal' + BEGIN SELECT RAISE(ABORT, 'App Update cleanup terminal operations are immutable'); END; -- Migration 0018: apps.default_channel_id (nullable FK to channels). -- SQLite ALTER TABLE ADD COLUMN is non-destructive; add inline so the -- test schema matches the migration shape. @@ -623,6 +663,7 @@ function makeMockDb() { event_type TEXT NOT NULL, event_id TEXT REFERENCES feedback_events(id) ON DELETE SET NULL, feedback_submission_event_id TEXT REFERENCES feedback_submission_events(id) ON DELETE SET NULL, + app_update_terminal_receipt_id TEXT REFERENCES app_update_cleanup_terminal_receipts(operation_id) ON DELETE RESTRICT, payload_json TEXT NOT NULL, signing_secret TEXT, signature_key_version TEXT NOT NULL DEFAULT 'legacy-v1', @@ -641,6 +682,9 @@ function makeMockDb() { ); CREATE UNIQUE INDEX idx_webhook_deliveries_event ON webhook_deliveries(webhook_id, event_id) WHERE event_id IS NOT NULL; + CREATE UNIQUE INDEX idx_webhook_deliveries_app_update_terminal + ON webhook_deliveries(webhook_id, app_update_terminal_receipt_id) + WHERE app_update_terminal_receipt_id IS NOT NULL; CREATE TABLE feedback_events ( id TEXT PRIMARY KEY, event_type TEXT NOT NULL, @@ -4901,6 +4945,495 @@ describe("quiver releases — draft lifecycle", () => { }); }); +describe("Android App Update cleanup terminal producer", () => { + let env: MockEnv; + const artifactDigest = "b".repeat(64); + const targetHash = "a".repeat(64); + + beforeEach(async () => { + env = makeMockEnv(); + const now = Date.now(); + await env.DB.prepare( + "INSERT INTO apps (id, org_id, slug, name, platform, created_at) VALUES (?, ?, ?, ?, ?, ?)", + ).bind("app-terminal", "default", "app-terminal", "App Terminal", "android", now).run(); + await env.DB.prepare( + "INSERT INTO channels (id, app_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)", + ).bind("channel-terminal", "app-terminal", "main", "Main", now).run(); + await env.DB.prepare( + `INSERT INTO builds + (id, app_id, channel_id, product_type, release_type, version_name, version_code, + source, status, build_metadata_json, parsed_metadata_json, should_force_update, + provenance_json, created_at, updated_at) + VALUES (?, ?, ?, 'android-apk', 'stable', '2.0.0', 200, 'ci', 'succeeded', + '{}', '{}', 0, '{}', ?, ?)`, + ).bind("build-terminal", "app-terminal", "channel-terminal", now, now).run(); + await env.DB.prepare( + `INSERT INTO build_assets + (id, build_id, artifact_kind, platform, arch, variant, filetype, r2_key, + file_hash, size_bytes, metadata_json, created_at) + VALUES (?, ?, 'installable', 'android', NULL, NULL, 'apk', ?, ?, 42, '{}', ?)`, + ).bind("asset-terminal", "build-terminal", "apps/terminal.apk", targetHash, now).run(); + await env.DB.prepare( + `INSERT INTO releases + (id, app_id, build_id, channel_id, product_type, release_type, status, + activated_at, revision, is_full, rollout_cohort_count, + rollout_target_cohorts_json, should_force_update, provenance_json, + created_by, created_at, updated_at) + VALUES (?, ?, ?, ?, 'android-apk', 'stable', 'cancelled', NULL, 4, 0, 100, + '[]', 0, '{}', 'tester', ?, ?)`, + ).bind( + "release-terminal", + "app-terminal", + "build-terminal", + "channel-terminal", + now, + now, + ).run(); + await env.DB.prepare( + `INSERT INTO device_groups (id, app_id, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?)`, + ).bind("group-terminal", "app-terminal", "Terminal target", now, now).run(); + await env.DB.prepare( + `INSERT INTO device_group_members (group_id, device_id, label, created_at) + VALUES (?, ?, ?, ?)`, + ).bind("group-terminal", "device-terminal", "fixture", now).run(); + await env.DB.prepare( + `INSERT INTO release_scopes (id, release_id, scope_type, scope_value, created_at) + VALUES (?, ?, 'device_group', ?, ?)`, + ).bind("scope-terminal", "release-terminal", "group-terminal", now).run(); + await env.DB.prepare( + `INSERT INTO webhooks + (id, org_id, app_id, url, secret, signature_key_version, events_json, + enabled, created_by, created_at, updated_at, archived_at) + VALUES (?, 'default', ?, ?, ?, 'key-v7', ?, 1, NULL, ?, ?, NULL)`, + ).bind( + "webhook-terminal", + "app-terminal", + "https://stamp.example/hooks/hands", + "frozen-signing-secret", + JSON.stringify(["app_update:cleanup_terminal"]), + now, + now, + ).run(); + }); + + function input(overrides: Record = {}) { + return { + operation_id: "operation-terminal-1", + run_case_id: "run-case-terminal-1", + attempt: 2, + artifact_bundle_digest: artifactDigest, + expected_release_revision: 4, + target_device_id: "device-terminal", + ...overrides, + }; + } + + function context(body: unknown, releaseId = "release-terminal") { + return { + env, + req: { + param: (name: string) => name === "appId" + ? "app-terminal" + : name === "releaseId" ? releaseId : "", + json: async () => body, + }, + get: (key: string) => key === "org_id" + ? "default" + : key === "admin_actor" ? "tester" : undefined, + json: (data: unknown, status = 200) => new Response(JSON.stringify(data), { + status, + headers: { "content-type": "application/json" }, + }), + } as any; + } + + async function counts() { + return { + operations: (await env.DB.prepare( + "SELECT COUNT(*) count FROM operation_logs WHERE kind = 'app-update-cleanup-terminal'", + ).first() as any).count, + receipts: (await env.DB.prepare( + "SELECT COUNT(*) count FROM app_update_cleanup_terminal_receipts", + ).first() as any).count, + deliveries: (await env.DB.prepare( + "SELECT COUNT(*) count FROM webhook_deliveries WHERE app_update_terminal_receipt_id IS NOT NULL", + ).first() as any).count, + }; + } + + async function seedActiveResolverWinner(options: { + releaseId: string; + buildId: string; + assetId: string; + versionCode: number; + artifactHash: string; + }) { + const now = Date.now() + 100; + await env.DB.prepare( + `INSERT INTO builds + (id, app_id, channel_id, product_type, release_type, version_name, version_code, + source, status, build_metadata_json, parsed_metadata_json, should_force_update, + provenance_json, created_at, updated_at) + VALUES (?, 'app-terminal', 'channel-terminal', 'android-apk', 'stable', ?, ?, + 'ci', 'succeeded', '{}', '{}', 0, '{}', ?, ?)`, + ).bind( + options.buildId, + `2.0.${options.versionCode}`, + options.versionCode, + now, + now, + ).run(); + await env.DB.prepare( + `INSERT INTO build_assets + (id, build_id, artifact_kind, platform, arch, variant, filetype, r2_key, + file_hash, size_bytes, metadata_json, created_at) + VALUES (?, ?, 'installable', 'android', NULL, NULL, 'apk', ?, ?, 42, '{}', ?)`, + ).bind( + options.assetId, + options.buildId, + `apps/${options.assetId}.apk`, + options.artifactHash, + now, + ).run(); + await env.DB.prepare( + `INSERT INTO releases + (id, app_id, build_id, channel_id, product_type, release_type, status, + activated_at, revision, is_full, rollout_cohort_count, + rollout_target_cohorts_json, should_force_update, provenance_json, + created_by, created_at, updated_at) + VALUES (?, 'app-terminal', ?, 'channel-terminal', 'android-apk', 'stable', + 'active', ?, 0, 0, 100, '[]', 0, '{}', 'tester', ?, ?)`, + ).bind(options.releaseId, options.buildId, now, now, now).run(); + await env.DB.prepare( + `INSERT INTO release_scopes (id, release_id, scope_type, scope_value, created_at) + VALUES (?, ?, 'device_group', 'group-terminal', ?)`, + ).bind(`scope-${options.releaseId}`, options.releaseId, now).run(); + } + + it("atomically freezes the exact event, resolver readback, subscriber, signing generation, and receipt", async () => { + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(201); + const body = await response.json() as any; + expect(body.replay).toBe(false); + expect(Object.keys(body.event).sort()).toEqual([ + "app_slug", + "artifact_bundle_digest", + "attempt", + "cancel_readback", + "channel", + "operation_id", + "receipt_digest", + "run_case_id", + "scope_deactivated", + "target_artifact_sha256", + "target_version_code", + ]); + expect(body.event).toMatchObject({ + operation_id: "operation-terminal-1", + run_case_id: "run-case-terminal-1", + attempt: 2, + artifact_bundle_digest: artifactDigest, + app_slug: "app-terminal", + channel: "main", + target_artifact_sha256: targetHash, + target_version_code: 200, + cancel_readback: "inactive", + scope_deactivated: true, + }); + expect(body.event.receipt_digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(body.receipt).toMatchObject({ + delivery_bindings_digest: expect.stringMatching(/^sha256:[0-9a-f]{64}$/), + delivery_count: 1, + }); + expect(body.readback).toMatchObject({ + source: "hands_current_release_resolver", + release_revision: 4, + release_status: "cancelled", + target_matched_preserved_scope: true, + target_release_reachable: false, + scope_inactive: true, + }); + + expect(await counts()).toEqual({ operations: 1, receipts: 1, deliveries: 1 }); + const delivery = await env.DB.prepare( + `SELECT event_type, payload_json, signing_secret, signature_key_version, + app_update_terminal_receipt_id, attempts, status + FROM webhook_deliveries WHERE webhook_id = ?`, + ).bind("webhook-terminal").first() as any; + expect(delivery).toMatchObject({ + event_type: "app_update:cleanup_terminal", + signing_secret: "frozen-signing-secret", + signature_key_version: "key-v7", + app_update_terminal_receipt_id: "operation-terminal-1", + attempts: 0, + status: "pending", + }); + expect(JSON.parse(delivery.payload_json).payload).toEqual(body.event); + + const stored = await env.DB.prepare( + `SELECT canonical_request_json, scope_readback_json + FROM app_update_cleanup_terminal_receipts WHERE operation_id = ?`, + ).bind("operation-terminal-1").first() as any; + expect(stored.canonical_request_json).not.toContain("device-terminal"); + expect(stored.scope_readback_json).not.toContain("device-terminal"); + }); + + it("replays response loss with the same receipt and zero new rows or signing drift", async () => { + const first = await handleEmitAppUpdateCleanupTerminal(context(input())); + const firstBody = await first.json() as any; + const frozen = await env.DB.prepare( + "SELECT payload_json, signing_secret, signature_key_version FROM webhook_deliveries", + ).first() as any; + + await env.DB.prepare( + `UPDATE webhooks SET secret = 'rotated-secret', signature_key_version = 'key-v8', + archived_at = ? + WHERE id = 'webhook-terminal'`, + ).bind(Date.now()).run(); + await env.DB.prepare( + "UPDATE releases SET status = 'active', revision = 5 WHERE id = 'release-terminal'", + ).run(); + const replay = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(replay.status).toBe(200); + const replayBody = await replay.json() as any; + expect(replayBody.replay).toBe(true); + expect(replayBody.event).toEqual(firstBody.event); + expect(await counts()).toEqual({ operations: 1, receipts: 1, deliveries: 1 }); + expect(await env.DB.prepare( + "SELECT payload_json, signing_secret, signature_key_version FROM webhook_deliveries", + ).first()).toEqual(frozen); + }); + + it("collapses concurrent identical requests onto one operation, receipt, and delivery", async () => { + const [left, right] = await Promise.all([ + handleEmitAppUpdateCleanupTerminal(context(input())), + handleEmitAppUpdateCleanupTerminal(context(input())), + ]); + expect([left.status, right.status].sort()).toEqual([200, 201]); + const bodies = await Promise.all([left.json(), right.json()]) as any[]; + expect(bodies[0].event).toEqual(bodies[1].event); + expect(bodies.map((body) => body.replay).sort()).toEqual([false, true]); + expect(await counts()).toEqual({ operations: 1, receipts: 1, deliveries: 1 }); + }); + + it("rejects cross-run, stale-attempt, different-bundle, and new-operation rebinding with zero duplicates", async () => { + expect((await handleEmitAppUpdateCleanupTerminal(context(input()))).status).toBe(201); + for (const changed of [ + { attempt: 3 }, + { artifact_bundle_digest: "c".repeat(64) }, + { run_case_id: "run-case-terminal-2" }, + { operation_id: "operation-terminal-2" }, + ]) { + const response = await handleEmitAppUpdateCleanupTerminal(context(input(changed))); + expect(response.status, JSON.stringify(changed)).toBe(409); + expect((await response.json() as any).code).toBe("APP_UPDATE_TERMINAL_BINDING_CONFLICT"); + } + expect(await counts()).toEqual({ operations: 1, receipts: 1, deliveries: 1 }); + }); + + it("does not treat cancelled status alone as target scope inactivity", async () => { + const response = await handleEmitAppUpdateCleanupTerminal(context(input({ + target_device_id: "another-device", + }))); + expect(response.status).toBe(409); + expect((await response.json() as any).code).toBe("APP_UPDATE_TARGET_OUTSIDE_RELEASE_SCOPE"); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + }); + + it("rejects when the canonical resolver winner still serves the same B version and artifact", async () => { + await seedActiveResolverWinner({ + releaseId: "release-active-same-b", + buildId: "build-active-same-b", + assetId: "asset-active-same-b", + versionCode: 200, + artifactHash: targetHash, + }); + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(409); + expect((await response.json() as any).code).toBe("APP_UPDATE_TARGET_STILL_REACHABLE"); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + expect((await env.DB.prepare( + "SELECT COUNT(*) count FROM audit_logs WHERE action='release.app_update_cleanup_terminal'", + ).first() as any).count).toBe(0); + }); + + it("repeats the canonical winner gate in the commit CAS", async () => { + const originalBatch = (env.DB as any).batch.bind(env.DB); + let injected = false; + (env.DB as any).batch = async (statements: unknown[]) => { + if (!injected) { + injected = true; + await seedActiveResolverWinner({ + releaseId: "release-race-same-b", + buildId: "build-race-same-b", + assetId: "asset-race-same-b", + versionCode: 200, + artifactHash: targetHash, + }); + } + return originalBatch(statements); + }; + + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(409); + expect((await response.json() as any).code).toBe( + "APP_UPDATE_TERMINAL_PRECONDITION_CHANGED", + ); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + expect((await env.DB.prepare( + "SELECT COUNT(*) count FROM audit_logs WHERE action='release.app_update_cleanup_terminal'", + ).first() as any).count).toBe(0); + }); + + it("allows a canonical active winner that serves a different artifact", async () => { + await seedActiveResolverWinner({ + releaseId: "release-active-different", + buildId: "build-active-different", + assetId: "asset-active-different", + versionCode: 201, + artifactHash: "e".repeat(64), + }); + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(201); + const body = await response.json() as any; + expect(body.readback).toMatchObject({ + resolver_candidate_count: 1, + resolver_winner_release_id: "release-active-different", + resolver_winner_version_code: 201, + target_release_reachable: false, + scope_inactive: true, + }); + expect(await counts()).toEqual({ operations: 1, receipts: 1, deliveries: 1 }); + }); + + it("rejects commit-time drift in the canonical winner build facts", async () => { + await seedActiveResolverWinner({ + releaseId: "release-active-build-drift", + buildId: "build-active-build-drift", + assetId: "asset-active-build-drift", + versionCode: 201, + artifactHash: "e".repeat(64), + }); + const originalBatch = (env.DB as any).batch.bind(env.DB); + (env.DB as any).batch = async (statements: unknown[]) => { + await env.DB.prepare( + "UPDATE builds SET version_code = 202 WHERE id = 'build-active-build-drift'", + ).run(); + return originalBatch(statements); + }; + + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(409); + expect((await response.json() as any).code).toBe( + "APP_UPDATE_TERMINAL_PRECONDITION_CHANGED", + ); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + }); + + it("rejects commit-time drift in the canonical winner APK identity", async () => { + await seedActiveResolverWinner({ + releaseId: "release-active-asset-drift", + buildId: "build-active-asset-drift", + assetId: "asset-active-asset-drift", + versionCode: 201, + artifactHash: "e".repeat(64), + }); + const originalBatch = (env.DB as any).batch.bind(env.DB); + (env.DB as any).batch = async (statements: unknown[]) => { + await env.DB.prepare( + `UPDATE build_assets SET file_hash = ? + WHERE id = 'asset-active-asset-drift'`, + ).bind("f".repeat(64)).run(); + return originalBatch(statements); + }; + + const response = await handleEmitAppUpdateCleanupTerminal(context(input())); + expect(response.status).toBe(409); + expect((await response.json() as any).code).toBe( + "APP_UPDATE_TERMINAL_PRECONDITION_CHANGED", + ); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + }); + + it("fails closed with zero side effects on stale generation, active release, no subscriber, or ambiguous target", async () => { + const cases: Array<{ + prepare: () => Promise; + body?: Record; + code: string; + }> = [ + { + prepare: async () => undefined, + body: { expected_release_revision: 3 }, + code: "RELEASE_REVISION_CONFLICT", + }, + { + prepare: async () => { + await env.DB.prepare("UPDATE releases SET status = 'active' WHERE id = 'release-terminal'").run(); + }, + code: "APP_UPDATE_RELEASE_NOT_INACTIVE", + }, + { + prepare: async () => { + await env.DB.prepare("UPDATE webhooks SET enabled = 0 WHERE id = 'webhook-terminal'").run(); + }, + code: "APP_UPDATE_TERMINAL_SUBSCRIBER_MISSING", + }, + { + prepare: async () => { + await env.DB.prepare( + `INSERT INTO build_assets + (id, build_id, artifact_kind, platform, arch, variant, filetype, r2_key, + file_hash, size_bytes, metadata_json, created_at) + VALUES ('asset-terminal-2', 'build-terminal', 'installable', 'android', 'arm64', + NULL, 'apk', 'apps/terminal-arm64.apk', ?, 42, '{}', ?)`, + ).bind("d".repeat(64), Date.now()).run(); + }, + code: "APP_UPDATE_TARGET_ARTIFACT_AMBIGUOUS", + }, + ]; + + for (const testCase of cases) { + await testCase.prepare(); + const response = await handleEmitAppUpdateCleanupTerminal(context(input(testCase.body))); + expect(response.status, testCase.code).toBe(409); + expect((await response.json() as any).code).toBe(testCase.code); + expect(await counts()).toEqual({ operations: 0, receipts: 0, deliveries: 0 }); + // Restore fixture state for the next independent failure. + await env.DB.prepare("UPDATE releases SET status = 'cancelled' WHERE id = 'release-terminal'").run(); + await env.DB.prepare("UPDATE webhooks SET enabled = 1 WHERE id = 'webhook-terminal'").run(); + await env.DB.prepare("DELETE FROM build_assets WHERE id = 'asset-terminal-2'").run(); + } + }); + + it("makes the terminal operation and receipt mechanically immutable", async () => { + expect((await handleEmitAppUpdateCleanupTerminal(context(input()))).status).toBe(201); + const { handleRetryOperation, handleDeleteOperation } = await import("../src/routes/operations"); + const operationContext = { + env, + req: { param: () => "operation-terminal-1" }, + json: (data: unknown, status = 200) => new Response(JSON.stringify(data), { + status, + headers: { "content-type": "application/json" }, + }), + } as any; + expect((await handleRetryOperation(operationContext)).status).toBe(400); + expect((await handleDeleteOperation(operationContext)).status).toBe(409); + await expect(env.DB.prepare( + "UPDATE operation_logs SET status = 'failed' WHERE id = ?", + ).bind("operation-terminal-1").run()).rejects.toThrow(/immutable/i); + await expect(env.DB.prepare( + "DELETE FROM operation_logs WHERE id = ?", + ).bind("operation-terminal-1").run()).rejects.toThrow(/immutable/i); + await expect(env.DB.prepare( + "UPDATE app_update_cleanup_terminal_receipts SET attempt = 99 WHERE operation_id = ?", + ).bind("operation-terminal-1").run()).rejects.toThrow(/immutable/i); + await expect(env.DB.prepare( + "DELETE FROM app_update_cleanup_terminal_receipts WHERE operation_id = ?", + ).bind("operation-terminal-1").run()).rejects.toThrow(/immutable/i); + }); +}); + // ============================================================================= // P3.3.2 — public API scope resolution (publish-architecture §5.4) // ============================================================================= diff --git a/worker/test/webhook_reaper.test.ts b/worker/test/webhook_reaper.test.ts index d5f1330..03f3e2d 100644 --- a/worker/test/webhook_reaper.test.ts +++ b/worker/test/webhook_reaper.test.ts @@ -18,6 +18,7 @@ function makeDb() { event_type TEXT NOT NULL, event_id TEXT, feedback_submission_event_id TEXT, + app_update_terminal_receipt_id TEXT, payload_json TEXT NOT NULL, signing_secret TEXT, status TEXT NOT NULL DEFAULT 'pending',