diff --git a/src/data/lobsterEncounters.ts b/src/data/lobsterEncounters.ts index afe7cc6..ce7f66a 100644 --- a/src/data/lobsterEncounters.ts +++ b/src/data/lobsterEncounters.ts @@ -5,6 +5,7 @@ import { type LobsterEncounter } from "../db/schema.js" import { commitProjection } from "../quilt/commit.js" +import { recordWalFailure } from "../quilt/ops.js" import { projectLobsterEncounter, projectLobsterPublication, @@ -144,7 +145,7 @@ const projectEncounterSafely = async ( (kernel) => projectLobsterEncounter(kernel, projection) ) } catch (error) { - console.warn("quilt wal encounter projection failed", error) + recordWalFailure("encounter_projection", error) } } @@ -518,7 +519,7 @@ export const bindLobsterMessage = async ( }) ) } catch (error) { - console.warn("quilt wal bind projection failed", error) + recordWalFailure("bind_projection", error) } } return { kind, encounter } @@ -581,7 +582,7 @@ export const markLobsterPublicationFailed = async ( }) ) } catch (error) { - console.warn("quilt wal publication projection failed", error) + recordWalFailure("publication_projection", error) } return { kind: "marked_failed", encounter } } @@ -687,7 +688,7 @@ export const recordLobsterResponse = async ( }) ) } catch (error) { - console.warn("quilt wal response projection failed", error) + recordWalFailure("response_projection", error) } } return { kind, encounter } diff --git a/src/data/nominations.ts b/src/data/nominations.ts index e2a02e1..8a3329e 100644 --- a/src/data/nominations.ts +++ b/src/data/nominations.ts @@ -1,4 +1,5 @@ import { commitNominationVoteProjection } from "../quilt/commit.js" +import { recordWalFailure } from "../quilt/ops.js" import { and, asc, @@ -246,7 +247,7 @@ const projectVoteToKernel = async ( try { await commitNominationVoteProjection(database.$client, input) } catch (error) { - console.warn("quilt wal projection failed", error) + recordWalFailure("nomination_projection", error) } } diff --git a/src/quilt/commit.ts b/src/quilt/commit.ts index 0779b00..5524b9a 100644 --- a/src/quilt/commit.ts +++ b/src/quilt/commit.ts @@ -14,6 +14,7 @@ import { buildWalRows, type WalRow } from "./projection.js" +import { recordWalCommit, recordWalFailure } from "./ops.js" export type WalClient = { batch>( @@ -97,9 +98,9 @@ export const commitProjection = async ( const holder = nextHolder(context.mutationId) const acquired = await claimWalLock(client, holder, new Date()) if (!acquired) { - console.warn( - "quilt wal lock not acquired; projection skipped", - context.mutationId + recordWalFailure( + "lock_exhausted", + new Error(`projection skipped for ${context.mutationId}`) ) return 0 } @@ -129,6 +130,7 @@ export const commitProjection = async ( }) if (rows.length === 0) return 0 + recordWalCommit(rows.length) await client.batch( rows.map((row: WalRow) => client diff --git a/src/quilt/ops.ts b/src/quilt/ops.ts new file mode 100644 index 0000000..50927aa --- /dev/null +++ b/src/quilt/ops.ts @@ -0,0 +1,96 @@ +// ops.ts — the ledger reports its own health. +// +// Review finding #2 (the half that is code, not ops): every WAL projection +// failure reduced to a console.warn, and nothing in production ever reads +// quilt_wal — so the mirror could die at deploy and nobody would know for +// a week. This module turns "logged and forgotten" into counted and +// exported: failure counters incremented at every catch site, a rows- +// committed gauge, and reconcileTick — a cron-ready wrapper around the +// reconciliation pass that returns a summary instead of throwing. +// Posture unchanged: the user path never sees an exception from the mirror. + +export type WalFailureKind = + | "nomination_projection" + | "encounter_projection" + | "bind_projection" + | "publication_projection" + | "response_projection" + | "lock_exhausted" + | "reconcile_tick" + +const failureCounts = new Map() +let rowsCommitted = 0 + +// Called from every silent catch. Increments the kind's counter and logs +// once — the log line is for humans tailing, the counter is for alerting. +export const recordWalFailure = ( + kind: WalFailureKind, + error: unknown +): void => { + failureCounts.set(kind, (failureCounts.get(kind) ?? 0) + 1) + console.warn(`quilt wal failure [${kind}]`, error) +} + +// Called from commitProjection on success — the mirror liveness gauge. +export const recordWalCommit = (rows: number): void => { + rowsCommitted += rows +} + +export const getWalFailureCounts = (): Record => + Object.fromEntries(failureCounts) + +export const getWalRowsCommitted = (): number => rowsCommitted + +// Test hook — production never resets. +export const resetWalOps = (): void => { + failureCounts.clear() + rowsCommitted = 0 +} + +export type ReconcileTickSummary = { + ranAt: string + ok: boolean + checked: { nominations: number; encounters: number } + mismatchCount: number + chainOk: boolean + breakAt?: number + failureCounts: Record + rowsCommitted: number +} + +// Cron-ready wrapper around the reconciliation pass. Scheduled handlers +// (wrangler: [triggers] crons = ["*/15 * * * *"]) call this; it never +// throws — a failed tick is itself a counted failure. Wire alerting on +// failureCounts.reconcile_tick > 0 or mismatchCount > 0. +export const reconcileTick = async ( + reconcile: () => Promise<{ + checked: { nominations: number; encounters: number } + mismatches: unknown[] + chain: { ok: boolean; breakAt?: number } + }> +): Promise => { + try { + const result = await reconcile() + return { + ranAt: new Date().toISOString(), + ok: true, + checked: result.checked, + mismatchCount: result.mismatches.length, + chainOk: result.chain.ok, + breakAt: result.chain.breakAt, + failureCounts: getWalFailureCounts(), + rowsCommitted: getWalRowsCommitted() + } + } catch (error) { + recordWalFailure("reconcile_tick", error) + return { + ranAt: new Date().toISOString(), + ok: false, + checked: { nominations: 0, encounters: 0 }, + mismatchCount: 0, + chainOk: false, + failureCounts: getWalFailureCounts(), + rowsCommitted: getWalRowsCommitted() + } + } +} diff --git a/tests/quiltKernelOps.test.ts b/tests/quiltKernelOps.test.ts new file mode 100644 index 0000000..059af13 --- /dev/null +++ b/tests/quiltKernelOps.test.ts @@ -0,0 +1,145 @@ +// quiltKernelOps.test.ts — the ledger reports its own health (review +// finding #2, code half). Every silent catch now counts; reconcileTick is +// the cron-ready observer that never throws. +import { Database } from "bun:sqlite" +import { describe, expect, it } from "bun:test" +import { readdirSync, readFileSync } from "node:fs" +import { drizzle } from "drizzle-orm/d1" +import * as schema from "../src/db/schema.js" +import { recordNominationVote } from "../src/data/nominations.js" +import { + getWalFailureCounts, + getWalRowsCommitted, + reconcileTick, + recordWalFailure, + resetWalOps +} from "../src/quilt/ops.js" +import { SqliteD1Database } from "./helpers/sqliteD1.js" + +const migrationPaths = readdirSync("drizzle") + .filter((file) => + /0002_.*\.sql|000[4-9]_.*\.sql|001[0134]_.*\.sql/.test(file) + ) + .sort() + +const applyMigrations = (database: Database) => { + for (const path of migrationPaths) { + const migration = readFileSync(`drizzle/${path}`, "utf8") + for (const statement of migration.split("--> statement-breakpoint")) { + const trimmed = statement.trim() + if (trimmed) { + database.run(trimmed) + } + } + } +} + +const createHarness = () => { + const owner = new SqliteD1Database() + applyMigrations(owner.database) + return { owner } +} + +describe("quilt kernel — ops", () => { + it("counts a real projection failure and the vote still records", async () => { + resetWalOps() + const { owner } = createHarness() + const database = drizzle(owner as never, { schema }) + owner.database + .query( + `insert into nominations ( + guild_id, channel_id, nominee_id, nominator_id, reason, + target_role_id, required_approvals, status, expires_at + ) values (?, ?, ?, ?, ?, ?, 1, 'submitted', ?)` + ) + .run("guild-1", "channel-1", "nominee-1", "nominator-1", "reason", "role-1", null) + const nominationId = Number( + owner.database.query("select max(id) as id from nominations").get()?.id + ) + // the deploy-time failure mode: WAL table missing in this env + owner.database.run(`drop table quilt_wal`) + + const result = await recordNominationVote( + nominationId, + "ct-1", + "approve", + new Date(), + database as never + ) + // user path untouched — the mirror failed, the vote did not + expect(result.kind).toBe("granting") + const counts = getWalFailureCounts() + expect(counts.nomination_projection).toBe(1) + owner.close() + }) + + it("counts lock exhaustion as its own failure kind", async () => { + resetWalOps() + const { owner } = createHarness() + owner.database + .query( + `insert into quilt_wal_lock (id, holder, acquired_at) + values (1, 'stuck', ?)` + ) + .run(new Date().toISOString()) + // a commit under an unbreakable lock returns 0 and counts + const { commitProjection } = await import("../src/quilt/commit.js") + const written = await commitProjection( + owner as never, + { mutationId: "m-x", ts: "2026-09-17T00:00:00.000Z" }, + (kernel) => { + kernel.bind("nomination.9", { id: 9, kind: "nomination" }) + } + ) + expect(written).toBe(0) + expect(getWalFailureCounts().lock_exhausted).toBe(1) + owner.close() + }) + + it("gauges rows committed on success", async () => { + resetWalOps() + const { owner } = createHarness() + const { commitProjection } = await import("../src/quilt/commit.js") + await commitProjection( + owner as never, + { mutationId: "m-gauge", ts: "2026-09-17T00:00:00.000Z" }, + (kernel) => { + kernel.bind("nomination.5", { id: 5, kind: "nomination" }) + kernel.bind("nomination.5.status", "submitted") + } + ) + expect(getWalRowsCommitted()).toBe(2) + expect(Object.keys(getWalFailureCounts())).toHaveLength(0) + owner.close() + }) + + it("reconcileTick carries the summary and never throws", async () => { + resetWalOps() + const good = await reconcileTick(async () => ({ + checked: { nominations: 3, encounters: 2 }, + mismatches: [{ kind: "status" }], + chain: { ok: true } + })) + expect(good.ok).toBe(true) + expect(good.checked).toEqual({ nominations: 3, encounters: 2 }) + expect(good.mismatchCount).toBe(1) + expect(good.chainOk).toBe(true) + + const bad = await reconcileTick(async () => { + throw new Error("d1 exploded") + }) + expect(bad.ok).toBe(false) + expect(bad.chainOk).toBe(false) + expect(getWalFailureCounts().reconcile_tick).toBe(1) + }) + + it("recordWalFailure is total over weird errors", () => { + resetWalOps() + recordWalFailure("reconcile_tick", new Error("x")) + recordWalFailure("reconcile_tick", "string error") + recordWalFailure("reconcile_tick", undefined) + expect(getWalFailureCounts().reconcile_tick).toBe(3) + resetWalOps() + expect(Object.keys(getWalFailureCounts())).toHaveLength(0) + }) +})