quilt kernel P5-ops: the ledger reports its own health

src/data/lobsterEncounters.ts

@@ -5,6 +5,7 @@ import {
 	type LobsterEncounter
 } from "../db/schema.js"
 import { commitProjection } from "../quilt/commit.js"
+import { recordWalFailure } from "../quilt/ops.js"
 import {
 	projectLobsterEncounter,
 	projectLobsterPublication,
@@ -144,7 +145,7 @@ const projectEncounterSafely = async (
 			(kernel) => projectLobsterEncounter(kernel, projection)
 		)
 	} catch (error) {
-		console.warn("quilt wal encounter projection failed", error)
+		recordWalFailure("encounter_projection", error)
 	}
 }
 
@@ -518,7 +519,7 @@ export const bindLobsterMessage = async (
 					})
 			)
 		} catch (error) {
-			console.warn("quilt wal bind projection failed", error)
+			recordWalFailure("bind_projection", error)
 		}
 	}
 	return { kind, encounter }
@@ -581,7 +582,7 @@ export const markLobsterPublicationFailed = async (
 					})
 			)
 		} catch (error) {
-			console.warn("quilt wal publication projection failed", error)
+			recordWalFailure("publication_projection", error)
 		}
 		return { kind: "marked_failed", encounter }
 	}
@@ -687,7 +688,7 @@ export const recordLobsterResponse = async (
 					})
 			)
 		} catch (error) {
-			console.warn("quilt wal response projection failed", error)
+			recordWalFailure("response_projection", error)
 		}
 	}
 	return { kind, encounter }

src/data/nominations.ts

@@ -1,4 +1,5 @@
 import { commitNominationVoteProjection } from "../quilt/commit.js"
+import { recordWalFailure } from "../quilt/ops.js"
 import {
 	and,
 	asc,
@@ -246,7 +247,7 @@ const projectVoteToKernel = async (
 	try {
 		await commitNominationVoteProjection(database.$client, input)
 	} catch (error) {
-		console.warn("quilt wal projection failed", error)
+		recordWalFailure("nomination_projection", error)
 	}
 }
 

src/quilt/commit.ts

@@ -14,6 +14,7 @@ import {
 	buildWalRows,
 	type WalRow
 } from "./projection.js"
+import { recordWalCommit, recordWalFailure } from "./ops.js"
 
 export type WalClient = {
 	batch<T = Record<string, unknown>>(
@@ -97,9 +98,9 @@ export const commitProjection = async (
 	const holder = nextHolder(context.mutationId)
 	const acquired = await claimWalLock(client, holder, new Date())
 	if (!acquired) {
-		console.warn(
-			"quilt wal lock not acquired; projection skipped",
-			context.mutationId
+		recordWalFailure(
+			"lock_exhausted",
+			new Error(`projection skipped for ${context.mutationId}`)
 		)
 		return 0
 	}
@@ -129,6 +130,7 @@ export const commitProjection = async (
 		})
 		if (rows.length === 0) return 0
 
+		recordWalCommit(rows.length)
 		await client.batch(
 			rows.map((row: WalRow) =>
 				client

src/quilt/ops.ts

@@ -0,0 +1,96 @@
+// ops.ts — the ledger reports its own health.
+//
+// Review finding #2 (the half that is code, not ops): every WAL projection
+// failure reduced to a console.warn, and nothing in production ever reads
+// quilt_wal — so the mirror could die at deploy and nobody would know for
+// a week. This module turns "logged and forgotten" into counted and
+// exported: failure counters incremented at every catch site, a rows-
+// committed gauge, and reconcileTick — a cron-ready wrapper around the
+// reconciliation pass that returns a summary instead of throwing.
+// Posture unchanged: the user path never sees an exception from the mirror.
+
+export type WalFailureKind =
+	| "nomination_projection"
+	| "encounter_projection"
+	| "bind_projection"
+	| "publication_projection"
+	| "response_projection"
+	| "lock_exhausted"
+	| "reconcile_tick"
+
+const failureCounts = new Map<WalFailureKind, number>()
+let rowsCommitted = 0
+
+// Called from every silent catch. Increments the kind's counter and logs
+// once — the log line is for humans tailing, the counter is for alerting.
+export const recordWalFailure = (
+	kind: WalFailureKind,
+	error: unknown
+): void => {
+	failureCounts.set(kind, (failureCounts.get(kind) ?? 0) + 1)
+	console.warn(`quilt wal failure [${kind}]`, error)
+}
+
+// Called from commitProjection on success — the mirror liveness gauge.
+export const recordWalCommit = (rows: number): void => {
+	rowsCommitted += rows
+}
+
+export const getWalFailureCounts = (): Record<string, number> =>
+	Object.fromEntries(failureCounts)
+
+export const getWalRowsCommitted = (): number => rowsCommitted
+
+// Test hook — production never resets.
+export const resetWalOps = (): void => {
+	failureCounts.clear()
+	rowsCommitted = 0
+}
+
+export type ReconcileTickSummary = {
+	ranAt: string
+	ok: boolean
+	checked: { nominations: number; encounters: number }
+	mismatchCount: number
+	chainOk: boolean
+	breakAt?: number
+	failureCounts: Record<string, number>
+	rowsCommitted: number
+}
+
+// Cron-ready wrapper around the reconciliation pass. Scheduled handlers
+// (wrangler: [triggers] crons = ["*/15 * * * *"]) call this; it never
+// throws — a failed tick is itself a counted failure. Wire alerting on
+// failureCounts.reconcile_tick > 0 or mismatchCount > 0.
+export const reconcileTick = async (
+	reconcile: () => Promise<{
+		checked: { nominations: number; encounters: number }
+		mismatches: unknown[]
+		chain: { ok: boolean; breakAt?: number }
+	}>
+): Promise<ReconcileTickSummary> => {
+	try {
+		const result = await reconcile()
+		return {
+			ranAt: new Date().toISOString(),
+			ok: true,
+			checked: result.checked,
+			mismatchCount: result.mismatches.length,
+			chainOk: result.chain.ok,
+			breakAt: result.chain.breakAt,
+			failureCounts: getWalFailureCounts(),
+			rowsCommitted: getWalRowsCommitted()
+		}
+	} catch (error) {
+		recordWalFailure("reconcile_tick", error)
+		return {
+			ranAt: new Date().toISOString(),
+			ok: false,
+			checked: { nominations: 0, encounters: 0 },
+			mismatchCount: 0,
+			chainOk: false,
+			failureCounts: getWalFailureCounts(),
+			rowsCommitted: getWalRowsCommitted()
+		}
+	}
+}

tests/quiltKernelOps.test.ts

@@ -0,0 +1,145 @@
+// quiltKernelOps.test.ts — the ledger reports its own health (review
+// finding #2, code half). Every silent catch now counts; reconcileTick is
+// the cron-ready observer that never throws.
+import { Database } from "bun:sqlite"
+import { describe, expect, it } from "bun:test"
+import { readdirSync, readFileSync } from "node:fs"
+import { drizzle } from "drizzle-orm/d1"
+import * as schema from "../src/db/schema.js"
+import { recordNominationVote } from "../src/data/nominations.js"
+import {
+	getWalFailureCounts,
+	getWalRowsCommitted,
+	reconcileTick,
+	recordWalFailure,
+	resetWalOps
+} from "../src/quilt/ops.js"
+import { SqliteD1Database } from "./helpers/sqliteD1.js"
+
+const migrationPaths = readdirSync("drizzle")
+	.filter((file) =>
+		/0002_.*\.sql|000[4-9]_.*\.sql|001[0134]_.*\.sql/.test(file)
+	)
+	.sort()
+
+const applyMigrations = (database: Database) => {
+	for (const path of migrationPaths) {
+		const migration = readFileSync(`drizzle/${path}`, "utf8")
+		for (const statement of migration.split("--> statement-breakpoint")) {
+			const trimmed = statement.trim()
+			if (trimmed) {
+				database.run(trimmed)
+			}
+		}
+	}
+}
+
+const createHarness = () => {
+	const owner = new SqliteD1Database()
+	applyMigrations(owner.database)
+	return { owner }
+}
+
+describe("quilt kernel — ops", () => {
+	it("counts a real projection failure and the vote still records", async () => {
+		resetWalOps()
+		const { owner } = createHarness()
+		const database = drizzle(owner as never, { schema })
+		owner.database
+			.query(
+				`insert into nominations (
+					guild_id, channel_id, nominee_id, nominator_id, reason,
+					target_role_id, required_approvals, status, expires_at
+				) values (?, ?, ?, ?, ?, ?, 1, 'submitted', ?)`
+			)
+			.run("guild-1", "channel-1", "nominee-1", "nominator-1", "reason", "role-1", null)
+		const nominationId = Number(
+			owner.database.query("select max(id) as id from nominations").get()?.id
+		)
+		// the deploy-time failure mode: WAL table missing in this env
+		owner.database.run(`drop table quilt_wal`)
+
+		const result = await recordNominationVote(
+			nominationId,
+			"ct-1",
+			"approve",
+			new Date(),
+			database as never
+		)
+		// user path untouched — the mirror failed, the vote did not
+		expect(result.kind).toBe("granting")
+		const counts = getWalFailureCounts()
+		expect(counts.nomination_projection).toBe(1)
+		owner.close()
+	})
+
+	it("counts lock exhaustion as its own failure kind", async () => {
+		resetWalOps()
+		const { owner } = createHarness()
+		owner.database
+			.query(
+				`insert into quilt_wal_lock (id, holder, acquired_at)
+				 values (1, 'stuck', ?)`
+			)
+			.run(new Date().toISOString())
+		// a commit under an unbreakable lock returns 0 and counts
+		const { commitProjection } = await import("../src/quilt/commit.js")
+		const written = await commitProjection(
+			owner as never,
+			{ mutationId: "m-x", ts: "2026-09-17T00:00:00.000Z" },
+			(kernel) => {
+				kernel.bind("nomination.9", { id: 9, kind: "nomination" })
+			}
+		)
+		expect(written).toBe(0)
+		expect(getWalFailureCounts().lock_exhausted).toBe(1)
+		owner.close()
+	})
+
+	it("gauges rows committed on success", async () => {
+		resetWalOps()
+		const { owner } = createHarness()
+		const { commitProjection } = await import("../src/quilt/commit.js")
+		await commitProjection(
+			owner as never,
+			{ mutationId: "m-gauge", ts: "2026-09-17T00:00:00.000Z" },
+			(kernel) => {
+				kernel.bind("nomination.5", { id: 5, kind: "nomination" })
+				kernel.bind("nomination.5.status", "submitted")
+			}
+		)
+		expect(getWalRowsCommitted()).toBe(2)
+		expect(Object.keys(getWalFailureCounts())).toHaveLength(0)
+		owner.close()
+	})
+
+	it("reconcileTick carries the summary and never throws", async () => {
+		resetWalOps()
+		const good = await reconcileTick(async () => ({
+			checked: { nominations: 3, encounters: 2 },
+			mismatches: [{ kind: "status" }],
+			chain: { ok: true }
+		}))
+		expect(good.ok).toBe(true)
+		expect(good.checked).toEqual({ nominations: 3, encounters: 2 })
+		expect(good.mismatchCount).toBe(1)
+		expect(good.chainOk).toBe(true)
+
+		const bad = await reconcileTick(async () => {
+			throw new Error("d1 exploded")
+		})
+		expect(bad.ok).toBe(false)
+		expect(bad.chainOk).toBe(false)
+		expect(getWalFailureCounts().reconcile_tick).toBe(1)
+	})
+
+	it("recordWalFailure is total over weird errors", () => {
+		resetWalOps()
+		recordWalFailure("reconcile_tick", new Error("x"))
+		recordWalFailure("reconcile_tick", "string error")
+		recordWalFailure("reconcile_tick", undefined)
+		expect(getWalFailureCounts().reconcile_tick).toBe(3)
+		resetWalOps()
+		expect(Object.keys(getWalFailureCounts())).toHaveLength(0)
+	})
+})