diff --git a/.env.example b/.env.example index aa2d29a..f0710ff 100644 --- a/.env.example +++ b/.env.example @@ -233,6 +233,12 @@ STELLAR_CORE_MAX_RESTARTS=2 # Only relevant when DATABASE_URL is set. # ----------------------------------------------------------------------------- +# Enable or disable the automatic retention scheduler. +# Set to "false" to prevent any rows from being pruned automatically. +# The npm run retention CLI script is unaffected by this flag. +# Default: true +RETENTION_ENABLED=true + # Number of days to keep Event rows in the hot PostgreSQL table. # Events older than this are exported to a gzip-compressed CSV file # (see ARCHIVE_OUTPUT_DIR) and then deleted from the database. diff --git a/lib/retention/__tests__/pruner.test.ts b/lib/retention/__tests__/pruner.test.ts new file mode 100644 index 0000000..0209226 --- /dev/null +++ b/lib/retention/__tests__/pruner.test.ts @@ -0,0 +1,232 @@ +/** + * lib/retention/__tests__/pruner.test.ts + * + * Unit tests for the retention pruner. + * + * Tests use a mock PrismaClient so no database is required. The mock + * simulates seeded rows at various ages so the pruner's selection logic + * can be exercised in isolation. + * + * Test coverage: + * 1. Age threshold: only rows past the cutoff date are deleted. + * 2. RETENTION_ENABLED=false: startRetentionScheduler returns undefined (no-op). + * 3. Batch-cap safety rail: no more than batchCap rows are deleted per run. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { pruneOldData } from "../pruner"; +import type { PruneOptions } from "../pruner"; +import type { PrismaClient } from "@prisma/client"; + +// --------------------------------------------------------------------------- +// Mock PrismaClient builder +// --------------------------------------------------------------------------- + +/** + * Builds a minimal Prisma mock that holds rows for the three retention + * tables in memory. Supports the subset of the Prisma API that the pruner + * uses: count, aggregate (_min/_max createdAt), findMany (with take + orderBy), + * and deleteMany (with id in). + */ +function buildMockDb(initialRows: { + events?: Array<{ id: string; createdAt: Date }>; + deadLetterEvents?: Array<{ id: string; createdAt: Date }>; + webhookDeliveries?: Array<{ id: string; createdAt: Date }>; +}) { + let events = [...(initialRows.events ?? [])]; + let deadLetterEvents = [...(initialRows.deadLetterEvents ?? [])]; + let webhookDeliveries = [...(initialRows.webhookDeliveries ?? [])]; + + function makeTableMock(getRows: () => Array<{ id: string; createdAt: Date }>) { + return { + count: vi.fn(({ where }: { where: { createdAt: { lt: Date } } }) => { + return Promise.resolve( + getRows().filter((r) => r.createdAt < where.createdAt.lt).length + ); + }), + aggregate: vi.fn(({ where, _min, _max }: any) => { + const matching = getRows().filter((r) => r.createdAt < where.createdAt.lt); + const dates = matching.map((r) => r.createdAt.getTime()); + return Promise.resolve({ + _min: _min ? { createdAt: dates.length ? new Date(Math.min(...dates)) : null } : undefined, + _max: _max ? { createdAt: dates.length ? new Date(Math.max(...dates)) : null } : undefined, + }); + }), + findMany: vi.fn(({ where, take, orderBy }: any) => { + let rows = getRows().filter((r) => r.createdAt < where.createdAt.lt); + // Apply orderBy createdAt asc + rows = [...rows].sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()); + return Promise.resolve(rows.slice(0, take).map((r) => ({ id: r.id }))); + }), + deleteMany: vi.fn(({ where }: { where: { id: { in: string[] } } }) => { + const ids = new Set(where.id.in); + const deleted = getRows().filter((r) => ids.has(r.id)).length; + // Mutate the array in place + const arr = getRows(); + const toRemove = arr.filter((r) => ids.has(r.id)); + toRemove.forEach((r) => arr.splice(arr.indexOf(r), 1)); + return Promise.resolve({ count: deleted }); + }), + }; + } + + const db = { + event: makeTableMock(() => events), + deadLetterEvent: makeTableMock(() => deadLetterEvents), + webhookDelivery: makeTableMock(() => webhookDeliveries), + } as unknown as PrismaClient; + + return { db, events, deadLetterEvents, webhookDeliveries }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function daysAgo(n: number): Date { + const d = new Date(); + d.setDate(d.getDate() - n); + return d; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("pruneOldData", () => { + beforeEach(() => { + vi.unstubAllEnvs(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // ------------------------------------------------------------------------- + // Test 1: Age threshold — only rows past the cutoff are deleted + // ------------------------------------------------------------------------- + it("deletes only rows older than the cutoff date and leaves newer rows intact", async () => { + const { db, events, deadLetterEvents } = buildMockDb({ + events: [ + { id: "evt-old-1", createdAt: daysAgo(200) }, // older than 180 days → should be deleted + { id: "evt-old-2", createdAt: daysAgo(190) }, // older than 180 days → should be deleted + { id: "evt-new-1", createdAt: daysAgo(10) }, // newer → should survive + { id: "evt-new-2", createdAt: daysAgo(1) }, // newer → should survive + ], + deadLetterEvents: [ + { id: "dle-old-1", createdAt: daysAgo(365) }, // older → should be deleted + { id: "dle-new-1", createdAt: daysAgo(5) }, // newer → should survive + ], + webhookDeliveries: [], + }); + + const cutoffDate = daysAgo(180); + const result = await pruneOldData({ db, cutoffDate, batchCap: 1000 }); + + // Three rows should have been deleted (2 events + 1 dead-letter). + expect(result.totalDeleted).toBe(3); + expect(result.tables.Event.deleted).toBe(2); + expect(result.tables.DeadLetterEvent.deleted).toBe(1); + expect(result.tables.WebhookDelivery.deleted).toBe(0); + + // Eligible counts match what was seeded past the cutoff. + expect(result.tables.Event.eligible).toBe(2); + expect(result.tables.DeadLetterEvent.eligible).toBe(1); + + // Newer rows must still be present in the in-memory arrays. + expect(events.map((r) => r.id)).toContain("evt-new-1"); + expect(events.map((r) => r.id)).toContain("evt-new-2"); + expect(events.map((r) => r.id)).not.toContain("evt-old-1"); + expect(events.map((r) => r.id)).not.toContain("evt-old-2"); + + expect(deadLetterEvents.map((r) => r.id)).toContain("dle-new-1"); + expect(deadLetterEvents.map((r) => r.id)).not.toContain("dle-old-1"); + }); + + // ------------------------------------------------------------------------- + // Test 2: RETENTION_ENABLED=false — see scheduler.test.ts + // (That test requires a top-level vi.mock of lib/db/client so it lives + // in its own file to allow Vitest to hoist the mock correctly.) + // ------------------------------------------------------------------------- + + // ------------------------------------------------------------------------- + // Test 3: Batch-cap safety rail + // ------------------------------------------------------------------------- + it("deletes no more than batchCap rows in total across all tables", async () => { + const { db } = buildMockDb({ + events: Array.from({ length: 20 }, (_, i) => ({ + id: `evt-${i}`, + createdAt: daysAgo(200), + })), + deadLetterEvents: Array.from({ length: 20 }, (_, i) => ({ + id: `dle-${i}`, + createdAt: daysAgo(200), + })), + webhookDeliveries: Array.from({ length: 20 }, (_, i) => ({ + id: `wh-${i}`, + createdAt: daysAgo(200), + })), + }); + + const cutoffDate = daysAgo(180); + const batchCap = 15; + + const result = await pruneOldData({ db, cutoffDate, batchCap }); + + // Total deleted must never exceed batchCap. + expect(result.totalDeleted).toBeLessThanOrEqual(batchCap); + expect(result.totalDeleted).toBe(batchCap); + + // All 60 rows are eligible; the cap is reported faithfully. + expect(result.totalEligible).toBe(60); + expect(result.batchCap).toBe(batchCap); + }); + + // ------------------------------------------------------------------------- + // Test 4: Dry-run mode — no rows touched, counts are accurate + // ------------------------------------------------------------------------- + it("dry-run mode reports the correct deletion counts without deleting anything", async () => { + const { db, events } = buildMockDb({ + events: [ + { id: "evt-1", createdAt: daysAgo(200) }, + { id: "evt-2", createdAt: daysAgo(190) }, + { id: "evt-3", createdAt: daysAgo(5) }, // too new + ], + deadLetterEvents: [], + webhookDeliveries: [], + }); + + const cutoffDate = daysAgo(180); + const result = await pruneOldData({ db, cutoffDate, batchCap: 1000, dryRun: true }); + + expect(result.dryRun).toBe(true); + expect(result.totalDeleted).toBe(2); // would delete 2 + expect(result.totalEligible).toBe(2); + + // Nothing was actually removed from the in-memory store. + expect(db.event.deleteMany).not.toHaveBeenCalled(); + expect(events).toHaveLength(3); + }); + + // ------------------------------------------------------------------------- + // Test 5: Boundary rows (exactly at cutoff) are not deleted + // ------------------------------------------------------------------------- + it("rows created exactly at the cutoff date are NOT deleted (lt, not lte)", async () => { + const cutoffDate = new Date("2024-01-01T00:00:00.000Z"); + + const { db, events } = buildMockDb({ + events: [ + // Exactly at cutoff — should NOT be deleted (pruner uses lt, not lte). + { id: "evt-boundary", createdAt: new Date("2024-01-01T00:00:00.000Z") }, + // One millisecond before cutoff — SHOULD be deleted. + { id: "evt-just-before", createdAt: new Date("2023-12-31T23:59:59.999Z") }, + ], + }); + + const result = await pruneOldData({ db, cutoffDate, batchCap: 1000 }); + + expect(result.totalDeleted).toBe(1); + expect(events.map((r) => r.id)).toContain("evt-boundary"); + expect(events.map((r) => r.id)).not.toContain("evt-just-before"); + }); +}); diff --git a/lib/retention/__tests__/scheduler.test.ts b/lib/retention/__tests__/scheduler.test.ts new file mode 100644 index 0000000..4d09dff --- /dev/null +++ b/lib/retention/__tests__/scheduler.test.ts @@ -0,0 +1,47 @@ +/** + * lib/retention/__tests__/scheduler.test.ts + * + * Unit tests for the retention scheduler. + * + * The scheduler imports lib/db/client which instantiates a PrismaClient. + * Because the Prisma generated artefacts may not exist in CI (no database), + * we mock the db/client module at the top level so Prisma is never loaded. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Top-level mock — must appear before any import of the scheduler so Vitest +// can hoist it ahead of module evaluation. +vi.mock("../../db/client", () => ({ + db: {}, +})); + +// Now safe to import the scheduler. +import { startRetentionScheduler } from "../scheduler"; + +describe("startRetentionScheduler", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("returns undefined and never starts a cron job when RETENTION_ENABLED=false", () => { + vi.stubEnv("RETENTION_ENABLED", "false"); + + const task = startRetentionScheduler(); + + expect(task).toBeUndefined(); + }); + + it("returns a scheduled task when RETENTION_ENABLED is not set (defaults to enabled)", () => { + vi.stubEnv("RETENTION_ENABLED", "true"); + // Use a valid cron that won't actually fire during the test. + vi.stubEnv("RETENTION_CRON_SCHEDULE", "0 3 * * *"); + + const task = startRetentionScheduler(); + + // A ScheduledTask is returned and can be stopped. + expect(task).toBeDefined(); + // Clean up to prevent the task from outliving the test. + task?.stop(); + }); +}); diff --git a/lib/retention/index.ts b/lib/retention/index.ts new file mode 100644 index 0000000..dc56db7 --- /dev/null +++ b/lib/retention/index.ts @@ -0,0 +1,14 @@ +/** + * lib/retention/index.ts + * + * Public surface of the retention module. + * + * - `startRetentionScheduler` — call once at server startup; starts the + * node-cron job that prunes old rows on a configurable schedule. + * - `pruneOldData` / `logPruneResult` — lower-level exports for the CLI + * script and tests. + */ + +export { startRetentionScheduler } from "./scheduler"; +export { pruneOldData, logPruneResult } from "./pruner"; +export type { PruneOptions, PruneResult, TablePruneResult } from "./pruner"; diff --git a/lib/retention/pruner.ts b/lib/retention/pruner.ts new file mode 100644 index 0000000..af715fc --- /dev/null +++ b/lib/retention/pruner.ts @@ -0,0 +1,319 @@ +/** + * lib/retention/pruner.ts + * + * Core retention pruning logic. Deletes rows older than a configurable age + * threshold from Event, DeadLetterEvent, and WebhookDelivery tables. + * + * Design goals: + * - Pure function over a PrismaClient, so it can be called by the scheduler + * OR directly from the CLI script without any side effects. + * - Dry-run mode: counts what *would* be deleted without touching the DB. + * - Batch-delete safety rail: never deletes more than `batchCap` rows total + * per invocation to avoid long-running transactions on large tables. + * - Structured log output for operator auditing. + */ + +import type { PrismaClient } from "@prisma/client"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface PruneOptions { + /** Prisma client to use. Injected so tests can pass a mock. */ + db: PrismaClient; + /** Rows older than this Date will be targeted for deletion. */ + cutoffDate: Date; + /** + * Maximum number of rows to delete across all tables in one invocation. + * When `dryRun` is true this is used only to cap the count that is + * reported, not to limit any actual work. + * Default: 1000 + */ + batchCap?: number; + /** + * When true, count affected rows but do not delete anything. + * Default: false + */ + dryRun?: boolean; +} + +export interface TablePruneResult { + /** Total rows targeted (before batchCap is applied). */ + eligible: number; + /** Rows actually deleted (or that would be deleted in dry-run). */ + deleted: number; + /** createdAt of the oldest eligible row, or null when none found. */ + oldest: Date | null; + /** createdAt of the newest eligible row, or null when none found. */ + newest: Date | null; +} + +export interface PruneResult { + dryRun: boolean; + cutoffDate: Date; + batchCap: number; + tables: { + Event: TablePruneResult; + DeadLetterEvent: TablePruneResult; + WebhookDelivery: TablePruneResult; + }; + totalEligible: number; + totalDeleted: number; + durationMs: number; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Count rows and find oldest/newest timestamps without deleting anything. */ +async function countAndBoundaries( + db: PrismaClient, + table: "event" | "deadLetterEvent" | "webhookDelivery", + cutoffDate: Date +): Promise<{ count: number; oldest: Date | null; newest: Date | null }> { + // Prisma does not expose a generic aggregate with min/max on arbitrary + // models via a shared interface, so we handle each table explicitly. + if (table === "event") { + const [count, bounds] = await Promise.all([ + db.event.count({ where: { createdAt: { lt: cutoffDate } } }), + db.event.aggregate({ + where: { createdAt: { lt: cutoffDate } }, + _min: { createdAt: true }, + _max: { createdAt: true }, + }), + ]); + return { + count, + oldest: bounds._min.createdAt ?? null, + newest: bounds._max.createdAt ?? null, + }; + } + + if (table === "deadLetterEvent") { + const [count, bounds] = await Promise.all([ + db.deadLetterEvent.count({ where: { createdAt: { lt: cutoffDate } } }), + db.deadLetterEvent.aggregate({ + where: { createdAt: { lt: cutoffDate } }, + _min: { createdAt: true }, + _max: { createdAt: true }, + }), + ]); + return { + count, + oldest: bounds._min.createdAt ?? null, + newest: bounds._max.createdAt ?? null, + }; + } + + // webhookDelivery + const [count, bounds] = await Promise.all([ + db.webhookDelivery.count({ where: { createdAt: { lt: cutoffDate } } }), + db.webhookDelivery.aggregate({ + where: { createdAt: { lt: cutoffDate } }, + _min: { createdAt: true }, + _max: { createdAt: true }, + }), + ]); + return { + count, + oldest: bounds._min.createdAt ?? null, + newest: bounds._max.createdAt ?? null, + }; +} + +/** + * Delete at most `limit` rows from a table older than `cutoffDate`. + * + * Prisma does not support `deleteMany` with a `take` (LIMIT) clause, so we + * first select the IDs of the rows to delete and then delete by ID. This + * keeps the transaction short and predictable. + */ +async function deleteBatch( + db: PrismaClient, + table: "event" | "deadLetterEvent" | "webhookDelivery", + cutoffDate: Date, + limit: number +): Promise { + if (table === "event") { + const rows = await db.event.findMany({ + where: { createdAt: { lt: cutoffDate } }, + select: { id: true }, + take: limit, + orderBy: { createdAt: "asc" }, + }); + if (rows.length === 0) return 0; + const result = await db.event.deleteMany({ + where: { id: { in: rows.map((r) => r.id) } }, + }); + return result.count; + } + + if (table === "deadLetterEvent") { + const rows = await db.deadLetterEvent.findMany({ + where: { createdAt: { lt: cutoffDate } }, + select: { id: true }, + take: limit, + orderBy: { createdAt: "asc" }, + }); + if (rows.length === 0) return 0; + const result = await db.deadLetterEvent.deleteMany({ + where: { id: { in: rows.map((r) => r.id) } }, + }); + return result.count; + } + + // webhookDelivery + const rows = await db.webhookDelivery.findMany({ + where: { createdAt: { lt: cutoffDate } }, + select: { id: true }, + take: limit, + orderBy: { createdAt: "asc" }, + }); + if (rows.length === 0) return 0; + const result = await db.webhookDelivery.deleteMany({ + where: { id: { in: rows.map((r) => r.id) } }, + }); + return result.count; +} + +// --------------------------------------------------------------------------- +// Main export +// --------------------------------------------------------------------------- + +/** + * Prune rows older than `cutoffDate` from the three retention-eligible tables. + * + * The total number of rows deleted across all tables is capped at `batchCap` + * (default 1000). In `dryRun` mode, no rows are deleted; the function only + * counts what would be removed. + */ +export async function pruneOldData(options: PruneOptions): Promise { + const { db, cutoffDate, batchCap = 1000, dryRun = false } = options; + const startedAt = Date.now(); + + // Count eligible rows and find boundaries for all three tables in parallel. + const [eventStats, deadLetterStats, webhookStats] = await Promise.all([ + countAndBoundaries(db, "event", cutoffDate), + countAndBoundaries(db, "deadLetterEvent", cutoffDate), + countAndBoundaries(db, "webhookDelivery", cutoffDate), + ]); + + const totalEligible = + eventStats.count + deadLetterStats.count + webhookStats.count; + + // In dry-run mode we stop here. + if (dryRun) { + const eventDryDeleted = Math.min(eventStats.count, batchCap); + const remainingAfterEvents = Math.max(0, batchCap - eventDryDeleted); + const deadLetterDryDeleted = Math.min(deadLetterStats.count, remainingAfterEvents); + const remainingAfterDead = Math.max(0, remainingAfterEvents - deadLetterDryDeleted); + const webhookDryDeleted = Math.min(webhookStats.count, remainingAfterDead); + const totalDeleted = eventDryDeleted + deadLetterDryDeleted + webhookDryDeleted; + + return { + dryRun: true, + cutoffDate, + batchCap, + tables: { + Event: { + eligible: eventStats.count, + deleted: eventDryDeleted, + oldest: eventStats.oldest, + newest: eventStats.newest, + }, + DeadLetterEvent: { + eligible: deadLetterStats.count, + deleted: deadLetterDryDeleted, + oldest: deadLetterStats.oldest, + newest: deadLetterStats.newest, + }, + WebhookDelivery: { + eligible: webhookStats.count, + deleted: webhookDryDeleted, + oldest: webhookStats.oldest, + newest: webhookStats.newest, + }, + }, + totalEligible, + totalDeleted, + durationMs: Date.now() - startedAt, + }; + } + + // Live delete — distribute the batchCap budget across tables in order. + let remainingBudget = batchCap; + + const eventDeleted = remainingBudget > 0 + ? await deleteBatch(db, "event", cutoffDate, remainingBudget) + : 0; + remainingBudget = Math.max(0, remainingBudget - eventDeleted); + + const deadLetterDeleted = remainingBudget > 0 + ? await deleteBatch(db, "deadLetterEvent", cutoffDate, remainingBudget) + : 0; + remainingBudget = Math.max(0, remainingBudget - deadLetterDeleted); + + const webhookDeleted = remainingBudget > 0 + ? await deleteBatch(db, "webhookDelivery", cutoffDate, remainingBudget) + : 0; + + const totalDeleted = eventDeleted + deadLetterDeleted + webhookDeleted; + + return { + dryRun: false, + cutoffDate, + batchCap, + tables: { + Event: { + eligible: eventStats.count, + deleted: eventDeleted, + oldest: eventStats.oldest, + newest: eventStats.newest, + }, + DeadLetterEvent: { + eligible: deadLetterStats.count, + deleted: deadLetterDeleted, + oldest: deadLetterStats.oldest, + newest: deadLetterStats.newest, + }, + WebhookDelivery: { + eligible: webhookStats.count, + deleted: webhookDeleted, + oldest: webhookStats.oldest, + newest: webhookStats.newest, + }, + }, + totalEligible, + totalDeleted, + durationMs: Date.now() - startedAt, + }; +} + +/** + * Emit a structured log entry for a completed prune run. + * This is separated from `pruneOldData` so callers can suppress output + * or redirect it (e.g. to pino, a metrics sink, etc.). + */ +export function logPruneResult(result: PruneResult): void { + const mode = result.dryRun ? "[DRY-RUN]" : "[PRUNED]"; + const cutoff = result.cutoffDate.toISOString(); + + console.log( + JSON.stringify({ + level: "info", + msg: `retention ${mode}`, + cutoffDate: cutoff, + batchCap: result.batchCap, + totalEligible: result.totalEligible, + totalDeleted: result.totalDeleted, + durationMs: result.durationMs, + tables: { + Event: result.tables.Event, + DeadLetterEvent: result.tables.DeadLetterEvent, + WebhookDelivery: result.tables.WebhookDelivery, + }, + }) + ); +} diff --git a/lib/retention/scheduler.ts b/lib/retention/scheduler.ts new file mode 100644 index 0000000..3187789 --- /dev/null +++ b/lib/retention/scheduler.ts @@ -0,0 +1,128 @@ +/** + * lib/retention/scheduler.ts + * + * Wraps `pruneOldData` in a node-cron job. Reads configuration from + * environment variables so the schedule and thresholds can be tuned + * without code changes. + * + * Environment variables (all optional with documented defaults): + * + * RETENTION_ENABLED — set to "false" to disable the scheduler + * entirely. Any other value (or unset) is + * treated as enabled. Default: true. + * + * RETENTION_DAYS — rows older than this many days are pruned. + * Default: 180. + * + * RETENTION_CRON_SCHEDULE — node-cron expression for when to run. + * Default: "0 3 * * *" (daily at 03:00 UTC) + * + * ARCHIVE_BATCH_SIZE — maximum rows deleted per run across all + * tables. Default: 1000. + * + * Call `startRetentionScheduler()` once at server startup. It returns the + * scheduled task so callers can stop it (e.g. during tests or graceful + * shutdown). + */ + +import cron from "node-cron"; +import { db } from "../db/client"; +import { pruneOldData, logPruneResult } from "./pruner"; + +// --------------------------------------------------------------------------- +// Config helpers +// --------------------------------------------------------------------------- + +/** Parse an integer env var, falling back to `defaultValue` on invalid input. */ +function envInt(name: string, defaultValue: number): number { + const raw = process.env[name]; + if (!raw) return defaultValue; + const parsed = parseInt(raw, 10); + return Number.isFinite(parsed) ? parsed : defaultValue; +} + +/** Returns true unless RETENTION_ENABLED is explicitly set to "false". */ +function isRetentionEnabled(): boolean { + const val = process.env.RETENTION_ENABLED; + return val !== "false"; +} + +// --------------------------------------------------------------------------- +// Scheduler +// --------------------------------------------------------------------------- + +/** + * Start the retention scheduler. + * + * - When `RETENTION_ENABLED=false` this is a no-op; it logs a single info + * line and returns undefined so callers do not need to branch. + * - Otherwise, schedules a node-cron job according to + * `RETENTION_CRON_SCHEDULE` (default: every day at 03:00 UTC). + * + * @returns The cron.ScheduledTask, or undefined when retention is disabled. + */ +export function startRetentionScheduler(): cron.ScheduledTask | undefined { + if (!isRetentionEnabled()) { + console.log( + JSON.stringify({ + level: "info", + msg: "retention scheduler disabled (RETENTION_ENABLED=false)", + }) + ); + return undefined; + } + + const retentionDays = envInt("RETENTION_DAYS", 180); + const batchCap = envInt("ARCHIVE_BATCH_SIZE", 1000); + const schedule = process.env.RETENTION_CRON_SCHEDULE ?? "0 3 * * *"; + + if (!cron.validate(schedule)) { + console.error( + JSON.stringify({ + level: "error", + msg: `retention scheduler: invalid cron expression "${schedule}" — scheduler not started`, + }) + ); + return undefined; + } + + console.log( + JSON.stringify({ + level: "info", + msg: "retention scheduler started", + schedule, + retentionDays, + batchCap, + }) + ); + + const task = cron.schedule(schedule, async () => { + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + + console.log( + JSON.stringify({ + level: "info", + msg: "retention run starting", + cutoffDate: cutoffDate.toISOString(), + retentionDays, + batchCap, + }) + ); + + try { + const result = await pruneOldData({ db, cutoffDate, batchCap }); + logPruneResult(result); + } catch (err) { + console.error( + JSON.stringify({ + level: "error", + msg: "retention run failed", + error: err instanceof Error ? err.message : String(err), + }) + ); + } + }); + + return task; +} diff --git a/package-lock.json b/package-lock.json index 3b3ce1e..ec3f40e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5234,6 +5234,7 @@ "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -5353,7 +5354,8 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/bull": { "version": "4.10.4", @@ -10959,6 +10961,7 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -12236,6 +12239,7 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -12251,6 +12255,7 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -12569,7 +12574,8 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-kapsule": { "version": "2.6.0", diff --git a/package.json b/package.json index 0755e9a..14ff23b 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,9 @@ "db:migrate": "prisma migrate dev", "db:generate": "prisma generate", "db:seed": "ts-node prisma/seed.ts", - "db:studio": "prisma studio" + "db:studio": "prisma studio", + "retention": "tsx scripts/retention.ts", + "retention:dry-run": "tsx scripts/retention.ts --dry-run" }, "dependencies": { "@clickhouse/client": "^1.0.0", diff --git a/scripts/retention.ts b/scripts/retention.ts new file mode 100644 index 0000000..3ab0763 --- /dev/null +++ b/scripts/retention.ts @@ -0,0 +1,99 @@ +#!/usr/bin/env tsx +/** + * scripts/retention.ts + * + * CLI script for manually running the retention pruner. + * + * Usage: + * npm run retention # live delete + * npm run retention:dry-run # report only, no deletions + * + * Or directly with tsx: + * tsx scripts/retention.ts [--dry-run] [--days ] [--batch-cap ] + * + * Flags: + * --dry-run Report what would be deleted without touching the DB. + * --days Retention threshold in days (default: $RETENTION_DAYS ?? 180). + * --batch-cap Max rows deleted per run (default: $ARCHIVE_BATCH_SIZE ?? 1000). + */ + +import "dotenv/config"; +import { Command } from "commander"; +import { db } from "../lib/db/client"; +import { pruneOldData, logPruneResult } from "../lib/retention/pruner"; + +const program = new Command(); + +program + .name("retention") + .description("Manually prune old Event, DeadLetterEvent, and WebhookDelivery rows.") + .option("--dry-run", "Report intended deletions without deleting anything.", false) + .option( + "--days ", + "Delete rows older than this many days.", + String(parseInt(process.env.RETENTION_DAYS ?? "180", 10)) + ) + .option( + "--batch-cap ", + "Maximum rows to delete across all tables per run.", + String(parseInt(process.env.ARCHIVE_BATCH_SIZE ?? "1000", 10)) + ) + .action(async (opts: { dryRun: boolean; days: string; batchCap: string }) => { + const dryRun = opts.dryRun; + const days = parseInt(opts.days, 10); + const batchCap = parseInt(opts.batchCap, 10); + + if (!Number.isFinite(days) || days < 0) { + console.error(`--days must be a non-negative integer (got: ${opts.days})`); + process.exit(1); + } + if (!Number.isFinite(batchCap) || batchCap <= 0) { + console.error(`--batch-cap must be a positive integer (got: ${opts.batchCap})`); + process.exit(1); + } + + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - days); + + console.log( + JSON.stringify({ + level: "info", + msg: dryRun ? "retention dry-run starting" : "retention run starting", + cutoffDate: cutoffDate.toISOString(), + retentionDays: days, + batchCap, + }) + ); + + try { + const result = await pruneOldData({ db, cutoffDate, batchCap, dryRun }); + logPruneResult(result); + + if (dryRun) { + console.log( + `\nDry-run summary: ${result.totalEligible} rows eligible, ` + + `${result.totalDeleted} would be deleted (cap: ${batchCap}).\n` + + "Run without --dry-run to apply." + ); + } else { + console.log( + `\nDone: ${result.totalDeleted} rows deleted ` + + `(${result.totalEligible} were eligible, cap: ${batchCap}).` + ); + } + } catch (err) { + console.error( + JSON.stringify({ + level: "error", + msg: "retention script failed", + error: err instanceof Error ? err.message : String(err), + stack: err instanceof Error ? err.stack : undefined, + }) + ); + process.exit(1); + } finally { + await db.$disconnect(); + } + }); + +program.parse(process.argv); diff --git a/server.ts b/server.ts index 544cd17..aaa1320 100644 --- a/server.ts +++ b/server.ts @@ -43,6 +43,9 @@ const handle = app.getRequestHandler(); app.prepare().then(async () => { await startTelemetry(); + // Start the data-retention scheduler (no-op when RETENTION_ENABLED=false). + startRetentionScheduler(); + const httpServer = createServer((req, res) => { applyContentSecurityPolicy(res); const parsedUrl = parse(req.url ?? "/", true);