From 878e1288d547fed9f458bf998b3cadec7db68718 Mon Sep 17 00:00:00 2001 From: Harry Randall Date: Tue, 22 Sep 2026 12:08:44 +1000 Subject: [PATCH 1/2] feat: replace catalogue imports with record syncs --- apps/web/.env.example | 22 +- apps/web/app/admin/courses/imports/error.tsx | 2 - .../web/app/admin/courses/imports/loading.tsx | 5 - apps/web/app/admin/courses/imports/page.tsx | 8 - apps/web/app/admin/majors/imports/error.tsx | 2 - apps/web/app/admin/majors/imports/loading.tsx | 5 - apps/web/app/admin/majors/imports/page.tsx | 8 - apps/web/app/admin/minors/imports/error.tsx | 2 - apps/web/app/admin/minors/imports/loading.tsx | 5 - apps/web/app/admin/minors/imports/page.tsx | 8 - apps/web/app/admin/page.tsx | 4 +- .../app/admin/programmes/imports/error.tsx | 2 - .../app/admin/programmes/imports/loading.tsx | 5 - .../web/app/admin/programmes/imports/page.tsx | 8 - .../admin/specialisations/imports/error.tsx | 2 - .../admin/specialisations/imports/loading.tsx | 5 - .../admin/specialisations/imports/page.tsx | 10 - .../api/admin/catalogue-directory/route.ts | 4 +- .../artifacts/[artifactId]/route.ts | 47 - .../app/api/admin/catalogue-imports/route.ts | 106 -- .../app/api/admin/catalogue-syncs/route.ts | 73 ++ .../app/api/queues/catalogue-import/route.ts | 7 - .../app/api/queues/catalogue-sync/route.ts | 7 + apps/web/lib/admin/settings-actions.ts | 10 +- apps/web/lib/assistant/model-actions.ts | 4 +- apps/web/lib/auth/viewer.ts | 4 +- apps/web/lib/catalogue-import/apply-review.ts | 179 --- apps/web/lib/catalogue-import/directory.ts | 52 +- apps/web/lib/catalogue-import/import-store.ts | 697 ----------- .../catalogue-import/kinds/course/adapter.ts | 4 +- .../kinds/structure/adapter.ts | 4 +- apps/web/lib/catalogue-import/queue.ts | 287 ----- .../lib/catalogue-import/version-content.ts | 7 +- .../artifact-store.ts | 64 +- .../kind-adapter.ts | 23 +- .../persist-source-version.ts} | 287 ++--- .../process-sync.ts} | 244 ++-- apps/web/lib/catalogue-sync/sync-queue.ts | 195 +++ apps/web/lib/catalogue-sync/sync-service.ts | 52 + apps/web/lib/catalogue-sync/sync-store.ts | 463 +++++++ apps/web/lib/catalogue/drafts.ts | 55 +- .../lib/coursemap/admin-catalogue-actions.ts | 122 +- .../lib/coursemap/admin-catalogue-record.ts | 164 +-- apps/web/lib/coursemap/admin-catalogue.ts | 827 +++---------- apps/web/lib/coursemap/catalogue-kinds.ts | 162 +-- .../lib/coursemap/requisite-search-actions.ts | 4 +- apps/web/playwright/catalogue-review.spec.ts | 2 +- .../scripts/catalogue/lib/local-database.mjs | 14 +- apps/web/scripts/fixtures/local-preview.sql | 20 +- apps/web/tests/breadcrumbs.test.tsx | 16 +- ...atalogue-import-pipeline-database.test.mjs | 463 ------- apps/web/tests/catalogue-review-state.test.ts | 276 ----- apps/web/tests/catalogue-sync-button.test.tsx | 61 + .../tests/catalogue-sync-database.test.mjs | 259 ++++ apps/web/tests/catalogue-sync-queue.test.ts | 34 + apps/web/tests/notifications-menu.test.tsx | 14 +- apps/web/tests/value-diff.test.tsx | 69 -- apps/web/types/database.ts | 887 +++++++------ apps/web/ui/admin/catalogue/anu-source.ts | 2 +- apps/web/ui/admin/catalogue/artefact-data.ts | 49 - .../ui/admin/catalogue/artefact-viewer.tsx | 162 --- .../ui/admin/catalogue/artefact-viewport.tsx | 31 - .../admin/catalogue/catalogue-directory.tsx | 31 +- .../ui/admin/catalogue/catalogue-pages.tsx | 95 +- .../web/ui/admin/catalogue/catalogue-tabs.tsx | 63 - .../ui/admin/catalogue/catalogue-value.tsx | 44 - apps/web/ui/admin/catalogue/import-runs.tsx | 782 ------------ apps/web/ui/admin/catalogue/record-header.tsx | 31 +- .../web/ui/admin/catalogue/record-history.tsx | 12 +- apps/web/ui/admin/catalogue/record-page.tsx | 46 +- apps/web/ui/admin/catalogue/review-panel.tsx | 1100 ----------------- apps/web/ui/admin/catalogue/review-state.ts | 361 ------ .../ui/admin/catalogue/source-code.module.css | 48 - apps/web/ui/admin/catalogue/source-code.tsx | 46 - apps/web/ui/admin/catalogue/sync-button.tsx | 79 ++ apps/web/ui/admin/catalogue/use-artefact.ts | 56 - apps/web/ui/admin/catalogue/value-diff.tsx | 192 --- .../web/ui/admin/catalogue/workflow-badge.tsx | 114 -- apps/web/ui/shell/breadcrumbs.tsx | 2 +- apps/web/ui/shell/notifications-menu.tsx | 4 +- apps/web/vercel.json | 4 +- docs/architecture.md | 59 +- docs/catalogue-operations.md | 155 +-- docs/conventions.md | 2 +- .../20260922100000_catalogue_record_sync.sql | 488 ++++++++ .../database/catalogue_legacy_cleanup.sql | 10 +- supabase/tests/database/catalogue_sync.sql | 79 ++ supabase/tests/database/notifications.sql | 128 +- 88 files changed, 3056 insertions(+), 7591 deletions(-) delete mode 100644 apps/web/app/admin/courses/imports/error.tsx delete mode 100644 apps/web/app/admin/courses/imports/loading.tsx delete mode 100644 apps/web/app/admin/courses/imports/page.tsx delete mode 100644 apps/web/app/admin/majors/imports/error.tsx delete mode 100644 apps/web/app/admin/majors/imports/loading.tsx delete mode 100644 apps/web/app/admin/majors/imports/page.tsx delete mode 100644 apps/web/app/admin/minors/imports/error.tsx delete mode 100644 apps/web/app/admin/minors/imports/loading.tsx delete mode 100644 apps/web/app/admin/minors/imports/page.tsx delete mode 100644 apps/web/app/admin/programmes/imports/error.tsx delete mode 100644 apps/web/app/admin/programmes/imports/loading.tsx delete mode 100644 apps/web/app/admin/programmes/imports/page.tsx delete mode 100644 apps/web/app/admin/specialisations/imports/error.tsx delete mode 100644 apps/web/app/admin/specialisations/imports/loading.tsx delete mode 100644 apps/web/app/admin/specialisations/imports/page.tsx delete mode 100644 apps/web/app/api/admin/catalogue-imports/artifacts/[artifactId]/route.ts delete mode 100644 apps/web/app/api/admin/catalogue-imports/route.ts create mode 100644 apps/web/app/api/admin/catalogue-syncs/route.ts delete mode 100644 apps/web/app/api/queues/catalogue-import/route.ts create mode 100644 apps/web/app/api/queues/catalogue-sync/route.ts delete mode 100644 apps/web/lib/catalogue-import/apply-review.ts delete mode 100644 apps/web/lib/catalogue-import/import-store.ts delete mode 100644 apps/web/lib/catalogue-import/queue.ts rename apps/web/lib/{catalogue-import => catalogue-sync}/artifact-store.ts (71%) rename apps/web/lib/{catalogue-import => catalogue-sync}/kind-adapter.ts (78%) rename apps/web/lib/{catalogue-import/persist-version.ts => catalogue-sync/persist-source-version.ts} (76%) rename apps/web/lib/{catalogue-import/process-target.ts => catalogue-sync/process-sync.ts} (71%) create mode 100644 apps/web/lib/catalogue-sync/sync-queue.ts create mode 100644 apps/web/lib/catalogue-sync/sync-service.ts create mode 100644 apps/web/lib/catalogue-sync/sync-store.ts delete mode 100644 apps/web/tests/catalogue-import-pipeline-database.test.mjs delete mode 100644 apps/web/tests/catalogue-review-state.test.ts create mode 100644 apps/web/tests/catalogue-sync-button.test.tsx create mode 100644 apps/web/tests/catalogue-sync-database.test.mjs create mode 100644 apps/web/tests/catalogue-sync-queue.test.ts delete mode 100644 apps/web/tests/value-diff.test.tsx delete mode 100644 apps/web/ui/admin/catalogue/artefact-data.ts delete mode 100644 apps/web/ui/admin/catalogue/artefact-viewer.tsx delete mode 100644 apps/web/ui/admin/catalogue/artefact-viewport.tsx delete mode 100644 apps/web/ui/admin/catalogue/catalogue-tabs.tsx delete mode 100644 apps/web/ui/admin/catalogue/catalogue-value.tsx delete mode 100644 apps/web/ui/admin/catalogue/import-runs.tsx delete mode 100644 apps/web/ui/admin/catalogue/review-panel.tsx delete mode 100644 apps/web/ui/admin/catalogue/review-state.ts delete mode 100644 apps/web/ui/admin/catalogue/source-code.module.css delete mode 100644 apps/web/ui/admin/catalogue/source-code.tsx create mode 100644 apps/web/ui/admin/catalogue/sync-button.tsx delete mode 100644 apps/web/ui/admin/catalogue/use-artefact.ts delete mode 100644 apps/web/ui/admin/catalogue/value-diff.tsx delete mode 100644 apps/web/ui/admin/catalogue/workflow-badge.tsx create mode 100644 supabase/migrations/20260922100000_catalogue_record_sync.sql create mode 100644 supabase/tests/database/catalogue_sync.sql diff --git a/apps/web/.env.example b/apps/web/.env.example index 505d18a4..5b6c0b1a 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -32,10 +32,10 @@ NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY= NEXT_PUBLIC_SITE_URL=http://localhost:3000 # ============================================================================= -# 2. Optional feature: catalogue imports +# 2. Optional feature: catalogue synchronisation # ============================================================================= # Ordinary catalogue browsing and student planning do not need these credentials. -# Configure them when running the corresponding import operations below. +# Configure them when checking detailed ANU source material. # Privileged Supabase server key for local preview and browser test user setup. # dev:local supplies the local server key automatically. @@ -50,18 +50,18 @@ NEXT_PUBLIC_SITE_URL=http://localhost:3000 # Used by: lib/catalogue-import/openrouter.ts. # OPENROUTER_API_KEY= -# Direct Postgres connection for the import worker on Vercel. Development and +# Direct Postgres connection for the sync worker on Vercel. Development and # the local test server use the loopback database automatically, so leave this # unset locally. Use the Supabase connection pooler URL for the hosted project. -# Used by: lib/catalogue-import/import-store.ts. -# COURSEMAP_IMPORT_DATABASE_URL= +# Used by: lib/catalogue-sync/sync-store.ts. +# COURSEMAP_SYNC_DATABASE_URL= -# Set to exactly "true" to publish import targets to Vercel Queues -# (topic catalogue-import-v1, consumed by app/api/queues/catalogue-import). -# Anything else processes targets in the web process after the request, which -# is how local development completes an import without a queue. -# Used by: lib/catalogue-import/queue.ts. -# COURSEMAP_QUEUE_IMPORTS_ENABLED=false +# Set to exactly "true" to publish catalogue syncs to Vercel Queues +# (topic catalogue-sync-v1, consumed by app/api/queues/catalogue-sync). +# Anything else processes the record sync in the web process after the request, +# which is how local development completes a sync without a queue. +# Used by: lib/catalogue-sync/sync-queue.ts. +# COURSEMAP_QUEUE_SYNCS_ENABLED=false # ============================================================================= # 3. Optional overrides: Room Finder services diff --git a/apps/web/app/admin/courses/imports/error.tsx b/apps/web/app/admin/courses/imports/error.tsx deleted file mode 100644 index 22a72703..00000000 --- a/apps/web/app/admin/courses/imports/error.tsx +++ /dev/null @@ -1,2 +0,0 @@ -"use client"; -export { CatalogueError as default } from "@/ui/admin/catalogue-table/catalogue-error"; diff --git a/apps/web/app/admin/courses/imports/loading.tsx b/apps/web/app/admin/courses/imports/loading.tsx deleted file mode 100644 index 49ab30e6..00000000 --- a/apps/web/app/admin/courses/imports/loading.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ImportRecordsLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; - -export default function Loading() { - return ; -} diff --git a/apps/web/app/admin/courses/imports/page.tsx b/apps/web/app/admin/courses/imports/page.tsx deleted file mode 100644 index 2eb638ee..00000000 --- a/apps/web/app/admin/courses/imports/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; -import { CatalogueImportsPage } from "@/ui/admin/catalogue/catalogue-pages"; - -export const dynamic = "force-dynamic"; - -export default function Page({ searchParams }: { searchParams: SearchParams }) { - return ; -} diff --git a/apps/web/app/admin/majors/imports/error.tsx b/apps/web/app/admin/majors/imports/error.tsx deleted file mode 100644 index 22a72703..00000000 --- a/apps/web/app/admin/majors/imports/error.tsx +++ /dev/null @@ -1,2 +0,0 @@ -"use client"; -export { CatalogueError as default } from "@/ui/admin/catalogue-table/catalogue-error"; diff --git a/apps/web/app/admin/majors/imports/loading.tsx b/apps/web/app/admin/majors/imports/loading.tsx deleted file mode 100644 index 49ab30e6..00000000 --- a/apps/web/app/admin/majors/imports/loading.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ImportRecordsLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; - -export default function Loading() { - return ; -} diff --git a/apps/web/app/admin/majors/imports/page.tsx b/apps/web/app/admin/majors/imports/page.tsx deleted file mode 100644 index aeea62a0..00000000 --- a/apps/web/app/admin/majors/imports/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; -import { CatalogueImportsPage } from "@/ui/admin/catalogue/catalogue-pages"; - -export const dynamic = "force-dynamic"; - -export default function Page({ searchParams }: { searchParams: SearchParams }) { - return ; -} diff --git a/apps/web/app/admin/minors/imports/error.tsx b/apps/web/app/admin/minors/imports/error.tsx deleted file mode 100644 index 22a72703..00000000 --- a/apps/web/app/admin/minors/imports/error.tsx +++ /dev/null @@ -1,2 +0,0 @@ -"use client"; -export { CatalogueError as default } from "@/ui/admin/catalogue-table/catalogue-error"; diff --git a/apps/web/app/admin/minors/imports/loading.tsx b/apps/web/app/admin/minors/imports/loading.tsx deleted file mode 100644 index 49ab30e6..00000000 --- a/apps/web/app/admin/minors/imports/loading.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ImportRecordsLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; - -export default function Loading() { - return ; -} diff --git a/apps/web/app/admin/minors/imports/page.tsx b/apps/web/app/admin/minors/imports/page.tsx deleted file mode 100644 index 49e8131c..00000000 --- a/apps/web/app/admin/minors/imports/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; -import { CatalogueImportsPage } from "@/ui/admin/catalogue/catalogue-pages"; - -export const dynamic = "force-dynamic"; - -export default function Page({ searchParams }: { searchParams: SearchParams }) { - return ; -} diff --git a/apps/web/app/admin/page.tsx b/apps/web/app/admin/page.tsx index 7ae3f438..595d17a7 100644 --- a/apps/web/app/admin/page.tsx +++ b/apps/web/app/admin/page.tsx @@ -2,7 +2,7 @@ import { UsersRound } from "lucide-react"; import { ImportModelCard } from "@/ui/admin/imports/import-model-card"; import { loadImportModelSetting } from "@/lib/admin/settings"; import { loadAdminUserSummary } from "@/lib/admin/users"; -import { canManageCourseImports } from "@/lib/auth/viewer"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; import { AppShell } from "@/ui/shell"; import { StatTile } from "@/ui/common/stat-tile"; @@ -12,7 +12,7 @@ export default async function AdminOverviewPage() { const [users, importModel, canManageImports] = await Promise.all([ loadAdminUserSummary(), loadImportModelSetting(), - canManageCourseImports(), + canManageCatalogueSources(), ]); return ( diff --git a/apps/web/app/admin/programmes/imports/error.tsx b/apps/web/app/admin/programmes/imports/error.tsx deleted file mode 100644 index 22a72703..00000000 --- a/apps/web/app/admin/programmes/imports/error.tsx +++ /dev/null @@ -1,2 +0,0 @@ -"use client"; -export { CatalogueError as default } from "@/ui/admin/catalogue-table/catalogue-error"; diff --git a/apps/web/app/admin/programmes/imports/loading.tsx b/apps/web/app/admin/programmes/imports/loading.tsx deleted file mode 100644 index 49ab30e6..00000000 --- a/apps/web/app/admin/programmes/imports/loading.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ImportRecordsLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; - -export default function Loading() { - return ; -} diff --git a/apps/web/app/admin/programmes/imports/page.tsx b/apps/web/app/admin/programmes/imports/page.tsx deleted file mode 100644 index 33900b37..00000000 --- a/apps/web/app/admin/programmes/imports/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; -import { CatalogueImportsPage } from "@/ui/admin/catalogue/catalogue-pages"; - -export const dynamic = "force-dynamic"; - -export default function Page({ searchParams }: { searchParams: SearchParams }) { - return ; -} diff --git a/apps/web/app/admin/specialisations/imports/error.tsx b/apps/web/app/admin/specialisations/imports/error.tsx deleted file mode 100644 index 22a72703..00000000 --- a/apps/web/app/admin/specialisations/imports/error.tsx +++ /dev/null @@ -1,2 +0,0 @@ -"use client"; -export { CatalogueError as default } from "@/ui/admin/catalogue-table/catalogue-error"; diff --git a/apps/web/app/admin/specialisations/imports/loading.tsx b/apps/web/app/admin/specialisations/imports/loading.tsx deleted file mode 100644 index 49ab30e6..00000000 --- a/apps/web/app/admin/specialisations/imports/loading.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { ImportRecordsLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; - -export default function Loading() { - return ; -} diff --git a/apps/web/app/admin/specialisations/imports/page.tsx b/apps/web/app/admin/specialisations/imports/page.tsx deleted file mode 100644 index 2bc631d2..00000000 --- a/apps/web/app/admin/specialisations/imports/page.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; -import { CatalogueImportsPage } from "@/ui/admin/catalogue/catalogue-pages"; - -export const dynamic = "force-dynamic"; - -export default function Page({ searchParams }: { searchParams: SearchParams }) { - return ( - - ); -} diff --git a/apps/web/app/api/admin/catalogue-directory/route.ts b/apps/web/app/api/admin/catalogue-directory/route.ts index 33b9923f..d758d7c7 100644 --- a/apps/web/app/api/admin/catalogue-directory/route.ts +++ b/apps/web/app/api/admin/catalogue-directory/route.ts @@ -1,4 +1,4 @@ -import { canManageCourseImports } from "@/lib/auth/viewer"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; import { refreshCatalogueDirectory } from "@/lib/catalogue-import/directory"; import { isCatalogueKind } from "@/lib/catalogue/content"; @@ -20,7 +20,7 @@ function eventResponse(data: unknown, status: number) { /** Streams directory refresh progress as server-sent events. */ export async function POST(request: Request) { - if (!(await canManageCourseImports())) { + if (!(await canManageCatalogueSources())) { return eventResponse( { type: "error", message: "Import permission is required." }, 403, diff --git a/apps/web/app/api/admin/catalogue-imports/artifacts/[artifactId]/route.ts b/apps/web/app/api/admin/catalogue-imports/artifacts/[artifactId]/route.ts deleted file mode 100644 index c5f0ae31..00000000 --- a/apps/web/app/api/admin/catalogue-imports/artifacts/[artifactId]/route.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { canManageCourseImports } from "@/lib/auth/viewer"; -import { - type ImportArtifactLocator, - readImportArtifact, -} from "@/lib/catalogue-import/artifact-store"; -import { createClient } from "@/lib/supabase/server"; - -export const runtime = "nodejs"; - -/** Serves a stored artefact as inert text for the admin viewer. */ -export async function GET( - _request: Request, - { params }: { params: Promise<{ artifactId: string }> }, -) { - if (!(await canManageCourseImports())) { - return new Response("Import permission is required.", { status: 403 }); - } - const { artifactId } = await params; - const supabase = await createClient(); - const { data, error } = await supabase - .from("catalogue_import_artifacts") - .select("media_type,content_sha256,byte_size,storage_bucket,storage_path") - .eq("id", artifactId) - .maybeSingle(); - if (error || !data) - return new Response("Artefact not found.", { status: 404 }); - try { - const body = await readImportArtifact({ - artifact: { - bucket: data.storage_bucket as ImportArtifactLocator["bucket"], - path: data.storage_path, - mediaType: data.media_type, - contentSha256: data.content_sha256, - byteSize: data.byte_size, - }, - }); - return new Response(body, { - headers: { - "content-type": "text/plain; charset=utf-8", - "cache-control": "private, no-store", - "x-content-type-options": "nosniff", - }, - }); - } catch { - return new Response("The artefact could not be read.", { status: 502 }); - } -} diff --git a/apps/web/app/api/admin/catalogue-imports/route.ts b/apps/web/app/api/admin/catalogue-imports/route.ts deleted file mode 100644 index d6b3cb81..00000000 --- a/apps/web/app/api/admin/catalogue-imports/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { after } from "next/server"; -import { canManageCourseImports } from "@/lib/auth/viewer"; -import { - dispatchImportRun, - processImportRunInline, -} from "@/lib/catalogue-import/queue"; -import { adapterForKind } from "@/lib/catalogue-import/process-target"; -import { isCatalogueKind } from "@/lib/catalogue/content"; -import { loadImportModelSetting } from "@/lib/admin/settings"; -import { createClient } from "@/lib/supabase/server"; - -export const runtime = "nodejs"; -export const maxDuration = 60; - -type StartRequest = { - kind?: unknown; - academicYear?: unknown; - codes?: unknown; - requestedModel?: unknown; -}; - -function json(data: unknown, status = 200) { - return Response.json(data, { status }); -} - -/** Creates a run and starts processing. Inline runs continue after the response. */ -export async function POST(request: Request) { - if (!(await canManageCourseImports())) { - return json({ error: "Import permission is required." }, 403); - } - let payload: StartRequest; - try { - payload = (await request.json()) as StartRequest; - } catch { - return json({ error: "Invalid import request." }, 400); - } - const kind = payload.kind; - if (!isCatalogueKind(kind)) { - return json({ error: "The catalogue kind is not recognised." }, 400); - } - const academicYear = Number(payload.academicYear); - if (!Number.isInteger(academicYear)) { - return json({ error: "The academic year is required." }, 400); - } - const codes = Array.isArray(payload.codes) - ? payload.codes.filter((code): code is string => typeof code === "string") - : []; - if (codes.length === 0) { - return json({ error: "Select at least one record to import." }, 400); - } - const setting = await loadImportModelSetting(); - const requestedModel = - typeof payload.requestedModel === "string" && payload.requestedModel.trim() - ? payload.requestedModel.trim() - : setting.model; - if (!requestedModel) { - return json({ error: "Choose an import model first." }, 400); - } - - const adapter = adapterForKind(kind); - const supabase = await createClient(); - const { data, error } = await supabase.rpc("start_catalogue_import", { - p_academic_year: academicYear, - p_kind: kind, - p_codes: codes, - p_requested_model: requestedModel, - p_parser_version: adapter.parserVersion, - p_prompt_version: adapter.promptVersion, - p_schema_version: adapter.schemaVersion, - }); - if (error) return json({ error: error.message }, 400); - const run = data as { runId: string; targets: Array<{ targetId: string }> }; - const targetIds = run.targets.map((target) => target.targetId); - - const dispatch = await dispatchImportRun({ runId: run.runId, targetIds }); - if (dispatch.mode === "inline") { - after(() => processImportRunInline({ runId: run.runId })); - } - return json({ - runId: run.runId, - targets: targetIds.length, - mode: dispatch.mode, - }); -} - -/** Stops a run's unfinished targets. */ -export async function DELETE(request: Request) { - if (!(await canManageCourseImports())) { - return json({ error: "Import permission is required." }, 403); - } - let payload: { runId?: unknown }; - try { - payload = (await request.json()) as typeof payload; - } catch { - return json({ error: "Invalid request." }, 400); - } - if (typeof payload.runId !== "string") { - return json({ error: "A run identifier is required." }, 400); - } - const supabase = await createClient(); - const { data, error } = await supabase.rpc("cancel_catalogue_import", { - p_run_id: payload.runId, - }); - if (error) return json({ error: error.message }, 400); - return json({ cancelled: data }); -} diff --git a/apps/web/app/api/admin/catalogue-syncs/route.ts b/apps/web/app/api/admin/catalogue-syncs/route.ts new file mode 100644 index 00000000..1a46034d --- /dev/null +++ b/apps/web/app/api/admin/catalogue-syncs/route.ts @@ -0,0 +1,73 @@ +import { after } from "next/server"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { isCatalogueKind } from "@/lib/catalogue/content"; +import { processCatalogueSyncInline } from "@/lib/catalogue-sync/sync-queue"; +import { startCatalogueSync } from "@/lib/catalogue-sync/sync-service"; +import { createClient } from "@/lib/supabase/server"; + +export const runtime = "nodejs"; +export const maxDuration = 60; + +type StartRequest = { recordId?: unknown; kind?: unknown }; + +function json(data: unknown, status = 200) { + return Response.json(data, { status }); +} + +/** Creates one record sync. Inline processing continues after the response. */ +export async function POST(request: Request) { + if (!(await canManageCatalogueSources())) { + return json({ error: "Catalogue sync permission is required." }, 403); + } + let payload: StartRequest; + try { + payload = (await request.json()) as StartRequest; + } catch { + return json({ error: "Invalid sync request." }, 400); + } + const recordId = Number(payload.recordId); + if (!Number.isInteger(recordId) || !isCatalogueKind(payload.kind)) { + return json({ error: "A catalogue record is required." }, 400); + } + try { + const result = await startCatalogueSync({ + recordId, + trigger: "manual", + kind: payload.kind, + }); + if (result.mode === "inline") { + after(() => processCatalogueSyncInline({ syncId: result.syncId })); + } + return json(result); + } catch (error) { + return json( + { + error: + error instanceof Error ? error.message : "The sync could not start.", + }, + 400, + ); + } +} + +/** Stops an unfinished record sync. */ +export async function DELETE(request: Request) { + if (!(await canManageCatalogueSources())) { + return json({ error: "Catalogue sync permission is required." }, 403); + } + let payload: { syncId?: unknown }; + try { + payload = (await request.json()) as typeof payload; + } catch { + return json({ error: "Invalid request." }, 400); + } + if (typeof payload.syncId !== "string") { + return json({ error: "A sync identifier is required." }, 400); + } + const supabase = await createClient(); + const { data, error } = await supabase.rpc("cancel_catalogue_sync", { + p_sync_id: payload.syncId, + }); + if (error) return json({ error: error.message }, 400); + return json({ cancelled: data }); +} diff --git a/apps/web/app/api/queues/catalogue-import/route.ts b/apps/web/app/api/queues/catalogue-import/route.ts deleted file mode 100644 index 0980d0a4..00000000 --- a/apps/web/app/api/queues/catalogue-import/route.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { createImportQueueConsumer } from "@/lib/catalogue-import/queue"; - -export const runtime = "nodejs"; -// Structure imports wait up to 150 s on the model; the budget leaves room to record the outcome. -export const maxDuration = 300; - -export const POST = createImportQueueConsumer(); diff --git a/apps/web/app/api/queues/catalogue-sync/route.ts b/apps/web/app/api/queues/catalogue-sync/route.ts new file mode 100644 index 00000000..e0f673b5 --- /dev/null +++ b/apps/web/app/api/queues/catalogue-sync/route.ts @@ -0,0 +1,7 @@ +import { createSyncQueueConsumer } from "@/lib/catalogue-sync/sync-queue"; + +export const runtime = "nodejs"; +// Structure syncs wait up to 150 s on the model; the budget leaves room to record the outcome. +export const maxDuration = 300; + +export const POST = createSyncQueueConsumer(); diff --git a/apps/web/lib/admin/settings-actions.ts b/apps/web/lib/admin/settings-actions.ts index 09f12347..1b2a4ef2 100644 --- a/apps/web/lib/admin/settings-actions.ts +++ b/apps/web/lib/admin/settings-actions.ts @@ -1,7 +1,7 @@ "use server"; import { revalidatePath } from "next/cache"; -import { canManageCourseImports } from "@/lib/auth/viewer"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; import { createClient } from "@/lib/supabase/server"; import { IMPORT_MODEL_SETTING_KEY } from "@/lib/admin/settings"; import { fetchCatalogueModel } from "@/lib/admin/model-catalogue"; @@ -20,7 +20,7 @@ function refreshImportPages() { export async function setImportModel( model: string, ): Promise { - if (!(await canManageCourseImports())) + if (!(await canManageCatalogueSources())) return { ok: false, model, @@ -61,7 +61,7 @@ export async function saveImportModel( model: string, refreshOnly = false, ): Promise { - if (!(await canManageCourseImports())) + if (!(await canManageCatalogueSources())) return { ok: false, model, @@ -103,7 +103,7 @@ export async function saveImportModel( export async function removeImportModel( model: string, ): Promise { - if (!(await canManageCourseImports())) + if (!(await canManageCatalogueSources())) return { ok: false, model, @@ -139,7 +139,7 @@ export async function setImportModelVisibility( model: string, visible: boolean, ): Promise { - if (!(await canManageCourseImports())) + if (!(await canManageCatalogueSources())) return { ok: false, model, diff --git a/apps/web/lib/assistant/model-actions.ts b/apps/web/lib/assistant/model-actions.ts index e84b304f..e1af6c71 100644 --- a/apps/web/lib/assistant/model-actions.ts +++ b/apps/web/lib/assistant/model-actions.ts @@ -1,6 +1,6 @@ "use server"; -import { canManageCourseImports } from "@/lib/auth/viewer"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; import { loadImportModelSetting } from "@/lib/admin/settings"; import type { ImportModel } from "@/lib/admin/import-model"; @@ -9,7 +9,7 @@ export async function loadAssistantModels(): Promise<{ defaultModel: string; error: string | null; }> { - if (!(await canManageCourseImports())) { + if (!(await canManageCatalogueSources())) { return { models: [], defaultModel: "", diff --git a/apps/web/lib/auth/viewer.ts b/apps/web/lib/auth/viewer.ts index 31bb7026..f8d095e3 100644 --- a/apps/web/lib/auth/viewer.ts +++ b/apps/web/lib/auth/viewer.ts @@ -73,8 +73,8 @@ export async function canManageCatalogueImports() { return currentUserHasPermission("imports.manage"); } -/** Course-only name for the shared import-worker permission. */ -export async function canManageCourseImports() { +/** Check the shared permission for catalogue sources and extraction models. */ +export async function canManageCatalogueSources() { return currentUserHasPermission("imports.manage"); } diff --git a/apps/web/lib/catalogue-import/apply-review.ts b/apps/web/lib/catalogue-import/apply-review.ts deleted file mode 100644 index 37185fa5..00000000 --- a/apps/web/lib/catalogue-import/apply-review.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { applyAcceptedChanges } from "./changes.ts"; -import { type ImportSql, withImportDatabaseClient } from "./import-store.ts"; -import { insertVersionContent } from "./persist-version.ts"; -import { - contentHashForCatalogueContent, - readVersionContent, -} from "./version-content.ts"; -import type { CatalogueKind } from "../catalogue/content.ts"; - -export class ApplyReviewError extends Error { - readonly code: string; - - constructor(message: string, code: string) { - super(message); - this.name = "ApplyReviewError"; - this.code = code; - } -} - -/** - * Turns a reviewed target into a new immutable version: the baseline with every accepted - * change applied, rejected changes left as they were. Refuses when changes - * are still open or the record gained another meaningful version since the import ran. - */ -export async function applyImportReview({ - targetId, - userId, - sql, -}: { - targetId: string; - userId: string; - sql?: ImportSql; -}) { - const work = async (client: ImportSql) => - client.begin(async (tx) => { - const [target] = await tx` - select targets.id, targets.kind, targets.record_id, targets.academic_year_id, - targets.baseline_version_id, targets.candidate_version_id, targets.source_page_id, - targets.status, targets.applied_at, - item_years.archived_at, current_version.id as current_version_id - from public.catalogue_import_targets as targets - join public.catalogue_records as item_years on item_years.id = targets.record_id - left join lateral ( - select versions.id - from public.catalogue_versions as versions - left join public.catalogue_import_targets as version_targets - on version_targets.id = versions.import_target_id - where versions.record_id = item_years.id - and versions.sealed_at is not null - and ( - versions.import_target_id is null - or version_targets.applied_version_id = versions.id - ) - order by versions.created_at desc, versions.id desc - limit 1 - ) as current_version on true - where targets.id = ${targetId}::uuid - for update of targets, item_years - `; - if (!target) - throw new ApplyReviewError( - "The import target no longer exists.", - "P0002", - ); - if (target.applied_at) - throw new ApplyReviewError( - "This review has already been applied.", - "55000", - ); - if (target.status !== "ready" || !target.candidate_version_id) { - throw new ApplyReviewError( - "Only targets that are ready for review can be applied.", - "55000", - ); - } - if (target.archived_at) - throw new ApplyReviewError("The record is archived.", "55000"); - - const currentBaseline = - target.current_version_id === null - ? null - : Number(target.current_version_id); - const importBaseline = - target.baseline_version_id === null - ? null - : Number(target.baseline_version_id); - if (currentBaseline !== importBaseline) { - throw new ApplyReviewError( - "The record changed since this import ran. Start a new import to review against the current version.", - "STALE_BASELINE", - ); - } - - const entries = await tx` - select field_path, status from public.catalogue_import_changes - where target_id = ${targetId}::uuid and entry_kind = 'change' - `; - if (entries.some((entry) => entry.status === "open")) { - throw new ApplyReviewError( - "Accept or reject every change before applying.", - "55000", - ); - } - const accepted = new Set( - entries - .filter((entry) => entry.status === "accepted") - .map((entry) => String(entry.field_path)), - ); - const candidateId = Number(target.candidate_version_id); - const kind = target.kind as CatalogueKind; - const recordId = Number(target.record_id); - - // Everything accepted: the candidate itself becomes the applied version. - if (importBaseline === null || accepted.size === entries.length) { - await tx` - update public.catalogue_import_targets - set applied_version_id = ${candidateId}, applied_at = now() - where id = ${targetId}::uuid - `; - return { versionId: candidateId, reusedCandidate: true }; - } - - const baselineWrite = await readVersionContent(tx, importBaseline); - const candidateWrite = await readVersionContent(tx, candidateId); - if (!baselineWrite || !candidateWrite) { - throw new ApplyReviewError( - "The snapshots for this review could not be read.", - "P0002", - ); - } - // Nothing accepted: the baseline stays current and the review is closed. - if (accepted.size === 0) { - await tx` - update public.catalogue_import_targets - set applied_version_id = ${importBaseline}, applied_at = now() - where id = ${targetId}::uuid - `; - return { versionId: importBaseline, reusedCandidate: false }; - } - - const merged = applyAcceptedChanges( - baselineWrite, - candidateWrite, - accepted, - ); - merged.contentHash = contentHashForCatalogueContent(merged); - const [snapshot] = await tx` - insert into public.catalogue_versions ( - record_id, kind, academic_year_id, origin, based_on_version_id, source_page_id, - content_hash, import_target_id, created_by - ) values ( - ${recordId}, ${kind}, ${Number(target.academic_year_id)}, 'import', ${importBaseline}, - ${target.source_page_id}, ${merged.contentHash}, ${targetId}::uuid, ${userId}::uuid - ) - returning id - `; - const snapshotId = Number(snapshot.id); - await insertVersionContent(tx, { - snapshotId, - kind, - academicYearId: Number(target.academic_year_id), - sourcePageId: - target.source_page_id === null ? null : Number(target.source_page_id), - write: merged, - }); - await tx` - update public.catalogue_versions - set sealed_at = greatest(statement_timestamp(), created_at) - where id = ${snapshotId} - `; - await tx` - update public.catalogue_import_targets - set applied_version_id = ${snapshotId}, applied_at = now() - where id = ${targetId}::uuid - `; - return { versionId: snapshotId, reusedCandidate: false }; - }); - return sql ? work(sql) : withImportDatabaseClient(work); -} diff --git a/apps/web/lib/catalogue-import/directory.ts b/apps/web/lib/catalogue-import/directory.ts index b63a89c4..8bcbf5f7 100644 --- a/apps/web/lib/catalogue-import/directory.ts +++ b/apps/web/lib/catalogue-import/directory.ts @@ -3,11 +3,10 @@ import { fetchAnuCourseDirectory } from "./anu-course-directory.ts"; import type { ImportDiagnostic } from "./import-source.ts"; import { ensureAnuSourceId, - recordSourcePage, - type ImportSql, - type ImportTransactionSql, - withImportDatabaseClient, -} from "./import-store.ts"; + type SyncSql, + type SyncTransactionSql, + withSyncDatabaseClient, +} from "../catalogue-sync/sync-store.ts"; import type { AcademicStructureKind } from "./kinds/structure/contract.ts"; import { type CatalogueKind, isCatalogueKind } from "../catalogue/content.ts"; @@ -38,6 +37,38 @@ export class DirectoryRefreshError extends Error { } } +async function recordDiscoverySourcePage( + sql: SyncSql, + input: { + sourceId: number; + academicYearId: number; + externalKey: string; + canonicalUrl: string; + mediaType: string; + contentSha256: string; + httpStatus: number | null; + httpEtag: string | null; + sourceLastModified: string | null; + fetchedAt: string; + byteSize: number; + }, +) { + const [row] = await sql` + insert into public.catalogue_source_pages ( + source_id, academic_year_id, kind, external_key, canonical_url, media_type, + content_sha256, http_status, http_etag, source_last_modified, fetched_at, + byte_size, storage_bucket, storage_path + ) values (${input.sourceId}, ${input.academicYearId}, 'directory', + ${input.externalKey}, ${input.canonicalUrl}, ${input.mediaType}, + ${input.contentSha256}, ${input.httpStatus}, ${input.httpEtag}, + ${input.sourceLastModified}, ${input.fetchedAt}, ${input.byteSize}, null, null) + on conflict (source_id, academic_year_id, kind, external_key, content_sha256) + do update set fetched_at = excluded.fetched_at + returning id + `; + return Number(row.id); +} + type DirectoryEntryInput = { code: string; title: string | null; @@ -112,7 +143,7 @@ async function fetchDirectoryEntries( } export async function reconcileCatalogueListings( - tx: ImportTransactionSql, + tx: SyncTransactionSql, { academicYearId, kind, @@ -192,7 +223,7 @@ export async function reconcileCatalogueListings( } async function setDirectoryStatus( - sql: ImportSql | ImportTransactionSql, + sql: SyncSql | SyncTransactionSql, { academicYearId, kind, @@ -253,7 +284,7 @@ export async function refreshCatalogueDirectory({ "INVALID_YEAR", ); } - return withImportDatabaseClient(async (sql) => { + return withSyncDatabaseClient(async (sql) => { const [year] = await sql` select id from public.academic_years where year = ${academicYear} `; @@ -285,10 +316,9 @@ export async function refreshCatalogueDirectory({ const sourceId = await ensureAnuSourceId(sql); const sourcePageIds = new Map(); for (const page of fetched.sourcePages) { - const sourcePageId = await recordSourcePage(sql, { + const sourcePageId = await recordDiscoverySourcePage(sql, { sourceId, academicYearId, - kind: "directory", externalKey: page.externalKey, canonicalUrl: page.sourceUrl, mediaType: page.mediaType, @@ -298,8 +328,6 @@ export async function refreshCatalogueDirectory({ sourceLastModified: page.sourceLastModified, fetchedAt: page.fetchedAt, byteSize: page.byteSize, - storageBucket: null, - storagePath: null, }); sourcePageIds.set(page.externalKey, sourcePageId); await sql` diff --git a/apps/web/lib/catalogue-import/import-store.ts b/apps/web/lib/catalogue-import/import-store.ts deleted file mode 100644 index d1b7fdb7..00000000 --- a/apps/web/lib/catalogue-import/import-store.ts +++ /dev/null @@ -1,697 +0,0 @@ -import type postgres from "postgres"; -import { - createHostedImportDatabaseClient, - createLocalDatabaseClient, -} from "../../scripts/catalogue/lib/local-database.mjs"; -import type { - ImportArtifactKind, - ImportArtifactLocator, -} from "./artifact-store.ts"; -import { ANU_PROGRAMS_AND_COURSES_SOURCE } from "./import-source.ts"; -import type { CatalogueKind } from "../catalogue/content.ts"; - -export type ImportStageName = - | "source_fetch" - | "html_capture" - | "markdown_normalise" - | "model_input_prepare" - | "deterministic_extract" - | "model_extract" - | "schema_validate" - | "domain_validate" - | "database_project" - | "snapshot_persist"; - -export type ImportSql = Awaited>; -export type ImportTransactionSql = postgres.TransactionSql; -type AnyImportSql = ImportSql | ImportTransactionSql; - -export class ImportStoreError extends Error { - readonly code: string; - - constructor(message: string, code: string) { - super(message); - this.name = "ImportStoreError"; - this.code = code; - } -} - -export class ImportDatabaseConfigurationError extends Error { - constructor() { - super( - "Configure COURSEMAP_IMPORT_DATABASE_URL before running durable imports on Vercel.", - ); - this.name = "ImportDatabaseConfigurationError"; - } -} - -/** - * Development and the local test server use the loopback database (the - * latter through COURSEMAP_DATABASE_URL); every other environment needs the - * hosted worker URL. - */ -export async function createImportDatabaseClient() { - // A loopback COURSEMAP_DATABASE_URL marks the local test server, which must - // never reach the hosted database even when .env.local names one. - if ( - process.env.NODE_ENV === "development" || - process.env.COURSEMAP_DATABASE_URL?.trim() - ) { - return createLocalDatabaseClient(); - } - const configured = process.env.COURSEMAP_IMPORT_DATABASE_URL?.trim(); - if (configured) return createHostedImportDatabaseClient(configured); - throw new ImportDatabaseConfigurationError(); -} - -export async function withImportDatabaseClient( - callback: (sql: ImportSql) => Promise, -) { - const sql = await createImportDatabaseClient(); - try { - return await callback(sql); - } finally { - await sql.end({ timeout: 5 }); - } -} - -export type ClaimedImportTarget = { - runId: string; - targetId: string; - kind: CatalogueKind; - code: string; - academicYear: number; - academicYearId: number; - itemId: number; - recordId: number; - directoryEntryId: number | null; - baselineVersionId: number | null; - requestedModel: string; - parserVersion: string; - promptVersion: string; - schemaVersion: string; - sourceId: number; - attemptCount: number; - lockVersion: number; - leaseExpiresAt: string; -}; - -function numberOrNull(value: unknown) { - return value === null || value === undefined ? null : Number(value); -} - -export async function ensureAnuSourceId(sql: AnyImportSql) { - const [existing] = await sql` - select id from public.catalogue_sources - where kind = ${ANU_PROGRAMS_AND_COURSES_SOURCE.kind} - and base_url = ${ANU_PROGRAMS_AND_COURSES_SOURCE.baseUrl} - `; - if (existing) return Number(existing.id); - const [inserted] = await sql` - insert into public.catalogue_sources (name, kind, base_url, is_active) - values ( - ${ANU_PROGRAMS_AND_COURSES_SOURCE.name}, - ${ANU_PROGRAMS_AND_COURSES_SOURCE.kind}, - ${ANU_PROGRAMS_AND_COURSES_SOURCE.baseUrl}, - true - ) - on conflict (kind, base_url) do update set is_active = true - returning id - `; - return Number(inserted.id); -} - -/** - * Takes a lease on a queued target, or on a running target whose lease has - * expired. Returns null when the target is not claimable, including when it - * has already finished or been cancelled. - */ -export async function claimImportTarget( - sql: ImportSql, - { - runId, - targetId, - workerId, - leaseSeconds = 120, - }: { - runId: string; - targetId: string; - workerId: string; - leaseSeconds?: number; - }, -): Promise { - return sql.begin(async (tx) => { - const sourceId = await ensureAnuSourceId(tx); - const [row] = await tx` - update public.catalogue_import_targets as targets - set - status = 'running', - attempt_count = targets.attempt_count + 1, - lock_version = targets.lock_version + 1, - worker_id = ${workerId}::uuid, - lease_expires_at = now() + make_interval(secs => ${leaseSeconds}) - from public.catalogue_import_runs as runs - join public.academic_years on academic_years.id = runs.academic_year_id - where targets.id = ${targetId}::uuid - and targets.run_id = ${runId}::uuid - and runs.id = targets.run_id - and targets.attempt_count < 5 - and ( - targets.status = 'queued' - or (targets.status = 'running' and targets.lease_expires_at < now()) - ) - returning - targets.id, - targets.run_id, - targets.kind, - targets.code, - academic_years.year as academic_year, - targets.academic_year_id, - targets.code_id, - targets.record_id, - targets.directory_entry_id, - targets.baseline_version_id, - runs.requested_model, - runs.parser_version, - runs.prompt_version, - runs.schema_version, - targets.attempt_count, - targets.lock_version, - targets.lease_expires_at - `; - if (!row) return null; - await tx`select private.refresh_catalogue_import_run(${runId}::uuid)`; - return { - runId: String(row.run_id), - targetId: String(row.id), - kind: row.kind as CatalogueKind, - code: String(row.code), - academicYear: Number(row.academic_year), - academicYearId: Number(row.academic_year_id), - itemId: Number(row.code_id), - recordId: Number(row.record_id), - directoryEntryId: numberOrNull(row.directory_entry_id), - baselineVersionId: numberOrNull(row.baseline_version_id), - requestedModel: String(row.requested_model), - parserVersion: String(row.parser_version), - promptVersion: String(row.prompt_version), - schemaVersion: String(row.schema_version), - sourceId, - attemptCount: Number(row.attempt_count), - lockVersion: Number(row.lock_version), - leaseExpiresAt: new Date(row.lease_expires_at as string).toISOString(), - }; - }); -} - -export async function getImportTargetStatus( - sql: AnyImportSql, - { runId, targetId }: { runId: string; targetId: string }, -) { - const [row] = await sql` - select status from public.catalogue_import_targets - where id = ${targetId}::uuid and run_id = ${runId}::uuid - `; - return row ? { status: String(row.status) } : null; -} - -export async function startImportStage( - sql: AnyImportSql, - { - targetId, - stageName, - attemptNumber, - }: { targetId: string; stageName: ImportStageName; attemptNumber: number }, -) { - const [row] = await sql` - insert into public.catalogue_import_stages (target_id, stage_name, attempt_number) - values (${targetId}::uuid, ${stageName}, ${attemptNumber}) - on conflict (target_id, stage_name, attempt_number) do update - set status = 'running', started_at = statement_timestamp(), - completed_at = null, error_code = null, error_summary = null - returning id - `; - return String(row.id); -} - -export async function finishImportStage( - sql: AnyImportSql, - { stageId }: { stageId: string }, -) { - await sql` - update public.catalogue_import_stages - set status = 'completed', completed_at = statement_timestamp() - where id = ${stageId}::uuid - `; -} - -export async function failImportStage( - sql: AnyImportSql, - { - stageId, - errorCode, - errorSummary, - }: { stageId: string; errorCode: string; errorSummary: string }, -) { - await sql` - update public.catalogue_import_stages - set status = 'failed', completed_at = statement_timestamp(), - error_code = ${errorCode}, error_summary = ${errorSummary} - where id = ${stageId}::uuid - `; -} - -export type ImportArtifactRecord = ImportArtifactLocator & { - id: string; - kind: ImportArtifactKind; -}; - -export async function recordImportArtifact( - sql: AnyImportSql, - { - targetId, - stageId, - kind, - attemptNumber, - mediaType, - contentSha256, - byteSize, - storageBucket, - storagePath, - }: { - targetId: string; - stageId: string; - kind: ImportArtifactKind; - attemptNumber: number; - mediaType: string; - contentSha256: string; - byteSize: number; - storageBucket: string; - storagePath: string; - }, -): Promise { - const [row] = await sql` - insert into public.catalogue_import_artifacts ( - target_id, stage_id, kind, attempt_number, media_type, content_sha256, - byte_size, storage_bucket, storage_path - ) values ( - ${targetId}::uuid, ${stageId}::uuid, ${kind}, ${attemptNumber}, ${mediaType}, - ${contentSha256}, ${byteSize}, ${storageBucket}, ${storagePath} - ) - returning id - `; - return { - id: String(row.id), - kind, - bucket: storageBucket as ImportArtifactLocator["bucket"], - path: storagePath, - mediaType, - contentSha256, - byteSize, - }; -} - -/** Records a fetched page once; the same content hash for the same key is reused. */ -export async function recordSourcePage( - sql: AnyImportSql, - { - sourceId, - academicYearId, - kind, - externalKey, - canonicalUrl, - contentSha256, - httpStatus, - httpEtag, - sourceLastModified, - fetchedAt, - byteSize, - mediaType, - storageBucket, - storagePath, - }: { - sourceId: number; - academicYearId: number; - kind: CatalogueKind | "directory"; - externalKey: string; - canonicalUrl: string; - contentSha256: string; - httpStatus: number | null; - httpEtag: string | null; - sourceLastModified: string | null; - fetchedAt: string; - byteSize: number | null; - mediaType?: string; - storageBucket: string | null; - storagePath: string | null; - }, -) { - await sql` - insert into public.catalogue_source_pages ( - source_id, academic_year_id, kind, external_key, canonical_url, media_type, - content_sha256, http_status, http_etag, source_last_modified, fetched_at, - byte_size, storage_bucket, storage_path - ) values ( - ${sourceId}, ${academicYearId}, ${kind}, ${externalKey}, ${canonicalUrl}, ${mediaType ?? "text/html"}, - ${contentSha256}, ${httpStatus}, ${httpEtag}, ${sourceLastModified}, ${fetchedAt}, - ${byteSize}, ${storageBucket}, ${storagePath} - ) - on conflict (source_id, academic_year_id, kind, external_key, content_sha256) do nothing - `; - const [row] = await sql` - select id from public.catalogue_source_pages - where source_id = ${sourceId} and academic_year_id = ${academicYearId} - and kind = ${kind} and external_key = ${externalKey} - and content_sha256 = ${contentSha256} - `; - return Number(row.id); -} - -export type ReusableExtraction = { - id: string; - targetId: string; - responseArtifact: ImportArtifactLocator; -}; - -/** - * A recorded response for identical input can be reused without a paid call. - * Validation is deterministic, so an invalid response stays invalid and the - * merge falls back to deterministic data exactly as it did the first time. - */ -export async function findReusableExtraction( - sql: AnyImportSql, - { fingerprint }: { fingerprint: string }, -): Promise { - const [row] = await sql` - select extractions.id, extractions.target_id, - artifacts.media_type, artifacts.content_sha256, artifacts.byte_size, - artifacts.storage_bucket, artifacts.storage_path - from public.catalogue_extractions as extractions - join public.catalogue_import_artifacts as artifacts - on artifacts.id = extractions.response_artifact_id - where extractions.fingerprint = ${fingerprint} - and extractions.completed_at is not null - order by extractions.completed_at desc - limit 1 - `; - if (!row) return null; - return { - id: String(row.id), - targetId: String(row.target_id), - responseArtifact: { - bucket: row.storage_bucket as ImportArtifactLocator["bucket"], - path: String(row.storage_path), - mediaType: String(row.media_type), - contentSha256: String(row.content_sha256), - byteSize: Number(row.byte_size), - }, - }; -} - -export type ExtractionReservation = { - id: string; - created: boolean; - responseArtifactId: string | null; -}; - -/** - * Reserves the paid model call for this attempt. When a reservation for the - * same fingerprint already exists on the target without a recorded response, - * the caller must not issue a second paid call. - */ -export async function reserveExtraction( - sql: AnyImportSql, - { - targetId, - extractionNumber, - requestedModel, - fingerprint, - promptVersion, - schemaVersion, - requestArtifactId, - }: { - targetId: string; - extractionNumber: number; - requestedModel: string; - fingerprint: string; - promptVersion: string; - schemaVersion: string; - requestArtifactId: string; - }, -): Promise { - const [existing] = await sql` - select id, response_artifact_id from public.catalogue_extractions - where target_id = ${targetId}::uuid and fingerprint = ${fingerprint} - order by started_at desc limit 1 - `; - if (existing) { - return { - id: String(existing.id), - created: false, - responseArtifactId: - existing.response_artifact_id === null - ? null - : String(existing.response_artifact_id), - }; - } - const [row] = await sql` - insert into public.catalogue_extractions ( - target_id, extraction_number, requested_model, fingerprint, prompt_version, - schema_version, request_artifact_id - ) values ( - ${targetId}::uuid, ${extractionNumber}, ${requestedModel}, ${fingerprint}, - ${promptVersion}, ${schemaVersion}, ${requestArtifactId}::uuid - ) - returning id - `; - return { id: String(row.id), created: true, responseArtifactId: null }; -} - -export async function attachExtractionResponse( - sql: AnyImportSql, - { - extractionId, - responseArtifactId, - resolvedModel, - reusedFromExtractionId, - providerRequestId, - finishReason, - inputTokens, - cachedInputTokens, - outputTokens, - reasoningTokens, - costUsd, - costSource, - latencyMs, - }: { - extractionId: string; - responseArtifactId: string; - resolvedModel: string; - reusedFromExtractionId: string | null; - providerRequestId: string | null; - finishReason: string | null; - inputTokens: number; - cachedInputTokens: number; - outputTokens: number; - reasoningTokens: number; - costUsd: number; - costSource: "provider" | "cache" | "unknown"; - latencyMs: number; - }, -) { - await sql` - update public.catalogue_extractions - set response_artifact_id = ${responseArtifactId}::uuid, - resolved_model = ${resolvedModel}, - reused_from_extraction_id = ${reusedFromExtractionId}::uuid, - provider_request_id = ${providerRequestId}, - finish_reason = ${finishReason}, - input_tokens = ${inputTokens}, - cached_input_tokens = ${cachedInputTokens}, - output_tokens = ${outputTokens}, - reasoning_tokens = ${reasoningTokens}, - cost_usd = ${costUsd}, - cost_source = ${costSource}, - latency_ms = ${Math.round(latencyMs)} - where id = ${extractionId}::uuid - `; -} - -export async function completeExtraction( - sql: AnyImportSql, - { - extractionId, - validatedArtifactId, - schemaValid, - domainValid, - warningCount, - errorCount, - errorSummary, - }: { - extractionId: string; - validatedArtifactId: string | null; - schemaValid: boolean; - domainValid: boolean; - warningCount: number; - errorCount: number; - errorSummary: string | null; - }, -) { - await sql` - update public.catalogue_extractions - set validated_artifact_id = ${validatedArtifactId}::uuid, - validation_status = ${schemaValid && domainValid ? "valid" : "invalid"}, - schema_valid = ${schemaValid}, - domain_valid = ${domainValid}, - warning_count = ${warningCount}, - error_count = ${errorCount}, - error_summary = ${errorSummary}, - completed_at = now() - where id = ${extractionId}::uuid - `; -} - -function assertLeaseHeld(count: number) { - if (count !== 1) { - throw new ImportStoreError( - "The import target lease was lost before its result could be recorded.", - "LEASE_LOST", - ); - } -} - -export async function finishImportTarget( - sql: ImportSql, - { - runId, - targetId, - workerId, - expectedLockVersion, - status, - changeKind, - sourcePageId, - candidateVersionId, - errorCode = null, - errorMessage = null, - }: { - runId: string; - targetId: string; - workerId: string; - expectedLockVersion: number; - status: "ready" | "unchanged" | "failed"; - changeKind: "new" | "changed" | "unchanged" | null; - sourcePageId: number | null; - candidateVersionId: number | null; - errorCode?: string | null; - errorMessage?: string | null; - }, -) { - await sql.begin(async (tx) => { - const updated = await tx` - update public.catalogue_import_targets - set status = ${status}, - change_kind = ${changeKind}, - source_page_id = ${sourcePageId}, - candidate_version_id = ${candidateVersionId}, - error_code = ${errorCode}, - error_message = ${errorMessage}, - worker_id = null, - lease_expires_at = null, - completed_at = now() - where id = ${targetId}::uuid - and run_id = ${runId}::uuid - and status = 'running' - and worker_id = ${workerId}::uuid - and lock_version = ${expectedLockVersion} - returning id - `; - assertLeaseHeld(updated.length); - await tx`select private.refresh_catalogue_import_run(${runId}::uuid)`; - }); -} - -export async function releaseImportTargetForRetry( - sql: ImportSql, - { - runId, - targetId, - workerId, - expectedLockVersion, - errorCode, - errorMessage, - }: { - runId: string; - targetId: string; - workerId: string; - expectedLockVersion: number; - errorCode: string; - errorMessage: string; - }, -) { - await sql.begin(async (tx) => { - const updated = await tx` - update public.catalogue_import_targets - set status = 'queued', - worker_id = null, - lease_expires_at = null, - error_code = ${errorCode}, - error_message = ${errorMessage} - where id = ${targetId}::uuid - and run_id = ${runId}::uuid - and status = 'running' - and worker_id = ${workerId}::uuid - and lock_version = ${expectedLockVersion} - returning id - `; - assertLeaseHeld(updated.length); - await tx`select private.refresh_catalogue_import_run(${runId}::uuid)`; - }); -} - -export async function recordImportDispatch( - sql: ImportSql, - { - runId, - dispatched, - failedTargetIds, - errorMessage = "The queue did not accept this target.", - }: { - runId: string; - dispatched: Array<{ targetId: string; messageId: string | null }>; - failedTargetIds: readonly string[]; - errorMessage?: string; - }, -) { - await sql.begin(async (tx) => { - for (const target of dispatched) { - await tx` - update public.catalogue_import_targets - set dispatched_at = coalesce(dispatched_at, now()), - queue_message_id = coalesce(queue_message_id, ${target.messageId}) - where id = ${target.targetId}::uuid and run_id = ${runId}::uuid and status = 'queued' - `; - } - if (failedTargetIds.length > 0) { - await tx` - update public.catalogue_import_targets - set status = 'failed', - error_code = 'QUEUE_DISPATCH_FAILED', - error_message = ${errorMessage}, - completed_at = now() - where run_id = ${runId}::uuid - and id = any(${tx.array([...failedTargetIds])}::uuid[]) - and status = 'queued' - `; - } - await tx`select private.refresh_catalogue_import_run(${runId}::uuid)`; - }); -} - -export async function listQueuedTargetIds(sql: AnyImportSql, runId: string) { - const rows = await sql` - select id from public.catalogue_import_targets - where run_id = ${runId}::uuid and status = 'queued' - order by created_at - `; - return rows.map((row) => String(row.id)); -} diff --git a/apps/web/lib/catalogue-import/kinds/course/adapter.ts b/apps/web/lib/catalogue-import/kinds/course/adapter.ts index 231178ce..28e7fed0 100644 --- a/apps/web/lib/catalogue-import/kinds/course/adapter.ts +++ b/apps/web/lib/catalogue-import/kinds/course/adapter.ts @@ -1,4 +1,4 @@ -import type { CatalogueKindAdapter } from "../../kind-adapter.ts"; +import type { CatalogueSyncAdapter } from "../../../catalogue-sync/kind-adapter.ts"; import { courseCatalogueContent } from "../../../catalogue/content.ts"; import { COURSE_EXTRACTION_JSON_SCHEMA, @@ -25,7 +25,7 @@ import { } from "./prompt.ts"; import { fetchAnuCoursePage } from "./source.ts"; -export const courseKindAdapter: CatalogueKindAdapter = { +export const courseKindAdapter: CatalogueSyncAdapter = { kinds: ["course"], parserVersion: COURSE_IMPORT_PARSER_VERSION, promptVersion: COURSE_IMPORT_PROMPT_VERSION, diff --git a/apps/web/lib/catalogue-import/kinds/structure/adapter.ts b/apps/web/lib/catalogue-import/kinds/structure/adapter.ts index 62baccf4..9a335971 100644 --- a/apps/web/lib/catalogue-import/kinds/structure/adapter.ts +++ b/apps/web/lib/catalogue-import/kinds/structure/adapter.ts @@ -1,4 +1,4 @@ -import type { CatalogueKindAdapter } from "../../kind-adapter.ts"; +import type { CatalogueSyncAdapter } from "../../../catalogue-sync/kind-adapter.ts"; import { structureCatalogueContent } from "../../../catalogue/content.ts"; import { ACADEMIC_STRUCTURE_EXTRACTION_JSON_SCHEMA, @@ -34,7 +34,7 @@ function structureKind(kind: string): AcademicStructureKind { return kind as AcademicStructureKind; } -export const structureKindAdapter: CatalogueKindAdapter = +export const structureKindAdapter: CatalogueSyncAdapter = { kinds: ["programme", "major", "minor", "specialisation"], parserVersion: ACADEMIC_STRUCTURE_IMPORT_PARSER_VERSION, diff --git a/apps/web/lib/catalogue-import/queue.ts b/apps/web/lib/catalogue-import/queue.ts deleted file mode 100644 index b9f7819f..00000000 --- a/apps/web/lib/catalogue-import/queue.ts +++ /dev/null @@ -1,287 +0,0 @@ -import type { MessageMetadata, RetryDirective } from "@vercel/queue"; -import { - listQueuedTargetIds, - recordImportDispatch, - withImportDatabaseClient, -} from "./import-store.ts"; -import { - processImportTarget, - type ProcessImportTargetInput, -} from "./process-target.ts"; - -export const IMPORT_QUEUE_TOPIC = "catalogue-import-v1"; -export const IMPORT_QUEUE_MESSAGE_VERSION = 1 as const; -export const IMPORT_QUEUE_RETENTION_SECONDS = 24 * 60 * 60; -export const IMPORT_QUEUE_MAX_DELIVERIES = 5; -export const IMPORT_QUEUE_MAX_CALLBACK_DELIVERIES = 12; -export const IMPORT_QUEUE_VISIBILITY_TIMEOUT_SECONDS = 600; -export const IMPORT_QUEUE_DELIVERY_BUDGET_MS = 290_000; -export const MAX_TARGETS_PER_IMPORT_RUN = 10; - -const UUID_PATTERN = - /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -export type ImportQueueMessage = { - version: typeof IMPORT_QUEUE_MESSAGE_VERSION; - runId: string; - targetId: string; -}; - -export class ImportQueueMessageError extends TypeError { - constructor(message: string) { - super(message); - this.name = "ImportQueueMessageError"; - } -} - -export class ImportQueueDispatchError extends Error { - readonly dispatched: Array<{ targetId: string; messageId: string | null }>; - readonly failedTargetIds: readonly string[]; - - constructor({ - dispatched, - failedTargetIds, - }: { - dispatched: Array<{ targetId: string; messageId: string | null }>; - failedTargetIds: readonly string[]; - }) { - super( - `Queued ${dispatched.length} import target${dispatched.length === 1 ? "" : "s"}; ${failedTargetIds.length} could not be queued.`, - ); - this.name = "ImportQueueDispatchError"; - this.dispatched = dispatched; - this.failedTargetIds = failedTargetIds; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -export function parseImportQueueMessage(value: unknown): ImportQueueMessage { - if (!isRecord(value)) { - throw new ImportQueueMessageError("Import queue messages must be objects."); - } - if (Object.keys(value).sort().join(",") !== "runId,targetId,version") { - throw new ImportQueueMessageError( - "Import queue message fields do not match version 1.", - ); - } - if (value.version !== IMPORT_QUEUE_MESSAGE_VERSION) { - throw new ImportQueueMessageError( - "Unsupported import queue message version.", - ); - } - if (typeof value.runId !== "string" || !UUID_PATTERN.test(value.runId)) { - throw new ImportQueueMessageError("Import queue runId must be a UUID."); - } - if ( - typeof value.targetId !== "string" || - !UUID_PATTERN.test(value.targetId) - ) { - throw new ImportQueueMessageError("Import queue targetId must be a UUID."); - } - return { - version: IMPORT_QUEUE_MESSAGE_VERSION, - runId: value.runId, - targetId: value.targetId, - }; -} - -export function createImportQueueMessage(input: { - runId: string; - targetId: string; -}) { - return parseImportQueueMessage({ - version: IMPORT_QUEUE_MESSAGE_VERSION, - ...input, - }); -} - -export function createImportQueueIdempotencyKey(message: ImportQueueMessage) { - return `catalogue-import:v${message.version}:${message.runId}:${message.targetId}`; -} - -/** Only the exact value "true" publishes to Vercel Queues. */ -export function importQueueEnabled( - value = process.env.COURSEMAP_QUEUE_IMPORTS_ENABLED, -) { - return value === "true"; -} - -export type ImportQueueSend = ( - topic: string, - message: ImportQueueMessage, - options: { idempotencyKey: string; retentionSeconds: number }, -) => Promise<{ messageId: string | null }>; - -async function sendWithVercelQueue( - topic: string, - message: ImportQueueMessage, - options: { idempotencyKey: string; retentionSeconds: number }, -) { - // The queue SDK is loaded only when publishing so Next.js page collection - // never constructs its region-aware client. - const { send } = await import("@vercel/queue"); - return send(topic, message, options); -} - -export async function enqueueImportTargets( - { runId, targetIds }: { runId: string; targetIds: readonly string[] }, - send: ImportQueueSend = sendWithVercelQueue, -) { - if (targetIds.length === 0 || targetIds.length > MAX_TARGETS_PER_IMPORT_RUN) { - throw new RangeError( - `An import run must contain 1 to ${MAX_TARGETS_PER_IMPORT_RUN} targets.`, - ); - } - if (new Set(targetIds).size !== targetIds.length) { - throw new TypeError("An import target may only be queued once."); - } - const messages = targetIds.map((targetId) => - createImportQueueMessage({ runId, targetId }), - ); - const results = await Promise.allSettled( - messages.map(async (message) => { - const result = await send(IMPORT_QUEUE_TOPIC, message, { - idempotencyKey: createImportQueueIdempotencyKey(message), - retentionSeconds: IMPORT_QUEUE_RETENTION_SECONDS, - }); - return { targetId: message.targetId, messageId: result.messageId }; - }), - ); - const dispatched = results.flatMap((result) => - result.status === "fulfilled" ? [result.value] : [], - ); - const failedTargetIds = results.flatMap((result, index) => - result.status === "rejected" ? [messages[index]!.targetId] : [], - ); - if (failedTargetIds.length > 0) { - throw new ImportQueueDispatchError({ dispatched, failedTargetIds }); - } - return dispatched; -} - -/** - * Starts processing for a run's queued targets. With the queue enabled each - * target becomes a message; otherwise the targets run in this process one - * after another, which is how development and tests complete an import. - */ -export async function dispatchImportRun({ - runId, - targetIds, - send, -}: { - runId: string; - targetIds: readonly string[]; - send?: ImportQueueSend; -}) { - if (importQueueEnabled()) { - try { - const dispatched = await enqueueImportTargets({ runId, targetIds }, send); - await withImportDatabaseClient((sql) => - recordImportDispatch(sql, { runId, dispatched, failedTargetIds: [] }), - ); - return { mode: "queue" as const, dispatched: dispatched.length }; - } catch (error) { - if (error instanceof ImportQueueDispatchError) { - await withImportDatabaseClient((sql) => - recordImportDispatch(sql, { - runId, - dispatched: error.dispatched, - failedTargetIds: error.failedTargetIds, - }), - ); - } - throw error; - } - } - - await withImportDatabaseClient((sql) => - recordImportDispatch(sql, { - runId, - dispatched: targetIds.map((targetId) => ({ targetId, messageId: null })), - failedTargetIds: [], - }), - ); - return { mode: "inline" as const, dispatched: targetIds.length }; -} - -/** - * Processes every queued target of a run in this process. A retryable failure - * leaves the target queued and continues with the next one; repeated passes - * give each target up to its attempt limit. - */ -export async function processImportRunInline({ - runId, - process = processImportTarget, - signal, -}: { - runId: string; - process?: (input: ProcessImportTargetInput) => Promise; - signal?: AbortSignal; -}) { - let completed = 0; - for (let pass = 1; pass <= IMPORT_QUEUE_MAX_DELIVERIES; pass += 1) { - const queued = await withImportDatabaseClient((sql) => - listQueuedTargetIds(sql, runId), - ); - if (queued.length === 0) break; - for (const targetId of queued) { - signal?.throwIfAborted(); - try { - await process({ - runId, - targetId, - deliveryCount: pass, - maxDeliveries: IMPORT_QUEUE_MAX_DELIVERIES, - signal, - }); - completed += 1; - } catch { - // The processor already recorded the failure or requeued the target. - } - } - } - return { completed }; -} - -function retryImportQueueMessage( - error: unknown, - metadata: MessageMetadata, -): RetryDirective { - if (error instanceof ImportQueueMessageError) return { acknowledge: true }; - if (metadata.deliveryCount >= IMPORT_QUEUE_MAX_CALLBACK_DELIVERIES) { - return { acknowledge: true }; - } - return { afterSeconds: Math.min(300, 5 * 2 ** (metadata.deliveryCount - 1)) }; -} - -export const importQueueInternals = { retryImportQueueMessage }; - -export function createImportQueueConsumer( - process: ( - input: ProcessImportTargetInput, - ) => void | Promise = processImportTarget, -) { - return async (request: Request) => { - const { handleCallback } = await import("@vercel/queue"); - const consume = handleCallback( - async (value, metadata) => { - const message = parseImportQueueMessage(value); - await process({ - runId: message.runId, - targetId: message.targetId, - deliveryCount: metadata.deliveryCount, - maxDeliveries: IMPORT_QUEUE_MAX_DELIVERIES, - signal: AbortSignal.timeout(IMPORT_QUEUE_DELIVERY_BUDGET_MS), - }); - }, - { - visibilityTimeoutSeconds: IMPORT_QUEUE_VISIBILITY_TIMEOUT_SECONDS, - retry: retryImportQueueMessage, - }, - ); - return consume(request); - }; -} diff --git a/apps/web/lib/catalogue-import/version-content.ts b/apps/web/lib/catalogue-import/version-content.ts index 64aa64a4..9eb1e5c8 100644 --- a/apps/web/lib/catalogue-import/version-content.ts +++ b/apps/web/lib/catalogue-import/version-content.ts @@ -1,6 +1,9 @@ import type postgres from "postgres"; import { stableFingerprint } from "./canonical.ts"; -import type { ImportSql, ImportTransactionSql } from "./import-store.ts"; +import type { + SyncSql, + SyncTransactionSql, +} from "../catalogue-sync/sync-store.ts"; import type { CatalogueKind, CatalogueContent, @@ -12,7 +15,7 @@ import type { StructureContentWrite, } from "../catalogue/content.ts"; -type Sql = ImportSql | ImportTransactionSql | postgres.Sql; +type Sql = SyncSql | SyncTransactionSql | postgres.Sql; function num(value: unknown) { return value === null || value === undefined ? null : Number(value); diff --git a/apps/web/lib/catalogue-import/artifact-store.ts b/apps/web/lib/catalogue-sync/artifact-store.ts similarity index 71% rename from apps/web/lib/catalogue-import/artifact-store.ts rename to apps/web/lib/catalogue-sync/artifact-store.ts index d2ed1dc1..a1ab764b 100644 --- a/apps/web/lib/catalogue-import/artifact-store.ts +++ b/apps/web/lib/catalogue-sync/artifact-store.ts @@ -1,10 +1,10 @@ import { createHash } from "node:crypto"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; -export const IMPORT_ARTIFACT_BUCKET = "course-import-artifacts"; -export const MAX_IMPORT_ARTIFACT_BYTES = 5 * 1024 * 1024; +export const SYNC_ARTIFACT_BUCKET = "course-import-artifacts"; +export const MAX_SYNC_ARTIFACT_BYTES = 5 * 1024 * 1024; -export type ImportArtifactKind = +export type SyncArtifactKind = | "raw_html" | "normalised_markdown" | "model_input" @@ -13,8 +13,7 @@ export type ImportArtifactKind = | "model_response" | "validated_json" | "validation_report" - | "database_projection" - | "change_set"; + | "content_projection"; const ALLOWED_MEDIA_TYPES = new Set([ "application/json", @@ -30,22 +29,22 @@ const EXTENSION_BY_MEDIA_TYPE: Record = { "text/plain": "txt", }; -export type StoredImportArtifact = { - bucket: typeof IMPORT_ARTIFACT_BUCKET; +export type StoredSyncArtifact = { + bucket: typeof SYNC_ARTIFACT_BUCKET; path: string; mediaType: string; byteSize: number; contentSha256: string; }; -export type ImportArtifactLocator = StoredImportArtifact; +export type SyncArtifactLocator = StoredSyncArtifact; -export class ImportArtifactConfigurationError extends Error { +export class SyncArtifactConfigurationError extends Error { constructor() { super( - "Configure NEXT_PUBLIC_SUPABASE_URL and the server-only SUPABASE_SECRET_KEY before running durable imports.", + "Configure NEXT_PUBLIC_SUPABASE_URL and the server-only SUPABASE_SECRET_KEY before running durable catalogue syncs.", ); - this.name = "ImportArtifactConfigurationError"; + this.name = "SyncArtifactConfigurationError"; } } @@ -75,7 +74,7 @@ function isDuplicateStorageError(error: unknown) { function configuredStorageClient(env: NodeJS.ProcessEnv = process.env) { const url = env.NEXT_PUBLIC_SUPABASE_URL?.trim(); const secretKey = env.SUPABASE_SECRET_KEY?.trim(); - if (!url || !secretKey) throw new ImportArtifactConfigurationError(); + if (!url || !secretKey) throw new SyncArtifactConfigurationError(); return createClient(url, secretKey, { auth: { autoRefreshToken: false, persistSession: false }, }); @@ -85,10 +84,9 @@ function configuredStorageClient(env: NodeJS.ProcessEnv = process.env) { * Stores an immutable content-addressed artefact. A duplicate upload is safe on * worker redelivery because the hash is part of the object path. */ -export async function storeImportArtifact({ +export async function storeSyncArtifact({ academicYear, - runId, - targetId, + syncId, stage, kind, mediaType, @@ -96,14 +94,13 @@ export async function storeImportArtifact({ client = configuredStorageClient(), }: { academicYear: number; - runId: string; - targetId: string; + syncId: string; stage: string; - kind: ImportArtifactKind; + kind: SyncArtifactKind; mediaType: string; body: string | Uint8Array; client?: SupabaseClient; -}): Promise { +}): Promise { if ( !Number.isInteger(academicYear) || academicYear < 2000 || @@ -115,27 +112,28 @@ export async function storeImportArtifact({ } if (!ALLOWED_MEDIA_TYPES.has(mediaType)) { throw new TypeError( - `Unsupported course import artefact media type: ${mediaType}`, + `Unsupported catalogue sync artefact media type: ${mediaType}`, ); } const bytes = typeof body === "string" ? Buffer.from(body, "utf8") : body; - if (bytes.byteLength > MAX_IMPORT_ARTIFACT_BYTES) { - throw new RangeError("The course import artefact exceeds the 5 MiB limit."); + if (bytes.byteLength > MAX_SYNC_ARTIFACT_BYTES) { + throw new RangeError( + "The catalogue sync artefact exceeds the 5 MiB limit.", + ); } const contentSha256 = sha256(bytes); const extension = EXTENSION_BY_MEDIA_TYPE[mediaType]!; const path = [ String(academicYear), - safePathPart(runId, "runId"), - safePathPart(targetId, "targetId"), + safePathPart(syncId, "syncId"), safePathPart(stage, "stage"), `${safePathPart(kind, "kind")}-${contentSha256}.${extension}`, ].join("/"); const { error } = await client.storage - .from(IMPORT_ARTIFACT_BUCKET) + .from(SYNC_ARTIFACT_BUCKET) .upload(path, bytes, { cacheControl: "31536000", contentType: mediaType, @@ -144,7 +142,7 @@ export async function storeImportArtifact({ if (error && !isDuplicateStorageError(error)) throw error; return { - bucket: IMPORT_ARTIFACT_BUCKET, + bucket: SYNC_ARTIFACT_BUCKET, path, mediaType, byteSize: bytes.byteLength, @@ -156,21 +154,21 @@ export async function storeImportArtifact({ * Reads an immutable private artefact and verifies it against the database * metadata before a retried worker trusts the contents. */ -export async function readImportArtifact({ +export async function readSyncArtifact({ artifact, client = configuredStorageClient(), }: { - artifact: ImportArtifactLocator; + artifact: SyncArtifactLocator; client?: SupabaseClient; }) { - if (artifact.bucket !== IMPORT_ARTIFACT_BUCKET) { + if (artifact.bucket !== SYNC_ARTIFACT_BUCKET) { throw new TypeError( - "The course import artefact uses an unexpected bucket.", + "The catalogue sync artefact uses an unexpected bucket.", ); } if (!ALLOWED_MEDIA_TYPES.has(artifact.mediaType)) { throw new TypeError( - `Unsupported course import artefact media type: ${artifact.mediaType}`, + `Unsupported catalogue sync artefact media type: ${artifact.mediaType}`, ); } @@ -178,14 +176,14 @@ export async function readImportArtifact({ .from(artifact.bucket) .download(artifact.path); if (error) throw error; - if (!data) throw new Error("The course import artefact was not downloaded."); + if (!data) throw new Error("The catalogue sync artefact was not downloaded."); const bytes = new Uint8Array(await data.arrayBuffer()); if ( bytes.byteLength !== artifact.byteSize || sha256(bytes) !== artifact.contentSha256 ) { - throw new Error("The course import artefact failed its integrity check."); + throw new Error("The catalogue sync artefact failed its integrity check."); } return Buffer.from(bytes).toString("utf8"); } diff --git a/apps/web/lib/catalogue-import/kind-adapter.ts b/apps/web/lib/catalogue-sync/kind-adapter.ts similarity index 78% rename from apps/web/lib/catalogue-import/kind-adapter.ts rename to apps/web/lib/catalogue-sync/kind-adapter.ts index 1099b90c..935f6992 100644 --- a/apps/web/lib/catalogue-import/kind-adapter.ts +++ b/apps/web/lib/catalogue-sync/kind-adapter.ts @@ -1,4 +1,4 @@ -import type { ClaimedImportTarget } from "./import-store.ts"; +import type { ClaimedCatalogueSync } from "./sync-store.ts"; import type { CatalogueKind, CatalogueContent } from "../catalogue/content.ts"; export type FetchedSourcePage = { @@ -29,8 +29,7 @@ export type MergeOutcome = { report: unknown; /** * Set when the model output was discarded. The processor records it on the - * target so a snapshot built from deterministic parsing alone says so, - * rather than finishing `ready` with no error at all. + * sync so a source version built from deterministic parsing alone says so. */ errorCode?: string | null; /** The reason, for `catalogue_extractions.error_summary`. */ @@ -38,12 +37,12 @@ export type MergeOutcome = { }; /** - * Everything kind-specific about an import: where the page lives, how it + * Everything kind-specific about a sync: where the page lives, how it * becomes Markdown and model input, the deterministic parser, the model - * contract and how a merged extraction becomes snapshot rows. The processor + * contract and how a merged extraction becomes version rows. The processor * owns stages, artefacts, leases and persistence. */ -export type CatalogueKindAdapter = { +export type CatalogueSyncAdapter = { kinds: readonly CatalogueKind[]; parserVersion: string; promptVersion: string; @@ -54,27 +53,27 @@ export type CatalogueKindAdapter = { requestTimeoutMs: number; extractionJsonSchema: Record; fetchSource( - claim: ClaimedImportTarget, + claim: ClaimedCatalogueSync, options: { signal?: AbortSignal }, ): Promise; /** Normalised Markdown for the audit trail and the trimmed model input. */ prepareInput( - claim: ClaimedImportTarget, + claim: ClaimedCatalogueSync, page: FetchedSourcePage, ): { markdown: string; modelInput: string }; buildSystemPrompt(): string; - buildUserPrompt(claim: ClaimedImportTarget, modelInput: string): string; + buildUserPrompt(claim: ClaimedCatalogueSync, modelInput: string): string; extractDeterministic( - claim: ClaimedImportTarget, + claim: ClaimedCatalogueSync, page: FetchedSourcePage, ): Extraction; /** Strict validation of raw model output against the extraction contract. */ validateModelOutput( - claim: ClaimedImportTarget, + claim: ClaimedCatalogueSync, value: unknown, ): ValidationOutcome; merge(input: { - claim: ClaimedImportTarget; + claim: ClaimedCatalogueSync; deterministic: Extraction; model: unknown; modelValid: boolean; diff --git a/apps/web/lib/catalogue-import/persist-version.ts b/apps/web/lib/catalogue-sync/persist-source-version.ts similarity index 76% rename from apps/web/lib/catalogue-import/persist-version.ts rename to apps/web/lib/catalogue-sync/persist-source-version.ts index 23994b85..13dcce64 100644 --- a/apps/web/lib/catalogue-import/persist-version.ts +++ b/apps/web/lib/catalogue-sync/persist-source-version.ts @@ -1,34 +1,20 @@ import type postgres from "postgres"; -import { - diffSnapshotWrites, - isBlockingFlag, - type SnapshotChange, -} from "./changes.ts"; -import type { ClaimedImportTarget, ImportSql } from "./import-store.ts"; -import { readVersionContent } from "./version-content.ts"; +import type { ClaimedCatalogueSync, SyncSql } from "./sync-store.ts"; import type { CatalogueKind, CatalogueContent, RequirementWrite, } from "../catalogue/content.ts"; +import { + CATALOGUE_CONTENT_SCHEMA_VERSION, + emptyCatalogueContent, +} from "../catalogue/content.ts"; +import { contentHashForCatalogueContent } from "../catalogue-import/version-content.ts"; -export type SnapshotChangeKind = "new" | "changed" | "unchanged"; - -export type PersistedSnapshotCandidate = { - changeKind: SnapshotChangeKind; - candidateVersionId: number | null; - baselineVersionId: number | null; - becameDraft: boolean; - changeSet: { - changeKind: SnapshotChangeKind; - contentHash: string; - baselineVersionId: number | null; - baselineContentHash: string | null; - candidateVersionId: number | null; - becameDraft: boolean; - changes: SnapshotChange[]; - flags: CatalogueContent["flags"]; - }; +export type PersistedSourceVersion = { + status: "unchanged" | "review_required" | "applied"; + sourceVersionId: number; + populatedDraft: boolean; }; type Tx = postgres.TransactionSql; @@ -480,189 +466,156 @@ export async function insertVersionContent( } } -/** Records the review entries for a target: one row per change and per flag. */ -export async function insertImportChanges( - tx: Tx, - { - targetId, - changes, - flags, - acceptAll, - }: { - targetId: string; - changes: SnapshotChange[]; - flags: CatalogueContent["flags"]; - acceptAll: boolean; - }, -) { - await tx`delete from public.catalogue_import_changes where target_id = ${targetId}::uuid`; - let position = 0; - for (const change of changes) { - await tx` - insert into public.catalogue_import_changes ( - target_id, entry_kind, field_path, old_value, new_value, summary, source_locator, - source_excerpt, status, resolved_at, position - ) values ( - ${targetId}::uuid, 'change', ${change.fieldPath}, - ${tx.json(change.oldValue as never)}, ${tx.json(change.newValue as never)}, - ${change.summary}, ${change.sourceLocator}, ${change.sourceExcerpt}, - ${acceptAll ? "accepted" : "open"}, ${acceptAll ? tx`now()` : null}, ${position} - ) - `; - position += 1; - } - for (const flag of flags) { - await tx` - insert into public.catalogue_import_changes ( - target_id, entry_kind, field_path, severity, is_blocking, issue_code, summary, - source_excerpt, position - ) values ( - ${targetId}::uuid, 'flag', ${flag.fieldPath ?? "snapshot"}, ${flag.severity}, - ${isBlockingFlag(flag)}, ${flag.code}, ${flag.message}, ${flag.sourceExcerpt}, ${position} - ) - `; - position += 1; - } -} - -/** - * Assembles a candidate version for an import target. Returns `unchanged` - * without writing when the content hash matches the baseline. A first import - * for a record becomes its applied version immediately with every change accepted; - * otherwise the changes stay open for review. - */ -export async function persistVersionCandidate( - sql: ImportSql, +/** Persists one semantic ANU observation without changing local content. */ +export async function persistSourceVersion( + sql: SyncSql, { claim, - sourcePageId, + sourceDocumentId, write, }: { - claim: ClaimedImportTarget; - sourcePageId: number | null; + claim: ClaimedCatalogueSync; + sourceDocumentId: number; write: CatalogueContent; }, -): Promise { +): Promise { if ( write.kind !== claim.kind || write.code !== claim.code || write.academicYear !== claim.academicYear ) { throw new TypeError( - "The snapshot content does not match its import target.", + "The source content does not match its catalogue sync.", ); } return sql.begin(async (tx) => { - const [itemYear] = await tx` - select id - from public.catalogue_records - where id = ${claim.recordId} - for update + const [record] = await tx` + select records.id, records.published_version_id, records.latest_source_version_id, + codes.code, years.year, listings.title as listing_title + from public.catalogue_records as records + join public.catalogue_codes as codes on codes.id = records.code_id + join public.academic_years as years on years.id = records.academic_year_id + left join public.catalogue_listings as listings on listings.code_id = records.code_id + and listings.academic_year_id = records.academic_year_id + and listings.kind = records.kind + where records.id = ${claim.recordId} + for update of records `; - if (!itemYear) throw new Error("The catalogue item year was not resolved."); - const baselineVersionId = claim.baselineVersionId; - const [baseline] = baselineVersionId - ? await tx`select content_hash from public.catalogue_versions where id = ${baselineVersionId}` - : []; - const baselineContentHash = baseline ? String(baseline.content_hash) : null; - - if (baselineContentHash === write.contentHash) { - await insertImportChanges(tx, { - targetId: claim.targetId, - changes: [], - flags: write.flags, - acceptAll: true, - }); + if (!record) throw new Error("The catalogue record was not resolved."); + const [existing] = await tx` + select versions.id + from public.catalogue_versions as versions + where versions.sync_id = ${claim.syncId}::uuid + limit 1 + `; + if (existing) { + const sourceVersionId = Number(existing.id); + const [draftFromSource] = await tx` + select 1 from public.catalogue_drafts + where record_id = ${claim.recordId} and base_version_id = ${sourceVersionId} + `; return { - changeKind: "unchanged" as const, - candidateVersionId: null, - baselineVersionId, - becameDraft: false, - changeSet: { - changeKind: "unchanged" as const, - contentHash: write.contentHash, - baselineVersionId, - baselineContentHash, - candidateVersionId: null, - becameDraft: false, - changes: [], - flags: write.flags, - }, + status: draftFromSource ? "applied" : "review_required", + sourceVersionId, + populatedDraft: Boolean(draftFromSource), }; } - - const baselineWrite = baselineVersionId - ? await readVersionContent(tx, baselineVersionId) - : null; - const changes = diffSnapshotWrites(baselineWrite, write); - if (claim.directoryEntryId !== null) { - await tx` - update public.catalogue_listings - set code_id = ${claim.itemId} - where id = ${claim.directoryEntryId} and code_id is null - `; + const previousSourceVersionId = + record.latest_source_version_id === null + ? null + : Number(record.latest_source_version_id); + const [previous] = previousSourceVersionId + ? await tx`select content_hash from public.catalogue_versions where id = ${previousSourceVersionId}` + : []; + if (previous && String(previous.content_hash) === write.contentHash) { + await tx`update public.catalogue_records set source_checked_at = now() + where id = ${claim.recordId}`; + await tx`insert into public.catalogue_change_events ( + record_id, event_kind, origin, actor_id, version_id + ) values (${claim.recordId}, 'source_checked', 'source', null, ${previousSourceVersionId})`; + return { + status: "unchanged", + sourceVersionId: previousSourceVersionId!, + populatedDraft: false, + }; } - const [snapshot] = await tx` + const [version] = await tx` insert into public.catalogue_versions ( - record_id, kind, academic_year_id, origin, based_on_version_id, source_page_id, - content_hash, import_target_id + record_id, kind, academic_year_id, origin, based_on_version_id, + source_document_id, content_hash, sync_id ) values ( - ${claim.recordId}, ${claim.kind}, ${claim.academicYearId}, 'import', - ${baselineVersionId}, ${sourcePageId}, ${write.contentHash}, ${claim.targetId}::uuid + ${claim.recordId}, ${claim.kind}, ${claim.academicYearId}, 'source', + ${previousSourceVersionId}, ${sourceDocumentId}, ${write.contentHash}, ${claim.syncId}::uuid ) returning id `; - const snapshotId = Number(snapshot.id); + const sourceVersionId = Number(version.id); await insertVersionContent(tx, { - snapshotId, + snapshotId: sourceVersionId, kind: claim.kind, academicYearId: claim.academicYearId, - sourcePageId, + sourcePageId: null, write, }); + await tx`update public.catalogue_version_provenance + set source_document_id = ${sourceDocumentId} + where version_id = ${sourceVersionId}`; await tx` update public.catalogue_versions set sealed_at = greatest(statement_timestamp(), created_at) - where id = ${snapshotId} + where id = ${sourceVersionId} `; + await tx`update public.catalogue_records set + latest_source_version_id = ${sourceVersionId}, source_checked_at = now() + where id = ${claim.recordId}`; - const becameDraft = baselineVersionId === null; - await insertImportChanges(tx, { - targetId: claim.targetId, - changes, - flags: write.flags, - acceptAll: becameDraft, + const [draft] = await tx`select content_hash from public.catalogue_drafts + where record_id = ${claim.recordId} for update`; + const empty = emptyCatalogueContent({ + kind: claim.kind, + code: claim.code, + academicYear: claim.academicYear, + title: + record.listing_title === null ? null : String(record.listing_title), }); - if (becameDraft) { - // A first import has nothing to compare against, so its changes are - // recorded as already accepted and the candidate becomes the applied version - // without anyone pressing Apply. Recording that here keeps the target - // honest: it was applied, and leaving applied_version_id null made a - // published record still read "Ready for review". - await tx` - update public.catalogue_import_targets - set applied_version_id = ${snapshotId}, applied_at = now() - where id = ${claim.targetId}::uuid - `; + const hasMeaningfulLocalContent = + record.published_version_id !== null || + (draft && + String(draft.content_hash) !== contentHashForCatalogueContent(empty)); + const populateDraft = + previousSourceVersionId === null && !hasMeaningfulLocalContent; + if (populateDraft) { + await tx`insert into public.catalogue_drafts ( + record_id, base_version_id, content, content_hash, content_schema_version, + revision, updated_by + ) values (${claim.recordId}, ${sourceVersionId}, ${tx.json(write as never)}, + ${write.contentHash}, ${CATALOGUE_CONTENT_SCHEMA_VERSION}, 0, null) + on conflict (record_id) do update set base_version_id = excluded.base_version_id, + content = excluded.content, content_hash = excluded.content_hash, + content_schema_version = excluded.content_schema_version, + revision = public.catalogue_drafts.revision + 1, updated_by = null, + updated_at = now()`; + await tx`delete from public.catalogue_draft_provenance where record_id = ${claim.recordId}`; + await tx`insert into public.catalogue_draft_provenance ( + record_id, field_path, origin, source_version_id, source_evidence_id + ) select ${claim.recordId}, field_path, method, ${sourceVersionId}, id + from public.catalogue_version_provenance where version_id = ${sourceVersionId}`; + await tx`insert into public.catalogue_change_events ( + record_id, draft_revision, event_kind, origin, version_id + ) select ${claim.recordId}, revision, 'source_draft_created', 'source', ${sourceVersionId} + from public.catalogue_drafts where record_id = ${claim.recordId}`; + return { status: "applied", sourceVersionId, populatedDraft: true }; } - const changeKind: SnapshotChangeKind = becameDraft ? "new" : "changed"; + + await tx`insert into public.catalogue_change_events ( + record_id, event_kind, origin, version_id + ) values (${claim.recordId}, 'source_changed', 'source', ${sourceVersionId})`; return { - changeKind, - candidateVersionId: snapshotId, - baselineVersionId, - becameDraft, - changeSet: { - changeKind, - contentHash: write.contentHash, - baselineVersionId, - baselineContentHash, - candidateVersionId: snapshotId, - becameDraft, - changes, - flags: write.flags, - }, + status: "review_required", + sourceVersionId, + populatedDraft: false, }; }); } diff --git a/apps/web/lib/catalogue-import/process-target.ts b/apps/web/lib/catalogue-sync/process-sync.ts similarity index 71% rename from apps/web/lib/catalogue-import/process-target.ts rename to apps/web/lib/catalogue-sync/process-sync.ts index 034ae274..1556d139 100644 --- a/apps/web/lib/catalogue-import/process-target.ts +++ b/apps/web/lib/catalogue-sync/process-sync.ts @@ -1,94 +1,99 @@ import { randomUUID } from "node:crypto"; import { - ImportArtifactConfigurationError, - type ImportArtifactKind, - readImportArtifact, - storeImportArtifact, + SyncArtifactConfigurationError, + type SyncArtifactKind, + readSyncArtifact, + storeSyncArtifact, } from "./artifact-store.ts"; -import { stableFingerprint, stableStringify } from "./canonical.ts"; import { - type ClaimedImportTarget, - type ImportSql, - type ImportStageName, + stableFingerprint, + stableStringify, +} from "../catalogue-import/canonical.ts"; +import { + type ClaimedCatalogueSync, + type SyncSql, + type SyncStageName, attachExtractionResponse, - claimImportTarget, + claimCatalogueSync, completeExtraction, - failImportStage, + failSyncStage, findReusableExtraction, - finishImportStage, - finishImportTarget, - getImportTargetStatus, - recordImportArtifact, - recordSourcePage, - releaseImportTargetForRetry, + finishCatalogueSync, + finishSyncStage, + getCatalogueSyncStatus, + recordSourceDocument, + recordSyncArtifact, + releaseCatalogueSyncForRetry, reserveExtraction, - startImportStage, - withImportDatabaseClient, -} from "./import-store.ts"; -import type { CatalogueKindAdapter } from "./kind-adapter.ts"; -import { courseKindAdapter } from "./kinds/course/adapter.ts"; -import { structureKindAdapter } from "./kinds/structure/adapter.ts"; + startSyncStage, + withSyncDatabaseClient, +} from "./sync-store.ts"; +import type { CatalogueSyncAdapter } from "./kind-adapter.ts"; +import { courseKindAdapter } from "../catalogue-import/kinds/course/adapter.ts"; +import { structureKindAdapter } from "../catalogue-import/kinds/structure/adapter.ts"; import { OpenRouterConfigurationError, OpenRouterRequestError, buildOpenRouterRequestBody, extractWithOpenRouter, restoreOpenRouterExtraction, -} from "./openrouter.ts"; -import { persistVersionCandidate } from "./persist-version.ts"; +} from "../catalogue-import/openrouter.ts"; +import { persistSourceVersion } from "./persist-source-version.ts"; import type { CatalogueKind } from "../catalogue/content.ts"; -const TERMINAL_TARGET_STATUSES = new Set([ - "ready", +const TERMINAL_SYNC_STATUSES = new Set([ + "applied", + "review_required", "unchanged", "failed", "cancelled", ]); -export const CATALOGUE_KIND_ADAPTERS: readonly CatalogueKindAdapter[] = [ - courseKindAdapter as CatalogueKindAdapter, - structureKindAdapter as CatalogueKindAdapter, +export const CATALOGUE_SYNC_ADAPTERS: readonly CatalogueSyncAdapter[] = [ + courseKindAdapter as CatalogueSyncAdapter, + structureKindAdapter as CatalogueSyncAdapter, ]; -export function adapterForKind(kind: CatalogueKind): CatalogueKindAdapter { - const adapter = CATALOGUE_KIND_ADAPTERS.find((candidate) => +export function syncAdapterForKind(kind: CatalogueKind): CatalogueSyncAdapter { + const adapter = CATALOGUE_SYNC_ADAPTERS.find((candidate) => candidate.kinds.includes(kind), ); - if (!adapter) throw new TypeError(`No import adapter handles ${kind}.`); + if (!adapter) + throw new TypeError(`No catalogue sync adapter handles ${kind}.`); return adapter; } -export class ImportPaidOutcomeUncertainError extends Error { +export class SyncPaidOutcomeUncertainError extends Error { constructor(cause: unknown) { super( "An OpenRouter request may have reached the provider, but its response was not durably recorded. Coursemap will not issue an automatic second paid call.", { cause }, ); - this.name = "ImportPaidOutcomeUncertainError"; + this.name = "SyncPaidOutcomeUncertainError"; } } -export class ImportVersionMismatchError extends TypeError { - readonly code = "IMPORT_VERSION_UNSUPPORTED"; +export class SyncVersionMismatchError extends TypeError { + readonly code = "SYNC_VERSION_UNSUPPORTED"; constructor() { super( - "The queued import was created for a different pipeline version. Start a new import with the deployed worker.", + "The queued sync was created for a different pipeline version. Start a new sync with the deployed worker.", ); - this.name = "ImportVersionMismatchError"; + this.name = "SyncVersionMismatchError"; } } function assertCurrentVersions( - adapter: CatalogueKindAdapter, - claim: ClaimedImportTarget, + adapter: CatalogueSyncAdapter, + claim: ClaimedCatalogueSync, ) { if ( claim.parserVersion !== adapter.parserVersion || claim.promptVersion !== adapter.promptVersion || claim.schemaVersion !== adapter.schemaVersion ) { - throw new ImportVersionMismatchError(); + throw new SyncVersionMismatchError(); } } @@ -100,8 +105,7 @@ export function safeErrorSummary(error: unknown) { ? ` Cause: ${error.cause.message}` : ""; const source = - (error instanceof Error ? error.message : "Catalogue import failed.") + - cause; + (error instanceof Error ? error.message : "Catalogue sync failed.") + cause; return source .replace(/postgres(?:ql)?:\/\/[^\s]+/gi, "[database URL redacted]") .replace(/Bearer\s+[^\s]+/gi, "Bearer [redacted]") @@ -109,8 +113,8 @@ export function safeErrorSummary(error: unknown) { .slice(0, 1_500); } -export function importErrorCode(error: unknown) { - if (error instanceof ImportPaidOutcomeUncertainError) +export function syncErrorCode(error: unknown) { + if (error instanceof SyncPaidOutcomeUncertainError) return "OPENROUTER_OUTCOME_UNCERTAIN"; if (error instanceof OpenRouterConfigurationError) return "OPENROUTER_NOT_CONFIGURED"; @@ -125,10 +129,10 @@ export function importErrorCode(error: unknown) { ) { return error.code.trim().slice(0, 120); } - return error instanceof TypeError ? "INVALID_PIPELINE_DATA" : "IMPORT_FAILED"; + return error instanceof TypeError ? "INVALID_PIPELINE_DATA" : "SYNC_FAILED"; } -export function isRetryableImportError(error: unknown) { +export function isRetryableSyncError(error: unknown) { if ( typeof error === "object" && error !== null && @@ -137,7 +141,7 @@ export function isRetryableImportError(error: unknown) { ) { return error.retryable; } - // A definitive HTTP failure is safe to report, but retrying the same target + // A definitive HTTP failure is safe to report, but retrying the same sync // would be misread as an uncertain paid outcome by the reservation check. if (error instanceof OpenRouterRequestError) return false; // Constraint and data errors from Postgres repeat identically on retry. @@ -152,8 +156,8 @@ export function isRetryableImportError(error: unknown) { } if ( error instanceof OpenRouterConfigurationError || - error instanceof ImportPaidOutcomeUncertainError || - error instanceof ImportArtifactConfigurationError || + error instanceof SyncPaidOutcomeUncertainError || + error instanceof SyncArtifactConfigurationError || error instanceof TypeError ) { return false; @@ -161,36 +165,34 @@ export function isRetryableImportError(error: unknown) { return true; } -export type ProcessImportTargetInput = { - runId: string; - targetId: string; +export type ProcessCatalogueSyncInput = { + syncId: string; deliveryCount?: number; maxDeliveries?: number; signal?: AbortSignal; }; /** - * Processes one target end to end under a worker lease. Retryable failures - * return the target to the queue until the final delivery; everything else - * records a failed target so the queue can acknowledge the message. + * Processes one sync end to end under a worker lease. Retryable failures + * return it to the queue until the final delivery; everything else records a + * failed sync so the queue can acknowledge the message. */ -export async function processImportTarget({ - runId, - targetId, +export async function processCatalogueSync({ + syncId, deliveryCount = 1, maxDeliveries = 5, signal, -}: ProcessImportTargetInput): Promise { - await withImportDatabaseClient(async (sql) => { +}: ProcessCatalogueSyncInput): Promise { + await withSyncDatabaseClient(async (sql) => { signal?.throwIfAborted(); const workerId = randomUUID(); - const claim = await claimImportTarget(sql, { runId, targetId, workerId }); + const claim = await claimCatalogueSync(sql, { syncId, workerId }); if (claim === null) { - const status = await getImportTargetStatus(sql, { runId, targetId }); - if (status && TERMINAL_TARGET_STATUSES.has(status.status)) return; - throw new Error("The import target could not be claimed."); + const status = await getCatalogueSyncStatus(sql, syncId); + if (status && TERMINAL_SYNC_STATUSES.has(status)) return; + throw new Error("The catalogue sync could not be claimed."); } - await processClaimedTarget({ + await processClaimedSync({ sql, claim, workerId, @@ -200,41 +202,41 @@ export async function processImportTarget({ }); } -async function processClaimedTarget({ +async function processClaimedSync({ sql, claim, workerId, finalDelivery, signal, }: { - sql: ImportSql; - claim: ClaimedImportTarget; + sql: SyncSql; + claim: ClaimedCatalogueSync; workerId: string; finalDelivery: boolean; signal?: AbortSignal; }) { - const adapter = adapterForKind(claim.kind); - let sourcePageId: number | null = null; + const adapter = syncAdapterForKind(claim.kind); + let sourceDocumentId: number | null = null; const runStage = async ( - stageName: ImportStageName, + stageName: SyncStageName, work: (stageId: string) => Promise, ) => { signal?.throwIfAborted(); - const stageId = await startImportStage(sql, { - targetId: claim.targetId, + const stageId = await startSyncStage(sql, { + syncId: claim.syncId, stageName, attemptNumber: claim.attemptCount, }); try { const value = await work(stageId); signal?.throwIfAborted(); - await finishImportStage(sql, { stageId }); + await finishSyncStage(sql, stageId); return value; } catch (error) { - await failImportStage(sql, { + await failSyncStage(sql, { stageId, - errorCode: importErrorCode(error), + errorCode: syncErrorCode(error), errorSummary: safeErrorSummary(error), }); throw error; @@ -249,23 +251,22 @@ async function processClaimedTarget({ body, }: { stageId: string; - stageName: ImportStageName; - kind: ImportArtifactKind; + stageName: SyncStageName; + kind: SyncArtifactKind; mediaType: string; body: string; }) => { signal?.throwIfAborted(); - const stored = await storeImportArtifact({ + const stored = await storeSyncArtifact({ academicYear: claim.academicYear, - runId: claim.runId, - targetId: claim.targetId, + syncId: claim.syncId, stage: stageName, kind, mediaType, body, }); - return recordImportArtifact(sql, { - targetId: claim.targetId, + return recordSyncArtifact(sql, { + syncId: claim.syncId, stageId, kind, attemptNumber: claim.attemptCount, @@ -292,8 +293,9 @@ async function processClaimedTarget({ mediaType: "text/html", body: page.html, }); - sourcePageId = await recordSourcePage(sql, { + sourceDocumentId = await recordSourceDocument(sql, { sourceId: claim.sourceId, + recordId: claim.recordId, academicYearId: claim.academicYearId, kind: claim.kind, externalKey: claim.code, @@ -378,7 +380,7 @@ async function processClaimedTarget({ body: stableStringify(requestBody), }); const reservation = await reserveExtraction(sql, { - targetId: claim.targetId, + syncId: claim.syncId, extractionNumber: claim.attemptCount, requestedModel: claim.requestedModel, fingerprint, @@ -386,7 +388,7 @@ async function processClaimedTarget({ schemaVersion: claim.schemaVersion, requestArtifactId: requestArtifact.id, }); - const reusable = await findReusableExtraction(sql, { fingerprint }); + const reusable = await findReusableExtraction(sql, fingerprint); let result; let responseArtifactId: string; @@ -395,7 +397,7 @@ async function processClaimedTarget({ if (reusable) { // Identical input already produced a validated response; reuse it // instead of paying for another call. - const body = await readImportArtifact({ + const body = await readSyncArtifact({ artifact: reusable.responseArtifact, }); result = restoreOpenRouterExtraction( @@ -414,14 +416,14 @@ async function processClaimedTarget({ reusable.id === reservation.id ? null : reusable.id; } else if (!reservation.created) { if (!reservation.responseArtifactId) { - throw new ImportPaidOutcomeUncertainError(null); + throw new SyncPaidOutcomeUncertainError(null); } responseArtifactId = reservation.responseArtifactId; const [artifact] = await sql` select media_type, content_sha256, byte_size, storage_bucket, storage_path - from public.catalogue_import_artifacts where id = ${responseArtifactId}::uuid + from public.catalogue_sync_artifacts where id = ${responseArtifactId}::uuid `; - const body = await readImportArtifact({ + const body = await readSyncArtifact({ artifact: { bucket: artifact.storage_bucket, path: String(artifact.storage_path), @@ -461,7 +463,7 @@ async function processClaimedTarget({ ) { throw error; } - throw new ImportPaidOutcomeUncertainError(error); + throw new SyncPaidOutcomeUncertainError(error); } } @@ -537,61 +539,49 @@ async function processClaimedTarget({ return outcome; }); - const write = await runStage("database_project", async (stageId) => { + const write = await runStage("content_project", async (stageId) => { const result = adapter.project(merged.extraction); await persistArtifact({ stageId, - stageName: "database_project", - kind: "database_projection", + stageName: "content_project", + kind: "content_projection", mediaType: "application/json", body: stableStringify(result), }); return result; }); - const persisted = await runStage("snapshot_persist", async (stageId) => { - const result = await persistVersionCandidate(sql, { + const persisted = await runStage("source_version_persist", async () => { + if (sourceDocumentId === null) { + throw new Error("The ANU source document was not preserved."); + } + return persistSourceVersion(sql, { claim, - sourcePageId, + sourceDocumentId, write, }); - await persistArtifact({ - stageId, - stageName: "snapshot_persist", - kind: "change_set", - mediaType: "application/json", - body: stableStringify(result.changeSet), - }); - return result; }); - await finishImportTarget(sql, { - runId: claim.runId, - targetId: claim.targetId, + await finishCatalogueSync(sql, { + syncId: claim.syncId, workerId, expectedLockVersion: claim.lockVersion, - status: persisted.changeKind === "unchanged" ? "unchanged" : "ready", - changeKind: persisted.changeKind, - sourcePageId, - candidateVersionId: persisted.candidateVersionId, - // Discarding the model extraction used to be silent: the target ended - // `ready` with no error code, and only catalogue_extractions recorded - // it. The blocking flag the merge emitted holds publication; this says - // why on the target itself. + status: persisted.status, + sourceDocumentId, + sourceVersionId: persisted.sourceVersionId, errorCode: merged.errorCode ?? null, errorMessage: merged.errorSummary ?? null, }); } catch (error) { - const code = importErrorCode(error); + const code = syncErrorCode(error); const summary = safeErrorSummary(error); if ( - isRetryableImportError(error) && + isRetryableSyncError(error) && !finalDelivery && claim.attemptCount < 5 ) { - await releaseImportTargetForRetry(sql, { - runId: claim.runId, - targetId: claim.targetId, + await releaseCatalogueSyncForRetry(sql, { + syncId: claim.syncId, workerId, expectedLockVersion: claim.lockVersion, errorCode: code, @@ -599,15 +589,13 @@ async function processClaimedTarget({ }); throw error; } - await finishImportTarget(sql, { - runId: claim.runId, - targetId: claim.targetId, + await finishCatalogueSync(sql, { + syncId: claim.syncId, workerId, expectedLockVersion: claim.lockVersion, status: "failed", - changeKind: null, - sourcePageId, - candidateVersionId: null, + sourceDocumentId, + sourceVersionId: null, errorCode: code, errorMessage: summary, }); diff --git a/apps/web/lib/catalogue-sync/sync-queue.ts b/apps/web/lib/catalogue-sync/sync-queue.ts new file mode 100644 index 00000000..cbb153e6 --- /dev/null +++ b/apps/web/lib/catalogue-sync/sync-queue.ts @@ -0,0 +1,195 @@ +import type { MessageMetadata, RetryDirective } from "@vercel/queue"; +import { recordSyncDispatch, withSyncDatabaseClient } from "./sync-store.ts"; +import { + processCatalogueSync, + type ProcessCatalogueSyncInput, +} from "./process-sync.ts"; + +export const SYNC_QUEUE_TOPIC = "catalogue-sync-v1"; +export const SYNC_QUEUE_MESSAGE_VERSION = 1 as const; +export const SYNC_QUEUE_RETENTION_SECONDS = 24 * 60 * 60; +export const SYNC_QUEUE_MAX_DELIVERIES = 5; +export const SYNC_QUEUE_MAX_CALLBACK_DELIVERIES = 12; +export const SYNC_QUEUE_VISIBILITY_TIMEOUT_SECONDS = 600; +export const SYNC_QUEUE_DELIVERY_BUDGET_MS = 290_000; + +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export type SyncQueueMessage = { + version: typeof SYNC_QUEUE_MESSAGE_VERSION; + syncId: string; +}; + +export class SyncQueueMessageError extends TypeError { + constructor(message: string) { + super(message); + this.name = "SyncQueueMessageError"; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function parseSyncQueueMessage(value: unknown): SyncQueueMessage { + if (!isRecord(value)) { + throw new SyncQueueMessageError("Sync queue messages must be objects."); + } + if (Object.keys(value).sort().join(",") !== "syncId,version") { + throw new SyncQueueMessageError( + "Sync queue message fields do not match version 1.", + ); + } + if (value.version !== SYNC_QUEUE_MESSAGE_VERSION) { + throw new SyncQueueMessageError("Unsupported sync queue message version."); + } + if (typeof value.syncId !== "string" || !UUID_PATTERN.test(value.syncId)) { + throw new SyncQueueMessageError("Sync queue syncId must be a UUID."); + } + return { + version: SYNC_QUEUE_MESSAGE_VERSION, + syncId: value.syncId, + }; +} + +export function createSyncQueueMessage(syncId: string) { + return parseSyncQueueMessage({ + version: SYNC_QUEUE_MESSAGE_VERSION, + syncId, + }); +} + +export function createSyncQueueIdempotencyKey(message: SyncQueueMessage) { + return `catalogue-sync:v${message.version}:${message.syncId}`; +} + +/** Only the exact value "true" publishes to Vercel Queues. */ +export function syncQueueEnabled( + value = process.env.COURSEMAP_QUEUE_SYNCS_ENABLED, +) { + return value === "true"; +} + +export type SyncQueueSend = ( + topic: string, + message: SyncQueueMessage, + options: { idempotencyKey: string; retentionSeconds: number }, +) => Promise<{ messageId: string | null }>; + +async function sendWithVercelQueue( + topic: string, + message: SyncQueueMessage, + options: { idempotencyKey: string; retentionSeconds: number }, +) { + // The queue SDK is loaded only when publishing so Next.js page collection + // never constructs its region-aware client. + const { send } = await import("@vercel/queue"); + return send(topic, message, options); +} + +export async function dispatchCatalogueSync({ + syncId, + send = sendWithVercelQueue, +}: { + syncId: string; + send?: SyncQueueSend; +}) { + if (syncQueueEnabled()) { + const message = createSyncQueueMessage(syncId); + try { + const result = await send(SYNC_QUEUE_TOPIC, message, { + idempotencyKey: createSyncQueueIdempotencyKey(message), + retentionSeconds: SYNC_QUEUE_RETENTION_SECONDS, + }); + await withSyncDatabaseClient((sql) => + recordSyncDispatch(sql, { syncId, messageId: result.messageId }), + ); + return { mode: "queue" as const }; + } catch (error) { + await withSyncDatabaseClient((sql) => + recordSyncDispatch(sql, { + syncId, + messageId: null, + errorMessage: + error instanceof Error + ? error.message + : "The queue did not accept this sync.", + }), + ); + throw error; + } + } + await withSyncDatabaseClient((sql) => + recordSyncDispatch(sql, { syncId, messageId: null }), + ); + return { mode: "inline" as const }; +} + +export async function processCatalogueSyncInline({ + syncId, + process = processCatalogueSync, + signal, +}: { + syncId: string; + process?: (input: ProcessCatalogueSyncInput) => Promise; + signal?: AbortSignal; +}) { + for ( + let deliveryCount = 1; + deliveryCount <= SYNC_QUEUE_MAX_DELIVERIES; + deliveryCount += 1 + ) { + signal?.throwIfAborted(); + try { + await process({ + syncId, + deliveryCount, + maxDeliveries: SYNC_QUEUE_MAX_DELIVERIES, + signal, + }); + return; + } catch { + if (deliveryCount === SYNC_QUEUE_MAX_DELIVERIES) return; + } + } +} + +function retrySyncQueueMessage( + error: unknown, + metadata: MessageMetadata, +): RetryDirective { + if (error instanceof SyncQueueMessageError) return { acknowledge: true }; + if (metadata.deliveryCount >= SYNC_QUEUE_MAX_CALLBACK_DELIVERIES) { + return { acknowledge: true }; + } + return { afterSeconds: Math.min(300, 5 * 2 ** (metadata.deliveryCount - 1)) }; +} + +export const syncQueueInternals = { retrySyncQueueMessage }; + +export function createSyncQueueConsumer( + process: ( + input: ProcessCatalogueSyncInput, + ) => void | Promise = processCatalogueSync, +) { + return async (request: Request) => { + const { handleCallback } = await import("@vercel/queue"); + const consume = handleCallback( + async (value, metadata) => { + const message = parseSyncQueueMessage(value); + await process({ + syncId: message.syncId, + deliveryCount: metadata.deliveryCount, + maxDeliveries: SYNC_QUEUE_MAX_DELIVERIES, + signal: AbortSignal.timeout(SYNC_QUEUE_DELIVERY_BUDGET_MS), + }); + }, + { + visibilityTimeoutSeconds: SYNC_QUEUE_VISIBILITY_TIMEOUT_SECONDS, + retry: retrySyncQueueMessage, + }, + ); + return consume(request); + }; +} diff --git a/apps/web/lib/catalogue-sync/sync-service.ts b/apps/web/lib/catalogue-sync/sync-service.ts new file mode 100644 index 00000000..7b73fe0a --- /dev/null +++ b/apps/web/lib/catalogue-sync/sync-service.ts @@ -0,0 +1,52 @@ +import "server-only"; +import { canManageCatalogueSources, getAuthViewer } from "@/lib/auth/viewer"; +import { loadImportModelSetting } from "@/lib/admin/settings"; +import { createClient } from "@/lib/supabase/server"; +import { dispatchCatalogueSync } from "./sync-queue"; +import { syncAdapterForKind } from "./process-sync"; +import type { CatalogueKind } from "../catalogue/content"; + +export class CatalogueSyncStartError extends Error { + constructor(message: string) { + super(message); + this.name = "CatalogueSyncStartError"; + } +} + +export async function startCatalogueSync({ + recordId, + trigger, + requestedBy, + kind, +}: { + recordId: number; + trigger: "manual" | "scheduled"; + requestedBy?: string; + kind: CatalogueKind; +}) { + if (!(await canManageCatalogueSources())) { + throw new CatalogueSyncStartError("Catalogue sync permission is required."); + } + const viewer = await getAuthViewer(); + if (!viewer || (requestedBy && requestedBy !== viewer.id)) { + throw new CatalogueSyncStartError("Authentication is required."); + } + const setting = await loadImportModelSetting(); + if (!setting.model) { + throw new CatalogueSyncStartError("Choose an extraction model first."); + } + const adapter = syncAdapterForKind(kind); + const supabase = await createClient(); + const { data, error } = await supabase.rpc("start_catalogue_sync", { + p_record_id: recordId, + p_trigger: trigger, + p_requested_model: setting.model, + p_parser_version: adapter.parserVersion, + p_prompt_version: adapter.promptVersion, + p_schema_version: adapter.schemaVersion, + }); + if (error) throw new CatalogueSyncStartError(error.message); + const syncId = data as string; + const dispatch = await dispatchCatalogueSync({ syncId }); + return { syncId, mode: dispatch.mode }; +} diff --git a/apps/web/lib/catalogue-sync/sync-store.ts b/apps/web/lib/catalogue-sync/sync-store.ts new file mode 100644 index 00000000..e712b57d --- /dev/null +++ b/apps/web/lib/catalogue-sync/sync-store.ts @@ -0,0 +1,463 @@ +import type postgres from "postgres"; +import { + createHostedSyncDatabaseClient, + createLocalDatabaseClient, +} from "../../scripts/catalogue/lib/local-database.mjs"; +import type { CatalogueKind } from "../catalogue/content.ts"; +import type { + SyncArtifactKind, + SyncArtifactLocator, +} from "./artifact-store.ts"; +import { ANU_PROGRAMS_AND_COURSES_SOURCE } from "../catalogue-import/import-source.ts"; + +export type SyncStageName = + | "source_fetch" + | "html_capture" + | "markdown_normalise" + | "model_input_prepare" + | "deterministic_extract" + | "model_extract" + | "schema_validate" + | "domain_validate" + | "content_project" + | "source_version_persist"; + +export type SyncSql = Awaited>; +export type SyncTransactionSql = postgres.TransactionSql; +type AnySyncSql = SyncSql | SyncTransactionSql; + +export class SyncStoreError extends Error { + readonly code: string; + constructor(message: string, code: string) { + super(message); + this.name = "SyncStoreError"; + this.code = code; + } +} + +export async function createSyncDatabaseClient() { + if ( + process.env.NODE_ENV === "development" || + process.env.COURSEMAP_DATABASE_URL?.trim() + ) { + return createLocalDatabaseClient(); + } + const configured = process.env.COURSEMAP_SYNC_DATABASE_URL?.trim(); + if (configured) return createHostedSyncDatabaseClient(configured); + throw new Error( + "Configure COURSEMAP_SYNC_DATABASE_URL before running durable catalogue syncs.", + ); +} + +export async function withSyncDatabaseClient( + callback: (sql: SyncSql) => Promise, +) { + const sql = await createSyncDatabaseClient(); + try { + return await callback(sql); + } finally { + await sql.end({ timeout: 5 }); + } +} + +export type ClaimedCatalogueSync = { + syncId: string; + kind: CatalogueKind; + code: string; + academicYear: number; + academicYearId: number; + recordId: number; + previousSourceVersionId: number | null; + requestedModel: string; + parserVersion: string; + promptVersion: string; + schemaVersion: string; + sourceId: number; + attemptCount: number; + lockVersion: number; +}; + +function numberOrNull(value: unknown) { + return value === null || value === undefined ? null : Number(value); +} + +export async function ensureAnuSourceId(sql: AnySyncSql) { + const [row] = await sql` + insert into public.catalogue_sources (name, kind, base_url, is_active) + values (${ANU_PROGRAMS_AND_COURSES_SOURCE.name}, ${ANU_PROGRAMS_AND_COURSES_SOURCE.kind}, + ${ANU_PROGRAMS_AND_COURSES_SOURCE.baseUrl}, true) + on conflict (kind, base_url) do update set is_active = true + returning id + `; + return Number(row.id); +} + +export async function claimCatalogueSync( + sql: SyncSql, + { + syncId, + workerId, + leaseSeconds = 120, + }: { + syncId: string; + workerId: string; + leaseSeconds?: number; + }, +): Promise { + return sql.begin(async (tx) => { + const sourceId = await ensureAnuSourceId(tx); + const [row] = await tx` + update public.catalogue_syncs as syncs + set status = 'running', attempt_count = syncs.attempt_count + 1, + lock_version = syncs.lock_version + 1, worker_id = ${workerId}::uuid, + lease_expires_at = now() + make_interval(secs => ${leaseSeconds}), + started_at = coalesce(syncs.started_at, now()) + from public.catalogue_records as records + join public.catalogue_codes as codes on codes.id = records.code_id + join public.academic_years as years on years.id = records.academic_year_id + where syncs.id = ${syncId}::uuid and records.id = syncs.record_id + and syncs.attempt_count < 5 + and (syncs.status = 'queued' + or (syncs.status = 'running' and syncs.lease_expires_at < now())) + returning syncs.id, records.kind, codes.code, years.year as academic_year, + records.academic_year_id, records.id as record_id, + syncs.previous_source_version_id, syncs.requested_model, + syncs.parser_version, syncs.prompt_version, syncs.schema_version, + syncs.attempt_count, syncs.lock_version + `; + if (!row) return null; + return { + syncId: String(row.id), + kind: row.kind as CatalogueKind, + code: String(row.code), + academicYear: Number(row.academic_year), + academicYearId: Number(row.academic_year_id), + recordId: Number(row.record_id), + previousSourceVersionId: numberOrNull(row.previous_source_version_id), + requestedModel: String(row.requested_model), + parserVersion: String(row.parser_version), + promptVersion: String(row.prompt_version), + schemaVersion: String(row.schema_version), + sourceId, + attemptCount: Number(row.attempt_count), + lockVersion: Number(row.lock_version), + }; + }); +} + +export async function getCatalogueSyncStatus(sql: AnySyncSql, syncId: string) { + const [row] = + await sql`select status from public.catalogue_syncs where id = ${syncId}::uuid`; + return row ? String(row.status) : null; +} + +export async function startSyncStage( + sql: AnySyncSql, + input: { + syncId: string; + stageName: SyncStageName; + attemptNumber: number; + }, +) { + const [row] = await sql` + insert into public.catalogue_sync_stages (sync_id, stage_name, attempt_number) + values (${input.syncId}::uuid, ${input.stageName}, ${input.attemptNumber}) + on conflict (sync_id, stage_name, attempt_number) do update set + status = 'running', started_at = statement_timestamp(), completed_at = null, + error_code = null, error_summary = null + returning id + `; + return String(row.id); +} + +export async function finishSyncStage(sql: AnySyncSql, stageId: string) { + await sql`update public.catalogue_sync_stages set status = 'completed', + completed_at = statement_timestamp() where id = ${stageId}::uuid`; +} + +export async function failSyncStage( + sql: AnySyncSql, + input: { + stageId: string; + errorCode: string; + errorSummary: string; + }, +) { + await sql`update public.catalogue_sync_stages set status = 'failed', + completed_at = statement_timestamp(), error_code = ${input.errorCode}, + error_summary = ${input.errorSummary} where id = ${input.stageId}::uuid`; +} + +export type SyncArtifactRecord = SyncArtifactLocator & { + id: string; + kind: SyncArtifactKind; +}; + +export async function recordSyncArtifact( + sql: AnySyncSql, + input: { + syncId: string; + stageId: string; + kind: SyncArtifactKind; + attemptNumber: number; + mediaType: string; + contentSha256: string; + byteSize: number; + storageBucket: string; + storagePath: string; + }, +): Promise { + const [row] = await sql` + insert into public.catalogue_sync_artifacts ( + sync_id, stage_id, kind, attempt_number, media_type, content_sha256, + byte_size, storage_bucket, storage_path + ) values (${input.syncId}::uuid, ${input.stageId}::uuid, ${input.kind}, + ${input.attemptNumber}, ${input.mediaType}, ${input.contentSha256}, + ${input.byteSize}, ${input.storageBucket}, ${input.storagePath}) returning id + `; + return { + id: String(row.id), + kind: input.kind, + bucket: input.storageBucket as SyncArtifactLocator["bucket"], + path: input.storagePath, + mediaType: input.mediaType, + contentSha256: input.contentSha256, + byteSize: input.byteSize, + }; +} + +export async function recordSourceDocument( + sql: AnySyncSql, + input: { + sourceId: number; + recordId: number; + academicYearId: number; + kind: CatalogueKind; + externalKey: string; + canonicalUrl: string; + contentSha256: string; + httpStatus: number | null; + httpEtag: string | null; + sourceLastModified: string | null; + fetchedAt: string; + byteSize: number; + storageBucket: string; + storagePath: string; + }, +) { + const [inserted] = await sql` + insert into public.catalogue_source_documents ( + source_id, record_id, academic_year_id, kind, external_key, canonical_url, + content_sha256, http_status, http_etag, source_last_modified, fetched_at, + byte_size, storage_bucket, storage_path + ) values (${input.sourceId}, ${input.recordId}, ${input.academicYearId}, + ${input.kind}, ${input.externalKey}, ${input.canonicalUrl}, ${input.contentSha256}, + ${input.httpStatus}, ${input.httpEtag}, ${input.sourceLastModified}, + ${input.fetchedAt}, ${input.byteSize}, ${input.storageBucket}, ${input.storagePath}) + on conflict (source_id, record_id, content_sha256) do nothing + returning id + `; + if (inserted) return Number(inserted.id); + const [row] = await sql` + select id from public.catalogue_source_documents + where source_id = ${input.sourceId} and record_id = ${input.recordId} + and content_sha256 = ${input.contentSha256} + `; + return Number(row.id); +} + +export type ReusableExtraction = { + id: string; + syncId: string; + responseArtifact: SyncArtifactLocator; +}; + +export async function findReusableExtraction( + sql: AnySyncSql, + fingerprint: string, +) { + const [row] = await sql` + select extractions.id, extractions.sync_id, artifacts.media_type, + artifacts.content_sha256, artifacts.byte_size, artifacts.storage_bucket, + artifacts.storage_path + from public.catalogue_extractions as extractions + join public.catalogue_sync_artifacts as artifacts + on artifacts.id = extractions.response_artifact_id + where extractions.fingerprint = ${fingerprint} + and extractions.validation_status = 'valid' + and extractions.completed_at is not null + order by extractions.completed_at desc limit 1 + `; + if (!row) return null; + return { + id: String(row.id), + syncId: String(row.sync_id), + responseArtifact: { + bucket: row.storage_bucket as SyncArtifactLocator["bucket"], + path: String(row.storage_path), + mediaType: String(row.media_type), + contentSha256: String(row.content_sha256), + byteSize: Number(row.byte_size), + }, + } satisfies ReusableExtraction; +} + +export async function reserveExtraction( + sql: AnySyncSql, + input: { + syncId: string; + extractionNumber: number; + requestedModel: string; + fingerprint: string; + promptVersion: string; + schemaVersion: string; + requestArtifactId: string; + }, +) { + const [existing] = + await sql`select id, response_artifact_id from public.catalogue_extractions + where sync_id = ${input.syncId}::uuid and fingerprint = ${input.fingerprint} + order by started_at desc limit 1`; + if (existing) + return { + id: String(existing.id), + created: false, + responseArtifactId: existing.response_artifact_id + ? String(existing.response_artifact_id) + : null, + }; + const [row] = await sql`insert into public.catalogue_extractions ( + sync_id, extraction_number, requested_model, fingerprint, prompt_version, + schema_version, request_artifact_id + ) values (${input.syncId}::uuid, ${input.extractionNumber}, ${input.requestedModel}, + ${input.fingerprint}, ${input.promptVersion}, ${input.schemaVersion}, + ${input.requestArtifactId}::uuid) returning id`; + return { id: String(row.id), created: true, responseArtifactId: null }; +} + +export async function attachExtractionResponse( + sql: AnySyncSql, + input: Record & { + extractionId: string; + responseArtifactId: string; + }, +) { + await sql`update public.catalogue_extractions set + response_artifact_id = ${input.responseArtifactId}::uuid, + resolved_model = ${input.resolvedModel as string}, + reused_from_extraction_id = ${input.reusedFromExtractionId as string | null}::uuid, + provider_request_id = ${input.providerRequestId as string | null}, + finish_reason = ${input.finishReason as string | null}, + input_tokens = ${input.inputTokens as number}, + cached_input_tokens = ${input.cachedInputTokens as number}, + output_tokens = ${input.outputTokens as number}, + reasoning_tokens = ${input.reasoningTokens as number}, + cost_usd = ${input.costUsd as number}, cost_source = ${input.costSource as string}, + latency_ms = ${Math.round(input.latencyMs as number)} + where id = ${input.extractionId}::uuid`; +} + +export async function completeExtraction( + sql: AnySyncSql, + input: Record & { + extractionId: string; + }, +) { + const valid = Boolean(input.schemaValid) && Boolean(input.domainValid); + await sql`update public.catalogue_extractions set + validated_artifact_id = ${input.validatedArtifactId as string | null}::uuid, + validation_status = ${valid ? "valid" : "invalid"}, + schema_valid = ${Boolean(input.schemaValid)}, domain_valid = ${Boolean(input.domainValid)}, + warning_count = ${input.warningCount as number}, error_count = ${input.errorCount as number}, + error_summary = ${input.errorSummary as string | null}, completed_at = now() + where id = ${input.extractionId}::uuid`; +} + +function assertLeaseHeld(count: number) { + if (count !== 1) + throw new SyncStoreError( + "The catalogue sync lease was lost before its result could be recorded.", + "LEASE_LOST", + ); +} + +export async function finishCatalogueSync( + sql: SyncSql, + input: { + syncId: string; + workerId: string; + expectedLockVersion: number; + status: "unchanged" | "review_required" | "applied" | "failed"; + sourceDocumentId: number | null; + sourceVersionId: number | null; + errorCode?: string | null; + errorMessage?: string | null; + }, +) { + const rows = await sql.begin(async (tx) => { + const finished = + await tx`update public.catalogue_syncs set status = ${input.status}, + source_document_id = ${input.sourceDocumentId}, source_version_id = ${input.sourceVersionId}, + checked_at = case when ${input.status} <> 'failed' then now() else checked_at end, + completed_at = now(), worker_id = null, lease_expires_at = null, + error_code = ${input.errorCode ?? null}, error_message = ${input.errorMessage ?? null} + where id = ${input.syncId}::uuid and status = 'running' + and worker_id = ${input.workerId}::uuid and lock_version = ${input.expectedLockVersion} + returning id, record_id, requested_by`; + if (finished.length === 1 && input.status === "failed") { + await tx`insert into public.catalogue_change_events ( + record_id, event_kind, origin, actor_id + ) values (${finished[0].record_id}, 'sync_failed', 'source', ${finished[0].requested_by})`; + } + return finished; + }); + assertLeaseHeld(rows.length); +} + +export async function releaseCatalogueSyncForRetry( + sql: SyncSql, + input: { + syncId: string; + workerId: string; + expectedLockVersion: number; + errorCode: string; + errorMessage: string; + }, +) { + const rows = await sql`update public.catalogue_syncs set status = 'queued', + worker_id = null, lease_expires_at = null, error_code = ${input.errorCode}, + error_message = ${input.errorMessage} + where id = ${input.syncId}::uuid and status = 'running' + and worker_id = ${input.workerId}::uuid and lock_version = ${input.expectedLockVersion} + returning id`; + assertLeaseHeld(rows.length); +} + +export async function recordSyncDispatch( + sql: SyncSql, + input: { + syncId: string; + messageId: string | null; + errorMessage?: string; + }, +) { + if (input.errorMessage) { + const errorMessage = input.errorMessage; + await sql.begin(async (tx) => { + const [failed] = + await tx`update public.catalogue_syncs set status = 'failed', + error_code = 'QUEUE_DISPATCH_FAILED', error_message = ${errorMessage}, + completed_at = now() where id = ${input.syncId}::uuid and status = 'queued' + returning record_id, requested_by`; + if (failed) { + await tx`insert into public.catalogue_change_events ( + record_id, event_kind, origin, actor_id + ) values (${failed.record_id}, 'sync_failed', 'source', ${failed.requested_by})`; + } + }); + } else { + await sql`update public.catalogue_syncs set dispatched_at = coalesce(dispatched_at, now()), + queue_message_id = coalesce(queue_message_id, ${input.messageId}) + where id = ${input.syncId}::uuid and status = 'queued'`; + } +} diff --git a/apps/web/lib/catalogue/drafts.ts b/apps/web/lib/catalogue/drafts.ts index ef7a5605..826392b2 100644 --- a/apps/web/lib/catalogue/drafts.ts +++ b/apps/web/lib/catalogue/drafts.ts @@ -1,12 +1,12 @@ import "server-only"; import type { - ImportSql, - ImportTransactionSql, -} from "@/lib/catalogue-import/import-store"; -import { withImportDatabaseClient } from "@/lib/catalogue-import/import-store"; + SyncSql, + SyncTransactionSql, +} from "@/lib/catalogue-sync/sync-store"; +import { withSyncDatabaseClient } from "@/lib/catalogue-sync/sync-store"; import { diffSnapshotWrites } from "@/lib/catalogue-import/changes"; -import { insertVersionContent } from "@/lib/catalogue-import/persist-version"; +import { insertVersionContent } from "@/lib/catalogue-sync/persist-source-version"; import { contentHashForCatalogueContent, readVersionContent, @@ -19,7 +19,7 @@ import { type CatalogueKind, } from "@/lib/catalogue/content"; -type Sql = ImportSql | ImportTransactionSql; +type Sql = SyncSql | SyncTransactionSql; export type CatalogueDraft = { recordId: number; @@ -196,9 +196,9 @@ export async function createCatalogueDraft({ }: { recordId: number; userId: string; - sql?: ImportSql; + sql?: SyncSql; }) { - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); if (record.archived_at) @@ -213,11 +213,11 @@ export async function createCatalogueDraft({ ? draftFromRow(existing) : createDraftInTransaction(tx, record, userId); }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } export async function loadCatalogueDraft(recordId: number) { - return withImportDatabaseClient(async (sql) => { + return withSyncDatabaseClient(async (sql) => { const [row] = await sql` select * from public.catalogue_drafts where record_id = ${recordId} `; @@ -239,11 +239,11 @@ export async function saveCatalogueDraft({ content: unknown; userId: string; editingSessionId: string; - sql?: ImportSql; + sql?: SyncSql; }) { assertEditingSession(editingSessionId); const content = validateCatalogueContent(submitted); - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); if (record.archived_at) @@ -314,11 +314,11 @@ export async function saveCatalogueDraft({ changedPaths: changes.map((change) => change.fieldPath), }; }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } async function materialiseDraftVersion( - tx: ImportTransactionSql, + tx: SyncTransactionSql, { record, draft, @@ -373,7 +373,8 @@ async function materialiseDraftVersion( }); await tx` update public.catalogue_version_provenance as published - set source_page_id = source.source_page_id + set source_page_id = source.source_page_id, + source_document_id = source.source_document_id from public.catalogue_draft_provenance as draft join public.catalogue_version_provenance as source on source.id = draft.source_evidence_id @@ -400,10 +401,10 @@ export async function publishCatalogueDraft({ expectedRevision: number; userId: string; editingSessionId: string; - sql?: ImportSql; + sql?: SyncSql; }) { assertEditingSession(editingSessionId); - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); if (record.archived_at) @@ -457,7 +458,7 @@ export async function publishCatalogueDraft({ await tx`delete from public.catalogue_drafts where record_id = ${recordId}`; return { versionId }; }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } export async function unpublishCatalogueRecord({ @@ -469,10 +470,10 @@ export async function unpublishCatalogueRecord({ recordId: number; userId: string; editingSessionId: string; - sql?: ImportSql; + sql?: SyncSql; }) { assertEditingSession(editingSessionId); - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); if (record.published_version_id === null) @@ -497,7 +498,7 @@ export async function unpublishCatalogueRecord({ `; return { versionId }; }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } async function draftIsMeaningful( @@ -529,10 +530,10 @@ export async function discardCatalogueDraft({ expectedRevision: number; userId: string; editingSessionId: string; - sql?: ImportSql; + sql?: SyncSql; }) { assertEditingSession(editingSessionId); - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); const [row] = await tx` @@ -567,7 +568,7 @@ export async function discardCatalogueDraft({ await tx`delete from public.catalogue_drafts where record_id = ${recordId}`; return { checkpointVersionId, meaningful }; }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } export async function restoreCatalogueVersion({ @@ -585,10 +586,10 @@ export async function restoreCatalogueVersion({ replaceExistingDraft: boolean; userId: string; editingSessionId: string; - sql?: ImportSql; + sql?: SyncSql; }) { assertEditingSession(editingSessionId); - const work = (client: ImportSql) => + const work = (client: SyncSql) => client.begin(async (tx) => { const record = await recordForUpdate(tx, recordId); const content = await readVersionContent(tx, versionId); @@ -647,5 +648,5 @@ export async function restoreCatalogueVersion({ `; return { revision, content: restored }; }); - return sql ? work(sql) : withImportDatabaseClient(work); + return sql ? work(sql) : withSyncDatabaseClient(work); } diff --git a/apps/web/lib/coursemap/admin-catalogue-actions.ts b/apps/web/lib/coursemap/admin-catalogue-actions.ts index 22a1bc90..c0bbab46 100644 --- a/apps/web/lib/coursemap/admin-catalogue-actions.ts +++ b/apps/web/lib/coursemap/admin-catalogue-actions.ts @@ -1,15 +1,7 @@ "use server"; import { revalidatePath } from "next/cache"; -import { - canManageCourseImports, - canWriteCatalogue, - getAuthViewer, -} from "@/lib/auth/viewer"; -import { - ApplyReviewError, - applyImportReview, -} from "@/lib/catalogue-import/apply-review"; +import { canWriteCatalogue, getAuthViewer } from "@/lib/auth/viewer"; import type { CatalogueContent } from "@/lib/catalogue/content"; import { CatalogueDraftConflictError, @@ -20,7 +12,6 @@ import { saveCatalogueDraft, unpublishCatalogueRecord, } from "@/lib/catalogue/drafts"; -import { createClient } from "@/lib/supabase/server"; export type ActionResult = { ok: true; message?: string } | { ok: false; error: string }; @@ -34,13 +25,6 @@ export type DraftActionResult = currentRevision?: number; }; -function failure(error: unknown, fallback: string): ActionResult { - return { - ok: false, - error: error instanceof Error ? error.message : fallback, - }; -} - /** * The record page identifies itself with a URL carrying the academic year, but * revalidatePath matches a route path. Passing the query string made every @@ -51,110 +35,6 @@ function revalidateRecord(path: string) { revalidatePath(path.split("?")[0] ?? path); } -export async function resolveReviewEntryAction({ - entryId, - status, - note, - path, -}: { - entryId: number; - status: "open" | "accepted" | "rejected" | "acknowledged"; - note?: string; - path: string; -}): Promise { - if (!(await canManageCourseImports())) - return { ok: false, error: "Import permission is required." }; - const supabase = await createClient(); - const { error } = await supabase.rpc("resolve_catalogue_import_change", { - p_change_id: entryId, - p_status: status, - p_note: note ?? undefined, - }); - if (error) return { ok: false, error: error.message }; - revalidateRecord(path); - return { ok: true }; -} - -/** - * Several review entries at once: one group of changes, one kind of flag, or - * everything still open. The reviewer already has the entries on screen, so - * the ids come with the request rather than being looked up again, which keeps - * a bulk decision to exactly the rows the reviewer was shown. - */ -export async function resolveReviewEntriesAction({ - entryIds, - status, - note, - path, -}: { - entryIds: number[]; - status: "open" | "accepted" | "rejected" | "acknowledged"; - note?: string; - path: string; -}): Promise { - if (!(await canManageCourseImports())) - return { ok: false, error: "Import permission is required." }; - if (entryIds.length === 0) - return { ok: false, error: "There was nothing to decide." }; - const supabase = await createClient(); - let resolved = 0; - for (const entryId of entryIds) { - const { error } = await supabase.rpc("resolve_catalogue_import_change", { - p_change_id: entryId, - p_status: status, - p_note: note ?? undefined, - }); - // Report what did land, so a partial failure is not read as none at all. - if (error) - return { - ok: false, - error: - resolved === 0 - ? error.message - : `${resolved} of ${entryIds.length} were saved, then: ${error.message}`, - }; - resolved += 1; - } - revalidateRecord(path); - const verb = - status === "acknowledged" - ? "acknowledged" - : status === "open" - ? "reopened" - : status; - return { - ok: true, - message: `${resolved} ${resolved === 1 ? "entry" : "entries"} ${verb}.`, - }; -} - -export async function applyReviewAction({ - targetId, - path, -}: { - targetId: string; - path: string; -}): Promise { - if (!(await canManageCourseImports())) - return { ok: false, error: "Import permission is required." }; - const viewer = await getAuthViewer(); - if (!viewer) return { ok: false, error: "Authentication is required." }; - try { - const result = await applyImportReview({ targetId, userId: viewer.id }); - revalidateRecord(path); - return { - ok: true, - message: result.reusedCandidate - ? "The import is now the draft." - : "A new draft combines the current content with the accepted changes.", - }; - } catch (error) { - if (error instanceof ApplyReviewError) - return { ok: false, error: error.message }; - return failure(error, "The review could not be applied."); - } -} - function draftFailure(error: unknown, fallback: string): DraftActionResult { if (error instanceof CatalogueDraftConflictError) { return { diff --git a/apps/web/lib/coursemap/admin-catalogue-record.ts b/apps/web/lib/coursemap/admin-catalogue-record.ts index a6639f18..ee7adf68 100644 --- a/apps/web/lib/coursemap/admin-catalogue-record.ts +++ b/apps/web/lib/coursemap/admin-catalogue-record.ts @@ -1,41 +1,30 @@ import "server-only"; -import { withImportDatabaseClient } from "@/lib/catalogue-import/import-store"; import { readVersionContent } from "@/lib/catalogue-import/version-content"; +import { withSyncDatabaseClient } from "@/lib/catalogue-sync/sync-store"; import type { CatalogueContent } from "@/lib/catalogue/content"; import { createClient } from "@/lib/supabase/server"; import type { Json } from "@/types/database"; import type { CatalogueKind } from "./catalogue-kinds"; import { courseFromSnapshotProjection } from "./published-courses"; -export type ReviewEntry = { - id: number; - entryKind: "change" | "flag"; - fieldPath: string; - oldValue: unknown; - newValue: unknown; - severity: "warning" | "error" | null; - isBlocking: boolean; - issueCode: string | null; - summary: string | null; - sourceLocator: string | null; - sourceExcerpt: string | null; - status: "open" | "accepted" | "rejected" | "acknowledged"; - resolutionNote: string | null; - resolvedAt: string | null; -}; - -export type ReviewTarget = { +export type CatalogueSync = { id: string; - runId: string; - runNumber: number; - status: string; - changeKind: string | null; - createdAt: string; + status: + | "queued" + | "running" + | "unchanged" + | "review_required" + | "applied" + | "failed" + | "cancelled"; + trigger: "manual" | "scheduled"; + requestedAt: string; + checkedAt: string | null; completedAt: string | null; - appliedAt: string | null; - baselineVersionId: number | null; - candidateVersionId: number | null; - entries: ReviewEntry[]; + previousSourceVersionId: number | null; + sourceVersionId: number | null; + errorCode: string | null; + errorMessage: string | null; }; export type CatalogueVersion = { @@ -45,7 +34,7 @@ export type CatalogueVersion = { createdAt: string; sealedAt: string | null; basedOnVersionId: number | null; - importTargetId: string | null; + syncId: string | null; contentHash: string; }; @@ -59,6 +48,8 @@ export type CatalogueRecord = { title: string; currentVersionId: number | null; publishedVersionId: number | null; + latestSourceVersionId: number | null; + sourceCheckedAt: string | null; archivedAt: string | null; isListedByAnu: boolean | null; listingTitle: string | null; @@ -74,13 +65,22 @@ export type CatalogueRecord = { }>; changeEvents: Array<{ id: number; - eventKind: "edit" | "publish" | "unpublish" | "discard" | "restore"; + eventKind: + | "edit" + | "publish" + | "unpublish" + | "discard" + | "restore" + | "source_draft_created" + | "source_checked" + | "source_changed" + | "sync_failed"; draftRevision: number | null; editingSessionId: string | null; versionId: number | null; createdAt: string; }>; - reviews: ReviewTarget[]; + syncs: CatalogueSync[]; }; async function versionTitle( @@ -105,7 +105,7 @@ async function versionTitle( return data?.name ?? null; } -/** Everything the record page needs: pointers, history and every import review. */ +/** Everything the record page needs: local content and independent ANU sync state. */ export async function loadCatalogueRecord({ kind, code, @@ -119,7 +119,7 @@ export async function loadCatalogueRecord({ const { data: itemYear, error } = await supabase .from("catalogue_records") .select( - "id,public_id,code_id,published_version_id,archived_at,catalogue_codes!inner(code,kind),academic_years!inner(year)", + "id,public_id,code_id,published_version_id,latest_source_version_id,source_checked_at,archived_at,catalogue_codes!inner(code,kind),academic_years!inner(year)", ) .eq("kind", kind) .eq("catalogue_codes.code", code.toUpperCase()) @@ -131,7 +131,7 @@ export async function loadCatalogueRecord({ const [ versionsResult, publicationsResult, - targetsResult, + syncsResult, blockersResult, listingResult, changeEventsResult, @@ -139,7 +139,7 @@ export async function loadCatalogueRecord({ supabase .from("catalogue_versions") .select( - "id,public_id,origin,created_at,sealed_at,based_on_version_id,import_target_id,content_hash", + "id,public_id,origin,created_at,sealed_at,based_on_version_id,sync_id,content_hash", ) .eq("record_id", itemYear.id) .order("created_at", { ascending: false }), @@ -151,9 +151,9 @@ export async function loadCatalogueRecord({ .eq("record_id", itemYear.id) .order("published_at", { ascending: false }), supabase - .from("catalogue_import_targets") + .from("catalogue_syncs") .select( - "id,run_id,status,change_kind,created_at,completed_at,applied_at,applied_version_id,baseline_version_id,candidate_version_id,catalogue_import_runs!inner(run_number)", + "id,status,trigger,requested_at,checked_at,completed_at,previous_source_version_id,source_version_id,error_code,error_message", ) .eq("record_id", itemYear.id) .order("created_at", { ascending: false }), @@ -173,57 +173,14 @@ export async function loadCatalogueRecord({ ]); if (versionsResult.error) throw versionsResult.error; if (publicationsResult.error) throw publicationsResult.error; - if (targetsResult.error) throw targetsResult.error; + if (syncsResult.error) throw syncsResult.error; if (blockersResult.error) throw blockersResult.error; if (listingResult.error) throw listingResult.error; if (changeEventsResult.error) throw changeEventsResult.error; - const appliedVersionIds = new Set( - (targetsResult.data ?? []).flatMap((target) => - target.applied_version_id === null ? [] : [target.applied_version_id], - ), - ); - const currentVersionId = - (versionsResult.data ?? []).find( - (version) => - version.sealed_at !== null && - (version.id === itemYear.published_version_id || - (version.import_target_id !== null && - appliedVersionIds.has(version.id))), - )?.id ?? null; + const currentVersionId = itemYear.published_version_id; const title = await versionTitle(supabase, kind, currentVersionId); - const targetIds = (targetsResult.data ?? []).map((target) => target.id); - const { data: entries, error: entriesError } = targetIds.length - ? await supabase - .from("catalogue_import_changes") - .select("*") - .in("target_id", targetIds) - .order("position") - : { data: [], error: null }; - if (entriesError) throw entriesError; - const entriesByTarget = new Map(); - for (const entry of entries ?? []) { - const list = entriesByTarget.get(entry.target_id) ?? []; - list.push({ - id: entry.id, - entryKind: entry.entry_kind as ReviewEntry["entryKind"], - fieldPath: entry.field_path, - oldValue: entry.old_value, - newValue: entry.new_value, - severity: entry.severity as ReviewEntry["severity"], - isBlocking: entry.is_blocking, - issueCode: entry.issue_code, - summary: entry.summary, - sourceLocator: entry.source_locator, - sourceExcerpt: entry.source_excerpt, - status: entry.status as ReviewEntry["status"], - resolutionNote: entry.resolution_note, - resolvedAt: entry.resolved_at, - }); - entriesByTarget.set(entry.target_id, list); - } - return { kind, code: itemYear.catalogue_codes.code, @@ -234,6 +191,8 @@ export async function loadCatalogueRecord({ title: title ?? listingResult.data?.title ?? itemYear.catalogue_codes.code, currentVersionId, publishedVersionId: itemYear.published_version_id, + latestSourceVersionId: itemYear.latest_source_version_id, + sourceCheckedAt: itemYear.source_checked_at, archivedAt: itemYear.archived_at, isListedByAnu: listingResult.data?.is_current ?? null, listingTitle: listingResult.data?.title ?? null, @@ -246,7 +205,7 @@ export async function loadCatalogueRecord({ createdAt: version.created_at, sealedAt: version.sealed_at, basedOnVersionId: version.based_on_version_id, - importTargetId: version.import_target_id, + syncId: version.sync_id, contentHash: version.content_hash, })), publications: (publicationsResult.data ?? []).map((publication) => ({ @@ -265,39 +224,38 @@ export async function loadCatalogueRecord({ versionId: event.version_id, createdAt: event.created_at, })), - reviews: (targetsResult.data ?? []).map((target) => ({ - id: target.id, - runId: target.run_id, - runNumber: target.catalogue_import_runs.run_number, - status: target.status, - changeKind: target.change_kind, - createdAt: target.created_at, - completedAt: target.completed_at, - appliedAt: target.applied_at, - baselineVersionId: target.baseline_version_id, - candidateVersionId: target.candidate_version_id, - entries: entriesByTarget.get(target.id) ?? [], + syncs: (syncsResult.data ?? []).map((sync) => ({ + id: sync.id, + status: sync.status as CatalogueSync["status"], + trigger: sync.trigger as CatalogueSync["trigger"], + requestedAt: sync.requested_at, + checkedAt: sync.checked_at, + completedAt: sync.completed_at, + previousSourceVersionId: sync.previous_source_version_id, + sourceVersionId: sync.source_version_id, + errorCode: sync.error_code, + errorMessage: sync.error_message, })), }; } -/** The editable content of a snapshot, read through the import connection. */ -export async function loadSnapshotWrite( - snapshotId: number, +/** The editable content of an immutable catalogue version. */ +export async function loadVersionWrite( + versionId: number, ): Promise { - return withImportDatabaseClient((sql) => readVersionContent(sql, snapshotId)); + return withSyncDatabaseClient((sql) => readVersionContent(sql, versionId)); } -/** The student-facing course details for a snapshot, or null for structures. */ -export async function loadSnapshotCoursePreview(snapshotId: number) { +/** The student-facing course details for a version, or null for structures. */ +export async function loadVersionCoursePreview(versionId: number) { const supabase = await createClient(); const { data, error } = await supabase.rpc( "admin_catalogue_version_projection", { - p_version_id: snapshotId, + p_version_id: versionId, }, ); if (error) throw error; if (data === null) return null; - return courseFromSnapshotProjection(data as Json, snapshotId); + return courseFromSnapshotProjection(data as Json, versionId); } diff --git a/apps/web/lib/coursemap/admin-catalogue.ts b/apps/web/lib/coursemap/admin-catalogue.ts index f9b68a26..7cbda965 100644 --- a/apps/web/lib/coursemap/admin-catalogue.ts +++ b/apps/web/lib/coursemap/admin-catalogue.ts @@ -1,68 +1,38 @@ import "server-only"; import type { PostgrestError } from "@supabase/supabase-js"; import { createClient } from "@/lib/supabase/server"; -import { - type AdminCatalogueSummary, - CATALOGUE_KIND_LABELS, - type CatalogueDirectoryPage, - type CatalogueDirectoryRecord, - type CatalogueKind, - DEFAULT_IMPORT_RECORD_SORT, - type DirectoryFilter, - type DirectoryWorkflowStatus, - type ImportRecordSort, - type ImportRecordsPage, - type ImportRunProgress, - type ImportRunRow, - type ImportTargetDetail, +import type { + CatalogueDirectoryPage, + CatalogueDirectoryRecord, + CatalogueKind, } from "./catalogue-kinds"; export * from "./catalogue-kinds"; const PAGE_SIZE = 50; +const ROW_PAGE_SIZE = 1000; -function workflowFor(input: { - hasDraft: boolean; - isPublished: boolean; - latestStatus: string | null; -}): DirectoryWorkflowStatus { - if (input.latestStatus === "queued") return "queued"; - if (input.latestStatus === "running") return "running"; - if (input.latestStatus === "failed" && !input.isPublished && !input.hasDraft) - return "failed"; - if (input.isPublished && input.hasDraft) return "published_with_draft"; - if (input.isPublished) return "published"; - if (input.latestStatus === "ready") return "ready"; - if (input.hasDraft) return "draft"; - if (input.latestStatus === "failed") return "failed"; - return "not_imported"; -} - -function emptyCounts(): Record { - return { - not_imported: 0, - queued: 0, - running: 0, - ready: 0, - draft: 0, - published: 0, - published_with_draft: 0, - failed: 0, - }; +async function readAllRows( + readPage: ( + from: number, + to: number, + ) => PromiseLike<{ + data: Row[] | null; + error: PostgrestError | null; + }>, +) { + const rows: Row[] = []; + for (let from = 0; ; from += ROW_PAGE_SIZE) { + const { data, error } = await readPage(from, from + ROW_PAGE_SIZE - 1); + if (error) return { data: rows, error }; + rows.push(...(data ?? [])); + if ((data?.length ?? 0) < ROW_PAGE_SIZE) return { data: rows, error: null }; + } } -/** - * The years an administrator can choose. Every seeded academic year from 2020 - * to 2030 used to be offered, so the picker listed eleven years of which only - * one held any catalogue, tall enough to cover the page tabs when it opened. - * It now offers the years whose listing has been fetched, plus this year and - * next so the coming handbook can always be imported before it has any rows. - * The fetched years come from the directory statuses, one row per year and - * kind, rather than from the entries themselves, which run to thousands. - */ export async function loadCatalogueYears() { const supabase = await createClient(); - const [yearsResult, statusesResult] = await Promise.all([ + const [years, statuses] = await Promise.all([ supabase .from("academic_years") .select("id,year") @@ -70,13 +40,13 @@ export async function loadCatalogueYears() { .lte("year", 2030), supabase.from("catalogue_discovery_statuses").select("academic_year_id"), ]); - if (yearsResult.error) throw yearsResult.error; - if (statusesResult.error) throw statusesResult.error; + if (years.error) throw years.error; + if (statuses.error) throw statuses.error; const current = new Date().getFullYear(); const fetched = new Set( - (statusesResult.data ?? []).map((row) => row.academic_year_id), + (statuses.data ?? []).map((row) => row.academic_year_id), ); - return (yearsResult.data ?? []) + return (years.data ?? []) .filter( (row) => fetched.has(row.id) || row.year === current || row.year === current + 1, @@ -85,10 +55,6 @@ export async function loadCatalogueYears() { .sort((left, right) => right - left); } -/** - * The year an administrator most likely wants: the newest with catalogue - * content for the kind, else the current calendar year, else the newest. - */ export async function defaultCatalogueYear( kind: CatalogueKind, years: number[], @@ -101,60 +67,26 @@ export async function defaultCatalogueYear( .order("academic_year_id", { ascending: false }) .limit(1) .maybeSingle(); - const withContent = data?.academic_years?.year; - if (withContent && years.includes(withContent)) return withContent; + const available = data?.academic_years?.year; + if (available && years.includes(available)) return available; const current = new Date().getFullYear(); - if (years.includes(current)) return current; - return years[0] ?? current; + return years.includes(current) ? current : (years[0] ?? current); } -const ROW_PAGE_SIZE = 1000; - -/** - * PostgREST answers every request with at most 1,000 rows, whatever the query - * asks for, and does so without saying anything. The directory reads a whole - * year's listing to filter it in memory, so one request stopped at the - * thousandth code: everything after EMET1001, about two thirds of the course - * catalogue, could not be found, filtered or imported from the directory, and - * the footer reported a total of exactly 1,000. This reads every page. Each - * caller must order by a unique key so a row cannot move between pages. - */ -async function readAllRows( - readPage: ( - from: number, - to: number, - ) => PromiseLike<{ data: Row[] | null; error: PostgrestError | null }>, -): Promise<{ data: Row[]; error: PostgrestError | null }> { - const rows: Row[] = []; - for (let from = 0; ; from += ROW_PAGE_SIZE) { - const { data, error } = await readPage(from, from + ROW_PAGE_SIZE - 1); - if (error) return { data: rows, error }; - rows.push(...(data ?? [])); - if ((data?.length ?? 0) < ROW_PAGE_SIZE) return { data: rows, error: null }; - } -} - -/** - * Directory entries for one kind and year with each record's workflow state. - * Filtering by workflow happens in memory because the state derives from - * three tables; a directory holds a few thousand rows at most. - */ export async function loadCatalogueDirectoryPage({ kind, academicYear, query = "", - filter = "all", page = 1, }: { kind: CatalogueKind; academicYear: number; query?: string; - filter?: DirectoryFilter; page?: number; }): Promise { const supabase = await createClient(); const years = await loadCatalogueYears(); - const { data: yearRow } = await supabase + const { data: year } = await supabase .from("academic_years") .select("id") .eq("year", academicYear) @@ -168,615 +100,160 @@ export async function loadCatalogueDirectoryPage({ total: 0, page: 1, pageSize: PAGE_SIZE, - workflowCounts: emptyCounts(), }; - if (!yearRow) return empty; + if (!year) return empty; - const [statusResult, entriesResult, itemYearsResult, targetsResult] = - await Promise.all([ + const [status, listings, records] = await Promise.all([ + supabase + .from("catalogue_discovery_statuses") + .select("status,refreshed_at,message,entry_count") + .eq("academic_year_id", year.id) + .eq("kind", kind) + .maybeSingle(), + readAllRows((from, to) => supabase - .from("catalogue_discovery_statuses") - .select("status,refreshed_at,message,entry_count") - .eq("academic_year_id", yearRow.id) + .from("catalogue_listings") + .select("code,title,summary,code_id,is_current,last_seen_at") + .eq("academic_year_id", year.id) .eq("kind", kind) - .maybeSingle(), - // Codes are unique within a kind and year, so code alone orders the pages. - readAllRows((from, to) => - supabase - .from("catalogue_listings") - .select("code,title,summary,code_id,is_current,last_seen_at") - .eq("academic_year_id", yearRow.id) - .eq("kind", kind) - .order("code") - .range(from, to), - ), - readAllRows((from, to) => + .order("code") + .range(from, to), + ), + readAllRows((from, to) => + supabase + .from("catalogue_records") + .select("id,code_id,public_id,published_version_id,archived_at") + .eq("academic_year_id", year.id) + .eq("kind", kind) + .order("code_id") + .range(from, to), + ), + ]); + if (status.error) throw status.error; + if (listings.error) throw listings.error; + if (records.error) throw records.error; + + const recordIds = records.data.map((record) => record.id); + const [drafts, syncs] = recordIds.length + ? await Promise.all([ supabase - .from("catalogue_records") - .select("code_id,public_id,published_version_id,archived_at") - .eq("academic_year_id", yearRow.id) - .eq("kind", kind) - .order("code_id") - .range(from, to), - ), - // Newest first so the first target seen per item is its latest; id breaks - // ties between targets created in the same instant. - readAllRows((from, to) => + .from("catalogue_drafts") + .select("record_id") + .in("record_id", recordIds), supabase - .from("catalogue_import_targets") - .select( - "id,run_id,code_id,status,change_kind,error_message,completed_at,created_at,applied_version_id", - ) - .eq("academic_year_id", yearRow.id) - .eq("kind", kind) - .order("created_at", { ascending: false }) - .order("id") - .range(from, to), - ), - ]); - if (statusResult.error) throw statusResult.error; - if (entriesResult.error) throw entriesResult.error; - if (itemYearsResult.error) throw itemYearsResult.error; - if (targetsResult.error) throw targetsResult.error; - - const itemYearByItem = new Map( - (itemYearsResult.data ?? []).map((row) => [row.code_id, row]), + .from("catalogue_syncs") + .select("id,record_id,status,error_message,completed_at,created_at") + .in("record_id", recordIds) + .order("created_at", { ascending: false }), + ]) + : [ + { data: [], error: null }, + { data: [], error: null }, + ]; + if (drafts.error) throw drafts.error; + if (syncs.error) throw syncs.error; + const recordByCodeId = new Map( + records.data.map((record) => [record.code_id, record]), ); - const latestTargetByItem = new Map< - number, - (typeof targetsResult.data)[number] - >(); - for (const target of targetsResult.data ?? []) { - if (!latestTargetByItem.has(target.code_id)) { - latestTargetByItem.set(target.code_id, target); - } - } - - // Items imported directly (without a directory row) still appear so the - // administrator can see everything the year holds. - const entryCodes = new Set(entriesResult.data.map((row) => row.code)); - const entryItemIds = new Set(entriesResult.data.map((row) => row.code_id)); - const extraItemIds = [...itemYearByItem.keys()].filter( - (itemId) => !entryItemIds.has(itemId), + const draftIds = new Set((drafts.data ?? []).map((draft) => draft.record_id)); + const latestSync = new Map(); + for (const sync of syncs.data ?? []) + if (!latestSync.has(sync.record_id)) latestSync.set(sync.record_id, sync); + const listedIds = new Set( + listings.data.flatMap((listing) => + listing.code_id ? [listing.code_id] : [], + ), ); - const { data: extraItems } = extraItemIds.length + const extraIds = records.data + .map((record) => record.code_id) + .filter((id) => !listedIds.has(id)); + const { data: extraCodes } = extraIds.length ? await supabase .from("catalogue_codes") .select("id,code") - .in("id", extraItemIds) + .in("id", extraIds) : { data: [] as Array<{ id: number; code: string }> }; - const allRecords: CatalogueDirectoryRecord[] = [ - ...(entriesResult.data ?? []).map((entry) => ({ - code: entry.code, - title: entry.title, - summary: (entry.summary ?? {}) as Record, - itemId: entry.code_id, - isListedByAnu: entry.is_current, - lastSeenAt: entry.last_seen_at, + const rows = [ + ...listings.data.map((listing) => ({ + ...listing, + itemId: listing.code_id, })), - ...(extraItems ?? []) - .filter((item) => !entryCodes.has(item.code)) - .map((item) => ({ - code: item.code, - title: null, - summary: {}, - itemId: item.id, - isListedByAnu: null, - lastSeenAt: null, - })), - ] - .map(({ code, title, summary, itemId, isListedByAnu, lastSeenAt }) => { - const itemYear = itemId === null ? undefined : itemYearByItem.get(itemId); - const latest = - itemId === null ? undefined : latestTargetByItem.get(itemId); - const hasDraft = Boolean( - latest?.applied_version_id && - latest.applied_version_id !== itemYear?.published_version_id, - ); - const isPublished = - Boolean(itemYear?.published_version_id) && !itemYear?.archived_at; - return { - code, - title, - summary, - recordPublicId: itemYear?.public_id ?? null, - hasDraft, - isPublished, - isListedByAnu, - lastSeenAt, - workflow: workflowFor({ - hasDraft, - isPublished, - latestStatus: latest?.status ?? null, - }), - latestTarget: latest - ? { - id: latest.id, - runId: latest.run_id, - status: latest.status, - changeKind: latest.change_kind, - errorMessage: latest.error_message, - completedAt: latest.completed_at, - } - : null, - } satisfies CatalogueDirectoryRecord; - }) - .sort((left, right) => left.code.localeCompare(right.code)); - - const workflowCounts = emptyCounts(); - for (const record of allRecords) workflowCounts[record.workflow] += 1; - + ...(extraCodes ?? []).map((code) => ({ + code: code.code, + title: null, + summary: {}, + itemId: code.id, + is_current: null, + last_seen_at: null, + })), + ].map((listing) => { + const record = listing.itemId + ? recordByCodeId.get(listing.itemId) + : undefined; + const sync = record ? latestSync.get(record.id) : undefined; + const hasDraft = record ? draftIds.has(record.id) : false; + const isPublished = Boolean( + record?.published_version_id && !record.archived_at, + ); + const sourceState: CatalogueDirectoryRecord["sourceState"] = !sync + ? "never_synced" + : sync.status === "queued" || sync.status === "running" + ? "syncing" + : sync.status === "review_required" + ? "changes_available" + : sync.status === "failed" + ? "sync_failed" + : "up_to_date"; + return { + code: listing.code, + title: listing.title, + summary: (listing.summary ?? {}) as Record, + recordPublicId: record?.public_id ?? null, + hasDraft, + isPublished, + isListedByAnu: listing.is_current, + lastSeenAt: listing.last_seen_at, + sourceState, + latestSync: sync + ? { + id: sync.id, + status: sync.status, + errorMessage: sync.error_message, + completedAt: sync.completed_at, + } + : null, + } satisfies CatalogueDirectoryRecord; + }); const needle = query.trim().toUpperCase(); - // Code prefix matches come first, then other code matches, then titles. - const rank = (record: CatalogueDirectoryRecord) => - !needle - ? 0 - : record.code.startsWith(needle) - ? 0 - : record.code.includes(needle) - ? 1 - : 2; - const filtered = allRecords + const filtered = rows .filter( - (record) => - (filter === "all" || record.workflow === filter) && - (!needle || - record.code.includes(needle) || - (record.title ?? "").toUpperCase().includes(needle)), + (row) => + !needle || + row.code.includes(needle) || + (row.title ?? "").toUpperCase().includes(needle), ) - .sort( - (left, right) => - rank(left) - rank(right) || left.code.localeCompare(right.code), - ); + .sort((left, right) => left.code.localeCompare(right.code)); const safePage = Math.max( 1, Math.min(page, Math.ceil(filtered.length / PAGE_SIZE) || 1), ); const start = (safePage - 1) * PAGE_SIZE; - return { kind, academicYear, years, status: { state: - (statusResult.data - ?.status as CatalogueDirectoryPage["status"]["state"]) ?? "never", - refreshedAt: statusResult.data?.refreshed_at ?? null, - message: statusResult.data?.message ?? null, - entryCount: statusResult.data?.entry_count ?? 0, + (status.data?.status as CatalogueDirectoryPage["status"]["state"]) ?? + "never", + refreshedAt: status.data?.refreshed_at ?? null, + message: status.data?.message ?? null, + entryCount: status.data?.entry_count ?? 0, }, records: filtered.slice(start, start + PAGE_SIZE), total: filtered.length, page: safePage, pageSize: PAGE_SIZE, - workflowCounts, }; } - -const IMPORT_RECORD_PAGE_SIZE = 25; -const RECENT_RUN_LIMIT = 20; - -const RUN_COLUMNS = - "id,run_number,kind,status,requested_model,target_count,completed_count,failed_count,cost_usd,created_at,completed_at,academic_years(year)"; - -type RunRow = { - id: string; - run_number: number; - kind: string; - status: string; - requested_model: string; - target_count: number; - completed_count: number; - failed_count: number; - cost_usd: number | string; - created_at: string; - completed_at: string | null; - academic_years: { year: number } | null; -}; - -function runRow(run: RunRow): ImportRunRow { - return { - id: run.id, - runNumber: run.run_number, - kind: run.kind as CatalogueKind, - academicYear: run.academic_years?.year ?? 0, - status: run.status, - requestedModel: run.requested_model, - targetCount: run.target_count, - completedCount: run.completed_count, - failedCount: run.failed_count, - costUsd: Number(run.cost_usd), - createdAt: run.created_at, - completedAt: run.completed_at, - }; -} - -// The year is taken from the target's own column and resolved through the -// academic year table, rather than reached by a nested embed through the run: -// PostgREST resolves `catalogue_import_targets -> catalogue_records` by two -// foreign keys of the same name, so the embedded shape is not typed. -const RECORD_COLUMNS = - "id,code,academic_year_id,status,change_kind,attempt_count,error_code,error_message,applied_version_id,created_at,completed_at,run_id,directory_entry_id,record_id"; - -type RecordRow = { - id: string; - code: string; - academic_year_id: number; - status: string; - change_kind: string | null; - attempt_count: number; - error_code: string | null; - error_message: string | null; - applied_version_id: number | null; - created_at: string; - completed_at: string | null; - run_id: string; - directory_entry_id: number | null; - record_id: number; -}; - -/** - * A PostgREST `or` list is comma separated and parenthesised, so a needle - * carrying either character would change the shape of the filter rather than - * be matched by it. - */ -function safeNeedle(query: string) { - return query - .trim() - .replace(/[(),*]/g, " ") - .trim(); -} - -/** - * One page of imported records for a kind: a flat history across every run, - * narrowed by a search over code and title, by the record's own outcome and - * by the run that produced it. The run travels on each row, so a reader never - * has to pick a batch before seeing what was imported. - * - * The recent runs come back with the page because they are the run filter's - * options and, when one is chosen, the strip that carries its progress and - * its Stop control. - */ -export async function loadCatalogueImportRecords({ - kind, - query = "", - status = "", - runId = null, - sort = DEFAULT_IMPORT_RECORD_SORT, - page = 1, -}: { - kind: CatalogueKind; - query?: string; - status?: string; - runId?: string | null; - sort?: ImportRecordSort; - page?: number; -}): Promise { - const supabase = await createClient(); - const needle = safeNeedle(query); - - const { data: runData, error: runError } = await supabase - .from("catalogue_import_runs") - .select(RUN_COLUMNS) - .eq("kind", kind) - .order("created_at", { ascending: false }) - .limit(RECENT_RUN_LIMIT); - if (runError) throw runError; - const runs = ((runData ?? []) as RunRow[]).map(runRow); - - // A run chosen from an older page is not in the recent list, so it is read - // on its own rather than silently dropping the filter the reader applied. - let run = runs.find((candidate) => candidate.id === runId) ?? null; - if (runId && !run) { - const { data: single } = await supabase - .from("catalogue_import_runs") - .select(RUN_COLUMNS) - .eq("id", runId) - .eq("kind", kind) - .maybeSingle(); - run = single ? runRow(single as RunRow) : null; - } - - const empty: ImportRecordsPage = { - records: [], - page: 1, - pageSize: IMPORT_RECORD_PAGE_SIZE, - total: 0, - sort, - runs, - run, - }; - - let recordsQuery = supabase - .from("catalogue_import_targets") - .select(RECORD_COLUMNS, { count: "exact" }) - .eq("kind", kind); - if (status) recordsQuery = recordsQuery.eq("status", status); - if (runId) { - // An unknown run id would otherwise return every record, which reads as - // though the filter had been ignored. - if (!run) return empty; - recordsQuery = recordsQuery.eq("run_id", runId); - } - - if (needle) { - // The title lives on the directory entry, which PostgREST cannot reach - // from inside an `or`, so matching titles are resolved to entry ids first. - const { data: titleMatches, error: titleError } = await supabase - .from("catalogue_listings") - .select("id") - .eq("kind", kind) - .ilike("title", `%${needle}%`) - .limit(1000); - if (titleError) throw titleError; - const clauses = [`code.ilike.*${needle}*`]; - const entryIds = (titleMatches ?? []).map((match) => match.id); - if (entryIds.length) - clauses.push(`directory_entry_id.in.(${entryIds.join(",")})`); - recordsQuery = recordsQuery.or(clauses.join(",")); - } - - recordsQuery = - sort === "oldest" - ? recordsQuery.order("created_at", { ascending: true }) - : sort === "code-asc" - ? recordsQuery.order("code", { ascending: true }) - : sort === "code-desc" - ? recordsQuery.order("code", { ascending: false }) - : recordsQuery.order("created_at", { ascending: false }); - // Ties on code or on a shared run timestamp would otherwise order - // differently per page, so rows could repeat or go missing across pages. - recordsQuery = recordsQuery.order("id", { ascending: true }); - - const { count, error: countError } = await recordsQuery.range(0, 0); - if (countError) throw countError; - const total = count ?? 0; - const safePage = Math.max( - 1, - Math.min(page, Math.ceil(total / IMPORT_RECORD_PAGE_SIZE) || 1), - ); - const from = (safePage - 1) * IMPORT_RECORD_PAGE_SIZE; - const { data, error } = await recordsQuery.range( - from, - from + IMPORT_RECORD_PAGE_SIZE - 1, - ); - if (error) throw error; - const rows = (data ?? []) as RecordRow[]; - - // The labels a row needs are resolved by id over the page rather than by - // embedding them in the select above. One page is twenty-five rows, so this - // is four small reads, and it keeps the ambiguous embeds out: PostgREST - // reaches `catalogue_records` from a target through two foreign keys of - // the same name, which it will not resolve and cannot type. - const distinct = (values: Value[]) => [...new Set(values)]; - const entryIds = distinct( - rows.map((row) => row.directory_entry_id).filter((id) => id !== null), - ); - const yearIds = distinct(rows.map((row) => row.academic_year_id)); - const recordIds = distinct(rows.map((row) => row.record_id)); - // The recent runs are already loaded, so only a row from an older run than - // the list offers costs a read. - const olderRunIds = distinct(rows.map((row) => row.run_id)).filter( - (id) => !runs.some((candidate) => candidate.id === id), - ); - - const [entries, years, itemYears, olderRuns] = await Promise.all([ - entryIds.length - ? supabase - .from("catalogue_listings") - .select("id,title") - .in("id", entryIds) - : null, - yearIds.length - ? supabase.from("academic_years").select("id,year").in("id", yearIds) - : null, - recordIds.length - ? supabase - .from("catalogue_records") - .select("id,public_id") - .in("id", recordIds) - : null, - olderRunIds.length - ? supabase - .from("catalogue_import_runs") - .select("id,run_number") - .in("id", olderRunIds) - : null, - ]); - if (entries?.error) throw entries.error; - if (years?.error) throw years.error; - if (itemYears?.error) throw itemYears.error; - if (olderRuns?.error) throw olderRuns.error; - - const titleById = new Map( - (entries?.data ?? []).map((entry) => [entry.id, entry.title]), - ); - const yearById = new Map( - (years?.data ?? []).map((year) => [year.id, year.year]), - ); - const publicIdById = new Map( - (itemYears?.data ?? []).map((itemYear) => [ - itemYear.id, - itemYear.public_id, - ]), - ); - const runNumberById = new Map(runs.map((row) => [row.id, row.runNumber])); - for (const row of olderRuns?.data ?? []) - runNumberById.set(row.id, row.run_number); - - return { - records: rows.map((row) => ({ - id: row.id, - code: row.code, - title: - row.directory_entry_id === null - ? null - : (titleById.get(row.directory_entry_id) ?? null), - academicYear: yearById.get(row.academic_year_id) ?? 0, - status: row.status, - changeKind: row.change_kind, - attemptCount: row.attempt_count, - errorCode: row.error_code, - errorMessage: row.error_message, - appliedVersionId: row.applied_version_id, - recordPublicId: publicIdById.get(row.record_id) ?? null, - createdAt: row.created_at, - completedAt: row.completed_at, - runId: row.run_id, - runNumber: runNumberById.get(row.run_id) ?? 0, - })), - page: safePage, - pageSize: IMPORT_RECORD_PAGE_SIZE, - total, - sort, - runs, - run, - }; -} - -/** - * The counters of one run and nothing else. An active run is watched through - * this rather than by refetching the page, which previously reread every run - * and every target every four seconds to learn that one number had moved. - */ -export async function loadImportRunProgress( - runId: string, -): Promise { - const supabase = await createClient(); - const { data, error } = await supabase - .from("catalogue_import_runs") - .select("status,target_count,completed_count,failed_count") - .eq("id", runId) - .maybeSingle(); - if (error) throw error; - if (!data) return null; - return { - status: data.status, - targetCount: data.target_count, - completedCount: data.completed_count, - failedCount: data.failed_count, - }; -} - -export async function loadImportTargetDetail( - targetId: string, -): Promise { - const supabase = await createClient(); - const { data: target, error } = await supabase - .from("catalogue_import_targets") - .select("id,code,kind,status,attempt_count,error_code,error_message") - .eq("id", targetId) - .maybeSingle(); - if (error) throw error; - if (!target) return null; - const [stages, artifacts, extraction] = await Promise.all([ - supabase - .from("catalogue_import_stages") - .select( - "id,stage_name,attempt_number,status,started_at,completed_at,error_code,error_summary", - ) - .eq("target_id", targetId) - .order("attempt_number") - .order("started_at"), - supabase - .from("catalogue_import_artifacts") - .select("id,stage_id,kind,attempt_number,media_type,byte_size") - .eq("target_id", targetId) - .order("created_at"), - supabase - .from("catalogue_extractions") - .select( - "resolved_model,finish_reason,validation_status,input_tokens,output_tokens,cost_usd,latency_ms,warning_count,error_count,error_summary", - ) - .eq("target_id", targetId) - .order("started_at", { ascending: false }) - .limit(1) - .maybeSingle(), - ]); - if (stages.error) throw stages.error; - if (artifacts.error) throw artifacts.error; - if (extraction.error) throw extraction.error; - return { - id: target.id, - code: target.code, - kind: target.kind as CatalogueKind, - status: target.status, - attemptCount: target.attempt_count, - errorCode: target.error_code, - errorMessage: target.error_message, - stages: (stages.data ?? []).map((stage) => ({ - id: stage.id, - name: stage.stage_name, - attemptNumber: stage.attempt_number, - status: stage.status, - startedAt: stage.started_at, - completedAt: stage.completed_at, - errorCode: stage.error_code, - errorSummary: stage.error_summary, - })), - artifacts: (artifacts.data ?? []).map((artifact) => ({ - id: artifact.id, - stageId: artifact.stage_id, - kind: artifact.kind, - attemptNumber: artifact.attempt_number, - mediaType: artifact.media_type, - byteSize: artifact.byte_size, - })), - extraction: extraction.data - ? { - resolvedModel: extraction.data.resolved_model, - finishReason: extraction.data.finish_reason, - validationStatus: extraction.data.validation_status, - inputTokens: extraction.data.input_tokens, - outputTokens: extraction.data.output_tokens, - costUsd: Number(extraction.data.cost_usd), - latencyMs: extraction.data.latency_ms, - warningCount: extraction.data.warning_count, - errorCount: extraction.data.error_count, - errorSummary: extraction.data.error_summary, - } - : null, - }; -} - -export async function loadAdminCatalogueSummary(): Promise { - const supabase = await createClient(); - const [{ data: itemYears }, { data: items }, { data: appliedTargets }] = - await Promise.all([ - supabase - .from("catalogue_records") - .select("id,kind,published_version_id,archived_at"), - supabase.from("catalogue_codes").select("kind"), - supabase - .from("catalogue_import_targets") - .select("record_id,applied_version_id,created_at") - .not("applied_version_id", "is", null) - .order("created_at", { ascending: false }), - ]); - const summary = Object.fromEntries( - (Object.keys(CATALOGUE_KIND_LABELS) as CatalogueKind[]).map((kind) => [ - kind, - { published: 0, drafts: 0, identities: 0 }, - ]), - ) as AdminCatalogueSummary; - for (const item of items ?? []) - summary[item.kind as CatalogueKind].identities += 1; - const latestAppliedByRecord = new Map(); - for (const target of appliedTargets ?? []) { - if ( - !latestAppliedByRecord.has(target.record_id) && - target.applied_version_id - ) - latestAppliedByRecord.set(target.record_id, target.applied_version_id); - } - for (const year of itemYears ?? []) { - const bucket = summary[year.kind as CatalogueKind]; - if (year.published_version_id && !year.archived_at) bucket.published += 1; - const appliedVersionId = latestAppliedByRecord.get(year.id); - if (appliedVersionId && appliedVersionId !== year.published_version_id) - bucket.drafts += 1; - } - return summary; -} diff --git a/apps/web/lib/coursemap/catalogue-kinds.ts b/apps/web/lib/coursemap/catalogue-kinds.ts index 444093ad..d49c4bf1 100644 --- a/apps/web/lib/coursemap/catalogue-kinds.ts +++ b/apps/web/lib/coursemap/catalogue-kinds.ts @@ -45,16 +45,6 @@ export function publicCatalogueRecordPath( return `/${CATALOGUE_KIND_LABELS[kind].segment}/${year}/${encodeURIComponent(code.toLowerCase())}`; } -export type DirectoryWorkflowStatus = - | "not_imported" - | "queued" - | "running" - | "ready" - | "draft" - | "published" - | "published_with_draft" - | "failed"; - export type CatalogueDirectoryRecord = { code: string; title: string | null; @@ -64,12 +54,15 @@ export type CatalogueDirectoryRecord = { isPublished: boolean; isListedByAnu: boolean | null; lastSeenAt: string | null; - workflow: DirectoryWorkflowStatus; - latestTarget: { + sourceState: + | "never_synced" + | "syncing" + | "up_to_date" + | "changes_available" + | "sync_failed"; + latestSync: { id: string; - runId: string; status: string; - changeKind: string | null; errorMessage: string | null; completedAt: string | null; } | null; @@ -89,77 +82,6 @@ export type CatalogueDirectoryPage = { total: number; page: number; pageSize: number; - workflowCounts: Record; -}; - -export type DirectoryFilter = "all" | DirectoryWorkflowStatus; - -export type ImportRunSummary = { - id: string; - runNumber: number; - kind: CatalogueKind; - academicYear: number; - status: string; - requestedModel: string; - targetCount: number; - completedCount: number; - failedCount: number; - costUsd: number; - createdAt: string; - completedAt: string | null; - targets: Array<{ - id: string; - code: string; - title: string | null; - status: string; - changeKind: string | null; - attemptCount: number; - errorCode: string | null; - errorMessage: string | null; - candidateVersionId: number | null; - appliedVersionId: number | null; - recordPublicId: string | null; - }>; -}; - -export type ImportTargetDetail = { - id: string; - code: string; - kind: CatalogueKind; - status: string; - attemptCount: number; - errorCode: string | null; - errorMessage: string | null; - stages: Array<{ - id: string; - name: string; - attemptNumber: number; - status: string; - startedAt: string; - completedAt: string | null; - errorCode: string | null; - errorSummary: string | null; - }>; - artifacts: Array<{ - id: string; - stageId: string; - kind: string; - attemptNumber: number; - mediaType: string; - byteSize: number; - }>; - extraction: { - resolvedModel: string | null; - finishReason: string | null; - validationStatus: string; - inputTokens: number; - outputTokens: number; - costUsd: number; - latencyMs: number | null; - warningCount: number; - errorCount: number; - errorSummary: string | null; - } | null; }; export type AdminCatalogueSummary = Record< @@ -167,7 +89,7 @@ export type AdminCatalogueSummary = Record< { published: number; drafts: number; identities: number } >; -/** Reviewer-facing names for the field paths recorded by the import diff. */ +/** Administrator-facing names for stored catalogue content paths. */ export const FIELD_LABELS: Record = { "course.details.title": "Title", "course.details.unitValueKind": "Unit value kind", @@ -248,71 +170,3 @@ export function humaniseKey(key: string) { .trim(); return spaced.charAt(0).toUpperCase() + spaced.slice(1); } - -/** - * The imports list is read on the server and driven from the client, so its - * vocabulary lives here rather than beside the loader, which is server-only - * and would pull the Supabase client into the browser bundle. - */ -export const IMPORT_RECORD_SORTS = [ - "newest", - "oldest", - "code-asc", - "code-desc", -] as const; -export type ImportRecordSort = (typeof IMPORT_RECORD_SORTS)[number]; -export const DEFAULT_IMPORT_RECORD_SORT: ImportRecordSort = "newest"; - -/** `catalogue_import_targets.status`, as the check constraint defines it. */ -export const IMPORT_RECORD_STATUSES = [ - "queued", - "running", - "ready", - "unchanged", - "failed", - "cancelled", -] as const; - -/** A run as it appears beside the records it produced: counters, no targets. */ -export type ImportRunRow = Omit; - -/** - * One imported record. The run that produced it is carried on the row, because - * the list is a flat history of records rather than a list of batches. - */ -export type ImportRecordRow = { - id: string; - code: string; - title: string | null; - academicYear: number; - status: string; - changeKind: string | null; - attemptCount: number; - errorCode: string | null; - errorMessage: string | null; - appliedVersionId: number | null; - recordPublicId: string | null; - createdAt: string; - completedAt: string | null; - runId: string; - runNumber: number; -}; - -export type ImportRecordsPage = { - records: ImportRecordRow[]; - page: number; - pageSize: number; - total: number; - sort: ImportRecordSort; - /** Recent runs, offered as the run filter's options. */ - runs: ImportRunRow[]; - /** The run the list is narrowed to, when the reader has chosen one. */ - run: ImportRunRow | null; -}; - -export type ImportRunProgress = { - status: string; - targetCount: number; - completedCount: number; - failedCount: number; -}; diff --git a/apps/web/lib/coursemap/requisite-search-actions.ts b/apps/web/lib/coursemap/requisite-search-actions.ts index 9b7a0a6a..054f7ba3 100644 --- a/apps/web/lib/coursemap/requisite-search-actions.ts +++ b/apps/web/lib/coursemap/requisite-search-actions.ts @@ -2,7 +2,7 @@ import { canManageCatalogueImports, - canManageCourseImports, + canManageCatalogueSources, canWriteCourses, } from "@/lib/auth/viewer"; import { createClient } from "@/lib/supabase/server"; @@ -45,7 +45,7 @@ export async function searchRequisiteCourses( ): Promise { const term = query.trim().toUpperCase(); if (term.length < 2) return []; - if (!(await canWriteCourses()) && !(await canManageCourseImports())) { + if (!(await canWriteCourses()) && !(await canManageCatalogueSources())) { return []; } diff --git a/apps/web/playwright/catalogue-review.spec.ts b/apps/web/playwright/catalogue-review.spec.ts index 6c993fdc..945c9794 100644 --- a/apps/web/playwright/catalogue-review.spec.ts +++ b/apps/web/playwright/catalogue-review.spec.ts @@ -16,7 +16,7 @@ test("the changes route reports local unpublished state without source review", "true", ); await expect( - page.getByRole("heading", { name: "No unpublished changes" }), + page.getByRole("heading", { name: "No changes to review" }), ).toBeVisible(); await expect(page.getByRole("button", { name: /publish/i })).toHaveCount(0); await expect(page.getByRole("button", { name: /apply/i })).toHaveCount(0); diff --git a/apps/web/scripts/catalogue/lib/local-database.mjs b/apps/web/scripts/catalogue/lib/local-database.mjs index 679842e3..dcffccc3 100644 --- a/apps/web/scripts/catalogue/lib/local-database.mjs +++ b/apps/web/scripts/catalogue/lib/local-database.mjs @@ -171,7 +171,7 @@ export async function createLocalDatabaseClient(options = {}) { const connectionString = await discoverLocalDatabaseUrl(options); const sql = postgres(connectionString, { - application_name: "coursemap_import_runner", + application_name: "coursemap_catalogue_worker", connect_timeout: 5, idle_timeout: 5, max: 1, @@ -194,7 +194,7 @@ export function assertHostedSupabaseDatabaseUrl(connectionString) { const databaseUrl = parseDatabaseUrl(connectionString); if (!isHostedSupabaseDatabaseHost(databaseUrl.hostname)) { throw new Error( - "The hosted import runner only accepts a Supabase database connection URL.", + "The hosted catalogue worker only accepts a Supabase database connection URL.", ); } return databaseUrl; @@ -202,13 +202,13 @@ export function assertHostedSupabaseDatabaseUrl(connectionString) { /** * Create an explicitly configured hosted database client for an authenticated - * import runner. This is intentionally separate from the local-only client so - * routine CLI imports cannot accidentally target production. + * catalogue sync. This is intentionally separate from the local-only client so + * routine catalogue scripts cannot accidentally target production. */ -export function createHostedImportDatabaseClient(connectionString) { +export function createHostedSyncDatabaseClient(connectionString) { const databaseUrl = assertHostedSupabaseDatabaseUrl(connectionString); const sql = postgres(databaseUrl.toString(), { - application_name: "coursemap_import_runner", + application_name: "coursemap_catalogue_sync", connect_timeout: 10, idle_timeout: 5, max: 1, @@ -222,7 +222,7 @@ export function createHostedImportDatabaseClient(connectionString) { export function assertVerifiedImportDatabaseClient(sql) { if (!verifiedImportClients.has(sql)) { throw new Error( - "Imports require a client created by createLocalDatabaseClient() or createHostedImportDatabaseClient().", + "Catalogue writes require a verified local or hosted database client.", ); } } diff --git a/apps/web/scripts/fixtures/local-preview.sql b/apps/web/scripts/fixtures/local-preview.sql index a16ad7f1..3a438c4f 100644 --- a/apps/web/scripts/fixtures/local-preview.sql +++ b/apps/web/scripts/fixtures/local-preview.sql @@ -284,14 +284,12 @@ cross join (values where sources.kind = 'local_mock' on conflict (source_id, academic_year_id, kind, external_key, content_sha256) do nothing; --- One snapshot per item year. Course snapshots carry their source page; the --- structure fixtures are manual. +-- One manual version per annual record for the local preview. insert into public.catalogue_versions ( record_id, kind, academic_year_id, origin, - source_page_id, content_hash, created_by ) @@ -299,16 +297,12 @@ select item_years.id, item_years.kind, item_years.academic_year_id, - case when item_years.kind = 'course' then 'import' else 'manual' end, - pages.id, + 'manual', md5(items.code || ':2026:local-preview') || md5('published:' || items.code), '90000000-0000-4000-8000-000000000001'::uuid from public.catalogue_records as item_years join public.catalogue_codes as items on items.id = item_years.code_id -left join public.catalogue_source_pages as pages - on pages.academic_year_id = item_years.academic_year_id - and pages.kind = 'course' - and pages.external_key = items.code; +; insert into public.structure_version_details ( version_id, @@ -562,7 +556,7 @@ insert into public.course_offerings ( select snapshots.id, snapshots.academic_year_id, - snapshots.source_page_id, + null, 'In person', 'Acton' from public.catalogue_versions as snapshots @@ -591,7 +585,7 @@ select offerings.id, snapshots.id, snapshots.academic_year_id, - snapshots.source_page_id, + null, periods.id, 'S1', 'Semester 1', @@ -666,7 +660,7 @@ insert into public.catalogue_version_provenance ( select snapshots.id, snapshots.academic_year_id, - snapshots.source_page_id, + null, 'title', 'deterministic', 0.99, @@ -682,7 +676,7 @@ insert into public.requirement_rules ( select snapshots.id, snapshots.academic_year_id, - snapshots.source_page_id, + null, 'prerequisite', 'hard', 'You must have completed MATH1005.', diff --git a/apps/web/tests/breadcrumbs.test.tsx b/apps/web/tests/breadcrumbs.test.tsx index 784a1dd1..7aa8b49e 100644 --- a/apps/web/tests/breadcrumbs.test.tsx +++ b/apps/web/tests/breadcrumbs.test.tsx @@ -3,7 +3,7 @@ import { afterEach, expect, test, vi } from "vitest"; import { Breadcrumbs } from "@/ui/shell/breadcrumbs"; vi.mock("next/navigation", () => ({ - usePathname: () => "/admin/courses/imports/42", + usePathname: () => "/admin/courses/2026/infs1001", })); afterEach(() => { @@ -62,7 +62,7 @@ test("caps long trails at three positions even when there is room", () => { within(trail).queryByRole("link", { name: "Courses" }), ).not.toBeInTheDocument(); expect( - within(trail).queryByRole("link", { name: "Imports" }), + within(trail).queryByRole("link", { name: "2026" }), ).not.toBeInTheDocument(); resize(220); @@ -75,7 +75,7 @@ test("caps long trails at three positions even when there is room", () => { "truncate", ); expect( - within(trail).queryByRole("link", { name: "Imports" }), + within(trail).queryByRole("link", { name: "2026" }), ).not.toBeInTheDocument(); fireEvent.keyDown( within(trail).getByRole("button", { name: "Show hidden breadcrumbs" }), @@ -85,9 +85,9 @@ test("caps long trails at three positions even when there is room", () => { "href", "/admin/courses", ); - expect(screen.getByRole("menuitem", { name: "Imports" })).toHaveAttribute( + expect(screen.getByRole("menuitem", { name: "2026" })).toHaveAttribute( "href", - "/admin/courses/imports", + "/admin/courses/2026", ); fireEvent.keyDown(screen.getByRole("menu"), { key: "Escape" }); expect(screen.queryByRole("menu")).not.toBeInTheDocument(); @@ -115,7 +115,7 @@ test("keeps the base and current section when a page has a trailing tab", () => expect( within(trail).getByRole("button", { name: "Show hidden breadcrumbs" }), ).toBeVisible(); - expect(trail).not.toHaveTextContent("42"); + expect(trail).not.toHaveTextContent("infs1001"); }); test("reveals the hidden links on mouse hover", () => { @@ -128,13 +128,13 @@ test("reveals the hidden links on mouse hover", () => { Object.defineProperty(event, "pointerType", { value: "mouse" }); fireEvent(trigger, event); expect(screen.getByRole("menuitem", { name: "Courses" })).toBeVisible(); - expect(screen.getByRole("menuitem", { name: "Imports" })).toBeVisible(); + expect(screen.getByRole("menuitem", { name: "2026" })).toBeVisible(); }); test("shows three short breadcrumbs until width requires collapsing the middle", () => { const resize = measureAt(600); render( - , + , ); const trail = screen.getByRole("navigation", { name: "Breadcrumb" }); expect(within(trail).getByRole("link", { name: "Courses" })).toBeVisible(); diff --git a/apps/web/tests/catalogue-import-pipeline-database.test.mjs b/apps/web/tests/catalogue-import-pipeline-database.test.mjs deleted file mode 100644 index b4723ef8..00000000 --- a/apps/web/tests/catalogue-import-pipeline-database.test.mjs +++ /dev/null @@ -1,463 +0,0 @@ -import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; -import { afterAll, beforeAll, test } from "vitest"; - -import { processImportTarget } from "../lib/catalogue-import/process-target.ts"; -import { adapterForKind } from "../lib/catalogue-import/process-target.ts"; -import { processImportRunInline } from "../lib/catalogue-import/queue.ts"; -import { applyImportReview } from "../lib/catalogue-import/apply-review.ts"; -import { extractDeterministicCourse } from "../lib/catalogue-import/kinds/course/deterministic.ts"; -import { reconcileCatalogueListings } from "../lib/catalogue-import/directory.ts"; -import { createLocalDatabaseClient } from "../scripts/catalogue/lib/local-database.mjs"; -import { localTestEnvironment } from "../scripts/local/test-environment.mjs"; - -const ADMIN_ID = "98000000-0000-4000-8000-000000000001"; -const CODE = "COMP2400"; -const YEAR = 2026; -const sourceUrl = `https://programsandcourses.anu.edu.au/${YEAR}/course/${CODE}`; -const fixtureHtml = await readFile( - new URL( - "./fixtures/course-import/anu-2026-comp2400-rich.html", - import.meta.url, - ), - "utf8", -); - -// The canned model answer is the deterministic extraction itself, so the merge -// sees a valid, evidence-backed response without a paid request. -const deterministicAnswer = extractDeterministicCourse({ - html: fixtureHtml, - courseCode: CODE, - year: YEAR, - sourceUrl, -}); -const modelAnswer = { - ...deterministicAnswer, - evidence: deterministicAnswer.evidence.map((item) => ({ - ...item, - method: "model", - })), -}; -const modelAnswerRef = { current: modelAnswer }; - -let openRouterCalls = 0; -const realFetch = globalThis.fetch; - -const pageRef = { current: fixtureHtml }; - -function stubbedFetch(input, init) { - const url = typeof input === "string" ? input : input.url; - if (url.startsWith("https://programsandcourses.anu.edu.au/")) { - return Promise.resolve( - new Response(pageRef.current, { - status: 200, - headers: { "content-type": "text/html; charset=utf-8" }, - }), - ); - } - if (url.startsWith("https://openrouter.ai/")) { - openRouterCalls += 1; - return Promise.resolve( - Response.json({ - id: "gen-test", - model: "test/model", - choices: [ - { - finish_reason: "stop", - message: { content: JSON.stringify(modelAnswerRef.current) }, - }, - ], - usage: { - prompt_tokens: 100, - completion_tokens: 50, - total_tokens: 150, - cost: 0.001, - }, - }), - ); - } - return realFetch(input, init); -} - -let sql; - -beforeAll(async () => { - Object.assign(process.env, localTestEnvironment(), { - OPENROUTER_API_KEY: "sk-or-v1-test", - NODE_ENV: "development", - }); - globalThis.fetch = stubbedFetch; - sql = await createLocalDatabaseClient(); - await sql` - insert into auth.users (instance_id, id, aud, role, email, raw_app_meta_data, raw_user_meta_data, created_at, updated_at) - values ('00000000-0000-0000-0000-000000000000', ${ADMIN_ID}, 'authenticated', 'authenticated', - 'pipeline-admin@example.test', '{"provider":"email","providers":["email"]}'::jsonb, '{}'::jsonb, now(), now()) - on conflict (id) do nothing - `; - await sql` - insert into private.user_roles (user_id, role_id) - select ${ADMIN_ID}, id from private.app_roles where key = 'admin' - on conflict (user_id) do update set role_id = excluded.role_id - `; - await sql` - insert into public.import_models (id, name, provider, input_usd_per_million, output_usd_per_million) - values ('test/model', 'Test model', 'Test', 0.1, 0.2) - on conflict (id) do update set enabled = true, visible = true - `; - await removeFixtureData(); -}); - -/** - * Removes the fixture identities. Snapshots refuse deletes by design, so the - * cleanup briefly disables that trigger; production never deletes them. - */ -async function removeFixtureData() { - await sql`delete from public.catalogue_import_runs where requested_by = ${ADMIN_ID}`; - await sql`delete from public.catalogue_listings where code in (${CODE}, 'COMP2401')`; - await sql`alter table public.catalogue_versions disable trigger catalogue_versions_enforce_immutability`; - try { - await sql`delete from public.catalogue_codes where kind = 'course' and code in (${CODE}, 'COMP2401')`; - await sql` - delete from public.catalogue_codes - where kind = 'course' - and id not in (select code_id from public.catalogue_records) - and id not in (select code_id from public.requirement_conditions where code_id is not null) - and id not in (select code_id from public.requirement_condition_options where code_id is not null) - and id not in (select code_id from public.requirement_item_references) - and id not in (select related_course_id from public.course_related_courses) - and code not in ('MATH1005') - `; - } finally { - await sql`alter table public.catalogue_versions enable trigger catalogue_versions_enforce_immutability`; - } -} - -afterAll(async () => { - globalThis.fetch = realFetch; - if (sql) { - await removeFixtureData(); - await sql`delete from auth.users where id = ${ADMIN_ID}`; - await sql.end({ timeout: 5 }); - } -}); - -test("catalogue discovery creates records and only complete listings mark disappearances", async () => { - const [year] = - await sql`select id from public.academic_years where year = ${YEAR}`; - const entries = [ - { code: CODE, title: "Relational Databases", summary: {} }, - { code: "COMP2401", title: "Computer Systems", summary: {} }, - ]; - - const first = await sql.begin((tx) => - reconcileCatalogueListings(tx, { - academicYearId: Number(year.id), - kind: "course", - entries, - isComplete: true, - }), - ); - assert.equal(first.added, 2); - - const repeated = await sql.begin((tx) => - reconcileCatalogueListings(tx, { - academicYearId: Number(year.id), - kind: "course", - entries, - isComplete: true, - }), - ); - assert.equal(repeated.added, 0); - assert.equal(repeated.updated, 2); - - await sql.begin((tx) => - reconcileCatalogueListings(tx, { - academicYearId: Number(year.id), - kind: "course", - entries: entries.slice(0, 1), - isComplete: false, - }), - ); - let [missing] = await sql` - select listings.is_current, listings.record_id, records.archived_at, - records.published_version_id - from public.catalogue_listings as listings - join public.catalogue_records as records on records.id = listings.record_id - where listings.code = 'COMP2401' and listings.academic_year_id = ${year.id} - `; - assert.equal(missing.is_current, true); - - await sql.begin((tx) => - reconcileCatalogueListings(tx, { - academicYearId: Number(year.id), - kind: "course", - entries: entries.slice(0, 1), - isComplete: true, - }), - ); - [missing] = await sql` - select listings.is_current, listings.record_id, records.archived_at, - records.published_version_id - from public.catalogue_listings as listings - join public.catalogue_records as records on records.id = listings.record_id - where listings.code = 'COMP2401' and listings.academic_year_id = ${year.id} - `; - assert.equal(missing.is_current, false); - assert.ok(missing.record_id); - assert.equal(missing.archived_at, null); - assert.equal(missing.published_version_id, null); - - await sql.begin((tx) => - reconcileCatalogueListings(tx, { - academicYearId: Number(year.id), - kind: "course", - entries, - isComplete: true, - }), - ); - [missing] = await sql` - select is_current from public.catalogue_listings - where code = 'COMP2401' and academic_year_id = ${year.id} - `; - assert.equal(missing.is_current, true); -}); - -async function startRun(codes) { - const adapter = adapterForKind("course"); - const [row] = await sql.begin(async (tx) => { - await tx`select set_config('request.jwt.claim.sub', ${ADMIN_ID}, true)`; - return tx` - select public.start_catalogue_import( - ${YEAR}::smallint, 'course', ${tx.array(codes)}::text[], 'test/model', - ${adapter.parserVersion}, ${adapter.promptVersion}, ${adapter.schemaVersion} - ) as run - `; - }); - return row.run; -} - -test("a first import creates an applied version and a repeat import is unchanged", async () => { - const run = await startRun([CODE]); - assert.equal(run.targets.length, 1); - const targetId = run.targets[0].targetId; - - await processImportTarget({ runId: run.runId, targetId }); - - const [target] = await sql` - select status, change_kind, candidate_version_id, applied_version_id, error_message - from public.catalogue_import_targets where id = ${targetId}::uuid - `; - assert.equal(target.error_message, null); - assert.equal(target.status, "ready"); - assert.equal(target.change_kind, "new"); - assert.ok(target.candidate_version_id); - assert.equal( - Number(target.applied_version_id), - Number(target.candidate_version_id), - ); - assert.equal(openRouterCalls, 1); - - const [itemYear] = await sql` - select item_years.published_version_id - from public.catalogue_records as item_years - join public.catalogue_codes as items on items.id = item_years.code_id - where items.code = ${CODE} - `; - assert.equal(itemYear.published_version_id, null); - - const [details] = await sql` - select title, units, subject_code from public.course_version_details - where version_id = ${target.candidate_version_id} - `; - assert.equal(details.subject_code, "COMP"); - assert.ok(details.title.length > 0); - - const [{ count: stageCount }] = await sql` - select count(*)::int as count from public.catalogue_import_stages - where target_id = ${targetId}::uuid and status = 'completed' - `; - assert.equal(stageCount, 10); - - const [extraction] = await sql` - select validation_status, schema_valid, domain_valid, error_count, warning_count - from public.catalogue_extractions where target_id = ${targetId}::uuid - `; - assert.equal( - extraction.schema_valid, - true, - "the canned model answer passes the contract", - ); - - const [{ count: rules }] = await sql` - select count(*)::int as count from public.requirement_rules - where version_id = ${target.candidate_version_id} - `; - assert.ok(rules >= 1, "the fixture page carries requisite rules"); - - const [runRow] = await sql` - select status, completed_count, cost_usd from public.catalogue_import_runs where id = ${run.runId}::uuid - `; - assert.equal(runRow.status, "completed"); - assert.equal(runRow.completed_count, 1); - assert.equal(Number(runRow.cost_usd), 0.001); - - // A second run over identical content reuses the validated response and - // records no new version. - const secondRun = await startRun([CODE]); - const { completed } = await processImportRunInline({ - runId: secondRun.runId, - }); - assert.equal(completed, 1); - const [second] = await sql` - select status, change_kind, candidate_version_id from public.catalogue_import_targets - where run_id = ${secondRun.runId}::uuid - `; - assert.equal(second.status, "unchanged"); - assert.equal(second.change_kind, "unchanged"); - assert.equal(second.candidate_version_id, null); - assert.equal( - openRouterCalls, - 1, - "identical input does not pay for a second model call", - ); -}); - -test("a changed import records open changes, applies accepted ones and publishes", async () => { - // Alter the published fixture so the next import differs in the title only. - const [item] = await sql` - select item_years.id as record_id, item_years.published_version_id - from public.catalogue_records as item_years - join public.catalogue_codes as items on items.id = item_years.code_id - where items.code = ${CODE} - `; - await sql.begin(async (tx) => { - await tx`select set_config('request.jwt.claim.sub', ${ADMIN_ID}, true)`; - await tx`select public.publish_catalogue_version(${item.record_id})`; - }); - const [{ count: acceptedOnFirst }] = await sql` - select count(*)::int as count from public.catalogue_import_changes as changes - join public.catalogue_import_targets as targets on targets.id = changes.target_id - where targets.record_id = ${item.record_id} and changes.entry_kind = 'change' and changes.status = 'accepted' - `; - assert.ok( - acceptedOnFirst > 0, - "a first import records its fields as accepted changes", - ); - - // The revised page changes the title and the introduction; the canned model - // answer follows the page so the merge sees a consistent extraction. - const revisedHtml = fixtureHtml - .replaceAll("Relational Databases", "Relational Databases (revised)") - .replace( - "Students design, query and reason about relational databases.", - "Students design, query, tune and reason about relational databases.", - ); - const revisedDeterministic = extractDeterministicCourse({ - html: revisedHtml, - courseCode: CODE, - year: YEAR, - sourceUrl, - }); - const previousPage = pageRef.current; - const previousModel = modelAnswerRef.current; - pageRef.current = revisedHtml; - modelAnswerRef.current = { - ...revisedDeterministic, - evidence: revisedDeterministic.evidence.map((item) => ({ - ...item, - method: "model", - })), - }; - try { - const run = await startRun([CODE]); - await processImportRunInline({ runId: run.runId }); - const [target] = await sql` - select id, status, change_kind, candidate_version_id from public.catalogue_import_targets - where run_id = ${run.runId}::uuid - `; - assert.equal(target.status, "ready"); - assert.equal(target.change_kind, "changed"); - - const changes = await sql` - select id, field_path, status, old_value, new_value from public.catalogue_import_changes - where target_id = ${target.id}::uuid and entry_kind = 'change' order by position - `; - const paths = changes.map((change) => change.field_path); - assert.ok(paths.includes("course.details.title"), paths.join(",")); - assert.ok(paths.includes("course.details.introduction"), paths.join(",")); - assert.ok(changes.every((change) => change.status === "open")); - - // Publishing is blocked while changes are open. - const [{ blockers }] = await sql` - select public.catalogue_publish_blockers(${item.record_id}) as blockers - `; - assert.ok( - blockers.some((reason) => - /open changes|no version|already published/.test(reason), - ), - blockers.join(" "), - ); - - // Accept the title, reject the description. - await sql.begin(async (tx) => { - await tx`select set_config('request.jwt.claim.sub', ${ADMIN_ID}, true)`; - for (const change of changes) { - await tx` - select public.resolve_catalogue_import_change( - ${change.id}, ${change.field_path === "course.details.introduction" ? "rejected" : "accepted"} - ) - `; - } - }); - const applied = await applyImportReview({ - targetId: target.id, - userId: ADMIN_ID, - sql, - }); - assert.equal( - applied.reusedCandidate, - false, - "a partial acceptance builds a merged version", - ); - - const [merged] = await sql` - select details.title, details.introduction - from public.course_version_details as details - where details.version_id = ${applied.versionId} - `; - assert.equal(merged.title, revisedDeterministic.title); - assert.equal( - merged.introduction, - deterministicAnswer.introduction, - "the rejected change keeps the baseline value", - ); - - const [pointer] = await sql` - select published_version_id from public.catalogue_records where id = ${item.record_id} - `; - assert.notEqual(Number(pointer.published_version_id), applied.versionId); - - await assert.rejects( - applyImportReview({ targetId: target.id, userId: ADMIN_ID, sql }), - /already been applied/, - ); - - await sql.begin(async (tx) => { - await tx`select set_config('request.jwt.claim.sub', ${ADMIN_ID}, true)`; - await tx`select public.publish_catalogue_version(${item.record_id})`; - }); - const [published] = await sql` - select published_version_id from public.catalogue_records where id = ${item.record_id} - `; - assert.equal(Number(published.published_version_id), applied.versionId); - } finally { - pageRef.current = previousPage; - modelAnswerRef.current = previousModel; - } -}); - -test("a run refuses a second unfinished target for the same item year", async () => { - const run = await startRun(["COMP2401"]); - await assert.rejects(startRun(["COMP2401"]), /unfinished import/); - await sql`delete from public.catalogue_import_runs where id = ${run.runId}::uuid`; -}); diff --git a/apps/web/tests/catalogue-review-state.test.ts b/apps/web/tests/catalogue-review-state.test.ts deleted file mode 100644 index 982c0810..00000000 --- a/apps/web/tests/catalogue-review-state.test.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { expect, test } from "vitest"; -import type { - CatalogueRecord, - ReviewEntry, - ReviewTarget, -} from "@/lib/coursemap/admin-catalogue-record"; -import { anuSourceLocation } from "@/ui/admin/catalogue/anu-source"; -import { - flagFieldLabel, - groupChanges, - groupFlags, - recordNextStep, - reviewSummary, -} from "@/ui/admin/catalogue/review-state"; - -let nextId = 1; - -function entry(overrides: Partial): ReviewEntry { - return { - id: nextId++, - entryKind: "change", - fieldPath: "course.details.title", - oldValue: null, - newValue: null, - severity: null, - isBlocking: false, - issueCode: null, - summary: null, - sourceLocator: null, - sourceExcerpt: null, - status: "open", - resolutionNote: null, - resolvedAt: null, - ...overrides, - }; -} - -function target(overrides: Partial = {}): ReviewTarget { - return { - id: "target-1", - runId: "run-1", - runNumber: 3, - status: "ready", - changeKind: "changed", - createdAt: "2026-03-01T00:00:00Z", - completedAt: "2026-03-01T00:01:00Z", - appliedAt: null, - baselineVersionId: 10, - candidateVersionId: 11, - entries: [], - ...overrides, - }; -} - -function record(overrides: Partial = {}): CatalogueRecord { - return { - kind: "course", - code: "COMP3600", - academicYear: 2026, - codeId: 1, - recordId: 1, - recordPublicId: "iy_1", - title: "Algorithms", - currentVersionId: null, - publishedVersionId: null, - archivedAt: null, - isListedByAnu: true, - listingTitle: "Algorithms", - lastSeenAt: "2026-03-01T00:00:00Z", - publishBlockers: [], - versions: [], - publications: [], - changeEvents: [], - reviews: [], - ...overrides, - }; -} - -test("a flag's leaf key is named from the field label map", () => { - // Changes carry a full path, flags only the key the extractor used. - expect(flagFieldLabel("eftsl")).toBe("EFTSL"); - expect(flagFieldLabel("prescribedTexts")).toBe("Prescribed texts"); - expect(flagFieldLabel("course.details.units")).toBe("Units"); - // Nothing in the map ends in this key, so the key itself is made readable. - expect(flagFieldLabel("requisites.prerequisiteText")).toBe( - "Prerequisite text", - ); -}); - -test("blocking flags stay separate and repeated warnings collapse by kind", () => { - const flags = [ - entry({ entryKind: "flag", issueCode: "INVALID", isBlocking: true }), - ...["introduction", "description", "fees"].map((fieldPath) => - entry({ entryKind: "flag", issueCode: "CONFLICT", fieldPath }), - ), - entry({ entryKind: "flag", issueCode: "EVIDENCE_MISSING" }), - ]; - const { blocking, groups } = groupFlags(flags); - - expect(blocking).toHaveLength(1); - expect(groups.map((group) => [group.label, group.entries.length])).toEqual([ - ["Model disagreed with the parser", 3], - ["No supporting excerpt", 1], - ]); - expect(groups[0]?.open).toHaveLength(3); -}); - -test("changes group by the part of the record they belong to", () => { - const groups = groupChanges([ - entry({ fieldPath: "course.details.title" }), - entry({ fieldPath: "course.sessions" }), - entry({ fieldPath: "requirements.prerequisite" }), - entry({ fieldPath: "course.details.units", status: "accepted" }), - ]); - - expect(groups.map((group) => group.label)).toEqual([ - "Details", - "Requirements", - "Lists", - ]); - expect(groups[0]?.entries).toHaveLength(2); - expect(groups[0]?.open).toHaveLength(1); - // The whole-collection constraint is stated once, above the rows it binds. - expect(groups[2]?.note).toContain("whole list"); -}); - -test("a run reports what is left rather than what it contains", () => { - const summary = reviewSummary( - target({ - entries: [ - entry({ status: "accepted" }), - entry({ fieldPath: "course.fees" }), - entry({ entryKind: "flag", issueCode: "CONFLICT" }), - ], - }), - ); - - expect(summary.headline).toBe("1 of 2 changes still to decide"); - expect(summary.decided).toBe(1); - expect(summary.actionable).toBe(true); -}); - -test("a settled run asks for nothing and collapses", () => { - const summary = reviewSummary( - target({ - appliedAt: "2026-03-02T00:00:00Z", - entries: [entry({ status: "accepted" })], - }), - ); - - expect(summary.actionable).toBe(false); - expect(summary.reviewable).toBe(false); - expect(summary.headline).toBe("Applied to the draft"); -}); - -test("the record's next step walks from decisions to published", () => { - const open = entry({}); - const reviews = [target({ entries: [open, entry({ status: "accepted" })] })]; - - const outstanding = recordNextStep( - record({ - currentVersionId: 5, - reviews, - publishBlockers: ["The import review still has open changes."], - }), - ); - expect(outstanding.headline).toBe("1 decision outstanding"); - expect(outstanding.next).toBe("review"); - expect(outstanding.decided).toBe(1); - - const decided = recordNextStep( - record({ - currentVersionId: 5, - reviews: [target({ entries: [entry({ status: "accepted" })] })], - }), - ); - expect(decided.headline).toBe("Every change is decided"); - expect(decided.next).toBe("review"); - - const ready = recordNextStep( - record({ - currentVersionId: 5, - reviews: [ - target({ - appliedAt: "2026-03-02T00:00:00Z", - entries: [entry({ status: "accepted" })], - }), - ], - }), - ); - expect(ready.headline).toBe("Ready to publish"); - expect(ready.next).toBe("publish"); - - const live = recordNextStep(record({ publishedVersionId: 5 })); - expect(live.headline).toBe("Published"); - expect(live.next).toBe("none"); -}); - -test("a blocking flag holds publication and names itself", () => { - const step = recordNextStep( - record({ - currentVersionId: 5, - publishBlockers: ["A blocking flag on the import review is still open."], - reviews: [ - target({ - entries: [ - entry({ - entryKind: "flag", - isBlocking: true, - issueCode: "INVALID", - }), - ], - }), - ], - }), - ); - - expect(step.headline).toBe("1 flag blocking publication"); - expect(step.tone).toBe("warning"); -}); - -test("decisions from a superseded import do not hold a record back", () => { - // catalogue_publish_blockers reads the newest finished review and the one - // behind the draft, so the interface counts the same two and no others. - const step = recordNextStep( - record({ - currentVersionId: 5, - versions: [ - { - id: 5, - publicId: "sn_5", - origin: "import", - createdAt: "2026-03-02T00:00:00Z", - sealedAt: null, - basedOnVersionId: null, - importTargetId: "target-1", - contentHash: "abc", - }, - ], - reviews: [ - target({ - id: "target-1", - appliedAt: "2026-03-02T00:00:00Z", - entries: [entry({ status: "accepted" })], - }), - target({ id: "target-0", status: "failed", entries: [entry({})] }), - ], - }), - ); - - expect(step.headline).toBe("Ready to publish"); -}); - -test("an id locator deep-links the ANU page and a selector does not", () => { - const page = "https://programsandcourses.anu.edu.au/2026/course/COMP3600"; - - expect(anuSourceLocation(page, "#learning-outcomes")).toEqual({ - label: "Learning outcomes", - href: `${page}#learning-outcomes`, - }); - // A class or attribute selector names markup, which tells a reviewer nothing. - expect(anuSourceLocation(page, ".degree-summary")).toEqual({ - label: null, - href: page, - }); - expect(anuSourceLocation(page, 'meta[name="course-name"]')).toEqual({ - label: null, - href: page, - }); - // Structure evidence names a heading in the page's own words. - expect(anuSourceLocation(page, "Admission Requirements")).toEqual({ - label: "Admission Requirements", - href: page, - }); - expect(anuSourceLocation(page, null)).toEqual({ label: null, href: page }); -}); diff --git a/apps/web/tests/catalogue-sync-button.test.tsx b/apps/web/tests/catalogue-sync-button.test.tsx new file mode 100644 index 00000000..399cd671 --- /dev/null +++ b/apps/web/tests/catalogue-sync-button.test.tsx @@ -0,0 +1,61 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, expect, test, vi } from "vitest"; +import { CatalogueSyncButton } from "@/ui/admin/catalogue/sync-button"; + +const { refresh, success, failure } = vi.hoisted(() => ({ + refresh: vi.fn(), + success: vi.fn(), + failure: vi.fn(), +})); + +vi.mock("next/navigation", () => ({ useRouter: () => ({ refresh }) })); +vi.mock("sonner", () => ({ toast: { success, error: failure } })); + +beforeEach(() => { + vi.restoreAllMocks(); + refresh.mockReset(); + success.mockReset(); + failure.mockReset(); +}); + +test("starts one record-level ANU sync", async () => { + const request = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ syncId: "sync-1", mode: "inline" }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Sync from ANU" })); + + await waitFor(() => expect(request).toHaveBeenCalledTimes(1)); + expect(JSON.parse(String(request.mock.calls[0]?.[1]?.body))).toEqual({ + recordId: 42, + kind: "course", + }); + expect(await screen.findByRole("button")).toBeDisabled(); + expect(screen.getByRole("button")).toHaveTextContent("Syncing from ANU..."); +}); + +test("offers a retry after a failed sync", () => { + render( + , + ); + expect(screen.getByRole("button", { name: "Retry sync" })).toBeEnabled(); +}); diff --git a/apps/web/tests/catalogue-sync-database.test.mjs b/apps/web/tests/catalogue-sync-database.test.mjs new file mode 100644 index 00000000..e50a8c37 --- /dev/null +++ b/apps/web/tests/catalogue-sync-database.test.mjs @@ -0,0 +1,259 @@ +import assert from "node:assert/strict"; +import { afterAll, beforeAll, test } from "vitest"; + +import { + emptyCatalogueContent, + CATALOGUE_CONTENT_SCHEMA_VERSION, +} from "../lib/catalogue/content.ts"; +import { contentHashForCatalogueContent } from "../lib/catalogue-import/version-content.ts"; +import { persistSourceVersion } from "../lib/catalogue-sync/persist-source-version.ts"; +import { ensureAnuSourceId } from "../lib/catalogue-sync/sync-store.ts"; +import { createLocalDatabaseClient } from "../scripts/catalogue/lib/local-database.mjs"; +import { localTestEnvironment } from "../scripts/local/test-environment.mjs"; + +const YEAR = 2026; +const EMPTY_CODE = "TSTC9101"; +const MANUAL_CODE = "TSTC9102"; + +let sql; +let yearId; +let sourceId; +const records = new Map(); + +function sourceContent(code, title, description) { + const content = emptyCatalogueContent({ + kind: "course", + code, + academicYear: YEAR, + title, + }); + content.course.details.description = description; + content.contentHash = contentHashForCatalogueContent(content); + return content; +} + +async function removeFixtures() { + await sql`delete from public.catalogue_listings where code in (${EMPTY_CODE}, ${MANUAL_CODE})`; + await sql`alter table public.catalogue_source_documents disable trigger catalogue_source_documents_reject_mutation`; + await sql`alter table public.catalogue_versions disable trigger catalogue_versions_enforce_immutability`; + try { + await sql`delete from public.catalogue_codes where kind = 'course' and code in (${EMPTY_CODE}, ${MANUAL_CODE})`; + } finally { + await sql`alter table public.catalogue_versions enable trigger catalogue_versions_enforce_immutability`; + await sql`alter table public.catalogue_source_documents enable trigger catalogue_source_documents_reject_mutation`; + } +} + +async function createRecord(code, title) { + const [identity] = await sql` + insert into public.catalogue_codes (kind, code) + values ('course', ${code}) returning id + `; + const [record] = await sql` + insert into public.catalogue_records (code_id, kind, academic_year_id) + values (${identity.id}, 'course', ${yearId}) returning id + `; + await sql` + insert into public.catalogue_listings ( + academic_year_id, kind, code, title, code_id, record_id, is_current, + first_seen_at, last_seen_at + ) values ( + ${yearId}, 'course', ${code}, ${title}, ${identity.id}, ${record.id}, + true, now(), now() + ) + `; + records.set(code, Number(record.id)); + return Number(record.id); +} + +async function createSyncFixture(code, contentHash) { + const recordId = records.get(code); + const [record] = await sql` + select latest_source_version_id from public.catalogue_records where id = ${recordId} + `; + const [sync] = await sql` + insert into public.catalogue_syncs ( + record_id, trigger, status, requested_model, parser_version, + prompt_version, schema_version, previous_source_version_id + ) values ( + ${recordId}, 'manual', 'running', + (select id from public.import_models where enabled order by id limit 1), + 'test-parser', 'test-prompt', 'test-schema', ${record.latest_source_version_id} + ) returning id + `; + const [insertedDocument] = await sql` + insert into public.catalogue_source_documents ( + source_id, record_id, academic_year_id, kind, external_key, + canonical_url, content_sha256, http_status, fetched_at + ) values ( + ${sourceId}, ${recordId}, ${yearId}, 'course', ${code}, + ${`https://programsandcourses.anu.edu.au/course/${code}`}, + ${contentHash}, 200, now() + ) on conflict (source_id, record_id, content_sha256) do nothing + returning id + `; + const [document] = insertedDocument + ? [insertedDocument] + : await sql` + select id from public.catalogue_source_documents + where source_id = ${sourceId} and record_id = ${recordId} + and content_sha256 = ${contentHash} + `; + return { + documentId: Number(document.id), + claim: { + syncId: sync.id, + kind: "course", + code, + academicYear: YEAR, + academicYearId: Number(yearId), + recordId, + previousSourceVersionId: + record.latest_source_version_id === null + ? null + : Number(record.latest_source_version_id), + requestedModel: "test", + parserVersion: "test-parser", + promptVersion: "test-prompt", + schemaVersion: "test-schema", + sourceId: Number(sourceId), + attemptCount: 1, + lockVersion: 1, + }, + }; +} + +beforeAll(async () => { + Object.assign(process.env, localTestEnvironment(), { + NODE_ENV: "development", + }); + sql = await createLocalDatabaseClient(); + await removeFixtures(); + [{ id: yearId }] = await sql` + select id from public.academic_years where year = ${YEAR} + `; + sourceId = await ensureAnuSourceId(sql); + await createRecord(EMPTY_CODE, "Empty Source Record"); + await createRecord(MANUAL_CODE, "Manual Source Record"); +}); + +afterAll(async () => { + if (!sql) return; + await removeFixtures(); + await sql.end({ timeout: 5 }); +}); + +test("first, unchanged and changed source observations preserve local intent", async () => { + const emptyRecordId = records.get(EMPTY_CODE); + const emptyDraft = emptyCatalogueContent({ + kind: "course", + code: EMPTY_CODE, + academicYear: YEAR, + title: "Empty Source Record", + }); + emptyDraft.contentHash = contentHashForCatalogueContent(emptyDraft); + await sql` + insert into public.catalogue_drafts ( + record_id, content, content_hash, content_schema_version, revision + ) values (${emptyRecordId}, ${sql.json(emptyDraft)}, ${emptyDraft.contentHash}, + ${CATALOGUE_CONTENT_SCHEMA_VERSION}, 0) + `; + + const firstContent = sourceContent( + EMPTY_CODE, + "Empty Source Record", + "First ANU description.", + ); + const firstFixture = await createSyncFixture(EMPTY_CODE, "1".repeat(64)); + const first = await persistSourceVersion(sql, { + claim: firstFixture.claim, + sourceDocumentId: firstFixture.documentId, + write: firstContent, + }); + assert.equal(first.status, "applied"); + assert.equal(first.populatedDraft, true); + const replayedFirst = await persistSourceVersion(sql, { + claim: firstFixture.claim, + sourceDocumentId: firstFixture.documentId, + write: firstContent, + }); + assert.deepEqual(replayedFirst, first); + const [firstVersionCount] = await sql` + select count(*)::integer as count from public.catalogue_versions + where sync_id = ${firstFixture.claim.syncId} + `; + assert.equal(firstVersionCount.count, 1); + await sql`update public.catalogue_syncs set status = 'applied', completed_at = now() + where id = ${firstFixture.claim.syncId}`; + const [populated] = await sql` + select content, base_version_id from public.catalogue_drafts where record_id = ${emptyRecordId} + `; + assert.equal( + populated.content.course.details.description, + "First ANU description.", + ); + assert.equal(Number(populated.base_version_id), first.sourceVersionId); + + const unchangedFixture = await createSyncFixture(EMPTY_CODE, "1".repeat(64)); + const unchanged = await persistSourceVersion(sql, { + claim: unchangedFixture.claim, + sourceDocumentId: unchangedFixture.documentId, + write: firstContent, + }); + assert.equal(unchanged.status, "unchanged"); + assert.equal(unchanged.sourceVersionId, first.sourceVersionId); + await sql`update public.catalogue_syncs set status = 'unchanged', completed_at = now() + where id = ${unchangedFixture.claim.syncId}`; + + const changedContent = sourceContent( + EMPTY_CODE, + "Empty Source Record", + "Changed ANU description.", + ); + const changedFixture = await createSyncFixture(EMPTY_CODE, "2".repeat(64)); + const changed = await persistSourceVersion(sql, { + claim: changedFixture.claim, + sourceDocumentId: changedFixture.documentId, + write: changedContent, + }); + assert.equal(changed.status, "review_required"); + const [unchangedDraft] = await sql` + select content from public.catalogue_drafts where record_id = ${emptyRecordId} + `; + assert.equal( + unchangedDraft.content.course.details.description, + "First ANU description.", + ); + + const manualRecordId = records.get(MANUAL_CODE); + const manualDraft = sourceContent( + MANUAL_CODE, + "Manual Source Record", + "Locally authored description.", + ); + await sql` + insert into public.catalogue_drafts ( + record_id, content, content_hash, content_schema_version, revision + ) values (${manualRecordId}, ${sql.json(manualDraft)}, ${manualDraft.contentHash}, + ${CATALOGUE_CONTENT_SCHEMA_VERSION}, 1) + `; + const manualFirstContent = sourceContent( + MANUAL_CODE, + "Manual Source Record", + "ANU description.", + ); + const manualFixture = await createSyncFixture(MANUAL_CODE, "3".repeat(64)); + const manualFirst = await persistSourceVersion(sql, { + claim: manualFixture.claim, + sourceDocumentId: manualFixture.documentId, + write: manualFirstContent, + }); + assert.equal(manualFirst.status, "review_required"); + const [manualStillLocal] = await sql` + select content from public.catalogue_drafts where record_id = ${manualRecordId} + `; + assert.equal( + manualStillLocal.content.course.details.description, + "Locally authored description.", + ); +}); diff --git a/apps/web/tests/catalogue-sync-queue.test.ts b/apps/web/tests/catalogue-sync-queue.test.ts new file mode 100644 index 00000000..2133db80 --- /dev/null +++ b/apps/web/tests/catalogue-sync-queue.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + createSyncQueueIdempotencyKey, + createSyncQueueMessage, + parseSyncQueueMessage, +} from "@/lib/catalogue-sync/sync-queue"; + +const SYNC_ID = "10000000-0000-4000-8000-000000000001"; + +describe("catalogue sync queue messages", () => { + it("contains only the independently executable sync identifier", () => { + expect(createSyncQueueMessage(SYNC_ID)).toEqual({ + version: 1, + syncId: SYNC_ID, + }); + }); + + it("rejects obsolete run and target identifiers", () => { + expect(() => + parseSyncQueueMessage({ + version: 1, + syncId: SYNC_ID, + runId: SYNC_ID, + targetId: SYNC_ID, + }), + ).toThrow("Sync queue message fields do not match version 1."); + }); + + it("uses the sync as the idempotency boundary", () => { + expect(createSyncQueueIdempotencyKey(createSyncQueueMessage(SYNC_ID))).toBe( + `catalogue-sync:v1:${SYNC_ID}`, + ); + }); +}); diff --git a/apps/web/tests/notifications-menu.test.tsx b/apps/web/tests/notifications-menu.test.tsx index b907393d..9c077e39 100644 --- a/apps/web/tests/notifications-menu.test.tsx +++ b/apps/web/tests/notifications-menu.test.tsx @@ -12,10 +12,10 @@ vi.mock("@/lib/coursemap/notifications", () => inbox); function notification(overrides: Record = {}) { return { id: "run-1", - kind: "import_run", - title: "Import run #7 completed", - body: "3 records ready to review.", - href: "/admin/courses/imports?run=7", + kind: "published_change", + title: "ANU changes are ready to review", + body: "COMP1000 has a newer ANU source version.", + href: "/admin/courses/2026/comp1000?tab=changes", readAt: null, createdAt: new Date().toISOString(), ...overrides, @@ -69,7 +69,7 @@ test("the bell counts unread rows and opening does not mark them read", async () await user.click(bell); expect( - await screen.findByText("Import run #7 completed"), + await screen.findByText("ANU changes are ready to review"), ).toBeInTheDocument(); expect( screen.getByText("COMP2400 lost its Semester 2 offering"), @@ -90,7 +90,9 @@ test("opening a notification marks that row read and clears the count", async () await screen.findByRole("button", { name: "Notifications, 1 unread" }), ); await user.click( - await screen.findByRole("link", { name: /Import run #7 completed/ }), + await screen.findByRole("link", { + name: /ANU changes are ready to review/, + }), ); expect(inbox.markNotificationsRead).toHaveBeenCalledWith(["run-1"]); diff --git a/apps/web/tests/value-diff.test.tsx b/apps/web/tests/value-diff.test.tsx deleted file mode 100644 index c2c7d336..00000000 --- a/apps/web/tests/value-diff.test.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import { expect, test } from "vitest"; -import { render, screen } from "@testing-library/react"; -import { ValueDiff } from "@/ui/admin/catalogue/value-diff"; - -test("a scalar change reads as one pair of values", () => { - render( - , - ); - expect(screen.getByText("Algorithms")).toBeInTheDocument(); - expect(screen.getByText("Advanced Algorithms")).toBeInTheDocument(); -}); - -test("only the changed field of a collection row is shown", () => { - const before = [ - { classNumber: "1", deliveryMode: "In person", location: "Manning Clark" }, - { classNumber: "2", deliveryMode: "In person", location: "Hanna Neumann" }, - ]; - const after = [ - { classNumber: "1", deliveryMode: "Online", location: "Manning Clark" }, - { classNumber: "2", deliveryMode: "In person", location: "Hanna Neumann" }, - ]; - render( - , - ); - - // The one altered field is named, with its own before and after. - expect(screen.getByText("Item 1 · Delivery mode")).toBeInTheDocument(); - expect(screen.getByText("In person")).toBeInTheDocument(); - expect(screen.getByText("Online")).toBeInTheDocument(); - - // The untouched row never appears, and neither does a JSON dump of it. - expect(screen.queryByText(/Item 2/)).not.toBeInTheDocument(); - expect(screen.queryByText(/Hanna Neumann/)).not.toBeInTheDocument(); -}); - -test("an added row is labelled as added", () => { - render( - , - ); - expect(screen.getByText("Item 2 (added)")).toBeInTheDocument(); - expect(screen.getByText("International")).toBeInTheDocument(); -}); - -test("a value the walk cannot reduce falls back to the raw values", () => { - const wide = Array.from({ length: 80 }, (_, index) => ({ - key: `value-${index}`, - })); - render( - , - ); - // Nothing was dropped: the reviewer still sees both sides in full. - expect(screen.getByLabelText("Current course.sessions")).toBeInTheDocument(); - expect(screen.getByLabelText("Imported course.sessions")).toBeInTheDocument(); -}); diff --git a/apps/web/types/database.ts b/apps/web/types/database.ts index 62f6ed8d..5dfc93cc 100644 --- a/apps/web/types/database.ts +++ b/apps/web/types/database.ts @@ -1157,7 +1157,7 @@ export type Database = { schema_valid: boolean | null schema_version: string started_at: string - target_id: string + sync_id: string validated_artifact_id: string | null validation_status: string warning_count: number @@ -1188,7 +1188,7 @@ export type Database = { schema_valid?: boolean | null schema_version: string started_at?: string - target_id: string + sync_id: string validated_artifact_id?: string | null validation_status?: string warning_count?: number @@ -1219,45 +1219,45 @@ export type Database = { schema_valid?: boolean | null schema_version?: string started_at?: string - target_id?: string + sync_id?: string validated_artifact_id?: string | null validation_status?: string warning_count?: number } Relationships: [ { - foreignKeyName: "catalogue_extractions_request_artifact_fkey" + foreignKeyName: "catalogue_extractions_request_artifact_id_fkey" columns: ["request_artifact_id"] isOneToOne: false - referencedRelation: "catalogue_import_artifacts" + referencedRelation: "catalogue_sync_artifacts" referencedColumns: ["id"] }, { - foreignKeyName: "catalogue_extractions_response_artifact_fkey" + foreignKeyName: "catalogue_extractions_response_artifact_id_fkey" columns: ["response_artifact_id"] isOneToOne: false - referencedRelation: "catalogue_import_artifacts" + referencedRelation: "catalogue_sync_artifacts" referencedColumns: ["id"] }, { - foreignKeyName: "catalogue_extractions_reused_fkey" + foreignKeyName: "catalogue_extractions_reused_from_extraction_id_fkey" columns: ["reused_from_extraction_id"] isOneToOne: false referencedRelation: "catalogue_extractions" referencedColumns: ["id"] }, { - foreignKeyName: "catalogue_extractions_target_fkey" - columns: ["target_id"] + foreignKeyName: "catalogue_extractions_sync_id_fkey" + columns: ["sync_id"] isOneToOne: false - referencedRelation: "catalogue_import_targets" + referencedRelation: "catalogue_syncs" referencedColumns: ["id"] }, { - foreignKeyName: "catalogue_extractions_validated_artifact_fkey" + foreignKeyName: "catalogue_extractions_validated_artifact_id_fkey" columns: ["validated_artifact_id"] isOneToOne: false - referencedRelation: "catalogue_import_artifacts" + referencedRelation: "catalogue_sync_artifacts" referencedColumns: ["id"] }, ] @@ -1294,417 +1294,6 @@ export type Database = { }, ] } - catalogue_import_artifacts: { - Row: { - attempt_number: number - byte_size: number - content_sha256: string - created_at: string - id: string - kind: string - media_type: string - stage_id: string - storage_bucket: string - storage_path: string - target_id: string - } - Insert: { - attempt_number: number - byte_size: number - content_sha256: string - created_at?: string - id?: string - kind: string - media_type: string - stage_id: string - storage_bucket: string - storage_path: string - target_id: string - } - Update: { - attempt_number?: number - byte_size?: number - content_sha256?: string - created_at?: string - id?: string - kind?: string - media_type?: string - stage_id?: string - storage_bucket?: string - storage_path?: string - target_id?: string - } - Relationships: [ - { - foreignKeyName: "catalogue_import_artifacts_stage_fkey" - columns: ["stage_id"] - isOneToOne: false - referencedRelation: "catalogue_import_stages" - referencedColumns: ["id"] - }, - { - foreignKeyName: "catalogue_import_artifacts_target_fkey" - columns: ["target_id"] - isOneToOne: false - referencedRelation: "catalogue_import_targets" - referencedColumns: ["id"] - }, - ] - } - catalogue_import_changes: { - Row: { - created_at: string - entry_kind: string - field_path: string - id: number - is_blocking: boolean - issue_code: string | null - new_value: Json | null - old_value: Json | null - position: number - resolution_note: string | null - resolved_at: string | null - resolved_by: string | null - severity: string | null - source_excerpt: string | null - source_locator: string | null - status: string - summary: string | null - target_id: string - } - Insert: { - created_at?: string - entry_kind: string - field_path: string - id?: never - is_blocking?: boolean - issue_code?: string | null - new_value?: Json | null - old_value?: Json | null - position?: number - resolution_note?: string | null - resolved_at?: string | null - resolved_by?: string | null - severity?: string | null - source_excerpt?: string | null - source_locator?: string | null - status?: string - summary?: string | null - target_id: string - } - Update: { - created_at?: string - entry_kind?: string - field_path?: string - id?: never - is_blocking?: boolean - issue_code?: string | null - new_value?: Json | null - old_value?: Json | null - position?: number - resolution_note?: string | null - resolved_at?: string | null - resolved_by?: string | null - severity?: string | null - source_excerpt?: string | null - source_locator?: string | null - status?: string - summary?: string | null - target_id?: string - } - Relationships: [ - { - foreignKeyName: "catalogue_import_changes_target_fkey" - columns: ["target_id"] - isOneToOne: false - referencedRelation: "catalogue_import_targets" - referencedColumns: ["id"] - }, - ] - } - catalogue_import_runs: { - Row: { - academic_year_id: number - completed_at: string | null - completed_count: number - cost_usd: number - created_at: string - failed_count: number - id: string - input_tokens: number - kind: string - output_tokens: number - parser_version: string - prompt_version: string - requested_by: string | null - requested_model: string - run_number: number - schema_version: string - started_at: string | null - status: string - target_count: number - } - Insert: { - academic_year_id: number - completed_at?: string | null - completed_count?: number - cost_usd?: number - created_at?: string - failed_count?: number - id?: string - input_tokens?: number - kind: string - output_tokens?: number - parser_version: string - prompt_version: string - requested_by?: string | null - requested_model: string - run_number?: never - schema_version: string - started_at?: string | null - status?: string - target_count?: number - } - Update: { - academic_year_id?: number - completed_at?: string | null - completed_count?: number - cost_usd?: number - created_at?: string - failed_count?: number - id?: string - input_tokens?: number - kind?: string - output_tokens?: number - parser_version?: string - prompt_version?: string - requested_by?: string | null - requested_model?: string - run_number?: never - schema_version?: string - started_at?: string | null - status?: string - target_count?: number - } - Relationships: [ - { - foreignKeyName: "catalogue_import_runs_academic_year_fkey" - columns: ["academic_year_id"] - isOneToOne: false - referencedRelation: "academic_years" - referencedColumns: ["id"] - }, - { - foreignKeyName: "catalogue_import_runs_model_fkey" - columns: ["requested_model"] - isOneToOne: false - referencedRelation: "import_models" - referencedColumns: ["id"] - }, - ] - } - catalogue_import_stages: { - Row: { - attempt_number: number - completed_at: string | null - error_code: string | null - error_summary: string | null - id: string - stage_name: string - started_at: string - status: string - target_id: string - } - Insert: { - attempt_number: number - completed_at?: string | null - error_code?: string | null - error_summary?: string | null - id?: string - stage_name: string - started_at?: string - status?: string - target_id: string - } - Update: { - attempt_number?: number - completed_at?: string | null - error_code?: string | null - error_summary?: string | null - id?: string - stage_name?: string - started_at?: string - status?: string - target_id?: string - } - Relationships: [ - { - foreignKeyName: "catalogue_import_stages_target_fkey" - columns: ["target_id"] - isOneToOne: false - referencedRelation: "catalogue_import_targets" - referencedColumns: ["id"] - }, - ] - } - catalogue_import_targets: { - Row: { - academic_year_id: number - applied_at: string | null - applied_version_id: number | null - attempt_count: number - baseline_version_id: number | null - candidate_version_id: number | null - change_kind: string | null - code: string - code_id: number - completed_at: string | null - created_at: string - directory_entry_id: number | null - dispatched_at: string | null - error_code: string | null - error_message: string | null - id: string - kind: string - lease_expires_at: string | null - lock_version: number - queue_message_id: string | null - record_id: number - run_id: string - source_page_id: number | null - status: string - updated_at: string - worker_id: string | null - } - Insert: { - academic_year_id: number - applied_at?: string | null - applied_version_id?: number | null - attempt_count?: number - baseline_version_id?: number | null - candidate_version_id?: number | null - change_kind?: string | null - code: string - code_id: number - completed_at?: string | null - created_at?: string - directory_entry_id?: number | null - dispatched_at?: string | null - error_code?: string | null - error_message?: string | null - id?: string - kind: string - lease_expires_at?: string | null - lock_version?: number - queue_message_id?: string | null - record_id: number - run_id: string - source_page_id?: number | null - status?: string - updated_at?: string - worker_id?: string | null - } - Update: { - academic_year_id?: number - applied_at?: string | null - applied_version_id?: number | null - attempt_count?: number - baseline_version_id?: number | null - candidate_version_id?: number | null - change_kind?: string | null - code?: string - code_id?: number - completed_at?: string | null - created_at?: string - directory_entry_id?: number | null - dispatched_at?: string | null - error_code?: string | null - error_message?: string | null - id?: string - kind?: string - lease_expires_at?: string | null - lock_version?: number - queue_message_id?: string | null - record_id?: number - run_id?: string - source_page_id?: number | null - status?: string - updated_at?: string - worker_id?: string | null - } - Relationships: [ - { - foreignKeyName: "catalogue_import_targets_applied_fkey" - columns: ["applied_version_id", "record_id"] - isOneToOne: false - referencedRelation: "catalogue_versions" - referencedColumns: ["id", "record_id"] - }, - { - foreignKeyName: "catalogue_import_targets_baseline_fkey" - columns: ["baseline_version_id", "record_id"] - isOneToOne: false - referencedRelation: "catalogue_versions" - referencedColumns: ["id", "record_id"] - }, - { - foreignKeyName: "catalogue_import_targets_candidate_fkey" - columns: ["candidate_version_id", "record_id"] - isOneToOne: false - referencedRelation: "catalogue_versions" - referencedColumns: ["id", "record_id"] - }, - { - foreignKeyName: "catalogue_import_targets_directory_entry_fkey" - columns: ["directory_entry_id"] - isOneToOne: false - referencedRelation: "catalogue_directory_entries" - referencedColumns: ["id"] - }, - { - foreignKeyName: "catalogue_import_targets_directory_entry_fkey" - columns: ["directory_entry_id"] - isOneToOne: false - referencedRelation: "catalogue_listings" - referencedColumns: ["id"] - }, - { - foreignKeyName: "catalogue_import_targets_item_fkey" - columns: ["code_id", "kind"] - isOneToOne: false - referencedRelation: "catalogue_codes" - referencedColumns: ["id", "kind"] - }, - { - foreignKeyName: "catalogue_import_targets_item_year_fkey" - columns: ["record_id", "academic_year_id"] - isOneToOne: false - referencedRelation: "catalogue_records" - referencedColumns: ["id", "academic_year_id"] - }, - { - foreignKeyName: "catalogue_import_targets_item_year_fkey" - columns: ["record_id", "academic_year_id"] - isOneToOne: false - referencedRelation: "published_course_summaries" - referencedColumns: ["record_id", "academic_year_id"] - }, - { - foreignKeyName: "catalogue_import_targets_run_fkey" - columns: ["run_id"] - isOneToOne: false - referencedRelation: "catalogue_import_runs" - referencedColumns: ["id"] - }, - { - foreignKeyName: "catalogue_import_targets_source_page_fkey" - columns: ["source_page_id", "academic_year_id"] - isOneToOne: false - referencedRelation: "catalogue_source_pages" - referencedColumns: ["id", "academic_year_id"] - }, - ] - } catalogue_listings: { Row: { academic_year_id: number @@ -1846,8 +1435,10 @@ export type Database = { created_at: string id: number kind: string + latest_source_version_id: number | null public_id: string published_version_id: number | null + source_checked_at: string | null updated_at: string } Insert: { @@ -1857,8 +1448,10 @@ export type Database = { created_at?: string id?: never kind: string + latest_source_version_id?: number | null public_id?: string published_version_id?: number | null + source_checked_at?: string | null updated_at?: string } Update: { @@ -1868,8 +1461,10 @@ export type Database = { created_at?: string id?: never kind?: string + latest_source_version_id?: number | null public_id?: string published_version_id?: number | null + source_checked_at?: string | null updated_at?: string } Relationships: [ @@ -1877,22 +1472,142 @@ export type Database = { foreignKeyName: "catalogue_records_academic_year_id_fkey" columns: ["academic_year_id"] isOneToOne: false - referencedRelation: "academic_years" - referencedColumns: ["id"] + referencedRelation: "academic_years" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_records_code_kind_fkey" + columns: ["code_id", "kind"] + isOneToOne: false + referencedRelation: "catalogue_codes" + referencedColumns: ["id", "kind"] + }, + { + foreignKeyName: "catalogue_records_latest_source_version_fkey" + columns: ["latest_source_version_id", "id"] + isOneToOne: false + referencedRelation: "catalogue_versions" + referencedColumns: ["id", "record_id"] + }, + { + foreignKeyName: "catalogue_records_published_version_fkey" + columns: ["published_version_id", "id"] + isOneToOne: false + referencedRelation: "catalogue_versions" + referencedColumns: ["id", "record_id"] + }, + ] + } + catalogue_source_documents: { + Row: { + academic_year_id: number + byte_size: number | null + canonical_url: string + content_sha256: string + created_at: string + external_key: string + fetched_at: string + http_etag: string | null + http_status: number | null + id: number + kind: string + media_type: string + public_id: string + record_id: number + source_id: number + source_last_modified: string | null + storage_bucket: string | null + storage_path: string | null + } + Insert: { + academic_year_id: number + byte_size?: number | null + canonical_url: string + content_sha256: string + created_at?: string + external_key: string + fetched_at: string + http_etag?: string | null + http_status?: number | null + id?: never + kind: string + media_type?: string + public_id?: string + record_id: number + source_id: number + source_last_modified?: string | null + storage_bucket?: string | null + storage_path?: string | null + } + Update: { + academic_year_id?: number + byte_size?: number | null + canonical_url?: string + content_sha256?: string + created_at?: string + external_key?: string + fetched_at?: string + http_etag?: string | null + http_status?: number | null + id?: never + kind?: string + media_type?: string + public_id?: string + record_id?: number + source_id?: number + source_last_modified?: string | null + storage_bucket?: string | null + storage_path?: string | null + } + Relationships: [ + { + foreignKeyName: "catalogue_source_documents_academic_year_id_fkey" + columns: ["academic_year_id"] + isOneToOne: false + referencedRelation: "academic_years" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_source_documents_record_id_fkey" + columns: ["record_id"] + isOneToOne: false + referencedRelation: "catalogue_records" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_source_documents_record_id_fkey" + columns: ["record_id"] + isOneToOne: false + referencedRelation: "published_course_summaries" + referencedColumns: ["record_id"] }, { - foreignKeyName: "catalogue_records_code_kind_fkey" - columns: ["code_id", "kind"] + foreignKeyName: "catalogue_source_documents_record_kind_fkey" + columns: ["record_id", "kind"] isOneToOne: false - referencedRelation: "catalogue_codes" + referencedRelation: "catalogue_records" referencedColumns: ["id", "kind"] }, { - foreignKeyName: "catalogue_records_published_version_fkey" - columns: ["published_version_id", "id"] + foreignKeyName: "catalogue_source_documents_record_year_fkey" + columns: ["record_id", "academic_year_id"] isOneToOne: false - referencedRelation: "catalogue_versions" - referencedColumns: ["id", "record_id"] + referencedRelation: "catalogue_records" + referencedColumns: ["id", "academic_year_id"] + }, + { + foreignKeyName: "catalogue_source_documents_record_year_fkey" + columns: ["record_id", "academic_year_id"] + isOneToOne: false + referencedRelation: "published_course_summaries" + referencedColumns: ["record_id", "academic_year_id"] + }, + { + foreignKeyName: "catalogue_source_documents_source_id_fkey" + columns: ["source_id"] + isOneToOne: false + referencedRelation: "catalogue_sources" + referencedColumns: ["id"] }, ] } @@ -1998,6 +1713,240 @@ export type Database = { } Relationships: [] } + catalogue_sync_artifacts: { + Row: { + attempt_number: number + byte_size: number + content_sha256: string + created_at: string + id: string + kind: string + media_type: string + stage_id: string + storage_bucket: string + storage_path: string + sync_id: string + } + Insert: { + attempt_number: number + byte_size: number + content_sha256: string + created_at?: string + id?: string + kind: string + media_type: string + stage_id: string + storage_bucket: string + storage_path: string + sync_id: string + } + Update: { + attempt_number?: number + byte_size?: number + content_sha256?: string + created_at?: string + id?: string + kind?: string + media_type?: string + stage_id?: string + storage_bucket?: string + storage_path?: string + sync_id?: string + } + Relationships: [ + { + foreignKeyName: "catalogue_sync_artifacts_stage_id_fkey" + columns: ["stage_id"] + isOneToOne: false + referencedRelation: "catalogue_sync_stages" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_sync_artifacts_sync_id_fkey" + columns: ["sync_id"] + isOneToOne: false + referencedRelation: "catalogue_syncs" + referencedColumns: ["id"] + }, + ] + } + catalogue_sync_stages: { + Row: { + attempt_number: number + completed_at: string | null + error_code: string | null + error_summary: string | null + id: string + stage_name: string + started_at: string + status: string + sync_id: string + } + Insert: { + attempt_number: number + completed_at?: string | null + error_code?: string | null + error_summary?: string | null + id?: string + stage_name: string + started_at?: string + status?: string + sync_id: string + } + Update: { + attempt_number?: number + completed_at?: string | null + error_code?: string | null + error_summary?: string | null + id?: string + stage_name?: string + started_at?: string + status?: string + sync_id?: string + } + Relationships: [ + { + foreignKeyName: "catalogue_sync_stages_sync_id_fkey" + columns: ["sync_id"] + isOneToOne: false + referencedRelation: "catalogue_syncs" + referencedColumns: ["id"] + }, + ] + } + catalogue_syncs: { + Row: { + attempt_count: number + checked_at: string | null + completed_at: string | null + created_at: string + dispatched_at: string | null + error_code: string | null + error_message: string | null + id: string + lease_expires_at: string | null + lock_version: number + parser_version: string + previous_source_version_id: number | null + prompt_version: string + public_id: string + queue_message_id: string | null + record_id: number + requested_at: string + requested_by: string | null + requested_model: string + schema_version: string + source_document_id: number | null + source_version_id: number | null + started_at: string | null + status: string + trigger: string + updated_at: string + worker_id: string | null + } + Insert: { + attempt_count?: number + checked_at?: string | null + completed_at?: string | null + created_at?: string + dispatched_at?: string | null + error_code?: string | null + error_message?: string | null + id?: string + lease_expires_at?: string | null + lock_version?: number + parser_version: string + previous_source_version_id?: number | null + prompt_version: string + public_id?: string + queue_message_id?: string | null + record_id: number + requested_at?: string + requested_by?: string | null + requested_model: string + schema_version: string + source_document_id?: number | null + source_version_id?: number | null + started_at?: string | null + status?: string + trigger: string + updated_at?: string + worker_id?: string | null + } + Update: { + attempt_count?: number + checked_at?: string | null + completed_at?: string | null + created_at?: string + dispatched_at?: string | null + error_code?: string | null + error_message?: string | null + id?: string + lease_expires_at?: string | null + lock_version?: number + parser_version?: string + previous_source_version_id?: number | null + prompt_version?: string + public_id?: string + queue_message_id?: string | null + record_id?: number + requested_at?: string + requested_by?: string | null + requested_model?: string + schema_version?: string + source_document_id?: number | null + source_version_id?: number | null + started_at?: string | null + status?: string + trigger?: string + updated_at?: string + worker_id?: string | null + } + Relationships: [ + { + foreignKeyName: "catalogue_syncs_previous_source_version_fkey" + columns: ["previous_source_version_id", "record_id"] + isOneToOne: false + referencedRelation: "catalogue_versions" + referencedColumns: ["id", "record_id"] + }, + { + foreignKeyName: "catalogue_syncs_record_id_fkey" + columns: ["record_id"] + isOneToOne: false + referencedRelation: "catalogue_records" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_syncs_record_id_fkey" + columns: ["record_id"] + isOneToOne: false + referencedRelation: "published_course_summaries" + referencedColumns: ["record_id"] + }, + { + foreignKeyName: "catalogue_syncs_requested_model_fkey" + columns: ["requested_model"] + isOneToOne: false + referencedRelation: "import_models" + referencedColumns: ["id"] + }, + { + foreignKeyName: "catalogue_syncs_source_document_fkey" + columns: ["source_document_id", "record_id"] + isOneToOne: false + referencedRelation: "catalogue_source_documents" + referencedColumns: ["id", "record_id"] + }, + { + foreignKeyName: "catalogue_syncs_source_version_fkey" + columns: ["source_version_id", "record_id"] + isOneToOne: false + referencedRelation: "catalogue_versions" + referencedColumns: ["id", "record_id"] + }, + ] + } catalogue_version_provenance: { Row: { academic_year_id: number @@ -2006,6 +1955,7 @@ export type Database = { field_path: string id: number method: string + source_document_id: number | null source_excerpt: string | null source_locator: string | null source_page_id: number | null @@ -2018,6 +1968,7 @@ export type Database = { field_path: string id?: never method: string + source_document_id?: number | null source_excerpt?: string | null source_locator?: string | null source_page_id?: number | null @@ -2030,6 +1981,7 @@ export type Database = { field_path?: string id?: never method?: string + source_document_id?: number | null source_excerpt?: string | null source_locator?: string | null source_page_id?: number | null @@ -2043,6 +1995,13 @@ export type Database = { referencedRelation: "catalogue_versions" referencedColumns: ["id", "academic_year_id"] }, + { + foreignKeyName: "catalogue_version_provenance_source_document_id_fkey" + columns: ["source_document_id"] + isOneToOne: false + referencedRelation: "catalogue_source_documents" + referencedColumns: ["id"] + }, { foreignKeyName: "catalogue_version_provenance_source_page_fkey" columns: ["source_page_id", "academic_year_id"] @@ -2060,13 +2019,13 @@ export type Database = { created_at: string created_by: string | null id: number - import_target_id: string | null kind: string origin: string public_id: string record_id: number sealed_at: string | null - source_page_id: number | null + source_document_id: number | null + sync_id: string | null } Insert: { academic_year_id: number @@ -2075,13 +2034,13 @@ export type Database = { created_at?: string created_by?: string | null id?: never - import_target_id?: string | null kind: string origin: string public_id?: string record_id: number sealed_at?: string | null - source_page_id?: number | null + source_document_id?: number | null + sync_id?: string | null } Update: { academic_year_id?: number @@ -2090,13 +2049,13 @@ export type Database = { created_at?: string created_by?: string | null id?: never - import_target_id?: string | null kind?: string origin?: string public_id?: string record_id?: number sealed_at?: string | null - source_page_id?: number | null + source_document_id?: number | null + sync_id?: string | null } Relationships: [ { @@ -2106,13 +2065,6 @@ export type Database = { referencedRelation: "catalogue_versions" referencedColumns: ["id", "record_id"] }, - { - foreignKeyName: "catalogue_versions_import_target_fkey" - columns: ["import_target_id"] - isOneToOne: false - referencedRelation: "catalogue_import_targets" - referencedColumns: ["id"] - }, { foreignKeyName: "catalogue_versions_item_year_fkey" columns: ["record_id", "academic_year_id"] @@ -2135,11 +2087,18 @@ export type Database = { referencedColumns: ["id", "kind"] }, { - foreignKeyName: "catalogue_versions_source_page_fkey" - columns: ["source_page_id", "academic_year_id"] + foreignKeyName: "catalogue_versions_source_document_fkey" + columns: ["source_document_id", "record_id"] isOneToOne: false - referencedRelation: "catalogue_source_pages" - referencedColumns: ["id", "academic_year_id"] + referencedRelation: "catalogue_source_documents" + referencedColumns: ["id", "record_id"] + }, + { + foreignKeyName: "catalogue_versions_sync_fkey" + columns: ["sync_id"] + isOneToOne: false + referencedRelation: "catalogue_syncs" + referencedColumns: ["id"] }, ] } @@ -3960,7 +3919,7 @@ export type Database = { Args: { p_version_id: number } Returns: Json } - cancel_catalogue_import: { Args: { p_run_id: string }; Returns: number } + cancel_catalogue_sync: { Args: { p_sync_id: string }; Returns: boolean } catalogue_publish_blockers: { Args: { p_record_id: number } Returns: string[] @@ -3976,10 +3935,6 @@ export type Database = { Args: { required_permission: string } Returns: boolean } - delete_catalogue_item: { - Args: { p_code: string; p_kind: string } - Returns: undefined - } mark_notifications_read: { Args: { p_notification_ids?: string[] } Returns: number @@ -3993,10 +3948,6 @@ export type Database = { } Returns: undefined } - publish_catalogue_version: { - Args: { p_record_id: number } - Returns: number - } published_course_availability: { Args: { p_academic_year: number; p_course_code: string } Returns: { @@ -4042,19 +3993,10 @@ export type Database = { } Returns: string } - recover_catalogue_import_targets: { Args: never; Returns: number } - release_catalogue_import_target: { - Args: { p_target_id: string } - Returns: undefined - } remove_current_user_plan_item: { Args: { p_plan_item_id: string } Returns: boolean } - resolve_catalogue_import_change: { - Args: { p_change_id: number; p_note?: string; p_status: string } - Returns: undefined - } save_current_user_academic_result: { Args: { p_grade?: string @@ -4091,17 +4033,16 @@ export type Database = { Args: { p_role_key: string; p_user_id: string } Returns: string } - start_catalogue_import: { + start_catalogue_sync: { Args: { - p_academic_year: number - p_codes: string[] - p_kind: string p_parser_version: string p_prompt_version: string + p_record_id: number p_requested_model: string p_schema_version: string + p_trigger: string } - Returns: Json + Returns: string } unpublish_catalogue_record: { Args: { p_record_id: number } diff --git a/apps/web/ui/admin/catalogue/anu-source.ts b/apps/web/ui/admin/catalogue/anu-source.ts index c514b6ea..41b6b119 100644 --- a/apps/web/ui/admin/catalogue/anu-source.ts +++ b/apps/web/ui/admin/catalogue/anu-source.ts @@ -3,7 +3,7 @@ import type { CatalogueKind } from "@/lib/coursemap/catalogue-kinds"; const PROGRAMS_AND_COURSES = "https://programsandcourses.anu.edu.au"; /** - * The ANU Programs and Courses page a record was imported from. The site names + * The ANU Programs and Courses page used to sync a record. The site names * its own segments, so "programme" and "specialisation" cannot be used * verbatim. Kept here because the directory, the record header and the review * all need to point a reader back at the source they are judging. diff --git a/apps/web/ui/admin/catalogue/artefact-data.ts b/apps/web/ui/admin/catalogue/artefact-data.ts deleted file mode 100644 index 09ae1590..00000000 --- a/apps/web/ui/admin/catalogue/artefact-data.ts +++ /dev/null @@ -1,49 +0,0 @@ -export type ImportArtefact = { - id: string; - kind: string; - attemptNumber: number; - mediaType: string; -}; - -export const importArtefactLabels: Record = { - raw_html: "Raw HTML", - normalised_markdown: "Markdown", - model_input: "Model input", - deterministic_output: "Deterministic output", - model_request: "Model request", - model_response: "Model response", - validated_json: "Validated JSON", - validation_report: "Validation", - change_set: "Persistence decision", -}; - -export function groupImportArtefacts(artifacts: ImportArtefact[]) { - const order = Object.keys(importArtefactLabels); - const groups = new Map(); - // Projections live in Database rows, with the persisted records they describe. - for (const artifact of artifacts.filter( - (entry) => entry.kind !== "database_projection", - )) { - const group = groups.get(artifact.kind) ?? []; - group.push(artifact); - groups.set(artifact.kind, group); - } - return [...groups] - .map(([kind, attempts]) => ({ - kind, - attempts: attempts.sort((a, b) => b.attemptNumber - a.attemptNumber), - })) - .sort((a, b) => { - const position = (kind: string) => - order.includes(kind) ? order.indexOf(kind) : order.length; - return position(a.kind) - position(b.kind); - }); -} - -export function parseImportArtefact(content: string): unknown { - try { - return JSON.parse(content) as unknown; - } catch { - return null; - } -} diff --git a/apps/web/ui/admin/catalogue/artefact-viewer.tsx b/apps/web/ui/admin/catalogue/artefact-viewer.tsx deleted file mode 100644 index 1f10c0f1..00000000 --- a/apps/web/ui/admin/catalogue/artefact-viewer.tsx +++ /dev/null @@ -1,162 +0,0 @@ -"use client"; - -import { useMemo, useState } from "react"; -import { LoaderCircle } from "lucide-react"; -import { Alert, AlertDescription } from "@coursemap/ui/components/alert"; -import { Button } from "@coursemap/ui/primitives/button"; -import { - Tabs, - TabsContent, - TabsList, - TabsTrigger, -} from "@coursemap/ui/primitives/tabs"; -import { OptionPicker } from "@/ui/common/option-picker"; -import { JsonCode } from "@/ui/common/json-code"; -import { ArtefactViewport } from "./artefact-viewport"; -import { - groupImportArtefacts, - importArtefactLabels, - parseImportArtefact, - type ImportArtefact, -} from "./artefact-data"; -import { useImportArtefact } from "./use-artefact"; -import { SourceCode } from "./source-code"; -import navigationStyles from "./artefact-navigation.module.css"; - -export function ArtefactViewer({ - artifacts, - endpoint, -}: { - artifacts: ImportArtefact[]; - endpoint: string; -}) { - const grouped = useMemo(() => groupImportArtefacts(artifacts), [artifacts]); - const [activeKind, setActiveKind] = useState(""); - const [attempts, setAttempts] = useState>({}); - const group = - grouped.find((entry) => entry.kind === activeKind) ?? grouped[0]; - const artifact = - group?.attempts.find((entry) => entry.id === attempts[group.kind]) ?? - group?.attempts[0] ?? - null; - const { content, loading, error, retry } = useImportArtefact( - artifact, - endpoint, - ); - const label = artifact - ? (importArtefactLabels[artifact.kind] ?? - artifact.kind.replaceAll("_", " ")) - : "Artefact"; - const parsed = - content !== undefined && artifact?.mediaType === "application/json" - ? parseImportArtefact(content) - : null; - - if (!group || !artifact) - return ( -

- This attempt stored no artefacts. -

- ); - - return ( - -
-
- ({ - value: entry.kind, - label: - importArtefactLabels[entry.kind] ?? - entry.kind.replaceAll("_", " "), - }))} - /> -
- -
- -
- {group.attempts.length > 1 && ( -
- - setAttempts((current) => ({ ...current, [group.kind]: id })) - } - aria-label={`Choose ${label} attempt`} - className="w-44" - items={group.attempts.map((entry, index) => ({ - value: entry.id, - label: `Attempt ${entry.attemptNumber}${index === 0 ? " (latest)" : ""}`, - }))} - /> -
- )} - - {error ? ( -
- - {error} - - -
- ) : loading ? ( -
-
- ) : parsed !== null ? ( - - ) : ( - - )} - -
- - - ); -} diff --git a/apps/web/ui/admin/catalogue/artefact-viewport.tsx b/apps/web/ui/admin/catalogue/artefact-viewport.tsx deleted file mode 100644 index bc4d71a1..00000000 --- a/apps/web/ui/admin/catalogue/artefact-viewport.tsx +++ /dev/null @@ -1,31 +0,0 @@ -"use client"; - -import type { ReactNode } from "react"; - -export function ArtefactViewport({ - children, - label, - toolbar, -}: { - children: ReactNode; - label: string; - toolbar?: ReactNode; -}) { - return ( -
- {toolbar ? ( -
- {toolbar} -
- ) : null} -
- {children} -
-
- ); -} diff --git a/apps/web/ui/admin/catalogue/catalogue-directory.tsx b/apps/web/ui/admin/catalogue/catalogue-directory.tsx index d953de79..6494856e 100644 --- a/apps/web/ui/admin/catalogue/catalogue-directory.tsx +++ b/apps/web/ui/admin/catalogue/catalogue-directory.tsx @@ -37,6 +37,24 @@ function formatDate(value: string | null) { ); } +const SOURCE_STATE_LABELS = { + never_synced: "Never synced", + syncing: "Syncing", + up_to_date: "Up to date", + changes_available: "Changes available", + sync_failed: "Sync failed", +} as const; + +function sourceStateVariant( + state: keyof typeof SOURCE_STATE_LABELS, +): "outline" | "success-light" | "warning-light" | "destructive-light" { + if (state === "up_to_date") return "success-light"; + if (state === "changes_available" || state === "syncing") + return "warning-light"; + if (state === "sync_failed") return "destructive-light"; + return "outline"; +} + export function CatalogueDirectory({ page }: { page: CatalogueDirectoryPage }) { const router = useRouter(); const pathname = usePathname(); @@ -156,7 +174,8 @@ export function CatalogueDirectory({ page }: { page: CatalogueDirectoryPage }) { {labels.singular} Publication - ANU + ANU listing + ANU source @@ -202,6 +221,16 @@ export function CatalogueDirectory({ page }: { page: CatalogueDirectoryPage }) { )} + + + {SOURCE_STATE_LABELS[record.sourceState]} + + {record.latestSync?.completedAt ? ( + + {formatDate(record.latestSync.completedAt)} + + ) : null} + ); })} diff --git a/apps/web/ui/admin/catalogue/catalogue-pages.tsx b/apps/web/ui/admin/catalogue/catalogue-pages.tsx index 7f7a0c3b..a322c7bc 100644 --- a/apps/web/ui/admin/catalogue/catalogue-pages.tsx +++ b/apps/web/ui/admin/catalogue/catalogue-pages.tsx @@ -1,28 +1,14 @@ import { Suspense } from "react"; -import { canManageCourseImports } from "@/lib/auth/viewer"; +import { canManageCatalogueSources } from "@/lib/auth/viewer"; import { CATALOGUE_KIND_LABELS, - DEFAULT_IMPORT_RECORD_SORT, - IMPORT_RECORD_SORTS, type CatalogueKind, - type DirectoryFilter, - type ImportRecordSort, - type ImportRunProgress, - type ImportTargetDetail, - adminCataloguePath, loadCatalogueDirectoryPage, - loadCatalogueImportRecords, - loadImportRunProgress, - loadImportTargetDetail, } from "@/lib/coursemap/admin-catalogue"; import { AppShell } from "@/ui/shell"; import { AccessDeniedError } from "@/ui/errors/access-denied-error"; -import { - CatalogueTableLoading, - ImportRecordsSkeleton, -} from "@/ui/admin/catalogue-table/catalogue-loading"; +import { CatalogueTableLoading } from "@/ui/admin/catalogue-table/catalogue-loading"; import { CatalogueDirectory } from "./catalogue-directory"; -import { ImportRecords } from "./import-runs"; export type SearchParams = Promise< Record @@ -42,14 +28,13 @@ export async function CatalogueDirectoryPage({ academicYear: number; searchParams: SearchParams; }) { - if (!(await canManageCourseImports())) return ; + if (!(await canManageCatalogueSources())) return ; const params = await searchParams; const labels = CATALOGUE_KIND_LABELS[kind]; const page = loadCatalogueDirectoryPage({ kind, academicYear, query: first(params.q) ?? "", - filter: (first(params.status) as DirectoryFilter | undefined) ?? "all", page: Number(first(params.page)) || 1, }); return ( @@ -74,77 +59,3 @@ async function DirectoryContent({ const resolved = await page; return ; } - -export async function CatalogueImportsPage({ - kind, - searchParams, -}: { - kind: CatalogueKind; - searchParams: SearchParams; -}) { - if (!(await canManageCourseImports())) return ; - const params = await searchParams; - const labels = CATALOGUE_KIND_LABELS[kind]; - const requestedSort = first(params.sort) as ImportRecordSort | undefined; - const records = loadCatalogueImportRecords({ - kind, - query: first(params.q) ?? "", - status: first(params.status) ?? "", - runId: first(params.run) ?? null, - sort: - requestedSort && IMPORT_RECORD_SORTS.includes(requestedSort) - ? requestedSort - : DEFAULT_IMPORT_RECORD_SORT, - page: Number(first(params.page)) || 1, - }); - async function loadTarget(targetId: string) { - "use server"; - if (!(await canManageCourseImports())) return null; - return loadImportTargetDetail(targetId); - } - async function readRunProgress(runId: string) { - "use server"; - if (!(await canManageCourseImports())) return null; - return loadImportRunProgress(runId); - } - return ( - -

{labels.singular} imports

- }> - - - - ); -} - -async function ImportRecordsContent({ - records, - kind, - loadTarget, - readRunProgress, -}: { - records: ReturnType; - kind: CatalogueKind; - loadTarget: (targetId: string) => Promise; - readRunProgress: (runId: string) => Promise; -}) { - return ( - - ); -} diff --git a/apps/web/ui/admin/catalogue/catalogue-tabs.tsx b/apps/web/ui/admin/catalogue/catalogue-tabs.tsx deleted file mode 100644 index 0efe5961..00000000 --- a/apps/web/ui/admin/catalogue/catalogue-tabs.tsx +++ /dev/null @@ -1,63 +0,0 @@ -"use client"; - -import { Import, LibraryBig } from "lucide-react"; -import { Tabs } from "@coursemap/ui/primitives/tabs"; -import { usePathname, useRouter } from "next/navigation"; - -import { SectionTabs } from "@/ui/common/section-tabs"; - -/** - * The icons the directory tab bar can show. Naming them here keeps Lucide out - * of the server component that describes the tabs, and keeps the set closed: - * a tab bar with an icon per section only reads if the icons are chosen - * together. - */ -const TAB_ICONS = { - directory: LibraryBig, - imports: Import, -} as const; - -export type CatalogueTab = { - href: string; - icon: keyof typeof TAB_ICONS; - label: string; -}; - -/** Route-backed section tabs so the directory and imports each have a URL. */ -export function CatalogueTabs({ - tabs, - label, -}: { - tabs: readonly CatalogueTab[]; - label: string; -}) { - const pathname = usePathname(); - const router = useRouter(); - const active = - tabs.find((tab) => pathname === tab.href) ?? - [...tabs] - .sort((left, right) => right.href.length - left.href.length) - .find((tab) => pathname.startsWith(tab.href)) ?? - tabs[0]; - return ( - router.push(href)} - className="block" - > - { - const Icon = TAB_ICONS[tab.icon]; - // The label beside it is the accessible name, so the icon is - // decorative and must not be read out a second time. - return { - value: tab.href, - label: tab.label, - icon: - ); -} diff --git a/apps/web/ui/admin/catalogue/catalogue-value.tsx b/apps/web/ui/admin/catalogue/catalogue-value.tsx deleted file mode 100644 index fbdb1894..00000000 --- a/apps/web/ui/admin/catalogue/catalogue-value.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import { humaniseKey } from "@/lib/coursemap/catalogue-kinds"; - -/** - * A readable view of an extracted value. Reviewers compare content, not JSON, - * so objects become definition lists and arrays become stacks. Anything the - * pipeline cannot name still renders as text rather than being hidden. - */ -export function CatalogueValue({ value }: { value: unknown }) { - if (value === null || value === undefined || value === "") - return Not set; - if (typeof value === "boolean") return <>{value ? "Yes" : "No"}; - if (Array.isArray(value)) - return value.length ? ( -
    - {value.map((entry, index) => ( -
  • - -
    • - ))} -
- ) : ( - None - ); - if (typeof value === "object") - return ( -
- {Object.entries(value) - .filter(([, child]) => child !== null && child !== "") - .map(([key, child]) => ( -
-
- {humaniseKey(key)} -
-
- -
-
- ))} -
- ); - return ( - {String(value)} - ); -} diff --git a/apps/web/ui/admin/catalogue/import-runs.tsx b/apps/web/ui/admin/catalogue/import-runs.tsx deleted file mode 100644 index b77ad73f..00000000 --- a/apps/web/ui/admin/catalogue/import-runs.tsx +++ /dev/null @@ -1,782 +0,0 @@ -"use client"; - -import { Badge } from "@coursemap/ui/components/badge"; -import { Button } from "@coursemap/ui/primitives/button"; -import { Skeleton } from "@coursemap/ui/primitives/skeleton"; -import { - Alert, - AlertDescription, - AlertTitle, -} from "@coursemap/ui/components/alert"; -import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, -} from "@coursemap/ui/primitives/sheet"; -import { OctagonX, TriangleAlert, Workflow } from "lucide-react"; -import Link from "next/link"; -import { useRouter, useSearchParams } from "next/navigation"; -import { useEffect, useRef, useState } from "react"; -import { toast } from "sonner"; - -import { - CATALOGUE_KIND_LABELS, - DEFAULT_IMPORT_RECORD_SORT, - IMPORT_RECORD_STATUSES, - type CatalogueKind, - type ImportRecordRow, - type ImportRecordsPage, - type ImportRunProgress, - type ImportRunRow, - type ImportTargetDetail, -} from "@/lib/coursemap/catalogue-kinds"; -import { FilterBar } from "@/ui/common/filter-bar"; -import { Pagination } from "@/ui/common/pagination"; -import { SortMenu } from "@/ui/common/sort-menu"; -import { badgeVariantForTone, type Tone } from "@/lib/ui"; -import { CatalogueEmpty } from "@/ui/admin/catalogue-table/catalogue-empty"; -import { - CatalogueIdentity, - DataTableShell, - Table, - TableBody, - TableCaption, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/ui/admin/catalogue-table/catalogue-table"; -import { CatalogueRowActions } from "@/ui/admin/catalogue-table/catalogue-row-actions"; -import { LinkedTableRow } from "@/ui/common/linked-table-row"; -import { DataTableShell as PlainTableShell } from "@/ui/common/data-table"; -import { - Table as PlainTable, - TableBody as PlainTableBody, - TableCaption as PlainTableCaption, - TableCell as PlainTableCell, - TableHead as PlainTableHead, - TableHeader as PlainTableHeader, - TableRow as PlainTableRow, -} from "@coursemap/ui/primitives/table"; -import { ArtefactViewer } from "./artefact-viewer"; -import { - RunStatusBadge, - TARGET_STATUS, - TargetStatusBadge, -} from "./workflow-badge"; - -function readable(value: string) { - const words = value.replaceAll("_", " ").replaceAll("-", " "); - return words.charAt(0).toUpperCase() + words.slice(1); -} - -/** Milliseconds below a second, seconds above it. "12596ms" is not a span. */ -function elapsed(milliseconds: number) { - if (!Number.isFinite(milliseconds) || milliseconds < 0) return "-"; - if (milliseconds < 1_000) return `${milliseconds}ms`; - return `${(milliseconds / 1_000).toFixed(1)}s`; -} - -function duration(startedAt: string | null, completedAt: string | null) { - if (!startedAt || !completedAt) return "-"; - return elapsed( - new Date(completedAt).getTime() - new Date(startedAt).getTime(), - ); -} - -const STAGE_TONE: Record = { - completed: "success", - failed: "danger", - running: "info", - queued: "neutral", -}; - -const STAGE_LABELS: Record = { - source_fetch: "Fetch page", - html_capture: "Capture HTML", - markdown_normalise: "Normalise Markdown", - model_input_prepare: "Prepare model input", - deterministic_extract: "Deterministic parse", - model_extract: "Model extraction", - schema_validate: "Validate schema", - domain_validate: "Merge and validate", - database_project: "Project rows", - snapshot_persist: "Save snapshot", -}; - -function formatDateTime(value: string | null) { - if (!value) return "-"; - return new Intl.DateTimeFormat("en-AU", { - dateStyle: "medium", - timeStyle: "short", - }).format(new Date(value)); -} - -function formatCost(value: number) { - return value === 0 ? "No cost" : `US$${value.toFixed(4)}`; -} - -function runOptionLabel(run: ImportRunRow) { - return `#${run.runNumber} · ${run.academicYear} · ${formatDateTime(run.createdAt)}`; -} - -/** - * Every record an import has produced for one kind, newest first, across every - * run. The run is a column and a filter rather than a table of its own: a - * stacked run list over a run's records read as two disconnected pages, and an - * administrator looks for a record, not for the batch that carried it. - * - * Selecting a record opens its pipeline: the stages it ran, the model's cost - * and diagnostics, and the artefacts each stage saved. - */ -export function ImportRecords({ - page, - basePath, - kind, - loadTarget, - readRunProgress, -}: { - page: ImportRecordsPage; - basePath: string; - kind: CatalogueKind; - loadTarget: (targetId: string) => Promise; - readRunProgress: (runId: string) => Promise; -}) { - const router = useRouter(); - const searchParams = useSearchParams(); - const labels = CATALOGUE_KIND_LABELS[kind]; - const importsPath = `${basePath}/imports`; - const selectedTargetId = searchParams.get("target"); - const [detail, setDetail] = useState(null); - const [cancelling, setCancelling] = useState(false); - const [progress, setProgress] = useState<{ - runId: string; - value: ImportRunProgress; - } | null>(null); - - // The chosen run if there is one, else the newest run still working, so a - // list left unfiltered still keeps up with an import that is under way. - const activeRun = - page.run && (page.run.status === "queued" || page.run.status === "running") - ? page.run - : page.run - ? null - : (page.runs.find( - (run) => run.status === "queued" || run.status === "running", - ) ?? null); - const watchedRunId = activeRun?.id ?? null; - const lastCounters = useRef(null); - - // Watching one run's counters costs a single row. Refetching the whole page - // every four seconds reread every record to learn that one number had moved, - // and reset the reader's scroll position each time. The rows are reread only - // when a record actually finished, or when the run itself settled. - useEffect(() => { - if (!watchedRunId) return; - let cancelled = false; - lastCounters.current = null; - const tick = async () => { - const next = await readRunProgress(watchedRunId); - if (cancelled || !next) return; - setProgress({ runId: watchedRunId, value: next }); - const counters = `${next.completedCount}:${next.failedCount}`; - const settled = next.status !== "queued" && next.status !== "running"; - if ( - lastCounters.current !== null && - (counters !== lastCounters.current || settled) - ) { - router.refresh(); - } - lastCounters.current = counters; - }; - const timer = setInterval(() => void tick(), 4000); - void tick(); - return () => { - cancelled = true; - clearInterval(timer); - }; - }, [watchedRunId, readRunProgress, router]); - - /** The chosen run's counters as of the last poll, when it is the watched one. */ - const live = - progress && page.run && progress.runId === page.run.id - ? progress.value - : null; - - // Closing the sheet does not clear the detail. `visibleDetail` below shows - // one only while it belongs to the selected record, so the last one read - // stays cached for a reopen rather than costing a second round trip. - useEffect(() => { - if (!selectedTargetId) return; - let cancelled = false; - loadTarget(selectedTargetId).then((value) => { - if (!cancelled) setDetail(value); - }); - return () => { - cancelled = true; - }; - }, [selectedTargetId, loadTarget]); - const visibleDetail = - selectedTargetId && detail?.id === selectedTargetId ? detail : null; - - function select(params: Record) { - const next = new URLSearchParams(searchParams.toString()); - for (const [key, value] of Object.entries(params)) { - if (value === null) next.delete(key); - else next.set(key, value); - } - const query = next.toString(); - router.replace(query ? `${importsPath}?${query}` : importsPath, { - scroll: false, - }); - } - - async function cancelRun(runId: string) { - setCancelling(true); - try { - const response = await fetch("/api/admin/catalogue-imports", { - method: "DELETE", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ runId }), - }); - const body = (await response.json()) as { - error?: string; - cancelled?: number; - }; - if (!response.ok) - throw new Error(body.error ?? "The run could not be stopped."); - toast.success( - `Stopped ${body.cancelled ?? 0} record${body.cancelled === 1 ? "" : "s"}.`, - ); - router.refresh(); - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "The run could not be stopped.", - ); - } finally { - setCancelling(false); - } - } - - const chosenRun = page.run; - const filtered = Boolean( - searchParams.get("q") ?? - searchParams.get("status") ?? - searchParams.get("run"), - ); - - return ( -
-
-
- ({ - value, - label: TARGET_STATUS[value]?.label ?? readable(value), - })), - }, - ...(page.runs.length > 0 - ? [ - { - key: "run", - label: "Run", - allLabel: "Every run", - options: page.runs.map((run) => ({ - value: run.id, - label: runOptionLabel(run), - })), - }, - ] - : []), - ]} - /> -
- select({ sort: value, page: null })} - options={[ - { value: "newest", label: "Newest first", descending: true }, - { value: "oldest", label: "Oldest first" }, - { value: "code-asc", label: "Code, A to Z" }, - { value: "code-desc", label: "Code, Z to A", descending: true }, - ]} - value={page.sort} - /> -
- - {chosenRun ? ( - void cancelRun(chosenRun.id)} - onClear={() => select({ run: null, page: null })} - run={chosenRun} - /> - ) : null} - - - } - > - {page.records.length === 0 ? ( - - - - ) : ( - - - {labels.singular} imports, {SORT_CAPTIONS[page.sort]} - - - - Import - Year - Outcome - Change - Run - Started - - Actions - - - - - {page.records.map((record) => ( - select({ target: record.id })} - onSelectRun={() => - select({ run: record.runId, page: null, target: null }) - } - record={record} - /> - ))} - -
- )} - - - { - if (!open) select({ target: null }); - }} - > - - {visibleDetail ? ( - - ) : ( -
- Loading the pipeline - - - {Array.from({ length: 6 }, (_, index) => ( - - ))} -
- )} - - -
- ); -} - -const SORT_CAPTIONS: Record = { - newest: "newest first", - oldest: "oldest first", - "code-asc": "by code, A to Z", - "code-desc": "by code, Z to A", -}; - -function RecordRow({ - basePath, - kind, - onOpenPipeline, - onSelectRun, - record, -}: { - basePath: string; - kind: CatalogueKind; - onOpenPipeline: () => void; - onSelectRun: () => void; - record: ImportRecordRow; -}) { - const failure = - record.errorMessage || - (record.errorCode ? readable(record.errorCode) : null); - const reviewHref = - record.status === "ready" && record.recordPublicId - ? `${basePath}/${record.academicYear}/${record.code.toLowerCase()}/changes` - : undefined; - return ( - - - - - - {record.academicYear} - - -
- - {/* Why it failed, so a column of "Failed" badges can be told apart - without opening each pipeline in turn. The message is written for - a reader; the code is a machine token, so it is only the fallback - when nothing wrote a message. */} - {failure ? ( - - {failure} - - ) : null} -
- - - {record.changeKind ? ( - readable(record.changeKind) - ) : ( - None - )} - - - {/* The run narrows the same list rather than opening a second one. */} - - - - - - - , - onSelect: onOpenPipeline, - }, - ]} - links={[ - ...(reviewHref - ? [{ label: "Review import", href: reviewHref }] - : []), - ...(record.recordPublicId - ? [ - { - label: "Import history", - href: `${basePath}/${record.academicYear}/${record.code.toLowerCase()}/changelog`, - icon: "history" as const, - }, - ] - : []), - { - label: "Find in directory", - href: `${basePath}/${record.academicYear}?q=${encodeURIComponent(record.code)}`, - }, - ]} - /> - - - ); -} - -/** - * The chosen run, one line: what it cost, how far it got and how to stop it. - * It replaces the run table, and appears only while a run is the filter, so an - * unfiltered list is a single table rather than a master and a detail. - */ -function RunStrip({ - cancelling, - live, - onCancel, - onClear, - run, -}: { - cancelling: boolean; - live: ImportRunProgress | null; - onCancel: () => void; - onClear: () => void; - run: ImportRunRow; -}) { - const completed = live?.completedCount ?? run.completedCount; - const failed = live?.failedCount ?? run.failedCount; - const total = live?.targetCount ?? run.targetCount; - const status = live?.status ?? run.status; - return ( -
-

Run #{run.runNumber}

- - - {completed}/{total} done - {failed ? ( - · {failed} failed - ) : null} - - - {run.requestedModel} - - - {formatCost(run.costUsd)} - - -
- {status === "queued" || status === "running" ? ( - - ) : null} - -
-
- ); -} - -function TargetDetail({ detail }: { detail: ImportTargetDetail }) { - const latestAttempt = Math.max( - 1, - ...detail.stages.map((stage) => stage.attemptNumber), - ); - const stages = detail.stages.filter( - (stage) => stage.attemptNumber === latestAttempt, - ); - return ( - <> - - {detail.code} pipeline - - Attempt {latestAttempt} - {detail.extraction?.resolvedModel - ? ` · ${detail.extraction.resolvedModel}` - : ""} - - -
- {detail.extraction ? ( -
-
-
Input
-
- {detail.extraction.inputTokens.toLocaleString("en-AU")} -
-
-
-
Output
-
- {detail.extraction.outputTokens.toLocaleString("en-AU")} -
-
-
-
Cost
-
- {formatCost(detail.extraction.costUsd)} -
-
-
-
Latency
-
- {detail.extraction.latencyMs === null - ? "-" - : elapsed(detail.extraction.latencyMs)} -
-
-
-
Diagnostics
-
- {detail.extraction.warningCount} warnings ·{" "} - {detail.extraction.errorCount} errors -
-
-
-
Validation
-
- - {readable(detail.extraction.validationStatus)} - -
-
-
- ) : ( -

- No model extraction recorded. -

- )} - {detail.extraction?.finishReason === "length" ? ( - - - ) : null} - {detail.extraction?.errorSummary ? ( -
- Validation details -

{detail.extraction.errorSummary}

-
- ) : null} -
- - - - Import pipeline stages - - - - Step - Stage - Status - - Duration - - Error - - - - {stages.length === 0 ? ( - - - No stages were recorded for this attempt. - - - ) : null} - {stages.map((stage, index) => ( - - - {index + 1} - - - {STAGE_LABELS[stage.name] ?? readable(stage.name)} - - - - {readable(stage.status)} - - - - {duration(stage.startedAt, stage.completedAt)} - - - {stage.errorSummary ? ( - - {stage.errorCode ? `${stage.errorCode}: ` : ""} - {stage.errorSummary} - - ) : ( - {"-"} - )} - - - ))} - - - -
-
-

Artefacts

- artifact.attemptNumber === latestAttempt, - )} - endpoint="/api/admin/catalogue-imports/artifacts" - /> -
-
- - ); -} diff --git a/apps/web/ui/admin/catalogue/record-header.tsx b/apps/web/ui/admin/catalogue/record-header.tsx index ded388ab..1dd41bc1 100644 --- a/apps/web/ui/admin/catalogue/record-header.tsx +++ b/apps/web/ui/admin/catalogue/record-header.tsx @@ -2,8 +2,12 @@ import { Badge } from "@coursemap/ui/components/badge"; import { ExternalLink, TriangleAlert } from "lucide-react"; import Link from "next/link"; import type { CatalogueRecord } from "@/lib/coursemap/admin-catalogue-record"; -import { CATALOGUE_KIND_LABELS } from "@/lib/coursemap/catalogue-kinds"; +import { + CATALOGUE_KIND_LABELS, + adminCatalogueRecordPath, +} from "@/lib/coursemap/catalogue-kinds"; import { anuSourceUrl } from "./anu-source"; +import { CatalogueSyncButton } from "./sync-button"; function formatDate(value: string | null) { if (!value) return null; @@ -16,10 +20,12 @@ export function RecordHeader({ record, hasDraft, hasUnpublishedChanges, + canSync, }: { record: CatalogueRecord; hasDraft: boolean; hasUnpublishedChanges: boolean; + canSync: boolean; }) { const labels = CATALOGUE_KIND_LABELS[record.kind]; const publicationLabel = record.publishedVersionId @@ -72,8 +78,31 @@ export function RecordHeader({ View on ANU