From a1a6b3896afd899592dd379be64a77699ecec2a6 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Fri, 29 May 2026 21:35:30 -0700 Subject: [PATCH 1/2] fix(catalog): index physicalDescription, personality, role, motivations, significance in search_tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catalog full-text search currently misses bible-promoted characters' main narrative text — search_tsv only included description/notes/background/summary and bypassed the character canon fields physicalDescription + personality (and the type-specific role/motivations/significance fields). Postgres can't ALTER the expression of a STORED generated column, so the fix is DROP + re-ADD inside the idempotent ensureSchema DDL block. The block already runs on every boot (PR #542 made it unconditional), so the ALTERs apply to existing installs automatically and are no-ops on fresh installs because the new CREATE TABLE no longer declares search_tsv inline. Mirrored in server/scripts/init-db.sql so fresh-install schemas match what ensureSchema produces. PORTOS_SCHEMA_VERSIONS.catalog bumped to 2 — older peers still on 1 get a 412 from /api/catalog/sync/apply against newer peers, preventing them from pushing pre-expansion-shape rows that would mismatch the indexed expression. --- server/lib/db.js | 32 ++++++++++++++++++++--------- server/lib/schemaVersions.js | 21 +++++++++++++------ server/scripts/init-db.sql | 39 +++++++++++++++++++++++++----------- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/server/lib/db.js b/server/lib/db.js index 62b4023a1e..9f6c91b0e0 100644 --- a/server/lib/db.js +++ b/server/lib/db.js @@ -148,15 +148,6 @@ export async function ensureSchema() { tags TEXT[] DEFAULT '{}', embedding vector(768), embedding_model VARCHAR(100), - search_tsv tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(name, '')), 'A') || - setweight(to_tsvector('english', - coalesce(payload->>'description', '') || ' ' || - coalesce(payload->>'notes', '') || ' ' || - coalesce(payload->>'background', '') || ' ' || - coalesce(payload->>'summary', '') - ), 'B') - ) STORED, origin_instance_id VARCHAR(36), created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW(), @@ -164,6 +155,29 @@ export async function ensureSchema() { deleted_at TIMESTAMPTZ, sync_sequence BIGSERIAL )`, + // Postgres can't ALTER the expression of a STORED generated column, so the + // FTS column is DROPped and re-ADDed on every boot. The whole catalogDDL + // block is idempotent: on a fresh install the DROP is a no-op (column + // didn't exist) and ADD creates it; on an existing v1 install the DROP + // removes the narrow expression and ADD installs the expanded one that + // also indexes physicalDescription / personality / role / motivations / + // significance. Bumped PORTOS_SCHEMA_VERSIONS.catalog → 2 in lockstep. + `ALTER TABLE catalog_ingredients DROP COLUMN IF EXISTS search_tsv`, + `ALTER TABLE catalog_ingredients ADD COLUMN IF NOT EXISTS search_tsv tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(name, '')), 'A') || + setweight(to_tsvector('english', + coalesce(payload->>'description', '') || ' ' || + coalesce(payload->>'physicalDescription', '') || ' ' || + coalesce(payload->>'personality', '') || ' ' || + coalesce(payload->>'background', '') || ' ' || + coalesce(payload->>'summary', '') || ' ' || + coalesce(payload->>'notes', '') || ' ' || + coalesce(payload->>'role', '') || ' ' || + coalesce(payload->>'motivations', '') || ' ' || + coalesce(payload->>'significance', '') + ), 'B') + ) STORED`, `CREATE INDEX IF NOT EXISTS idx_catalog_ing_embedding ON catalog_ingredients USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64)`, diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.js index 6f4c0bd1e4..4804dc06cb 100644 --- a/server/lib/schemaVersions.js +++ b/server/lib/schemaVersions.js @@ -47,12 +47,21 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({ // pauses with old peers; issues/universes keep flowing. pipelineSeries: 2, mediaCollections: 1, - // v1 = creative ingredients catalog (Postgres tables: catalog_scraps, - // catalog_ingredients, catalog_ingredient_sources, catalog_ingredient_refs). - // Per-category gate so a new peer can sync its catalog independently of - // whether other categories are version-locked. `cat-ingredient` and - // `cat-scrap` record kinds map back here via RECORD_KIND_SCHEMA_CATEGORIES. - catalog: 1, + // v2 = `catalog_ingredients.search_tsv` expanded to also index the + // character canon fields (physicalDescription, personality) and the + // type-specific role/motivations/significance fields, so bible-promoted + // characters become searchable on their main narrative text. The schema + // is a DROP+re-ADD of the STORED generated column (Postgres can't ALTER + // its expression); applied in lockstep by `ensureSchema` in + // server/lib/db.js. Per-category gate so a new peer can sync its catalog + // independently of whether other categories are version-locked. An older + // v1 peer pushing to a v2 receiver is sender-behind on `catalog` (not + // ahead), so the receiver still accepts and re-derives `search_tsv` + // locally via the STORED expression. A v2 peer pushing to a v1 receiver + // is sender-ahead and gets 412 — the older code can't index the new + // payload fields. `cat-ingredient` and `cat-scrap` record kinds map back + // here via RECORD_KIND_SCHEMA_CATEGORIES. + catalog: 2, // NOTE: `videoHistory` is intentionally NOT listed here. The version gate // rejects the ENTIRE snapshot/push payload on ANY ahead-mismatch (the // comparator walks the union of keys), so declaring a brand-new key would diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sql index c4dfcb78c3..f3765ca6ac 100644 --- a/server/scripts/init-db.sql +++ b/server/scripts/init-db.sql @@ -161,18 +161,6 @@ CREATE TABLE IF NOT EXISTS catalog_ingredients ( tags TEXT[] DEFAULT '{}', embedding vector(768), embedding_model VARCHAR(100), - -- Weighted FTS column. Name carries the most weight (A); description/notes/ - -- background fall under B. Generated/stored so the GIN index stays fresh - -- without trigger code. - search_tsv tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(name, '')), 'A') || - setweight(to_tsvector('english', - coalesce(payload->>'description', '') || ' ' || - coalesce(payload->>'notes', '') || ' ' || - coalesce(payload->>'background', '') || ' ' || - coalesce(payload->>'summary', '') - ), 'B') - ) STORED, origin_instance_id VARCHAR(36), created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW(), @@ -180,6 +168,33 @@ CREATE TABLE IF NOT EXISTS catalog_ingredients ( deleted_at TIMESTAMPTZ, sync_sequence BIGSERIAL ); +-- Weighted FTS column. Name carries the most weight (A); the character canon +-- fields (description, physicalDescription, personality, background, summary, +-- notes) plus the role/motivations/significance type-specific fields fall under +-- B. Generated/stored so the GIN index stays fresh without trigger code. +-- Postgres can't ALTER the expression of a STORED generated column, so +-- ensureSchema (server/lib/db.js) DROPs + re-ADDs the column on every boot — +-- mirrored here as DROP IF EXISTS + ADD IF NOT EXISTS so a fresh-install run +-- of this script produces the same end-state as an existing-install boot +-- through ensureSchema. PORTOS_SCHEMA_VERSIONS.catalog is bumped to 2 in +-- lockstep so older peers can't push pre-expansion-shape rows that would +-- mismatch the indexed expression. +ALTER TABLE catalog_ingredients DROP COLUMN IF EXISTS search_tsv; +ALTER TABLE catalog_ingredients ADD COLUMN IF NOT EXISTS search_tsv tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(name, '')), 'A') || + setweight(to_tsvector('english', + coalesce(payload->>'description', '') || ' ' || + coalesce(payload->>'physicalDescription', '') || ' ' || + coalesce(payload->>'personality', '') || ' ' || + coalesce(payload->>'background', '') || ' ' || + coalesce(payload->>'summary', '') || ' ' || + coalesce(payload->>'notes', '') || ' ' || + coalesce(payload->>'role', '') || ' ' || + coalesce(payload->>'motivations', '') || ' ' || + coalesce(payload->>'significance', '') + ), 'B') + ) STORED; CREATE INDEX IF NOT EXISTS idx_catalog_ing_embedding ON catalog_ingredients USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64); From 39e9d8badf9c1cd9800c47f659be1c76e4b8e899 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Fri, 29 May 2026 21:48:33 -0700 Subject: [PATCH 2/2] address review (claude+codex): only rebuild search_tsv when v1 expression detected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously ensureSchema() unconditionally DROP+ADDed catalog_ingredients.search_tsv on every boot to swap the generation expression. Postgres treats that as a table rewrite under an AccessExclusive lock plus a GIN index rebuild — fine the first time but pure waste on every subsequent boot of an already-v2 install (and ensureSchema fires once from memoryBackend init and once from the catalog backfill chain). Gate the DROP on a pg_attrdef inspection: only drop when the existing generation expression is missing 'physicalDescription' (a v2-only token). Fresh installs skip the drop because the column doesn't exist yet; already-v2 installs skip it because the expression already matches; only an upgrading v1 install pays the rewrite. ADD COLUMN IF NOT EXISTS still does the right thing in all three cases. Mirrored byte-for-byte into init-db.sql. --- server/lib/db.js | 33 +++++++++++++++++++++++++-------- server/scripts/init-db.sql | 32 ++++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/server/lib/db.js b/server/lib/db.js index 9f6c91b0e0..e28c284430 100644 --- a/server/lib/db.js +++ b/server/lib/db.js @@ -155,14 +155,31 @@ export async function ensureSchema() { deleted_at TIMESTAMPTZ, sync_sequence BIGSERIAL )`, - // Postgres can't ALTER the expression of a STORED generated column, so the - // FTS column is DROPped and re-ADDed on every boot. The whole catalogDDL - // block is idempotent: on a fresh install the DROP is a no-op (column - // didn't exist) and ADD creates it; on an existing v1 install the DROP - // removes the narrow expression and ADD installs the expanded one that - // also indexes physicalDescription / personality / role / motivations / - // significance. Bumped PORTOS_SCHEMA_VERSIONS.catalog → 2 in lockstep. - `ALTER TABLE catalog_ingredients DROP COLUMN IF EXISTS search_tsv`, + // Postgres can't ALTER the expression of a STORED generated column, so when + // the v2 expansion needs to land we DROP and re-ADD `search_tsv`. The + // conditional below (executed after the table CREATE, before the + // ADD-only fallback) inspects pg_attrdef and rewrites the column ONLY + // when the current generation expression is missing a v2-only field + // (`physicalDescription`). That keeps boot O(1) on already-v2 installs — + // an unconditional DROP+ADD would AccessExclusive-lock the table, rewrite + // every row, and rebuild the GIN index on every server start. + // Fresh installs (no column yet) fall through to the ADD IF NOT EXISTS + // below and skip the DROP entirely. + `DO $$ + DECLARE + expr TEXT; + BEGIN + SELECT pg_get_expr(d.adbin, d.adrelid) + INTO expr + FROM pg_attribute a + JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = 'catalog_ingredients'::regclass + AND a.attname = 'search_tsv' + AND a.attgenerated = 's'; + IF expr IS NOT NULL AND position('physicalDescription' in expr) = 0 THEN + EXECUTE 'ALTER TABLE catalog_ingredients DROP COLUMN search_tsv'; + END IF; + END$$`, `ALTER TABLE catalog_ingredients ADD COLUMN IF NOT EXISTS search_tsv tsvector GENERATED ALWAYS AS ( setweight(to_tsvector('english', coalesce(name, '')), 'A') || diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sql index f3765ca6ac..6db256bf56 100644 --- a/server/scripts/init-db.sql +++ b/server/scripts/init-db.sql @@ -172,14 +172,30 @@ CREATE TABLE IF NOT EXISTS catalog_ingredients ( -- fields (description, physicalDescription, personality, background, summary, -- notes) plus the role/motivations/significance type-specific fields fall under -- B. Generated/stored so the GIN index stays fresh without trigger code. --- Postgres can't ALTER the expression of a STORED generated column, so --- ensureSchema (server/lib/db.js) DROPs + re-ADDs the column on every boot — --- mirrored here as DROP IF EXISTS + ADD IF NOT EXISTS so a fresh-install run --- of this script produces the same end-state as an existing-install boot --- through ensureSchema. PORTOS_SCHEMA_VERSIONS.catalog is bumped to 2 in --- lockstep so older peers can't push pre-expansion-shape rows that would --- mismatch the indexed expression. -ALTER TABLE catalog_ingredients DROP COLUMN IF EXISTS search_tsv; +-- Postgres can't ALTER the expression of a STORED generated column, so when +-- the v2 expansion needs to land we DROP and re-ADD the column. The DO block +-- below inspects pg_attrdef and only drops when the existing expression is +-- missing a v2-only field (`physicalDescription`) — fresh runs of this script +-- skip the drop entirely (column absent), already-v2 installs skip it too, and +-- only an upgrading v1 install pays the table-rewrite cost. ensureSchema in +-- server/lib/db.js mirrors the same gate. PORTOS_SCHEMA_VERSIONS.catalog is +-- bumped to 2 in lockstep so older peers can't push pre-expansion-shape rows +-- that would mismatch the indexed expression. +DO $$ + DECLARE + expr TEXT; + BEGIN + SELECT pg_get_expr(d.adbin, d.adrelid) + INTO expr + FROM pg_attribute a + JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = 'catalog_ingredients'::regclass + AND a.attname = 'search_tsv' + AND a.attgenerated = 's'; + IF expr IS NOT NULL AND position('physicalDescription' in expr) = 0 THEN + EXECUTE 'ALTER TABLE catalog_ingredients DROP COLUMN search_tsv'; + END IF; + END$$; ALTER TABLE catalog_ingredients ADD COLUMN IF NOT EXISTS search_tsv tsvector GENERATED ALWAYS AS ( setweight(to_tsvector('english', coalesce(name, '')), 'A') ||