From 24dfce7cdc5220d3d877bdabb05b0d6a1fe7a60d Mon Sep 17 00:00:00 2001 From: Bilko Date: Sun, 26 Jul 2026 22:20:48 -0700 Subject: [PATCH 1/3] fix(db): dampen search rank by title length to stop repeat-term blobs outranking exact matches FTS5's default bm25() rewards raw term frequency, so an authority whose title concatenates several child-entity names (repeating the query terms) could outrank an entity whose title matches the query once, cleanly (#25) --- packages/db/src/queries/search.ts | 28 +++--- packages/db/src/search-index-sql.test.ts | 106 +++++++++++++++++++++++ 2 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 packages/db/src/search-index-sql.test.ts diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 75dad268..ed603edb 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -98,6 +98,23 @@ interface HitRow { eik_valid: number | null; } +// FTS5's default bm25() rewards raw term frequency: a title that repeats the query terms several +// times (e.g. a municipality's name field concatenating several child-entity names — issue #25) +// can outscore a title that matches once, cleanly. bm25's own length normalization isn't enough to +// offset that within-document repetition, so we dampen `rank` by title length on top of it — the +// divisor grows by 1 per 20 chars, a soft enough curve that it reorders repeat-heavy blobs below +// clean short matches without sinking legitimate longer (but single-match) titles disproportionately. +const RANK_EXPR = `rank / (1.0 + LENGTH(search_index.title) / 20.0)`; + +export const SEARCH_HITS_SQL = `SELECT search_index.ref, search_index.title, search_index.ident, + search_index.subtitle, search_index.amount, + ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid +FROM search_index +LEFT JOIN company_totals ct + ON search_index.kind = 'company' AND ct.bidder_id = search_index.ref +WHERE search_index.kind = ? AND search_index MATCH ? +ORDER BY ${RANK_EXPR} LIMIT ?`; + export async function search(db: D1Database, rawQuery: string): Promise { const query = (rawQuery ?? '').trim(); const match = searchMatchQuery(query); @@ -120,16 +137,7 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const hits: SearchHit[] = results.map((r) => { diff --git a/packages/db/src/search-index-sql.test.ts b/packages/db/src/search-index-sql.test.ts new file mode 100644 index 00000000..3e8fc80a --- /dev/null +++ b/packages/db/src/search-index-sql.test.ts @@ -0,0 +1,106 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { SEARCH_HITS_SQL, searchMatchQuery } from './queries/search'; + +// Integration test for the REAL search ranking SQL (SEARCH_HITS_SQL, imported from queries/search.ts — +// not a hand-copied mirror) against a real SQLite FTS5 search_index built from the production +// migration. Regression coverage for issue #25: an authority/company whose title repeats the query +// terms several times (e.g. a municipality name field concatenating several child-entity names) must +// not outrank an entity whose title is an exact, single match. search.test.ts's unit tests fake D1 and +// never run real FTS5 ranking, so they can't catch this. Mirrors the sqlite3-CLI harness of +// amendments-sql.test.ts / competition-sql.test.ts. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); +} +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); +} +function withDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-search-index-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, migration0); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function insertAuthorityRows(dbPath: string, rows: [ref: string, title: string][]): void { + const values = rows + .map(([ref, title]) => `('authority', '${ref}', '${title.replace(/'/g, "''")}', '', NULL, 0)`) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) VALUES ${values};`, + ); +} + +// Runs the real production ORDER BY against a real FTS5 table and returns titles in ranked order. +function rankedTitles(dbPath: string, query: string): string[] { + const match = searchMatchQuery(query); + const sql = SEARCH_HITS_SQL.replace('?', "'authority'") + .replace('?', `'${match}'`) + .replace('?', '10'); + const out = sqlite(dbPath, sql); + if (out === '') return []; + // ref, title, ident, subtitle, amount, entity_kind, ownership_kind, eik_valid — title is column 2. + return out.split('\n').map((line) => line.split('|')[1] ?? ''); +} + +describe('search ranking SQL (real SQLite FTS5, SEARCH_HITS_SQL)', () => { + it('ranks exact, single-match titles above a title that repeats the query terms several times (#25)', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [ + [ + 'auth:blob', + 'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче', + ], + ['auth:clean1', 'Детска градина Слънце'], + ['auth:clean2', 'Детска градина Дъга'], + ]); + + const titles = rankedTitles(dbPath, 'детска градина'); + + expect(titles).toHaveLength(3); + const blobRank = titles.indexOf( + 'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче', + ); + const clean1Rank = titles.indexOf('Детска градина Слънце'); + const clean2Rank = titles.indexOf('Детска градина Дъга'); + + expect(clean1Rank).toBeLessThan(blobRank); + expect(clean2Rank).toBeLessThan(blobRank); + }); + }); + + it('does not sink a legitimate longer single-match title below an unrelated exact match', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [ + ['auth:long', 'Общинска детска градина за изкуство №5 към Столична община район Витоша'], + ['auth:short', 'Детска градина Дъга'], + ]); + + const titles = rankedTitles(dbPath, 'детска градина'); + + // Both are single, clean matches for the query; a longer descriptive title matching once + // should not be reordered to a wildly worse position than an equally clean shorter title — + // it should still be a top hit, not pushed out by the length dampening meant for repeat-blobs. + expect(titles).toEqual( + expect.arrayContaining([ + 'Детска градина Дъга', + 'Общинска детска градина за изкуство №5 към Столична община район Витоша', + ]), + ); + }); + }); +}); From de3e9b2bee3f64385330f5501bc739d93b6726df Mon Sep 17 00:00:00 2001 From: Bilko Date: Mon, 27 Jul 2026 00:23:32 -0700 Subject: [PATCH 2/3] build(deps): patch postcss/valibot CVEs, suppress react-router RSC-only CSRF postcss 8.5.15 -> 8.5.18+ fixes GHSA-r28c-9q8g-f849 (path traversal via sourceMappingURL auto-load); valibot 1.4.0 -> 1.4.2+ fixes GHSA-5qjj-4xww-7phc (flatten() crash on inherited-property keys). Both are non-breaking patch-level overrides. react-router 7.18.0's GHSA-qwww-vcr4-c8h2 CSRF only affects the unstable RSC code paths (verified via repo-wide grep, zero hits) and has no fix in the 7.x line - suppressed via osv-scanner.toml with a 2026-10-01 review date rather than forcing a major 7.x -> 8.x bump across this PR. --- osv-scanner.toml | 15 +++++++++++++++ pnpm-lock.yaml | 30 ++++++++++++++++-------------- pnpm-workspace.yaml | 6 ++++++ 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/osv-scanner.toml b/osv-scanner.toml index dad96e31..d7d33e5e 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -23,3 +23,18 @@ id = "GHSA-f88m-g3jw-g9cj" ignoreUntil = 2026-10-01T00:00:00Z reason = "sharp is a dev-only transitive of miniflare (local Workers simulator), pinned to 0.34.5 upstream and never bundled into the deployed Worker. Remove once miniflare/wrangler pins sharp >= 0.35.0 (pnpm why sharp)." + +# ── react-router 7.18.0 — GHSA-qwww-vcr4-c8h2 (High, CVSS 7.1), fixed in 8.3.0 ────────── +# WHY IGNORED: this CVE is a CSRF flaw in react-router's UNSTABLE RSC (React Server +# Components) code paths only — "this only affects your application if you are using the +# unstable RSC APIs" per the advisory. Verified via `git grep` across this repo for RSC +# usage (unstable_.*RSC, react-server, unstable_RSCPayload, unstable_routeRSCServerRequest): +# zero hits. This app does not use RSC. No fix exists in the 7.x line (introduced in 7.12.0, +# only patched in 8.3.0) — upgrading to react-router 8.x is a major, breaking version bump +# out of scope for a security patch to a code path this app never exercises. +# REMOVE WHEN: this app adopts react-router's RSC APIs (re-evaluate applicability first), or +# a deliberate, separately-planned major-version upgrade to react-router 8.x lands. +[[IgnoredVulns]] +id = "GHSA-qwww-vcr4-c8h2" +ignoreUntil = 2026-10-01T00:00:00Z +reason = "CSRF in react-router's unstable RSC code paths only (GHSA-qwww-vcr4-c8h2) - this app does not use RSC (verified via repo-wide grep for RSC APIs). No fix in the 7.x line; upgrading to 8.x is a major breaking change out of scope for a security patch to an unused code path." diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 150a51d1..11b2fdab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,8 @@ overrides: vite@8: ^8.0.16 undici: ^7.28.0 '@babel/core': ^7.29.6 + postcss: ^8.5.18 + valibot: ^1.4.2 importers: @@ -1555,8 +1557,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -1597,8 +1599,8 @@ packages: pkg-types@2.3.1: resolution: {integrity: sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==} - postcss@8.5.15: - resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} + postcss@8.5.23: + resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} engines: {node: ^10 || ^12 || >=14} prettier@3.8.3: @@ -1771,8 +1773,8 @@ packages: peerDependencies: browserslist: '>= 4.21.0' - valibot@1.4.0: - resolution: {integrity: sha512-iC/x7fVcSyOwlm/VSt7RlHnzNGLGvR9GnxdifUeWoCJo0q4ZZvrVkIHC6faTlkxG47I2Y4UrFquPuVHCrOnrLg==} + valibot@1.4.2: + resolution: {integrity: sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg==} peerDependencies: typescript: '>=5' peerDependenciesMeta: @@ -2547,7 +2549,7 @@ snapshots: react-router: 7.18.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) semver: 7.8.0 tinyglobby: 0.2.17 - valibot: 1.4.0(typescript@5.9.3) + valibot: 1.4.2(typescript@5.9.3) vite: 8.0.16(@types/node@22.19.19)(esbuild@0.28.1)(jiti@2.7.0) vite-node: 3.2.4(@types/node@22.19.19)(jiti@2.7.0)(lightningcss@1.32.0) optionalDependencies: @@ -3145,7 +3147,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.12: {} + nanoid@3.3.16: {} node-releases@2.0.45: {} @@ -3178,9 +3180,9 @@ snapshots: exsolve: 1.0.8 pathe: 2.0.3 - postcss@8.5.15: + postcss@8.5.23: dependencies: - nanoid: 3.3.12 + nanoid: 3.3.16 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -3382,7 +3384,7 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 - valibot@1.4.0(typescript@5.9.3): + valibot@1.4.2(typescript@5.9.3): optionalDependencies: typescript: 5.9.3 @@ -3412,7 +3414,7 @@ snapshots: esbuild: 0.28.1 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.15 + postcss: 8.5.23 rollup: 4.60.4 tinyglobby: 0.2.17 optionalDependencies: @@ -3425,7 +3427,7 @@ snapshots: dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 - postcss: 8.5.15 + postcss: 8.5.23 rolldown: 1.0.3 tinyglobby: 0.2.17 optionalDependencies: @@ -3438,7 +3440,7 @@ snapshots: dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 - postcss: 8.5.15 + postcss: 8.5.23 rolldown: 1.0.3 tinyglobby: 0.2.17 optionalDependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 96815cea..d62bde1d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -24,6 +24,12 @@ overrides: # @babel/core <7.29.6 — arbitrary file read via sourceMappingURL (GHSA-4x5r-pxfx-6jf8); # dev/build-time only (via @react-router/dev), never ships to the Worker. '@babel/core': '^7.29.6' + # postcss <8.5.18 — path traversal via sourceMappingURL auto-load + # (GHSA-r28c-9q8g-f849); patch-level fix. + # valibot <1.4.2 — flatten() crashes on inherited-property keys + # (GHSA-5qjj-4xww-7phc); patch-level fix. + postcss: '^8.5.18' + valibot: '^1.4.2' onlyBuiltDependencies: - esbuild From fc94c2e187ec495195a203bfbd916b64862e8220 Mon Sep 17 00:00:00 2001 From: Bilko Date: Thu, 30 Jul 2026 15:10:41 -0700 Subject: [PATCH 3/3] fix(db): drive search rank via FTS5's optimized ORDER BY rank path Land todorkolev's reviewed patch: two-stage query (inner ORDER BY rank LIMIT 50 using FTS5's rank-ordering index, outer re-rank over just those candidates) instead of ordering the whole match set by an expression over rank, which fell back to a full temp-B-tree sort. Also merge main to resolve the osv-scanner/pnpm conflict from #271. --- packages/db/src/queries/search.ts | 27 ++++++++++++++---- packages/db/src/search-index-sql.test.ts | 36 ++++++++++++++++++------ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index ed603edb..994f0026 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -104,15 +104,30 @@ interface HitRow { // offset that within-document repetition, so we dampen `rank` by title length on top of it — the // divisor grows by 1 per 20 chars, a soft enough curve that it reorders repeat-heavy blobs below // clean short matches without sinking legitimate longer (but single-match) titles disproportionately. -const RANK_EXPR = `rank / (1.0 + LENGTH(search_index.title) / 20.0)`; +const RANK_EXPR = `r / (1.0 + LENGTH(title) / 20.0)`; -export const SEARCH_HITS_SQL = `SELECT search_index.ref, search_index.title, search_index.ident, - search_index.subtitle, search_index.amount, +// Two stages, deliberately. `ORDER BY rank` is the ONLY form FTS5 optimizes: it drives the query with +// its rank-ordering index and pushes the LIMIT down (EXPLAIN: `VIRTUAL TABLE INDEX 32:M6`). Ordering by +// any expression OVER rank drops that (`INDEX 0:M6` + `USE TEMP B-TREE FOR ORDER BY`), so every row a +// common term matches — tens of thousands for e.g. „община" — gets materialized and sorted before the +// LIMIT. D1 bills rows read, so that is a real cost on the busiest query in the app. +// So: take the top CANDIDATES by the optimized rank path, then re-rank only those. The inner slice is +// wide enough that a repeat-heavy blob and the clean match it outranks are both inside it, and the +// sort in the outer query is over at most CANDIDATES rows, not the whole match set. +const CANDIDATES = 50; + +export const SEARCH_HITS_SQL = `SELECT h.ref, h.title, h.ident, h.subtitle, h.amount, ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid -FROM search_index +FROM ( + SELECT search_index.ref AS ref, search_index.title AS title, search_index.ident AS ident, + search_index.subtitle AS subtitle, search_index.amount AS amount, + search_index.kind AS kind, rank AS r + FROM search_index + WHERE search_index.kind = ? AND search_index MATCH ? + ORDER BY rank LIMIT ${CANDIDATES} +) h LEFT JOIN company_totals ct - ON search_index.kind = 'company' AND ct.bidder_id = search_index.ref -WHERE search_index.kind = ? AND search_index MATCH ? + ON h.kind = 'company' AND ct.bidder_id = h.ref ORDER BY ${RANK_EXPR} LIMIT ?`; export async function search(db: D1Database, rawQuery: string): Promise { diff --git a/packages/db/src/search-index-sql.test.ts b/packages/db/src/search-index-sql.test.ts index 3e8fc80a..178b35bf 100644 --- a/packages/db/src/search-index-sql.test.ts +++ b/packages/db/src/search-index-sql.test.ts @@ -92,15 +92,33 @@ describe('search ranking SQL (real SQLite FTS5, SEARCH_HITS_SQL)', () => { const titles = rankedTitles(dbPath, 'детска градина'); - // Both are single, clean matches for the query; a longer descriptive title matching once - // should not be reordered to a wildly worse position than an equally clean shorter title — - // it should still be a top hit, not pushed out by the length dampening meant for repeat-blobs. - expect(titles).toEqual( - expect.arrayContaining([ - 'Детска градина Дъга', - 'Общинска детска градина за изкуство №5 към Столична община район Витоша', - ]), - ); + // Both are single, clean matches. The dampening does put the shorter one first — that is its + // job — but the longer descriptive title must stay a top hit rather than being buried. Asserted + // as the EXACT order: `arrayContaining` would pass no matter how the two are ordered (both rows + // come back under LIMIT 10 regardless), so it could not detect the regression it guards against. + expect(titles).toEqual([ + 'Детска градина Дъга', + 'Общинска детска градина за изкуство №5 към Столична община район Витоша', + ]); + }); + }); + + // The ordering above can be satisfied by ordering the whole match set — which is exactly the + // regression to prevent. `ORDER BY rank` is the only form FTS5 optimizes (rank-ordering index + + // LIMIT pushdown); ordering by an expression over rank degrades to materializing and sorting every + // matching row. On „община"-class terms that is tens of thousands of rows per keystroke, and D1 + // bills rows read. Lock the plan so a later simplification back to a single-level query is caught. + it('drives the match with FTS5 rank ordering, not a full sort of every match', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [['auth:1', 'Община Ботевград']]); + const sql = SEARCH_HITS_SQL.replace('?', "'authority'") + .replace('?', `'${searchMatchQuery('ботевград')}'`) + .replace('?', '6'); + const plan = sqlite(dbPath, `EXPLAIN QUERY PLAN ${sql}`); + + // idx 32 is FTS5's "ordering by rank" flag; idx 0 means it fell back to an unordered scan. + expect(plan).toMatch(/VIRTUAL TABLE INDEX 32/); + expect(plan).not.toMatch(/VIRTUAL TABLE INDEX 0/); }); }); });