diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 75dad268..994f0026 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -98,6 +98,38 @@ interface HitRow { eik_valid: number | null; } +// FTS5's default bm25() rewards raw term frequency: a title that repeats the query terms several +// times (e.g. a municipality's name field concatenating several child-entity names — issue #25) +// can outscore a title that matches once, cleanly. bm25's own length normalization isn't enough to +// offset that within-document repetition, so we dampen `rank` by title length on top of it — the +// divisor grows by 1 per 20 chars, a soft enough curve that it reorders repeat-heavy blobs below +// clean short matches without sinking legitimate longer (but single-match) titles disproportionately. +const RANK_EXPR = `r / (1.0 + LENGTH(title) / 20.0)`; + +// Two stages, deliberately. `ORDER BY rank` is the ONLY form FTS5 optimizes: it drives the query with +// its rank-ordering index and pushes the LIMIT down (EXPLAIN: `VIRTUAL TABLE INDEX 32:M6`). Ordering by +// any expression OVER rank drops that (`INDEX 0:M6` + `USE TEMP B-TREE FOR ORDER BY`), so every row a +// common term matches — tens of thousands for e.g. „община" — gets materialized and sorted before the +// LIMIT. D1 bills rows read, so that is a real cost on the busiest query in the app. +// So: take the top CANDIDATES by the optimized rank path, then re-rank only those. The inner slice is +// wide enough that a repeat-heavy blob and the clean match it outranks are both inside it, and the +// sort in the outer query is over at most CANDIDATES rows, not the whole match set. +const CANDIDATES = 50; + +export const SEARCH_HITS_SQL = `SELECT h.ref, h.title, h.ident, h.subtitle, h.amount, + ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid +FROM ( + SELECT search_index.ref AS ref, search_index.title AS title, search_index.ident AS ident, + search_index.subtitle AS subtitle, search_index.amount AS amount, + search_index.kind AS kind, rank AS r + FROM search_index + WHERE search_index.kind = ? AND search_index MATCH ? + ORDER BY rank LIMIT ${CANDIDATES} +) h +LEFT JOIN company_totals ct + ON h.kind = 'company' AND ct.bidder_id = h.ref +ORDER BY ${RANK_EXPR} LIMIT ?`; + export async function search(db: D1Database, rawQuery: string): Promise { const query = (rawQuery ?? '').trim(); const match = searchMatchQuery(query); @@ -120,16 +152,7 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const hits: SearchHit[] = results.map((r) => { diff --git a/packages/db/src/search-index-sql.test.ts b/packages/db/src/search-index-sql.test.ts new file mode 100644 index 00000000..178b35bf --- /dev/null +++ b/packages/db/src/search-index-sql.test.ts @@ -0,0 +1,124 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { SEARCH_HITS_SQL, searchMatchQuery } from './queries/search'; + +// Integration test for the REAL search ranking SQL (SEARCH_HITS_SQL, imported from queries/search.ts — +// not a hand-copied mirror) against a real SQLite FTS5 search_index built from the production +// migration. Regression coverage for issue #25: an authority/company whose title repeats the query +// terms several times (e.g. a municipality name field concatenating several child-entity names) must +// not outrank an entity whose title is an exact, single match. search.test.ts's unit tests fake D1 and +// never run real FTS5 ranking, so they can't catch this. Mirrors the sqlite3-CLI harness of +// amendments-sql.test.ts / competition-sql.test.ts. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); +} +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); +} +function withDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-search-index-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, migration0); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function insertAuthorityRows(dbPath: string, rows: [ref: string, title: string][]): void { + const values = rows + .map(([ref, title]) => `('authority', '${ref}', '${title.replace(/'/g, "''")}', '', NULL, 0)`) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) VALUES ${values};`, + ); +} + +// Runs the real production ORDER BY against a real FTS5 table and returns titles in ranked order. +function rankedTitles(dbPath: string, query: string): string[] { + const match = searchMatchQuery(query); + const sql = SEARCH_HITS_SQL.replace('?', "'authority'") + .replace('?', `'${match}'`) + .replace('?', '10'); + const out = sqlite(dbPath, sql); + if (out === '') return []; + // ref, title, ident, subtitle, amount, entity_kind, ownership_kind, eik_valid — title is column 2. + return out.split('\n').map((line) => line.split('|')[1] ?? ''); +} + +describe('search ranking SQL (real SQLite FTS5, SEARCH_HITS_SQL)', () => { + it('ranks exact, single-match titles above a title that repeats the query terms several times (#25)', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [ + [ + 'auth:blob', + 'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче', + ], + ['auth:clean1', 'Детска градина Слънце'], + ['auth:clean2', 'Детска градина Дъга'], + ]); + + const titles = rankedTitles(dbPath, 'детска градина'); + + expect(titles).toHaveLength(3); + const blobRank = titles.indexOf( + 'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче', + ); + const clean1Rank = titles.indexOf('Детска градина Слънце'); + const clean2Rank = titles.indexOf('Детска градина Дъга'); + + expect(clean1Rank).toBeLessThan(blobRank); + expect(clean2Rank).toBeLessThan(blobRank); + }); + }); + + it('does not sink a legitimate longer single-match title below an unrelated exact match', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [ + ['auth:long', 'Общинска детска градина за изкуство №5 към Столична община район Витоша'], + ['auth:short', 'Детска градина Дъга'], + ]); + + const titles = rankedTitles(dbPath, 'детска градина'); + + // Both are single, clean matches. The dampening does put the shorter one first — that is its + // job — but the longer descriptive title must stay a top hit rather than being buried. Asserted + // as the EXACT order: `arrayContaining` would pass no matter how the two are ordered (both rows + // come back under LIMIT 10 regardless), so it could not detect the regression it guards against. + expect(titles).toEqual([ + 'Детска градина Дъга', + 'Общинска детска градина за изкуство №5 към Столична община район Витоша', + ]); + }); + }); + + // The ordering above can be satisfied by ordering the whole match set — which is exactly the + // regression to prevent. `ORDER BY rank` is the only form FTS5 optimizes (rank-ordering index + + // LIMIT pushdown); ordering by an expression over rank degrades to materializing and sorting every + // matching row. On „община"-class terms that is tens of thousands of rows per keystroke, and D1 + // bills rows read. Lock the plan so a later simplification back to a single-level query is caught. + it('drives the match with FTS5 rank ordering, not a full sort of every match', () => { + withDb((dbPath) => { + insertAuthorityRows(dbPath, [['auth:1', 'Община Ботевград']]); + const sql = SEARCH_HITS_SQL.replace('?', "'authority'") + .replace('?', `'${searchMatchQuery('ботевград')}'`) + .replace('?', '6'); + const plan = sqlite(dbPath, `EXPLAIN QUERY PLAN ${sql}`); + + // idx 32 is FTS5's "ordering by rank" flag; idx 0 means it fell back to an unordered scan. + expect(plan).toMatch(/VIRTUAL TABLE INDEX 32/); + expect(plan).not.toMatch(/VIRTUAL TABLE INDEX 0/); + }); + }); +});