fix(db): dampen search rank by title length to stop repeat-term blobs outranking exact matches

packages/db/src/queries/search.ts

@@ -98,6 +98,38 @@ interface HitRow {
   eik_valid: number | null;
 }
 
+// FTS5's default bm25() rewards raw term frequency: a title that repeats the query terms several
+// times (e.g. a municipality's name field concatenating several child-entity names — issue #25)
+// can outscore a title that matches once, cleanly. bm25's own length normalization isn't enough to
+// offset that within-document repetition, so we dampen `rank` by title length on top of it — the
+// divisor grows by 1 per 20 chars, a soft enough curve that it reorders repeat-heavy blobs below
+// clean short matches without sinking legitimate longer (but single-match) titles disproportionately.
+const RANK_EXPR = `r / (1.0 + LENGTH(title) / 20.0)`;
+
+// Two stages, deliberately. `ORDER BY rank` is the ONLY form FTS5 optimizes: it drives the query with
+// its rank-ordering index and pushes the LIMIT down (EXPLAIN: `VIRTUAL TABLE INDEX 32:M6`). Ordering by
+// any expression OVER rank drops that (`INDEX 0:M6` + `USE TEMP B-TREE FOR ORDER BY`), so every row a
+// common term matches — tens of thousands for e.g. „община" — gets materialized and sorted before the
+// LIMIT. D1 bills rows read, so that is a real cost on the busiest query in the app.
+// So: take the top CANDIDATES by the optimized rank path, then re-rank only those. The inner slice is
+// wide enough that a repeat-heavy blob and the clean match it outranks are both inside it, and the
+// sort in the outer query is over at most CANDIDATES rows, not the whole match set.
+const CANDIDATES = 50;
+
+export const SEARCH_HITS_SQL = `SELECT h.ref, h.title, h.ident, h.subtitle, h.amount,
+       ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid
+FROM (
+  SELECT search_index.ref AS ref, search_index.title AS title, search_index.ident AS ident,
+         search_index.subtitle AS subtitle, search_index.amount AS amount,
+         search_index.kind AS kind, rank AS r
+  FROM search_index
+  WHERE search_index.kind = ? AND search_index MATCH ?
+  ORDER BY rank LIMIT ${CANDIDATES}
+) h
+LEFT JOIN company_totals ct
+  ON h.kind = 'company' AND ct.bidder_id = h.ref
+ORDER BY ${RANK_EXPR} LIMIT ?`;
+
 export async function search(db: D1Database, rawQuery: string): Promise<SearchResults> {
   const query = (rawQuery ?? '').trim();
   const match = searchMatchQuery(query);
@@ -120,16 +152,7 @@ export async function search(db: D1Database, rawQuery: string): Promise<SearchRe
       const total = counts.get(g.kind) ?? 0;
       if (total === 0) return { kind: g.kind, label: g.label, total: 0, hits: [], moreHref: null };
       const { results } = await db
-        .prepare(
-          `SELECT search_index.ref, search_index.title, search_index.ident,
-                  search_index.subtitle, search_index.amount,
-                  ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid
-           FROM search_index
-           LEFT JOIN company_totals ct
-             ON search_index.kind = 'company' AND ct.bidder_id = search_index.ref
-           WHERE search_index.kind = ? AND search_index MATCH ?
-           ORDER BY rank LIMIT ?`,
-        )
+        .prepare(SEARCH_HITS_SQL)
         .bind(g.kind, match, g.limit)
         .all<HitRow>();
       const hits: SearchHit[] = results.map((r) => {

packages/db/src/search-index-sql.test.ts

@@ -0,0 +1,124 @@
+/// <reference types="node" />
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { SEARCH_HITS_SQL, searchMatchQuery } from './queries/search';
+
+// Integration test for the REAL search ranking SQL (SEARCH_HITS_SQL, imported from queries/search.ts —
+// not a hand-copied mirror) against a real SQLite FTS5 search_index built from the production
+// migration. Regression coverage for issue #25: an authority/company whose title repeats the query
+// terms several times (e.g. a municipality name field concatenating several child-entity names) must
+// not outrank an entity whose title is an exact, single match. search.test.ts's unit tests fake D1 and
+// never run real FTS5 ranking, so they can't catch this. Mirrors the sqlite3-CLI harness of
+// amendments-sql.test.ts / competition-sql.test.ts.
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql');
+
+function sqlite(dbPath: string, sql: string): string {
+  return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim();
+}
+function readScript(dbPath: string, path: string): void {
+  execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' });
+}
+function withDb<T>(fn: (dbPath: string) => T): T {
+  const dir = mkdtempSync(resolve(tmpdir(), 'sigma-search-index-'));
+  const dbPath = resolve(dir, 'test.sqlite');
+  try {
+    readScript(dbPath, migration0);
+    return fn(dbPath);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function insertAuthorityRows(dbPath: string, rows: [ref: string, title: string][]): void {
+  const values = rows
+    .map(([ref, title]) => `('authority', '${ref}', '${title.replace(/'/g, "''")}', '', NULL, 0)`)
+    .join(',\n');
+  sqlite(
+    dbPath,
+    `INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) VALUES ${values};`,
+  );
+}
+
+// Runs the real production ORDER BY against a real FTS5 table and returns titles in ranked order.
+function rankedTitles(dbPath: string, query: string): string[] {
+  const match = searchMatchQuery(query);
+  const sql = SEARCH_HITS_SQL.replace('?', "'authority'")
+    .replace('?', `'${match}'`)
+    .replace('?', '10');
+  const out = sqlite(dbPath, sql);
+  if (out === '') return [];
+  // ref, title, ident, subtitle, amount, entity_kind, ownership_kind, eik_valid — title is column 2.
+  return out.split('\n').map((line) => line.split('|')[1] ?? '');
+}
+
+describe('search ranking SQL (real SQLite FTS5, SEARCH_HITS_SQL)', () => {
+  it('ranks exact, single-match titles above a title that repeats the query terms several times (#25)', () => {
+    withDb((dbPath) => {
+      insertAuthorityRows(dbPath, [
+        [
+          'auth:blob',
+          'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче',
+        ],
+        ['auth:clean1', 'Детска градина Слънце'],
+        ['auth:clean2', 'Детска градина Дъга'],
+      ]);
+
+      const titles = rankedTitles(dbPath, 'детска градина');
+
+      expect(titles).toHaveLength(3);
+      const blobRank = titles.indexOf(
+        'Община Лясковец - детска градина Сладкопойна чучулига, детска градина Детелина, детска градина Славейче',
+      );
+      const clean1Rank = titles.indexOf('Детска градина Слънце');
+      const clean2Rank = titles.indexOf('Детска градина Дъга');
+
+      expect(clean1Rank).toBeLessThan(blobRank);
+      expect(clean2Rank).toBeLessThan(blobRank);
+    });
+  });
+
+  it('does not sink a legitimate longer single-match title below an unrelated exact match', () => {
+    withDb((dbPath) => {
+      insertAuthorityRows(dbPath, [
+        ['auth:long', 'Общинска детска градина за изкуство №5 към Столична община район Витоша'],
+        ['auth:short', 'Детска градина Дъга'],
+      ]);
+
+      const titles = rankedTitles(dbPath, 'детска градина');
+
+      // Both are single, clean matches. The dampening does put the shorter one first — that is its
+      // job — but the longer descriptive title must stay a top hit rather than being buried. Asserted
+      // as the EXACT order: `arrayContaining` would pass no matter how the two are ordered (both rows
+      // come back under LIMIT 10 regardless), so it could not detect the regression it guards against.
+      expect(titles).toEqual([
+        'Детска градина Дъга',
+        'Общинска детска градина за изкуство №5 към Столична община район Витоша',
+      ]);
+    });
+  });
+
+  // The ordering above can be satisfied by ordering the whole match set — which is exactly the
+  // regression to prevent. `ORDER BY rank` is the only form FTS5 optimizes (rank-ordering index +
+  // LIMIT pushdown); ordering by an expression over rank degrades to materializing and sorting every
+  // matching row. On „община"-class terms that is tens of thousands of rows per keystroke, and D1
+  // bills rows read. Lock the plan so a later simplification back to a single-level query is caught.
+  it('drives the match with FTS5 rank ordering, not a full sort of every match', () => {
+    withDb((dbPath) => {
+      insertAuthorityRows(dbPath, [['auth:1', 'Община Ботевград']]);
+      const sql = SEARCH_HITS_SQL.replace('?', "'authority'")
+        .replace('?', `'${searchMatchQuery('ботевград')}'`)
+        .replace('?', '6');
+      const plan = sqlite(dbPath, `EXPLAIN QUERY PLAN ${sql}`);
+
+      // idx 32 is FTS5's "ordering by rank" flag; idx 0 means it fell back to an unordered scan.
+      expect(plan).toMatch(/VIRTUAL TABLE INDEX 32/);
+      expect(plan).not.toMatch(/VIRTUAL TABLE INDEX 0/);
+    });
+  });
+});