fix: dictionaries plugin — inflected forms (plurals, tenses) resolve to their lemma definition

dictionaries/README.md

@@ -46,7 +46,10 @@ workflow in this repo
 ([`dictionaries-mirror.yml`](../.github/workflows/dictionaries-mirror.yml))
 generates them from [kaikki.org](https://kaikki.org)'s machine-readable
 Wiktionary extracts — matched to the reader's dictionary engine (index sort
-order, dictzip chunking, headword/definition size limits) — and hosts them as
+order, dictzip chunking, headword/definition size limits). Inflected forms
+("cats", "ran") resolve straight to the lemma's full definition instead of a
+bare "plural of cat" stub — the stub would otherwise shadow the reader's own
+stemming fallback. The files are hosted as
 assets on the rolling
 [`dictionaries` release](https://github.com/itsthisjustin/sd-plugins/releases/tag/dictionaries).
 The browse catalog under [`catalog/`](catalog/) is regenerated in the same

scripts/build-dictionaries.mjs

@@ -13,6 +13,10 @@
 //   - .idx sorted with asciiCaseCmp: bytewise, ASCII tolower per byte
 //   - headwords equal under that comparator share one merged definition
 //     (otherwise only the first of "march"/"March" is ever reachable)
+//   - inflected forms ("cats", carrying form_of/alt_of in the source) point
+//     their .idx rows at the lemma's definition bytes instead of keeping the
+//     bare "plural of cat" stub: a direct .idx hit preempts the reader's
+//     .syn and stemming fallbacks, so the stub would otherwise win
 //   - headwords < 256 bytes, definitions < 64 KB, 32-bit offsets
 //   - dictzip chunks decompress independently (raw deflate per chunk),
 //     chunk table <= 8192 entries (~460 MB uncompressed ceiling)
@@ -34,9 +38,9 @@ import { createGunzip, createDeflateRaw, gunzipSync, crc32, constants as zconst
 import { createInterface } from 'node:readline';
 import { join, basename } from 'node:path';
 import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
 
 const RELEASE_BASE = 'https://github.com/itsthisjustin/sd-plugins/releases/download/dictionaries/';
-const CATALOG_DIR = new URL('../dictionaries/catalog/', import.meta.url).pathname;
 const PAGE_SIZE = 8;
 
 // One entry per language: its own Wiktionary edition, so definitions are in
@@ -81,6 +85,19 @@ const flag = (name, fallback) => {
 const OUT = flag('out', 'dist');
 const ONLY = (flag('only', '') || '').split(',').filter(Boolean);
 const FROM_INDEX = args.includes('--from-index');
+// Overridable so tests can build into a scratch dir without touching the
+// committed catalog.
+const CATALOG_DIR = flag('catalog-dir', fileURLToPath(new URL('../dictionaries/catalog/', import.meta.url)));
+// --source id=url (comma-separated): override a source's URL, or add a new
+// kaikki-kind source. Mainly for testing — curl accepts file:// URLs.
+for (const spec of (flag('source', '') || '').split(',').filter(Boolean)) {
+  const eq = spec.indexOf('=');
+  const id = spec.slice(0, eq);
+  const url = spec.slice(eq + 1);
+  const existing = SOURCES.find((s) => s.id === id);
+  if (existing) existing.url = url;
+  else SOURCES.push({ id, title: id, kind: 'kaikki', url });
+}
 
 function curl(url, dest) {
   execFileSync('curl', ['-fsSL', '--retry', '3', '--retry-delay', '5', '-o', dest, url], { stdio: 'inherit' });
@@ -148,42 +165,107 @@ async function dictzip(dictBuf, dest) {
   writeFileSync(dest, Buffer.concat([header, extra, ...chunks, trailer]));
 }
 
-// entries: Map<headword, blockText>. Writes <id>.ifo/.idx/.dict.dz into dir
-// and returns catalog metadata.
+// entries: Map<headword, blockText | {text, refs, pure}> — refs are lemma
+// headwords this entry's form_of/alt_of senses point at; pure means every
+// sense is such a reference. Writes <id>.ifo/.idx/.dict.dz into dir and
+// returns catalog metadata.
 async function writeStardict(dir, id, title, entries) {
   // Group headwords that the reader's case-insensitive comparator cannot tell
   // apart; each gets its own .idx row pointing at the shared merged text.
   const groups = new Map();
-  for (const [word, text] of entries) {
+  for (const [word, entry] of entries) {
+    const e = typeof entry === 'string' ? { text: entry, refs: [], pure: false } : entry;
     const wordBuf = Buffer.from(word, 'utf8');
     if (wordBuf.length === 0 || wordBuf.length > MAX_HEADWORD_BYTES || wordBuf.includes(0)) continue;
     const key = foldKey(wordBuf).toString('latin1');
     let g = groups.get(key);
-    if (!g) groups.set(key, (g = { words: [], texts: [] }));
+    if (!g) groups.set(key, (g = { words: [], texts: [], refs: new Set(), pure: true }));
     g.words.push(wordBuf);
-    g.texts.push(text);
+    g.texts.push(e.text);
+    for (const r of e.refs || []) {
+      const refBuf = Buffer.from(r, 'utf8');
+      if (refBuf.length && refBuf.length <= MAX_HEADWORD_BYTES) {
+        g.refs.add(foldKey(refBuf).toString('latin1'));
+      }
+    }
+    if (!e.pure) g.pure = false;
+  }
+
+  // A pure form-of group with exactly one resolvable lemma becomes an alias:
+  // its .idx rows reuse the lemma's (offset, size) outright, so "cats" shows
+  // cat's definition at zero size cost. StarDict allows shared offsets and the
+  // reader only ever reads one (offset, size) pair per hit. Chains ("runnin'"
+  // -> "running" -> "run") are followed to the material end; a chain that
+  // loops keeps its own stub text instead.
+  const resolveAlias = (key) => {
+    let cur = key;
+    const seen = new Set([key]);
+    for (;;) {
+      const g = groups.get(cur);
+      const refs = [...g.refs].filter((r) => r !== cur && groups.has(r));
+      if (!g.pure || refs.length !== 1) return cur === key ? null : cur;  // material end
+      if (seen.has(refs[0])) return null;  // pure-ref cycle: keep the stub text
+      seen.add(refs[0]);
+      cur = refs[0];
+    }
+  };
+  const aliasOf = new Map();
+  for (const key of groups.keys()) {
+    const target = resolveAlias(key);
+    if (target) aliasOf.set(key, target);
   }
+
+  // Material groups with refs (mixed entries like "found": own sense plus
+  // past-of-find) append the referenced definitions after their own text.
+  const bodyCache = new Map();
+  const building = new Set();
+  const bodyText = (key) => {
+    const target = aliasOf.get(key) || key;
+    if (bodyCache.has(target)) return bodyCache.get(target);
+    const g = groups.get(target);
+    const own = g.texts.filter(Boolean).join('\n\n');
+    if (building.has(target)) return own;  // ref cycle: stop at own text
+    building.add(target);
+    let text = own;
+    for (const r of g.refs) {
+      if (!groups.has(r) || (aliasOf.get(r) || r) === target) continue;
+      const refText = bodyText(r);
+      if (refText) text += (text ? '\n\n' : '') + refText;
+    }
+    building.delete(target);
+    bodyCache.set(target, text);
+    return text;
+  };
+
   const keys = [...groups.keys()].sort((a, b) =>
     Buffer.compare(Buffer.from(a, 'latin1'), Buffer.from(b, 'latin1')));
 
+  // Lay out material bodies first (aliases contribute no bytes), then emit
+  // .idx rows in sorted order — alias rows borrow their target's placement,
+  // so offsets in the .idx are not monotonic, which StarDict permits.
   const dictParts = [];
-  const idxParts = [];
+  const placed = new Map();  // material key -> {offset, size}
   let offset = 0;
-  let wordCount = 0;
   for (const key of keys) {
-    const g = groups.get(key);
-    let body = Buffer.from(g.texts.join('\n\n'), 'utf8');
+    if (aliasOf.has(key)) continue;
+    let body = Buffer.from(bodyText(key), 'utf8');
     if (body.length > MAX_DEFINITION_BYTES) body = truncateUtf8(body, MAX_DEFINITION_BYTES);
     dictParts.push(body);
-    for (const wordBuf of g.words) {
+    placed.set(key, { offset, size: body.length });
+    offset += body.length;
+  }
+  const idxParts = [];
+  let wordCount = 0;
+  for (const key of keys) {
+    const loc = placed.get(aliasOf.get(key) || key);
+    for (const wordBuf of groups.get(key).words) {
       const row = Buffer.alloc(wordBuf.length + 9);
       wordBuf.copy(row);
-      row.writeUInt32BE(offset, wordBuf.length + 1);
-      row.writeUInt32BE(body.length, wordBuf.length + 5);
+      row.writeUInt32BE(loc.offset, wordBuf.length + 1);
+      row.writeUInt32BE(loc.size, wordBuf.length + 5);
       idxParts.push(row);
       wordCount++;
     }
-    offset += body.length;
   }
   if (offset > 0xffffffff) throw new Error('dict over 4 GB (32-bit offsets)');
 
@@ -234,7 +316,7 @@ async function buildKaikki(src, tmp) {
   const jsonl = join(tmp, src.id + '.jsonl.gz');
   curl(src.url, jsonl);
 
-  const blocks = new Map();  // word -> [ "word (pos)\n1. ...\n2. ...", ... ]
+  const blocks = new Map();  // word -> {texts: ["word (pos)\n1. ..."], refs, pure}
   const rl = createInterface({ input: createReadStream(jsonl).pipe(createGunzip()), crlfDelay: Infinity });
   let lines = 0;
   for await (const line of rl) {
@@ -243,23 +325,43 @@ async function buildKaikki(src, tmp) {
     try { e = JSON.parse(line); } catch (err) { continue; }
     if (!e.word || !Array.isArray(e.senses)) continue;
     const glosses = [];
+    const refs = [];
+    let ownSenses = 0;  // glossed senses that are NOT a form_of/alt_of pointer
     for (const sense of e.senses) {
       const g = Array.isArray(sense.glosses) ? sense.glosses.filter(Boolean).join('; ') : '';
-      if (g) glosses.push(g);
+      let isRef = false;
+      for (const f of [].concat(sense.form_of || [], sense.alt_of || [])) {
+        if (f && typeof f.word === 'string' && f.word && f.word !== e.word) {
+          refs.push(f.word);
+          isRef = true;
+        }
+      }
+      if (g) {
+        glosses.push(g);
+        if (!isRef) ownSenses++;
+      }
       if (glosses.length >= 30) break;
     }
     if (!glosses.length) continue;
     const header = e.word + (e.pos ? ` (${e.pos})` : '');
     const text = header + '\n' + glosses.map((g, i) => `${i + 1}. ${g}`).join('\n');
-    let list = blocks.get(e.word);
-    if (!list) blocks.set(e.word, (list = []));
-    if (list.length < 8) list.push(text);  // cap runaway homographs
+    let rec = blocks.get(e.word);
+    if (!rec) blocks.set(e.word, (rec = { texts: [], refs: new Set(), pure: true }));
+    if (rec.texts.length < 8) rec.texts.push(text);  // cap runaway homographs
+    for (const r of refs) { if (rec.refs.size < 3) rec.refs.add(r); }
+    if (ownSenses > 0) rec.pure = false;
   }
   rmSync(jsonl, { force: true });
   console.log(`  ${lines} lines -> ${blocks.size} headwords`);
 
   const entries = new Map();
-  for (const [word, list] of blocks) entries.set(word, list.join('\n\n'));
+  for (const [word, rec] of blocks) {
+    entries.set(word, {
+      text: rec.texts.join('\n\n'),
+      refs: [...rec.refs],
+      pure: rec.pure && rec.refs.size > 0,
+    });
+  }
   return writeStardict(join(OUT, 'assets'), src.id, src.title, entries);
 }
 

test/dictionaries.test.mjs

@@ -0,0 +1,101 @@
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { gunzipSync, gzipSync } from 'node:zlib';
+import test from 'node:test';
+
+// End-to-end build over a synthetic kaikki source: form_of/alt_of entries must
+// resolve to the lemma's definition (the reader's direct .idx hit preempts its
+// .syn and stemming fallbacks, so a bare "plural of cat" stub would otherwise
+// be all the user ever sees).
+
+const root = new URL('../', import.meta.url);
+const script = fileURLToPath(new URL('scripts/build-dictionaries.mjs', root));
+
+const kaikkiLines = [
+  { word: 'cat', pos: 'noun', senses: [{ glosses: ['small domesticated feline'] }] },
+  { word: 'cats', pos: 'noun', senses: [{ glosses: ['plural of cat'], form_of: [{ word: 'cat' }] }] },
+  // Lemma missing from the source: the stub is the best we have, keep it.
+  { word: 'mice', pos: 'noun', senses: [{ glosses: ['plural of mouse'], form_of: [{ word: 'mouse' }] }] },
+  // Mixed headword: own sense in one block, form-of in another.
+  { word: 'find', pos: 'verb', senses: [{ glosses: ['to locate something'] }] },
+  { word: 'found', pos: 'verb', senses: [{ glosses: ['simple past of find'], form_of: [{ word: 'find' }] }] },
+  { word: 'found', pos: 'verb', senses: [{ glosses: ['to establish an organization'] }] },
+  // Chain: runnin' -> running -> run.
+  { word: 'run', pos: 'verb', senses: [{ glosses: ['to move fast'] }] },
+  { word: 'running', pos: 'verb', senses: [{ glosses: ['present participle of run'], form_of: [{ word: 'run' }] }] },
+  { word: "runnin'", pos: 'verb', senses: [{ glosses: ['alternative spelling of running'], alt_of: [{ word: 'running' }] }] },
+  // Case-merged lemma group: marched must land on the merged march/March body.
+  { word: 'March', pos: 'noun', senses: [{ glosses: ['the third month'] }] },
+  { word: 'march', pos: 'verb', senses: [{ glosses: ['to walk in step'] }] },
+  { word: 'marched', pos: 'verb', senses: [{ glosses: ['simple past of march'], form_of: [{ word: 'march' }] }] },
+];
+
+const tmp = mkdtempSync(join(tmpdir(), 'dict-test-'));
+const jsonlGz = join(tmp, 'test.jsonl.gz');
+writeFileSync(jsonlGz, gzipSync(kaikkiLines.map((l) => JSON.stringify(l)).join('\n')));
+
+execFileSync(process.execPath, [
+  script,
+  '--only', 'test',
+  '--source', 'test=file://' + jsonlGz,
+  '--out', join(tmp, 'dist'),
+  '--catalog-dir', join(tmp, 'catalog'),
+], { stdio: 'pipe' });
+
+const idxBuf = readFileSync(join(tmp, 'dist', 'assets', 'test.idx'));
+const dictBuf = gunzipSync(readFileSync(join(tmp, 'dist', 'assets', 'test.dict.dz')));
+
+const index = new Map();  // headword -> {offset, size}
+{
+  let pos = 0;
+  while (pos < idxBuf.length) {
+    const nul = idxBuf.indexOf(0, pos);
+    index.set(idxBuf.subarray(pos, nul).toString('utf8'), {
+      offset: idxBuf.readUInt32BE(nul + 1),
+      size: idxBuf.readUInt32BE(nul + 5),
+    });
+    pos = nul + 9;
+  }
+}
+const body = (word) => {
+  const loc = index.get(word);
+  assert.ok(loc, `"${word}" missing from .idx`);
+  assert.ok(loc.offset + loc.size <= dictBuf.length, `"${word}" points out of bounds`);
+  return dictBuf.subarray(loc.offset, loc.offset + loc.size).toString('utf8');
+};
+
+test('pure form aliases the lemma definition at zero size cost', () => {
+  assert.match(body('cats'), /small domesticated feline/);
+  assert.doesNotMatch(body('cats'), /plural of cat/);
+  assert.deepEqual(index.get('cats'), index.get('cat'));
+});
+
+test('form whose lemma is missing keeps its stub', () => {
+  assert.match(body('mice'), /plural of mouse/);
+});
+
+test('mixed headword keeps its own senses and appends the lemma', () => {
+  assert.match(body('found'), /to establish an organization/);
+  assert.match(body('found'), /to locate something/);
+});
+
+test('alias chains resolve to the material end', () => {
+  assert.match(body("runnin'"), /to move fast/);
+  assert.deepEqual(index.get("runnin'"), index.get('run'));
+});
+
+test('alias lands on the case-merged lemma group', () => {
+  assert.match(body('marched'), /the third month/);
+  assert.match(body('marched'), /to walk in step/);
+});
+
+test('.ifo wordcount matches the .idx rows', () => {
+  const ifo = readFileSync(join(tmp, 'dist', 'assets', 'test.ifo'), 'utf8');
+  assert.equal(Number(ifo.match(/wordcount=(\d+)/)[1]), index.size);
+});
+
+test.after(() => rmSync(tmp, { recursive: true, force: true }));