From 224ca3b423e9cd0258e20b03983e8535181f1f82 Mon Sep 17 00:00:00 2001 From: SYMBIO <74346862+symbiodev@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:47:37 +0200 Subject: [PATCH] fix: resolve form_of/alt_of entries to their lemma definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inflected forms (plurals, verb tenses, ...) in the kaikki/Wiktionary source carry a form_of/alt_of pointer plus a stub gloss like "plural of cat" instead of a real definition. Because a hit on the word's own .idx row always wins before the reader falls back to .syn or stemming, every one of these stub entries permanently shadowed the real definition — looking up a plural noun, for instance, only ever showed "plural of X", never X's actual meaning. writeStardict() now folds a pure form-of/alt-of headword's .idx row onto its lemma's definition bytes (an alias, at zero extra size cost), appends the lemma definition after a mixed headword's own senses (e.g. "found": own "to establish" sense plus "simple past of find"), and follows alias chains to their material end while leaving a dead-end stub (lemma missing from the source) untouched. Rebuilding english with this fix turned 632k of 1.39M headwords (45%) from a bare stub into a real definition, at the same total dictionary size. Verified end to end in test/dictionaries.test.mjs against a synthetic kaikki source, and manually against a full rebuild of the english dictionary loaded onto a reader. --- dictionaries/README.md | 5 +- scripts/build-dictionaries.mjs | 142 ++++++++++++++++++++++++++++----- test/dictionaries.test.mjs | 101 +++++++++++++++++++++++ 3 files changed, 227 insertions(+), 21 deletions(-) create mode 100644 test/dictionaries.test.mjs diff --git a/dictionaries/README.md b/dictionaries/README.md index cfc1cf8..823e475 100644 --- a/dictionaries/README.md +++ b/dictionaries/README.md @@ -46,7 +46,10 @@ workflow in this repo ([`dictionaries-mirror.yml`](../.github/workflows/dictionaries-mirror.yml)) generates them from [kaikki.org](https://kaikki.org)'s machine-readable Wiktionary extracts — matched to the reader's dictionary engine (index sort -order, dictzip chunking, headword/definition size limits) — and hosts them as +order, dictzip chunking, headword/definition size limits). Inflected forms +("cats", "ran") resolve straight to the lemma's full definition instead of a +bare "plural of cat" stub — the stub would otherwise shadow the reader's own +stemming fallback. The files are hosted as assets on the rolling [`dictionaries` release](https://github.com/itsthisjustin/sd-plugins/releases/tag/dictionaries). The browse catalog under [`catalog/`](catalog/) is regenerated in the same diff --git a/scripts/build-dictionaries.mjs b/scripts/build-dictionaries.mjs index 0e3ba51..d3bf9a5 100644 --- a/scripts/build-dictionaries.mjs +++ b/scripts/build-dictionaries.mjs @@ -13,6 +13,10 @@ // - .idx sorted with asciiCaseCmp: bytewise, ASCII tolower per byte // - headwords equal under that comparator share one merged definition // (otherwise only the first of "march"/"March" is ever reachable) +// - inflected forms ("cats", carrying form_of/alt_of in the source) point +// their .idx rows at the lemma's definition bytes instead of keeping the +// bare "plural of cat" stub: a direct .idx hit preempts the reader's +// .syn and stemming fallbacks, so the stub would otherwise win // - headwords < 256 bytes, definitions < 64 KB, 32-bit offsets // - dictzip chunks decompress independently (raw deflate per chunk), // chunk table <= 8192 entries (~460 MB uncompressed ceiling) @@ -34,9 +38,9 @@ import { createGunzip, createDeflateRaw, gunzipSync, crc32, constants as zconst import { createInterface } from 'node:readline'; import { join, basename } from 'node:path'; import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; const RELEASE_BASE = 'https://github.com/itsthisjustin/sd-plugins/releases/download/dictionaries/'; -const CATALOG_DIR = new URL('../dictionaries/catalog/', import.meta.url).pathname; const PAGE_SIZE = 8; // One entry per language: its own Wiktionary edition, so definitions are in @@ -81,6 +85,19 @@ const flag = (name, fallback) => { const OUT = flag('out', 'dist'); const ONLY = (flag('only', '') || '').split(',').filter(Boolean); const FROM_INDEX = args.includes('--from-index'); +// Overridable so tests can build into a scratch dir without touching the +// committed catalog. +const CATALOG_DIR = flag('catalog-dir', fileURLToPath(new URL('../dictionaries/catalog/', import.meta.url))); +// --source id=url (comma-separated): override a source's URL, or add a new +// kaikki-kind source. Mainly for testing — curl accepts file:// URLs. +for (const spec of (flag('source', '') || '').split(',').filter(Boolean)) { + const eq = spec.indexOf('='); + const id = spec.slice(0, eq); + const url = spec.slice(eq + 1); + const existing = SOURCES.find((s) => s.id === id); + if (existing) existing.url = url; + else SOURCES.push({ id, title: id, kind: 'kaikki', url }); +} function curl(url, dest) { execFileSync('curl', ['-fsSL', '--retry', '3', '--retry-delay', '5', '-o', dest, url], { stdio: 'inherit' }); @@ -148,42 +165,107 @@ async function dictzip(dictBuf, dest) { writeFileSync(dest, Buffer.concat([header, extra, ...chunks, trailer])); } -// entries: Map. Writes .ifo/.idx/.dict.dz into dir -// and returns catalog metadata. +// entries: Map — refs are lemma +// headwords this entry's form_of/alt_of senses point at; pure means every +// sense is such a reference. Writes .ifo/.idx/.dict.dz into dir and +// returns catalog metadata. async function writeStardict(dir, id, title, entries) { // Group headwords that the reader's case-insensitive comparator cannot tell // apart; each gets its own .idx row pointing at the shared merged text. const groups = new Map(); - for (const [word, text] of entries) { + for (const [word, entry] of entries) { + const e = typeof entry === 'string' ? { text: entry, refs: [], pure: false } : entry; const wordBuf = Buffer.from(word, 'utf8'); if (wordBuf.length === 0 || wordBuf.length > MAX_HEADWORD_BYTES || wordBuf.includes(0)) continue; const key = foldKey(wordBuf).toString('latin1'); let g = groups.get(key); - if (!g) groups.set(key, (g = { words: [], texts: [] })); + if (!g) groups.set(key, (g = { words: [], texts: [], refs: new Set(), pure: true })); g.words.push(wordBuf); - g.texts.push(text); + g.texts.push(e.text); + for (const r of e.refs || []) { + const refBuf = Buffer.from(r, 'utf8'); + if (refBuf.length && refBuf.length <= MAX_HEADWORD_BYTES) { + g.refs.add(foldKey(refBuf).toString('latin1')); + } + } + if (!e.pure) g.pure = false; + } + + // A pure form-of group with exactly one resolvable lemma becomes an alias: + // its .idx rows reuse the lemma's (offset, size) outright, so "cats" shows + // cat's definition at zero size cost. StarDict allows shared offsets and the + // reader only ever reads one (offset, size) pair per hit. Chains ("runnin'" + // -> "running" -> "run") are followed to the material end; a chain that + // loops keeps its own stub text instead. + const resolveAlias = (key) => { + let cur = key; + const seen = new Set([key]); + for (;;) { + const g = groups.get(cur); + const refs = [...g.refs].filter((r) => r !== cur && groups.has(r)); + if (!g.pure || refs.length !== 1) return cur === key ? null : cur; // material end + if (seen.has(refs[0])) return null; // pure-ref cycle: keep the stub text + seen.add(refs[0]); + cur = refs[0]; + } + }; + const aliasOf = new Map(); + for (const key of groups.keys()) { + const target = resolveAlias(key); + if (target) aliasOf.set(key, target); } + + // Material groups with refs (mixed entries like "found": own sense plus + // past-of-find) append the referenced definitions after their own text. + const bodyCache = new Map(); + const building = new Set(); + const bodyText = (key) => { + const target = aliasOf.get(key) || key; + if (bodyCache.has(target)) return bodyCache.get(target); + const g = groups.get(target); + const own = g.texts.filter(Boolean).join('\n\n'); + if (building.has(target)) return own; // ref cycle: stop at own text + building.add(target); + let text = own; + for (const r of g.refs) { + if (!groups.has(r) || (aliasOf.get(r) || r) === target) continue; + const refText = bodyText(r); + if (refText) text += (text ? '\n\n' : '') + refText; + } + building.delete(target); + bodyCache.set(target, text); + return text; + }; + const keys = [...groups.keys()].sort((a, b) => Buffer.compare(Buffer.from(a, 'latin1'), Buffer.from(b, 'latin1'))); + // Lay out material bodies first (aliases contribute no bytes), then emit + // .idx rows in sorted order — alias rows borrow their target's placement, + // so offsets in the .idx are not monotonic, which StarDict permits. const dictParts = []; - const idxParts = []; + const placed = new Map(); // material key -> {offset, size} let offset = 0; - let wordCount = 0; for (const key of keys) { - const g = groups.get(key); - let body = Buffer.from(g.texts.join('\n\n'), 'utf8'); + if (aliasOf.has(key)) continue; + let body = Buffer.from(bodyText(key), 'utf8'); if (body.length > MAX_DEFINITION_BYTES) body = truncateUtf8(body, MAX_DEFINITION_BYTES); dictParts.push(body); - for (const wordBuf of g.words) { + placed.set(key, { offset, size: body.length }); + offset += body.length; + } + const idxParts = []; + let wordCount = 0; + for (const key of keys) { + const loc = placed.get(aliasOf.get(key) || key); + for (const wordBuf of groups.get(key).words) { const row = Buffer.alloc(wordBuf.length + 9); wordBuf.copy(row); - row.writeUInt32BE(offset, wordBuf.length + 1); - row.writeUInt32BE(body.length, wordBuf.length + 5); + row.writeUInt32BE(loc.offset, wordBuf.length + 1); + row.writeUInt32BE(loc.size, wordBuf.length + 5); idxParts.push(row); wordCount++; } - offset += body.length; } if (offset > 0xffffffff) throw new Error('dict over 4 GB (32-bit offsets)'); @@ -234,7 +316,7 @@ async function buildKaikki(src, tmp) { const jsonl = join(tmp, src.id + '.jsonl.gz'); curl(src.url, jsonl); - const blocks = new Map(); // word -> [ "word (pos)\n1. ...\n2. ...", ... ] + const blocks = new Map(); // word -> {texts: ["word (pos)\n1. ..."], refs, pure} const rl = createInterface({ input: createReadStream(jsonl).pipe(createGunzip()), crlfDelay: Infinity }); let lines = 0; for await (const line of rl) { @@ -243,23 +325,43 @@ async function buildKaikki(src, tmp) { try { e = JSON.parse(line); } catch (err) { continue; } if (!e.word || !Array.isArray(e.senses)) continue; const glosses = []; + const refs = []; + let ownSenses = 0; // glossed senses that are NOT a form_of/alt_of pointer for (const sense of e.senses) { const g = Array.isArray(sense.glosses) ? sense.glosses.filter(Boolean).join('; ') : ''; - if (g) glosses.push(g); + let isRef = false; + for (const f of [].concat(sense.form_of || [], sense.alt_of || [])) { + if (f && typeof f.word === 'string' && f.word && f.word !== e.word) { + refs.push(f.word); + isRef = true; + } + } + if (g) { + glosses.push(g); + if (!isRef) ownSenses++; + } if (glosses.length >= 30) break; } if (!glosses.length) continue; const header = e.word + (e.pos ? ` (${e.pos})` : ''); const text = header + '\n' + glosses.map((g, i) => `${i + 1}. ${g}`).join('\n'); - let list = blocks.get(e.word); - if (!list) blocks.set(e.word, (list = [])); - if (list.length < 8) list.push(text); // cap runaway homographs + let rec = blocks.get(e.word); + if (!rec) blocks.set(e.word, (rec = { texts: [], refs: new Set(), pure: true })); + if (rec.texts.length < 8) rec.texts.push(text); // cap runaway homographs + for (const r of refs) { if (rec.refs.size < 3) rec.refs.add(r); } + if (ownSenses > 0) rec.pure = false; } rmSync(jsonl, { force: true }); console.log(` ${lines} lines -> ${blocks.size} headwords`); const entries = new Map(); - for (const [word, list] of blocks) entries.set(word, list.join('\n\n')); + for (const [word, rec] of blocks) { + entries.set(word, { + text: rec.texts.join('\n\n'), + refs: [...rec.refs], + pure: rec.pure && rec.refs.size > 0, + }); + } return writeStardict(join(OUT, 'assets'), src.id, src.title, entries); } diff --git a/test/dictionaries.test.mjs b/test/dictionaries.test.mjs new file mode 100644 index 0000000..9bf3b53 --- /dev/null +++ b/test/dictionaries.test.mjs @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gunzipSync, gzipSync } from 'node:zlib'; +import test from 'node:test'; + +// End-to-end build over a synthetic kaikki source: form_of/alt_of entries must +// resolve to the lemma's definition (the reader's direct .idx hit preempts its +// .syn and stemming fallbacks, so a bare "plural of cat" stub would otherwise +// be all the user ever sees). + +const root = new URL('../', import.meta.url); +const script = fileURLToPath(new URL('scripts/build-dictionaries.mjs', root)); + +const kaikkiLines = [ + { word: 'cat', pos: 'noun', senses: [{ glosses: ['small domesticated feline'] }] }, + { word: 'cats', pos: 'noun', senses: [{ glosses: ['plural of cat'], form_of: [{ word: 'cat' }] }] }, + // Lemma missing from the source: the stub is the best we have, keep it. + { word: 'mice', pos: 'noun', senses: [{ glosses: ['plural of mouse'], form_of: [{ word: 'mouse' }] }] }, + // Mixed headword: own sense in one block, form-of in another. + { word: 'find', pos: 'verb', senses: [{ glosses: ['to locate something'] }] }, + { word: 'found', pos: 'verb', senses: [{ glosses: ['simple past of find'], form_of: [{ word: 'find' }] }] }, + { word: 'found', pos: 'verb', senses: [{ glosses: ['to establish an organization'] }] }, + // Chain: runnin' -> running -> run. + { word: 'run', pos: 'verb', senses: [{ glosses: ['to move fast'] }] }, + { word: 'running', pos: 'verb', senses: [{ glosses: ['present participle of run'], form_of: [{ word: 'run' }] }] }, + { word: "runnin'", pos: 'verb', senses: [{ glosses: ['alternative spelling of running'], alt_of: [{ word: 'running' }] }] }, + // Case-merged lemma group: marched must land on the merged march/March body. + { word: 'March', pos: 'noun', senses: [{ glosses: ['the third month'] }] }, + { word: 'march', pos: 'verb', senses: [{ glosses: ['to walk in step'] }] }, + { word: 'marched', pos: 'verb', senses: [{ glosses: ['simple past of march'], form_of: [{ word: 'march' }] }] }, +]; + +const tmp = mkdtempSync(join(tmpdir(), 'dict-test-')); +const jsonlGz = join(tmp, 'test.jsonl.gz'); +writeFileSync(jsonlGz, gzipSync(kaikkiLines.map((l) => JSON.stringify(l)).join('\n'))); + +execFileSync(process.execPath, [ + script, + '--only', 'test', + '--source', 'test=file://' + jsonlGz, + '--out', join(tmp, 'dist'), + '--catalog-dir', join(tmp, 'catalog'), +], { stdio: 'pipe' }); + +const idxBuf = readFileSync(join(tmp, 'dist', 'assets', 'test.idx')); +const dictBuf = gunzipSync(readFileSync(join(tmp, 'dist', 'assets', 'test.dict.dz'))); + +const index = new Map(); // headword -> {offset, size} +{ + let pos = 0; + while (pos < idxBuf.length) { + const nul = idxBuf.indexOf(0, pos); + index.set(idxBuf.subarray(pos, nul).toString('utf8'), { + offset: idxBuf.readUInt32BE(nul + 1), + size: idxBuf.readUInt32BE(nul + 5), + }); + pos = nul + 9; + } +} +const body = (word) => { + const loc = index.get(word); + assert.ok(loc, `"${word}" missing from .idx`); + assert.ok(loc.offset + loc.size <= dictBuf.length, `"${word}" points out of bounds`); + return dictBuf.subarray(loc.offset, loc.offset + loc.size).toString('utf8'); +}; + +test('pure form aliases the lemma definition at zero size cost', () => { + assert.match(body('cats'), /small domesticated feline/); + assert.doesNotMatch(body('cats'), /plural of cat/); + assert.deepEqual(index.get('cats'), index.get('cat')); +}); + +test('form whose lemma is missing keeps its stub', () => { + assert.match(body('mice'), /plural of mouse/); +}); + +test('mixed headword keeps its own senses and appends the lemma', () => { + assert.match(body('found'), /to establish an organization/); + assert.match(body('found'), /to locate something/); +}); + +test('alias chains resolve to the material end', () => { + assert.match(body("runnin'"), /to move fast/); + assert.deepEqual(index.get("runnin'"), index.get('run')); +}); + +test('alias lands on the case-merged lemma group', () => { + assert.match(body('marched'), /the third month/); + assert.match(body('marched'), /to walk in step/); +}); + +test('.ifo wordcount matches the .idx rows', () => { + const ifo = readFileSync(join(tmp, 'dist', 'assets', 'test.ifo'), 'utf8'); + assert.equal(Number(ifo.match(/wordcount=(\d+)/)[1]), index.size); +}); + +test.after(() => rmSync(tmp, { recursive: true, force: true }));