diff --git a/scripts/audit-anagram-pool.mjs b/scripts/audit-anagram-pool.mjs new file mode 100644 index 0000000..c695489 --- /dev/null +++ b/scripts/audit-anagram-pool.mjs @@ -0,0 +1,113 @@ +// Audit the anagram word pool: every word must be the ONLY dictionary word its +// letters spell. A word that shares its letters with another real word (star and +// rats, cloud and could) makes the challenge unanswerable-as-graded, because the +// grader accepts exactly one answer. +// +// Dev-only, and deliberately not a dependency: it reads a hunspell dictionary +// already on the machine and skips cleanly when there isn't one, so the runtime +// keeps its zero production dependencies. +// +// bun run scripts/audit-anagram-pool.mjs +// +// On Debian/Ubuntu the dictionary comes from `apt install hunspell-en-us`; +// on macOS, `brew install hunspell` plus an en_US dictionary. + +import { readFileSync, existsSync } from 'node:fs' + +const DICT_CANDIDATES = [ + '/usr/share/hunspell/en_US', + '/usr/share/myspell/en_US', + '/opt/homebrew/share/hunspell/en_US', + '/usr/local/share/hunspell/en_US', +] + +const base = DICT_CANDIDATES.find((p) => existsSync(`${p}.dic`) && existsSync(`${p}.aff`)) +if (!base) { + console.log('no hunspell en_US dictionary found; skipping audit') + console.log(`looked in: ${DICT_CANDIDATES.join(', ')}`) + process.exit(0) +} + +// Expand affix rules so inflections count. Without this, `rats` and `notes` look +// absent and the audit passes words that are in fact ambiguous. +function loadDictionary(prefix) { + const aff = readFileSync(`${prefix}.aff`, 'latin1').split('\n') + const rules = { SFX: new Map(), PFX: new Map() } + for (let i = 0; i < aff.length; i++) { + const p = aff[i].trim().split(/\s+/) + if ((p[0] === 'SFX' || p[0] === 'PFX') && /^\d+$/.test(p[3] ?? '')) { + const table = [] + for (let j = i + 1; j <= i + Number(p[3]) && j < aff.length; j++) { + const q = aff[j].trim().split(/\s+/) + if (q[0] === p[0] && q[1] === p[1]) { + table.push({ + strip: q[2] === '0' ? '' : q[2], + add: q[3] === '0' ? '' : (q[3] ?? '').split('/')[0], + cond: q[4] ?? '.', + }) + } + } + rules[p[0]].set(p[1], table) + i += Number(p[3]) + } + } + + const forms = new Set() + const dic = readFileSync(`${prefix}.dic`, 'latin1').split('\n').slice(1) + for (const line of dic) { + const [raw, flags = ''] = line.trim().split('/') + const word = (raw ?? '').trim().toLowerCase() + if (!word || !/^[a-z]+$/.test(word)) continue + forms.add(word) + for (const flag of flags.trim()) { + for (const { strip, add, cond } of rules.SFX.get(flag) ?? []) { + if (strip && !word.endsWith(strip)) continue + const stem = strip ? word.slice(0, -strip.length) : word + if (cond !== '.' && !new RegExp(`${cond}$`).test(stem)) continue + const form = stem + add + if (/^[a-z]+$/.test(form)) forms.add(form) + } + for (const { strip, add } of rules.PFX.get(flag) ?? []) { + if (strip) continue + const form = add + word + if (/^[a-z]+$/.test(form)) forms.add(form) + } + } + } + return forms +} + +const forms = loadDictionary(base) +const letters = (w) => [...w].sort().join('') + +const byLetters = new Map() +for (const w of forms) { + const k = letters(w) + if (!byLetters.has(k)) byLetters.set(k, []) + byLetters.get(k).push(w) +} + +// Read the pool straight out of the generator so the audit cannot drift from it. +const src = readFileSync(new URL('../src/generators/anagram.ts', import.meta.url), 'utf8') +const block = src.slice(src.indexOf('const WORDS'), src.indexOf('function lengthFor')) +const pool = [...block.matchAll(/'([a-z]+)'/g)].map((m) => m[1]) +if (pool.length === 0) { + console.error('could not read the word pool from src/generators/anagram.ts') + process.exit(1) +} + +let bad = 0 +for (const word of pool) { + const others = (byLetters.get(letters(word)) ?? []).filter((w) => w !== word) + if (others.length) { + console.error(` ${word} -> also spells: ${others.join(', ')}`) + bad++ + } +} + +console.log(`checked ${pool.length} words against ${forms.size.toLocaleString()} dictionary forms`) +if (bad) { + console.error(`\n${bad} word(s) have another valid answer. Replace them.`) + process.exit(1) +} +console.log('every word is the only dictionary word its letters spell') diff --git a/src/generators/anagram.ts b/src/generators/anagram.ts index 95ff253..7f4ee31 100644 --- a/src/generators/anagram.ts +++ b/src/generators/anagram.ts @@ -5,14 +5,22 @@ import { choice, shuffle } from '../rng.ts' import type { Generator, GeneratedChallenge } from '../types.ts' // Curated, lower-cased, grouped by length so difficulty maps to length. +// +// Every word here has exactly one arrangement that spells a dictionary word: +// itself. That invariant is what makes the challenge well posed, because the +// grader accepts one answer, so a word whose letters also spell another real +// word fails a solver who happened to think of the other one. `star` and `rats`, +// `cloud` and `could`, `garden` and `danger` were all in this list. +// +// Re-run `bun run scripts/audit-anagram-pool.mjs` after adding a word. const WORDS: Record = { - 3: ['cat', 'dog', 'sun', 'map', 'key', 'red', 'box', 'fox', 'cup', 'pen'], - 4: ['tree', 'book', 'fish', 'lamp', 'gold', 'rain', 'wind', 'star', 'door', 'leaf'], - 5: ['apple', 'house', 'water', 'plant', 'cloud', 'music', 'light', 'river', 'stone', 'bread'], - 6: ['garden', 'planet', 'silver', 'bridge', 'castle', 'forest', 'rocket', 'flower', 'orange', 'pencil'], - 7: ['journey', 'diamond', 'gravity', 'kitchen', 'machine', 'picture', 'rainbow', 'thunder', 'village', 'crystal'], - 8: ['mountain', 'elephant', 'sunshine', 'computer', 'hospital', 'language', 'treasure', 'umbrella', 'dinosaur', 'sandwich'], - 9: ['adventure', 'chocolate', 'telephone', 'butterfly', 'orchestra', 'discovery', 'wonderful', 'breakfast', 'invention', 'pineapple'], + 3: ['key', 'box', 'fox', 'pen', 'sky', 'toy', 'cow', 'log', 'ice', 'hat'], + 4: ['fish', 'gold', 'wind', 'bird', 'sand', 'cake', 'frog', 'milk', 'corn', 'desk'], + 5: ['house', 'water', 'plant', 'music', 'light', 'river', 'beach', 'bench', 'world', 'juice'], + 6: ['rocket', 'pencil', 'yellow', 'purple', 'summer', 'guitar', 'pocket', 'window', 'basket', 'hammer'], + 7: ['journey', 'diamond', 'gravity', 'machine', 'rainbow', 'thunder', 'village', 'crystal', 'morning', 'penguin'], + 8: ['mountain', 'elephant', 'sunshine', 'computer', 'hospital', 'language', 'umbrella', 'dinosaur', 'sandwich', 'airplane'], + 9: ['adventure', 'chocolate', 'butterfly', 'discovery', 'breakfast', 'invention', 'pineapple', 'crocodile', 'waterfall', 'astronaut'], } function lengthFor(d: number): number { diff --git a/test/anagram-pool.test.ts b/test/anagram-pool.test.ts new file mode 100644 index 0000000..efef54b --- /dev/null +++ b/test/anagram-pool.test.ts @@ -0,0 +1,83 @@ +// The anagram challenge grades against one answer, so the pool must not contain +// a word whose letters spell another real word. The full check needs a +// dictionary and lives in scripts/audit-anagram-pool.mjs; these are the parts +// that hold with no dictionary present, so they run everywhere CI does. + +import { expect, test, describe } from 'bun:test' +import { anagram } from '../src/generators/anagram.ts' +import { gradeDefault } from '../src/grader.ts' +import type { Generator } from '../src/types.ts' + +function gradeWith(gen: Generator, response: string, answer: string): boolean | Promise { + return gen.grade ? gen.grade(response, answer) : gradeDefault(response, answer) +} + +const letters = (w: string) => [...w].sort().join('') + +// Sample every level rather than reading the private WORDS table, so the tests +// exercise what a caller actually receives. +async function poolFromGenerator(): Promise> { + const seen = new Map() + for (let difficulty = 1; difficulty <= 10; difficulty++) { + for (let i = 0; i < 500; i++) { + const c = await anagram.generate(difficulty) + seen.set(c.answer, difficulty) + } + } + return seen +} + +describe('anagram pool', async () => { + const pool = await poolFromGenerator() + + test('samples a meaningful share of the pool', () => { + expect(pool.size).toBeGreaterThanOrEqual(60) + }) + + test('no two words in the pool are anagrams of each other', () => { + const byLetters = new Map() + for (const word of pool.keys()) { + const k = letters(word) + byLetters.set(k, [...(byLetters.get(k) ?? []), word]) + } + const collisions = [...byLetters.values()].filter((ws) => ws.length > 1) + expect(collisions).toEqual([]) + }) + + test('the scrambled prompt is never the answer itself', async () => { + // Otherwise the answer is printed in the question and any script passes. + for (let difficulty = 1; difficulty <= 10; difficulty++) { + for (let i = 0; i < 300; i++) { + const c = await anagram.generate(difficulty) + const shown = c.prompt.split(':').pop()!.trim().replace(/\s/g, '') + expect(await gradeWith(anagram, shown, c.answer)).toBe(false) + } + } + }) + + test('the source word is still accepted', async () => { + for (const word of pool.keys()) { + expect(await gradeWith(anagram, word, word)).toBe(true) + expect(await gradeWith(anagram, ` ${word.toUpperCase()} `, word)).toBe(true) + } + }) + + test('every word in a difficulty bucket is the same length', () => { + // Word length is what maps a word to a difficulty, so a word filed under the + // wrong length silently makes that level easier or harder than intended. + const byDifficulty = new Map>() + for (const [word, difficulty] of pool) { + const lengths = byDifficulty.get(difficulty) ?? new Set() + lengths.add(word.length) + byDifficulty.set(difficulty, lengths) + } + for (const [difficulty, lengths] of byDifficulty) { + expect({ difficulty, lengths: [...lengths] }).toEqual({ difficulty, lengths: [...lengths].slice(0, 1) }) + } + }) + + test('a different word is still rejected', async () => { + expect(await gradeWith(anagram, 'zebra', 'house')).toBe(false) + expect(await gradeWith(anagram, '', 'house')).toBe(false) + }) +})