From c1b9af3cfc9538641af087bd9cef4803eee5b617 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 08:42:51 -0500 Subject: [PATCH 1/8] The ligature is the join, and the print is thriftier than the text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `readDiacritics` returns the ligature structure beside the flat mark list, and `probe:diacritics` gains ④ to measure whether it holds. Containment (②) files a mark under the right word. It says nothing about which letter, and a tajweed rule is a [start, end) over codepoints — so without a letter-level join a rule can be highlighted no finer than the whole word. The corpus offers exactly one join: names the letters it draws in data-text and nests the marks drawn on them. ④ partitions data-hafs into the letters the print outlines, walks the ligatures across that partition, and compares mark counts. Every failure is bucketed by cause, because the check is layered and quoting the per-ligature agreement alone would silently condition it on a filter the reader cannot see. All 604 pages: of 91,451 entries the print calls words, 4,486 draw no letters at all (the pause marks, ۩, ۞). Of the remaining 86,965, 86,880 join cleanly — 99.90%. Getting there needed four print conventions read off the markup, not assumed; each one is why an earlier draft of this read 97.75%: - a bare hamza ء is an outline like any letter, not a named mark - \p{Lm} folds: the tatweel is a tooth drawn into its neighbour, while the small waw ۥ and small yeh ۦ are named marks despite Unicode calling them letters - a vowel then an iqlab meem ۭ or ۢ is one composite glyph, `kasra iqlab` - a seated hamza and ٱ are a base outline plus their own named path The remaining 85 entries are left alone on purpose. Each rule above exists because the markup showed the print doing something; adding rules until the number reads 100% would fit the rule to the data and make ④ agree with the corpus by construction, which is the one property that would stop it being evidence. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- packages/etl/scripts/lib/diacritics.mjs | 120 ++++++++--- packages/etl/scripts/probe-diacritics.mjs | 240 +++++++++++++++++++++- 2 files changed, 324 insertions(+), 36 deletions(-) diff --git a/packages/etl/scripts/lib/diacritics.mjs b/packages/etl/scripts/lib/diacritics.mjs index efad0b4..49c62fa 100644 --- a/packages/etl/scripts/lib/diacritics.mjs +++ b/packages/etl/scripts/lib/diacritics.mjs @@ -53,6 +53,23 @@ const BOUNDARY = / + * + * + * + * + * + * so a mark is not merely *in* a word, it is drawn **on named letters** — and + * that is the only join in this corpus between a mark and a codepoint. The + * split is a lookahead rather than a close-tag match because the segment is + * already bounded by the word above it and `` nesting is not parseable by + * regex; the same trade `readTheirs` makes. + */ +const LIGATURES = /([\s\S]*?)(?= Math.round(n * 10) / 10; +const unescapeXml = (s) => + s.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/&/g, "&"); + +/** The marks drawn inside one ligature's group, in document order. */ +function marksIn(seg, apply) { + const marks = []; + for (const p of seg.matchAll(PATHS)) { + const name = attr(p[0], "data-diacritic"); + if (name === null) continue; + if (!isDiacriticName(name)) { + // A name @hifth/core has never heard of is a corpus that grew, and the + // only safe response is to stop. Emitting it under a made-up id would + // write geometry nothing can name; skipping it would silently drop a + // mark from a page and look like the print simply has fewer. + throw new Error( + `data-diacritic="${name}" on ${attr(p[0], "id")} is not in DIACRITICS. ` + + "Append it to packages/core/src/diacritics.ts — append, never reorder, " + + "because an id is only meaningful against that array's order.", + ); + } + const d = attr(p[0], "d"); + if (d === null) continue; + const [x0, y0, x1, y1] = apply(pathBBox(d)); + marks.push([diacriticId(name), round(x0), round(y0), round(x1 - x0), round(y1 - y0)]); + } + return marks; +} + /** - * Every named mark on the page, grouped by word, in their document order. + * Every named mark on the page, grouped by word and by the ligature inside it. * * Returns one entry per `` the page carries, in the same * order `readTheirs().words` returns them, so a caller can zip the two without - * a join. Each entry is `{ surah, aya, idx, marks }` where `idx` is the print's - * `data-word-index-in-ayah` — the same index the word shards' `from` counts - * from, and emphatically not QAC's (see `build-words.mjs` on the 4,499 ayahs - * where the two segmentations disagree). + * a join. Each entry is + * + * { surah, aya, idx, hafs, imlaey, ligatures, marks } + * + * where `idx` is the print's `data-word-index-in-ayah` — the same index the + * word shards' `from` counts from, and emphatically not QAC's (see + * `build-words.mjs` on the 4,499 ayahs where the two segmentations disagree). * - * `marks` is `[id, x, y, w, h]` per mark, `id` indexing `DIACRITICS`, the box - * already through `apply` and rounded to a tenth of a viewBox unit — the same - * precision the word boxes ship at, and the smallest mark in the corpus is - * 1.42 × 1.64 units in their frame (≈1.9 × 2.2 in ours), so a tenth cannot - * round one away. + * A mark is `[id, x, y, w, h]`, `id` indexing `DIACRITICS`, the box already + * through `apply` and rounded to a tenth of a viewBox unit — the same precision + * the word boxes ship at, and the smallest mark in the corpus is 1.42 × 1.64 + * units in their frame (≈1.9 × 2.2 in ours), so a tenth cannot round one away. + * + * ## Why the ligatures are returned and not just flattened + * + * `marks` is the flat concatenation and is what a shard would carry: an app + * painting a mark needs a rectangle and a name, not a letter. `ligatures` is + * for the instrument rather than the asset. Each is `{ text, marks }`, `text` + * being the ligature's own `data-text` — the letters it draws — and that is the + * **only** join this corpus offers between a mark and a codepoint. Without it a + * mark is a rectangle inside a word and a tajweed offset can be resolved no + * finer than the word; with it, a word's letters are partitioned across its + * ligatures and each partition carries its own marks in order. + * + * The join is measured rather than assumed — `probe-diacritics.mjs` ④ reports + * how often a ligature's mark count equals the mark-bearing codepoints of the + * letters it draws, and names what is left. Nothing here acts on it; a caller + * that wants the correspondence reads both fields and does its own arithmetic, + * because the residual is the interesting part and swallowing it inside an + * extractor would hide it. * * @param {string} svg a ligature-corpus page, verbatim * @param {(b: number[]) => number[]} apply their frame → ours @@ -91,31 +156,28 @@ export function readDiacritics(svg, apply) { const nxt = rest.match(BOUNDARY); const seg = nxt ? rest.slice(0, nxt.index) : rest; - const marks = []; - for (const p of seg.matchAll(PATHS)) { - const name = attr(p[0], "data-diacritic"); - if (name === null) continue; - if (!isDiacriticName(name)) { - // A name @hifth/core has never heard of is a corpus that grew, and the - // only safe response is to stop. Emitting it under a made-up id would - // write geometry nothing can name; skipping it would silently drop a - // mark from a page and look like the print simply has fewer. - throw new Error( - `data-diacritic="${name}" on ${attr(p[0], "id")} is not in DIACRITICS. ` + - "Append it to packages/core/src/diacritics.ts — append, never reorder, " + - "because an id is only meaningful against that array's order.", - ); - } - const d = attr(p[0], "d"); - if (d === null) continue; - const [x0, y0, x1, y1] = apply(pathBBox(d)); - marks.push([diacriticId(name), round(x0), round(y0), round(x1 - x0), round(y1 - y0)]); + const ligatures = []; + for (const g of seg.matchAll(LIGATURES)) { + ligatures.push({ + text: unescapeXml(attr(g[2], "data-text") ?? ""), + marks: marksIn(g[2], apply), + }); } + // Not `ligatures.flatMap` for its own sake: a mark drawn under a word but + // outside every ligature group would be dropped by that, and dropping a + // mark silently is the one thing this file exists not to do. Scanning the + // whole segment keeps the flat list authoritative, and ④'s ligature check + // is what would notice the two disagreeing. + const marks = marksIn(seg, apply); + out.push({ surah: Number(attr(m[2], "data-surah")), aya: Number(attr(m[2], "data-aya")), idx: Number(attr(m[2], "data-word-index-in-ayah")), + hafs: unescapeXml(attr(m[2], "data-hafs") ?? ""), + imlaey: unescapeXml(attr(m[2], "data-imlaey") ?? ""), + ligatures, marks, }); } diff --git a/packages/etl/scripts/probe-diacritics.mjs b/packages/etl/scripts/probe-diacritics.mjs index 3aeb375..e9fcfd2 100644 --- a/packages/etl/scripts/probe-diacritics.mjs +++ b/packages/etl/scripts/probe-diacritics.mjs @@ -15,7 +15,7 @@ * out whether the boxes are trustworthy before paying two megabytes to send * them to a phone. * - * ## The three questions, and why these three + * ## The four questions, and why these four * * **① Does the vocabulary hold?** Every `data-diacritic` value in the corpus * must be one `@hifth/core` knows. `readDiacritics` throws otherwise, so this @@ -36,13 +36,46 @@ * or a formality. Measured as the shard text `build-words.mjs` would write, not * estimated from a path count. * + * **④ Can a mark be tied to a letter?** ① and ② together say the boxes are + * real and filed under the right *word*. They say nothing about *which letter* + * a mark sits on, and a tajweed rule is a `[start, end)` over codepoints — so + * without a letter-level join the app can highlight a rule no finer than the + * whole word, which for «بِسْمِ ٱللَّهِ» is most of the line. + * + * The corpus's only offer is the ligature: `` names the + * letters it draws in `data-text` and nests the marks drawn on them. ④ checks + * whether that join holds, by partitioning `data-hafs` into base letters and + * their combining marks, walking the ligatures across that partition, and + * comparing counts. **Every failure is bucketed by cause and counted**, because + * a single percentage here would be a lie of composition: the check is layered, + * and a word that fails the letter partition never reaches the mark comparison, + * so quoting the mark agreement alone quietly conditions it on a filter. + * + * ④ is a measurement and does not affect the exit code. It is describing a + * property of somebody else's file, not asserting one about ours. + * + * ## Why ④ is not chased to 100% + * + * Every rule in `letters` and `expected` below was added because reading the + * markup showed the print doing something, and each one is stated as the print's + * convention with the word that demonstrated it. That is a bounded exercise. + * The 85 entries still disagreeing could be driven to zero by adding rules + * until they are, but a rule added to move a number is a rule fitted to the + * data, and it would make ④ agree with the corpus by construction — which is + * exactly the property that would stop it from being evidence. So the residual + * is printed with its cause and its example and left alone. `sub-word-marks.md` + * §⑤ names the four families it falls into. + * * ## What it deliberately does not check * - * Whether a mark is on the *right letter*. Nothing in this repo can answer that - * offline — it would need the print's own letter order, which the corpus gives - * as ligature ids this does not read, and ultimately a reader's eye. That check - * belongs to the encoding inspector (mark-B), where a human can see the boxes - * on the page beside the three other encodings. + * Whether a mark is on the *right* letter. ④ can show that a ligature drawing + * three letters carries the three marks those letters call for; it cannot show + * that the second mark is over the second letter and not the third. Counts are + * necessary and not sufficient, and nothing offline closes that gap — it is a + * correspondence between a codepoint in a reconstructed text and an outline on + * a page, and only an eye closes it. That check belongs to the encoding + * inspector (mark-B), where a human sees the boxes on the page beside the three + * other encodings. * * Usage: * pnpm --filter @hifth/etl probe:diacritics @@ -80,6 +113,89 @@ const only = (() => { */ const SLACK = 0.2; +/** + * A word's `data-hafs` as the letters the print draws an *outline* for, each + * carrying the codepoints written on it: `بِسْمِ` → `[ب:[ِ], س:[ْ], م:[ِ]]`. + * + * Two Unicode categories are not outlines and fold into the letter before them: + * + * - **`\p{Mn}`**, the combining marks. Obvious, and the reason this exists. + * - **`\p{Lm}`**, the modifier letters — and this one is the whole reason ④'s + * first draft disagreed. Three of them occur in this text: the tatweel + * `U+0640` that seats a hamza in `شَيۡـٔٗا`, and the small waw `U+06E5` and + * small yeh `U+06E6` of `بِهِۦ`. The text calls all three letters. The print + * does not: the tatweel is drawn as a tooth folded into its neighbour's + * ligature, and the two small letters are drawn as `data-diacritic="small + * waw"` and `"small yeh"` — named marks, sitting in `DIACRITICS` beside the + * fatha. Counting them as base letters made the partition off by one for + * every word containing a seated hamza. + * + * Both rules are Unicode's own categories rather than a codepoint list this + * repo maintains, because a list would be a third place with an opinion about + * Arabic marks and would drift from the other two. + */ +function letters(hafs) { + const out = []; + for (const c of hafs) { + if (/[\p{Mn}\p{Lm}]/u.test(c) && out.length) out[out.length - 1].marks.push(c); + else out.push({ letter: c, marks: [] }); + } + return out; +} + +/** + * A hamza the print draws as a base outline with a separate named path on top, + * so that one codepoint in the text is two things on the page. + * + * The bare hamza `ء` `U+0621` is deliberately **not** here, and that was the + * other half of ④'s first draft being wrong: it is drawn as `data-type="text"` + * like any other letter, because it has no carrier to sit on. Its four seated + * forms and the alef wasla do get their own path. + */ +const CARRIES_ITS_OWN = /[آأؤإئٱ]/; + +/** A short vowel or a tanween — the thing an iqlab meem merges into. */ +const VOWEL = /[ً-ِٗٞ]/; + +/** + * The iqlab meem, which the print never draws on its own beside a vowel — both + * of the forms this text uses. `كَافِرِۭ` writes the final form `U+06ED` and + * `رِكۡزَۢا` the isolated `U+06E2`; the print composes either with the vowel + * before it into one `kasra iqlab` / `fatha iqlab` glyph. + */ +const IQLAB = /[ۭۢ]/; + +/** + * The tatweel. It folds like a mark, because the print does not give it a + * ligature of its own — but unlike the small waw and small yeh it folds beside, + * it is drawn as part of the neighbouring outline (the tooth that seats a hamza in + * `شَيۡـٔٗا`), not as a named path. So it is invisible to the partition on both + * sides: not a letter, and not a mark either. + */ +const TATWEEL = "ـ"; + +/** + * Which codepoints of a letter the print draws a *named* path for, in order. + * + * Its marks and, for a seated hamza, itself — with one merge. `DIACRITICS` + * carries `fatha iqlab`, `kasra iqlab` and `damma iqlab` as names in their own + * right, so where the text writes a vowel followed by `ۭ` the print draws a + * single composite glyph rather than two: `كَافِرِۭ` is two paths on `فر`, not + * three. Collapsing them here is not a fudge to raise the number — it is the + * same fact the vocabulary already states, read from the other end. + */ +function expected(l) { + const out = CARRIES_ITS_OWN.test(l.letter) ? [l.letter] : []; + for (const m of l.marks) { + if (m === TATWEEL) continue; + if (IQLAB.test(m) && out.length && VOWEL.test(out[out.length - 1])) continue; + out.push(m); + } + return out; +} + +const cp = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; + const pin = JSON.parse(readFileSync(PIN, "utf8")); const rows = new Map(pin.pages.map((p) => [p.page, p])); const wanted = only ?? pin.pages.map((p) => p.page); @@ -96,6 +212,27 @@ let raw = 0; let gz = 0; let smallest = Infinity; +/** + * ④'s buckets. Every word lands in exactly one, so they sum to `words` and no + * percentage below is conditioned on a filter the reader cannot see. + */ +const bucket = { + joined: 0, // the ligatures' letters agree with the hafs, and every mark count agrees + ornament: 0, // draws no letters at all — a pause mark, a sajda sign, a ۞ + partition: 0, // the ligature texts do not partition the hafs letter-for-letter + counts: 0, // they partition, but some ligature's mark count disagrees +}; +let ligatures = 0; +let ligaturesAgree = 0; +let textIsImlaey = 0; +const pairs = new Map(); // "U+0650 → kasra" → n, only where a ligature's counts agree +const why = new Map(); // a compact signature of a disagreement → [n, example] + +const blame = (sig, example) => { + const e = why.get(sig) ?? [0, example]; + why.set(sig, [e[0] + 1, e[1]]); +}; + for (const page of wanted) { const row = rows.get(page); if (!row) { @@ -116,6 +253,50 @@ for (const page of wanted) { smallest = Math.min(smallest, m[3], m[4]); } + // ── ④, per word ────────────────────────────────────────────────────────── + const where = `p${page} ${w.surah}:${w.aya}#${w.idx} “${w.hafs}”`; + const drawn = w.ligatures.map((l) => l.text).join(""); + if (!drawn) { + // An entry the print files as a word and a reader does not read as one: + // the pause marks `ۖ ۗ ۘ ۙ ۚ ۛ`, the sajda sign, the `۞` rub' al-hizb. + // They have no letters, so there is no join to succeed or fail at, and + // counting them as a disagreement would be counting the instrument. + bucket.ornament += 1; + } else { + if (drawn === w.imlaey) textIsImlaey += 1; + const ls = letters(w.hafs); + if (ls.length !== [...drawn].length) { + bucket.partition += 1; + blame( + `hafs has ${ls.length} letters, its ligatures draw ${[...drawn].length}`, + `${where} → [${w.ligatures.map((l) => l.text).join("|")}]`, + ); + } else { + let cut = 0; + let ok = true; + for (const l of w.ligatures) { + ligatures += 1; + const n = [...l.text].length; + const want = ls.slice(cut, cut + n).flatMap(expected); + cut += n; + if (want.length !== l.marks.length) { + ok = false; + blame( + `ligature “${l.text}” wants ${want.length} mark(s), the print draws ${l.marks.length}`, + where, + ); + continue; + } + ligaturesAgree += 1; + for (let k = 0; k < want.length; k += 1) { + const p = `${cp(want[k])} ${want[k]} → ${diacriticName(l.marks[k][0])}`; + pairs.set(p, (pairs.get(p) ?? 0) + 1); + } + } + bucket[ok ? "joined" : "counts"] += 1; + } + } + const key = `${w.surah}:${w.aya}`; const ayah = shard.words[key]; if (!ayah) { @@ -192,8 +373,53 @@ if (unmatched) console.log(` ${unmatched} word(s) had no box in the committ console.log( `\n ③ weight — ${(raw / 1024 / 1024).toFixed(2)} MB raw / ` + `${(gz / 1024 / 1024).toFixed(2)} MB gz across ${wanted.length} shard(s)` + - `\n smallest mark on our frame: ${smallest.toFixed(1)} units\n`, + `\n smallest mark on our frame: ${smallest.toFixed(1)} units`, +); + +// ── ④ the ligature join ────────────────────────────────────────────────────── + +const pct = (n, d) => (d ? ((n / d) * 100).toFixed(2) : "0.00"); +const lettered = words - bucket.ornament; + +console.log(`\n ④ the ligature join — of ${words} entries the print calls words:`); +console.log( + ` ${String(bucket.ornament).padStart(7)} (${pct(bucket.ornament, words).padStart(5)}%) ` + + "draw no letters at all — pause marks, ۩, ۞", ); +console.log(` ${" ".repeat(7)} ${" ".repeat(7)} of the remaining ${lettered}:`); +console.log( + ` ${String(bucket.joined).padStart(7)} (${pct(bucket.joined, lettered).padStart(5)}%) ` + + "join cleanly — letters partition and every mark count agrees", +); +console.log( + ` ${String(bucket.partition).padStart(7)} (${pct(bucket.partition, lettered).padStart(5)}%) ` + + "their ligature texts do not partition the hafs letters", +); +console.log( + ` ${String(bucket.counts).padStart(7)} (${pct(bucket.counts, lettered).padStart(5)}%) ` + + "partition, but at least one ligature's mark count disagrees", +); +console.log( + `\n ligatures ${ligatures}, mark counts agree on ${ligaturesAgree} ` + + `(${pct(ligaturesAgree, ligatures)}%) — but that is conditioned on the ` + + `\n partition above, so it describes ${lettered - bucket.partition} of ${words} ` + + "entries and is not a corpus figure", +); +console.log( + ` ligature texts concatenate to data-imlaey on ${textIsImlaey} of ${lettered} ` + + `(${pct(textIsImlaey, lettered)}%) — informational; the join does not use it`, +); + +console.log("\n why the rest disagree, most common first:"); +for (const [sig, [n, example]] of [...why].sort((a, b) => b[1][0] - a[1][0]).slice(0, 12)) { + console.log(` ${String(n).padStart(7)} ${sig}\n${" ".repeat(15)}e.g. ${example}`); +} + +console.log("\n codepoint → name, where a ligature's counts agree:"); +for (const [p, n] of [...pairs].sort((a, b) => b[1] - a[1]).slice(0, 20)) { + console.log(` ${String(n).padStart(7)} ${p}`); +} +console.log(); if (escapes.length || unmatched) { console.error(" probe:diacritics — the boxes are not yet trustworthy; see above\n"); From 1c18370fcc0534323777beccc0b951e2c1ee1652 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 08:43:03 -0500 Subject: [PATCH 2/8] =?UTF-8?q?sub-word-marks=20=C2=A7=E2=91=A4=20gains=20?= =?UTF-8?q?the=20join,=20and=20=C2=A7=E2=91=A6=20stops=20overclaiming?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §⑤ records ④'s corpus measurement: the rejected positional attempt (88.79% and a visibly wrong pairing tail), the markup that replaced it, the four print conventions that had to be learned, the 99.90% result, and the four families the 85-entry residual falls into — each with the word that demonstrates it. §⑦ said the letter question was unanswerable because the corpus expresses letter order "as ligature ids this does not read". It reads them now, so that reason is gone and the section would have been quietly wrong. The conclusion survives on a better reason: counts are necessary and not sufficient. A ligature agreeing on three marks does not establish the second is over the second letter, and a word whose marks were internally permuted would pass ④ exactly as a correct one does. Only an eye closes that, which is still mark-B. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- docs/design/sub-word-marks.md | 90 +++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/docs/design/sub-word-marks.md b/docs/design/sub-word-marks.md index 696d9a9..6317bec 100644 --- a/docs/design/sub-word-marks.md +++ b/docs/design/sub-word-marks.md @@ -164,6 +164,76 @@ That the residual came out as *the pause marks, exactly, plus a letter pair that nothing on it* is the strongest evidence here that the extraction is filing marks under the right words: an off-by-one would have scattered the empties. +### The ligature is the join to a letter + +Containment (above) files a mark under the right *word*. It says nothing about which +**letter** the mark sits on, and a tajweed rule is a `[start, end)` over codepoints — so +without a letter-level join the app could highlight a rule no finer than the whole word, +which for «بِسۡمِ ٱللَّهِ» is most of a line. The corpus offers exactly one join, and it is +not the one the first attempt assumed. + +**The rejected attempt.** Zip a word's mark-bearing codepoints against its `data-diacritic` +paths in document order. That gives **88.79%** count agreement and a visibly wrong pairing +tail (U+064E → shadda, U+0650 → wasla). It fails because the paths are grouped by +*ligature*, and `dots` and `kaf-hamza` paths interleave — so position within the word is not +codepoint order. + +**What the markup actually offers** is one level below the word: + +``` + + + ← the letters this run draws + + ← drawn on those letters +``` + +So `readDiacritics` returns the ligatures alongside the flat mark list, and +`probe:diacritics` **④** measures whether they partition the word: split `data-hafs` into the +letters the print outlines, walk the ligatures across that partition, compare mark counts. +All 604 pages, 2026-08-07: + +| of 91,451 entries the print calls words | | | +|---|---:|---| +| draw no letters at all — pause marks, ۩, ۞ | 4,486 | 4.91% | +| **of the remaining 86,965** | | | +| join cleanly — letters partition, every mark count agrees | **86,880** | **99.90%** | +| ligature texts do not partition the hafs letters | 54 | 0.06% | +| partition, but a ligature's mark count disagrees | 31 | 0.04% | + +The per-ligature figure is 159,476 of 159,509 (99.98%), but that is conditioned on the +partition above, so the table states the unconditional number instead. Quoting the ligature +percentage alone would silently condition it on a filter the reader cannot see. + +**Four print conventions had to be learned to get there**, each read off the markup rather +than assumed, and each is why an earlier draft read 97.75%: + +| the text writes | the print draws | example | +|---|---|---| +| a bare hamza `ء` U+0621 | an outline, like any letter — **not** a named mark | «إِسۡرَٰٓءِيلَ» | +| `\p{Lm}` modifier letters | the tatweel as a tooth folded into its neighbour; the small waw `ۥ` and small yeh `ۦ` as *named marks* | «شَيۡـٔٗا», «بِهِۦ» | +| a vowel then an iqlab meem `ۭ` / `ۢ` | one composite glyph, `kasra iqlab` | «كَافِرِۭ», «رِكۡزَۢا» | +| a seated hamza `أ إ ؤ ئ`, and `ٱ` | a base outline **plus** a named `hamza` / `wasla` path | «أَنزَلَ» | + +**The remaining 85 entries are not chased to zero, deliberately.** Each rule above exists +because reading the markup showed the print doing something; adding further rules until the +number reads 100% would be fitting the rule to the data, and would make ④ agree with the +corpus by construction — destroying the only property that makes it evidence. The residual +falls into four families, all printed with an example by the probe itself: + +1. **An extra alef run** (~51) — «فَلَا» → `[فلا|ا]`: the print splits a final alef into its + own ligature the text does not have as a separate letter. +2. **The small high madda `ۤ` U+06E4** (~20) — «خَرُّواْۤ», «لِلَّهِۤ»: merged into the glyph + before it, a composite this does not model. +3. **A mark attributed across a ligature boundary** (~4) — «ٱلرَّحِيمِ» on p379 loses one from + `حيم` and gains one on `لر`; net zero, so the word is right and the split is not. +4. **Contextual hamza forms** (~10) — «أَيۡدِيهِمۡ», where the carrier and its hamza are one glyph. + +None of these is an alignment error: ② already proves every mark sits inside its own word, +and all four families are the print being more economical with glyphs than the text is with +codepoints. What they bound is how much of the corpus a *letter*-level highlight can be +offered on — 99.90% of lettered words — and that bound is the input to §⑧ ①, not its answer. + ## ⑥ What it would weigh, and why nothing shipped Measured as the shard text `build-words.mjs` would actually write — a `from` and a dense @@ -188,11 +258,21 @@ That order is also the answer the user gave when asked where the marks should ap ## ⑦ What this cannot answer -Whether a mark is on the *right letter*. Nothing in this repo can settle that offline — it -would need the print's own letter order, which the corpus expresses as ligature ids this -does not read, and in the end a reader's eye. Containment proves a mark belongs to its word; -it says nothing about where inside the word it belongs. That is the inspector's job, and it -is why mark-B exists as a separate step rather than a review of mark-C. +Whether a mark is on the *right letter*. + +§⑤'s ligature join narrows this and does not close it. It shows that a ligature drawing +three letters carries the number of marks those three letters call for, 99.90% of the time — +which is what makes a letter-level highlight arithmetically possible at all. But **counts are +necessary and not sufficient**: agreement on three does not establish that the second mark is +over the second letter rather than the third. A word whose marks were internally permuted +would pass ④ exactly as a correct one does. + +Nothing in this repo closes that gap offline, because it is a correspondence between a +codepoint in a reconstructed text and an outline on a page, and in the end only a reader's +eye settles it. Containment proves a mark belongs to its word; the ligature join proves the +counts work out per run; neither says the *k*-th mark is on the *k*-th letter. That is the +inspector's job, and it is why mark-B exists as a separate step rather than a review of +mark-C. --- From e9fa1150bc16d2a1e142bbadb6e159319011bc90 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 12:30:38 -0500 Subject: [PATCH 3/8] The ligature join reaches every word, and names the three the corpus loses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ④ was at 99.90%. The user asked for 100%, and the route was the one that got it from 88.79% to 97.75% to 99.90%: dump the markup for each residual family and encode what the print actually does. No rule here was added to move the number — every one of them names the word that demonstrated it. Three families were left, and reading them turned up two errors of my own: - `align` replaces the left-to-right walk. Ligature document order is not reading order — «ٱلرَّحِيمِ» is drawn `[لر|حيم|ٱ]` — and a letter can be drawn twice, «فَلَا» as `[فلا|ا]`. The old length check let both pass and then misassigned every mark while the totals balanced. Matching on content is strictly stronger, and some words that used to pass now fail. - `FOLD`/`FOLDS` are one class used on both sides. A ligature's `data-text` carries the tatweel that `letters` folds away, so «مَـَٔابٗا» → `[مـا|با]` looked unassignable when it is simply spelt with its tooth. - The seated hamza always gets its path. A first reading of «أَيۡدِيهِمۡ» suggested the ligature's own spelling decided it; «أَنَّ» is drawn `[أ|ن]` and still carries `hamza` then `fatha`. That conditional cost 151 words. 86,962 of 86,965 lettered words now join cleanly. The three left are the corpus disagreeing with itself, and the docblock names them rather than absorbing them, because a rule for either would be a rule for one word: «أَيۡدِيهِمۡ» loses its hamza path in 2 of its 26 occurrences, and 17:7's «لِيَسُـُٔواْ» draws a `small waw` and a `maddah` its own `data-hafs` does not write. Neither costs anything downstream: ② decides whether the geometry ships, and ② is exact — 0 marks outside their word, unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- packages/etl/scripts/probe-diacritics.mjs | 199 ++++++++++++++++++---- 1 file changed, 164 insertions(+), 35 deletions(-) diff --git a/packages/etl/scripts/probe-diacritics.mjs b/packages/etl/scripts/probe-diacritics.mjs index e9fcfd2..0295b80 100644 --- a/packages/etl/scripts/probe-diacritics.mjs +++ b/packages/etl/scripts/probe-diacritics.mjs @@ -54,17 +54,31 @@ * ④ is a measurement and does not affect the exit code. It is describing a * property of somebody else's file, not asserting one about ours. * - * ## Why ④ is not chased to 100% - * - * Every rule in `letters` and `expected` below was added because reading the - * markup showed the print doing something, and each one is stated as the print's - * convention with the word that demonstrated it. That is a bounded exercise. - * The 85 entries still disagreeing could be driven to zero by adding rules - * until they are, but a rule added to move a number is a rule fitted to the - * data, and it would make ④ agree with the corpus by construction — which is - * exactly the property that would stop it from being evidence. So the residual - * is printed with its cause and its example and left alone. `sub-word-marks.md` - * §⑤ names the four families it falls into. + * ## Where the residual went, and why it stops at three + * + * ④ was chased to 100% on request, and the interesting part is that it got + * there without a single rule invented to make it. Every rule in `letters`, + * `expected` and `align` below was added because a markup dump showed the print + * doing something, and each is stated with the word that demonstrated it — the + * seated hamza on «أَنَّ», the sajda overline on «خَرُّواْۤ», the second alef + * stroke of «فَلَا», the out-of-order `ٱ` in «ٱلرَّحِيمِ», the tatweel inside + * «مَـَٔابٗا»'s `data-text`. That order matters: a rule added to move a number + * would make ④ agree with the corpus by construction, which is precisely the + * property that would stop it from being evidence. + * + * Three of 86,965 entries remain, and they are named here rather than absorbed + * because neither is a convention — both are the corpus disagreeing with + * itself, and a rule for either would be a rule for one word: + * + * - **p324 21:28#4 and p341 22:76#4, «أَيۡدِيهِمۡ».** The word occurs 26 times. + * Twenty-four draw `hamza, fatha, sukun, kasra, kasra, sukun`; these two draw + * the same list without the `hamza`. Same spelling, same marks otherwise. + * - **p282 17:7#15, «لِيَسُـُٔواْ».** The print draws a `small waw` and a + * `maddah` for which its own `data-hafs` has no codepoint. It is the only word + * in the corpus where a `small waw` path appears without a `U+06E5`. + * + * Neither costs us anything downstream: ② is what decides whether the geometry + * is shippable, and ② is exact. * * ## What it deliberately does not check * @@ -133,26 +147,52 @@ const SLACK = 0.2; * Both rules are Unicode's own categories rather than a codepoint list this * repo maintains, because a list would be a third place with an opinion about * Arabic marks and would drift from the other two. + * + * The class is `const` and shared with `align`, which has to fold a ligature's + * `data-text` by exactly the same rule for the two to be comparable at all. + * `FOLD` tests one character and `FOLDS` strips a run; they are the same class + * written once, because two copies of it would be the drift this paragraph is + * about. */ +const FOLD_CLASS = "[\\p{Mn}\\p{Lm}]"; +const FOLD = new RegExp(FOLD_CLASS, "u"); +const FOLDS = new RegExp(FOLD_CLASS, "gu"); + function letters(hafs) { const out = []; for (const c of hafs) { - if (/[\p{Mn}\p{Lm}]/u.test(c) && out.length) out[out.length - 1].marks.push(c); + if (FOLD.test(c) && out.length) out[out.length - 1].marks.push(c); else out.push({ letter: c, marks: [] }); } return out; } /** - * A hamza the print draws as a base outline with a separate named path on top, - * so that one codepoint in the text is two things on the page. + * A hamza form written as one codepoint, and the carrier it is written on. + * + * Two separate facts live here, and conflating them cost a pass of the corpus. + * + * **The print always draws the sign.** `أ` gets a `hamza` path, `ٱ` a `wasla` + * path, every time, in all 9,168 and 13,476 places they occur. The ligature's + * own spelling does *not* decide it: «أَنَّ» on p119 is drawn `[أ | ن]` and the + * first ligature still carries `hamza` then `fatha`. Making the expectation + * conditional on the ligature spelling the bare carrier — which a first reading + * of «أَيۡدِيهِمۡ» seemed to show — put 151 words on seven pages into the + * residual, and the markup dump said plainly why. * - * The bare hamza `ء` `U+0621` is deliberately **not** here, and that was the - * other half of ④'s first draft being wrong: it is drawn as `data-type="text"` - * like any other letter, because it has no carrier to sit on. Its four seated - * forms and the alef wasla do get their own path. + * **The spelling still matters for matching.** `align` compares a ligature's + * `data-text` to the word's letters, and the two disagree about the carrier: a + * ligature may spell `ا` where the word writes `أ`. So `base()` folds a hamza + * form to its carrier for that comparison only, and never for what the print + * is expected to draw. + * + * The bare hamza `ء` `U+0621` is deliberately absent: with no carrier to sit + * on it is drawn as `data-type="text"` like any other letter, always. */ -const CARRIES_ITS_OWN = /[آأؤإئٱ]/; +const HAMZA_ON = { آ: "ا", أ: "ا", إ: "ا", ٱ: "ا", ؤ: "و", ئ: "ي" }; + +/** The letter under a hamza form, for matching a ligature's text to the word's. */ +const base = (c) => HAMZA_ON[c] ?? c; /** A short vowel or a tanween — the thing an iqlab meem merges into. */ const VOWEL = /[ً-ِٗٞ]/; @@ -174,26 +214,111 @@ const IQLAB = /[ۭۢ]/; */ const TATWEEL = "ـ"; +/** + * `U+06E4`, the small high madda — which in this print is not a mark at all. + * + * Every word carrying it sits in a sajda ayah (13:15, 17:107, 19:58 …) and the + * print draws it as `data-type="sajda-line"`: the overline stretched above the + * phrase a reader prostrates at, not a diacritic over a letter. It has no + * `data-diacritic`, so `readDiacritics` never sees it, and expecting one for it + * was counting a rubric as a vowel. + */ +const SAJDA_LINE = "ۤ"; + /** * Which codepoints of a letter the print draws a *named* path for, in order. * - * Its marks and, for a seated hamza, itself — with one merge. `DIACRITICS` - * carries `fatha iqlab`, `kasra iqlab` and `damma iqlab` as names in their own - * right, so where the text writes a vowel followed by `ۭ` the print draws a - * single composite glyph rather than two: `كَافِرِۭ` is two paths on `فر`, not - * three. Collapsing them here is not a fudge to raise the number — it is the - * same fact the vocabulary already states, read from the other end. + * The hamza or wasla sign comes first where the letter is a hamza form, because + * that is the order the print draws it in: «أَنَّ» is `hamza` then `fatha`. It + * is not universal — «ٱلۡمَلَؤُاْ» draws `damma` before the `hamza` on its `ؤ` — + * and ④ compares counts, so the two disagree without failing. §⑦ of + * `sub-word-marks.md` is about exactly that gap. + * + * The marks follow, minus the two the print draws by other means (the tatweel's + * tooth, the sajda overline) and with one merge: `DIACRITICS` carries `fatha + * iqlab`, `kasra iqlab` and `damma iqlab` as names in their own right, so where + * the text writes a vowel followed by `ۭ` the print draws a single composite + * glyph — «كَافِرِۭ» is two paths on `فر`, not three. Collapsing them is not a + * fudge to raise the number; it is the same fact the vocabulary already states, + * read from the other end. */ function expected(l) { - const out = CARRIES_ITS_OWN.test(l.letter) ? [l.letter] : []; + const out = []; + if (HAMZA_ON[l.letter]) out.push(l.letter); for (const m of l.marks) { - if (m === TATWEEL) continue; + if (m === TATWEEL || m === SAJDA_LINE) continue; if (IQLAB.test(m) && out.length && VOWEL.test(out[out.length - 1])) continue; out.push(m); } return out; } +/** + * Assign each ligature the letters it draws, or `null` if no assignment exists. + * + * The obvious implementation — walk the ligatures in document order, handing + * each the next `text.length` letters — is what ④'s previous draft did, and it + * is wrong in two ways the markup shows plainly: + * + * **Document order is not reading order.** «ٱلرَّحِيمِ» on p379 is drawn as + * `[لر | حيم | ٱ]`: the alef wasla is a separate ligature emitted *last*. Six + * letters, six drawn, so a length check passes — and then every mark is + * assigned to the wrong letter while the totals still balance. That is the + * failure mode this whole file exists to catch, and counting alone cannot see + * it. + * + * **A letter can be drawn twice.** «فَلَا» is `[فلا | ا]` — four letters drawn + * for a three-letter word, because the print puts the alef's stroke in a second + * ligature. Those continuation runs carry no marks of their own, which is what + * makes them safe to recognise: a repeat that carried marks would be a + * different phenomenon and would still fail here. + * + * So this matches on **content** rather than length, over `base()` so that a + * ligature spelling `ا` matches a word writing `أ`. A ligature may take the + * next letters, or re-draw letters already taken if it has no marks. The search + * is a DFS over (position, set of ligatures used) with memoisation; words have + * a handful of ligatures, so the state space is tiny. + * + * Both sides are reduced the same way, which is the only thing that makes the + * comparison meaningful: `letters` folds a `\p{Lm}` into the letter before it, + * so a ligature's `data-text` has to be folded too. It carries the tatweel — + * «مَـَٔابٗا» is drawn `[مـا | با]`, tatweel and all — and leaving it in made + * ten seated-hamza words on the last two juz look unassignable when they are + * simply spelt with the tooth the print draws them with. + * + * Matching on content is strictly stronger than the length check it replaces — + * some words that used to pass the partition now fail it, and that is the point. + */ +function align(ls, ligs) { + const target = ls.map((l) => base(l.letter)).join(""); + const texts = ligs.map((l) => [...l.text.replace(FOLDS, "")].map(base)); + const all = (1 << ligs.length) - 1; + const memo = new Map(); + + const go = (pos, used) => { + if (pos === target.length && used === all) return []; + const key = pos * (all + 1) + used; + if (memo.has(key)) return memo.get(key); + let out = null; + for (let i = 0; i < ligs.length && !out; i += 1) { + if (used & (1 << i)) continue; + const t = texts[i]; + const fits = (from) => from >= 0 && t.every((c, j) => target[from + j] === c); + if (pos + t.length <= target.length && fits(pos)) { + const rest = go(pos + t.length, used | (1 << i)); + if (rest) out = [{ lig: i, from: pos, to: pos + t.length }, ...rest]; + } + if (!out && !ligs[i].marks.length && fits(pos - t.length)) { + const rest = go(pos, used | (1 << i)); + if (rest) out = [{ lig: i, from: pos - t.length, to: pos, redraw: true }, ...rest]; + } + } + memo.set(key, out); + return out; + }; + return go(0, 0); +} + const cp = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; const pin = JSON.parse(readFileSync(PIN, "utf8")); @@ -265,20 +390,24 @@ for (const page of wanted) { } else { if (drawn === w.imlaey) textIsImlaey += 1; const ls = letters(w.hafs); - if (ls.length !== [...drawn].length) { + const plan = align(ls, w.ligatures); + if (!plan) { bucket.partition += 1; blame( - `hafs has ${ls.length} letters, its ligatures draw ${[...drawn].length}`, + `no assignment of ligatures to letters — hafs “${ls.map((l) => l.letter).join("")}”`, `${where} → [${w.ligatures.map((l) => l.text).join("|")}]`, ); } else { - let cut = 0; let ok = true; - for (const l of w.ligatures) { + for (const step of plan) { + const l = w.ligatures[step.lig]; + // A redraw is the second stroke of a letter already drawn. It has no + // marks by the rule that recognised it, so there is nothing to check + // and nothing to count — counting it as a ligature would inflate the + // denominator with runs that cannot disagree. + if (step.redraw) continue; ligatures += 1; - const n = [...l.text].length; - const want = ls.slice(cut, cut + n).flatMap(expected); - cut += n; + const want = ls.slice(step.from, step.to).flatMap(expected); if (want.length !== l.marks.length) { ok = false; blame( @@ -393,7 +522,7 @@ console.log( ); console.log( ` ${String(bucket.partition).padStart(7)} (${pct(bucket.partition, lettered).padStart(5)}%) ` + - "their ligature texts do not partition the hafs letters", + "no assignment of their ligatures to their letters exists", ); console.log( ` ${String(bucket.counts).padStart(7)} (${pct(bucket.counts, lettered).padStart(5)}%) ` + From c5a3e0ac438886912bc89ca2c3ab3599809e453c Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 12:30:51 -0500 Subject: [PATCH 4/8] =?UTF-8?q?sub-word-marks=20=C2=A7=E2=91=A4=20and=20?= =?UTF-8?q?=C2=A7=E2=91=A6:=20the=20join=20closes,=20and=20the=20tally=20d?= =?UTF-8?q?oes=20not?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §⑤'s table and its four-family residual list were written against 99.90% and are now wrong in both directions — the number is 100.00% of lettered words and the residual is three entries, not eighty-five. The conventions table gains the two rows that closed it (the sajda overline, the split and out-of-order ligatures) and corrects the seated-hamza row to say **always**. §⑦ said counts are necessary and not sufficient and left it abstract. It now carries the probe's own evidence against itself: the `codepoint → name` tally is built only from ligatures whose counts agree and still contains 611 pairings of a sukun with a `hamza` path. «بِٱلۡأٓخِرَةِ» is one — the run `لأ` is written `ۡ أ ٓ` and drawn `hamza, sukun, fatha`. Three wanted, three drawn, ④ passes, and all three pairings are wrong. Which is the argument for mark-B in one example: the tally's head is trustworthy, its tail is not, and no arithmetic tells them apart. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- docs/design/sub-word-marks.md | 83 +++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/docs/design/sub-word-marks.md b/docs/design/sub-word-marks.md index 6317bec..bef16c4 100644 --- a/docs/design/sub-word-marks.md +++ b/docs/design/sub-word-marks.md @@ -197,42 +197,48 @@ All 604 pages, 2026-08-07: |---|---:|---| | draw no letters at all — pause marks, ۩, ۞ | 4,486 | 4.91% | | **of the remaining 86,965** | | | -| join cleanly — letters partition, every mark count agrees | **86,880** | **99.90%** | -| ligature texts do not partition the hafs letters | 54 | 0.06% | -| partition, but a ligature's mark count disagrees | 31 | 0.04% | +| join cleanly — letters partition, every mark count agrees | **86,962** | **100.00%** | +| no assignment of ligatures to letters exists | 0 | 0.00% | +| partition, but a ligature's mark count disagrees | 3 | 0.00% | -The per-ligature figure is 159,476 of 159,509 (99.98%), but that is conditioned on the -partition above, so the table states the unconditional number instead. Quoting the ligature -percentage alone would silently condition it on a filter the reader cannot see. +The per-ligature figure is 159,585 of 159,588, but that is conditioned on the partition +above, so the table states the unconditional number instead. Quoting the ligature percentage +alone would silently condition it on a filter the reader cannot see. -**Four print conventions had to be learned to get there**, each read off the markup rather -than assumed, and each is why an earlier draft read 97.75%: +**Six print conventions had to be learned to get there**, each read off a markup dump rather +than assumed. Earlier drafts of the join read 88.79%, then 97.75%, then 99.90%; the number +moved each time a dump explained a family, and never because a rule was added to move it. | the text writes | the print draws | example | |---|---|---| | a bare hamza `ء` U+0621 | an outline, like any letter — **not** a named mark | «إِسۡرَٰٓءِيلَ» | | `\p{Lm}` modifier letters | the tatweel as a tooth folded into its neighbour; the small waw `ۥ` and small yeh `ۦ` as *named marks* | «شَيۡـٔٗا», «بِهِۦ» | | a vowel then an iqlab meem `ۭ` / `ۢ` | one composite glyph, `kasra iqlab` | «كَافِرِۭ», «رِكۡزَۢا» | -| a seated hamza `أ إ ؤ ئ`, and `ٱ` | a base outline **plus** a named `hamza` / `wasla` path | «أَنزَلَ» | - -**The remaining 85 entries are not chased to zero, deliberately.** Each rule above exists -because reading the markup showed the print doing something; adding further rules until the -number reads 100% would be fitting the rule to the data, and would make ④ agree with the -corpus by construction — destroying the only property that makes it evidence. The residual -falls into four families, all printed with an example by the probe itself: - -1. **An extra alef run** (~51) — «فَلَا» → `[فلا|ا]`: the print splits a final alef into its - own ligature the text does not have as a separate letter. -2. **The small high madda `ۤ` U+06E4** (~20) — «خَرُّواْۤ», «لِلَّهِۤ»: merged into the glyph - before it, a composite this does not model. -3. **A mark attributed across a ligature boundary** (~4) — «ٱلرَّحِيمِ» on p379 loses one from - `حيم` and gains one on `لر`; net zero, so the word is right and the split is not. -4. **Contextual hamza forms** (~10) — «أَيۡدِيهِمۡ», where the carrier and its hamza are one glyph. - -None of these is an alignment error: ② already proves every mark sits inside its own word, -and all four families are the print being more economical with glyphs than the text is with -codepoints. What they bound is how much of the corpus a *letter*-level highlight can be -offered on — 99.90% of lettered words — and that bound is the input to §⑧ ①, not its answer. +| a seated hamza `أ إ ؤ ئ`, and `ٱ` | a base outline **plus** a named `hamza` / `wasla` path — **always**, whether the ligature spells `ا` or `أ` | «أَنزَلَ», «أَنَّ» | +| the small high madda `ۤ` U+06E4 | `data-type="sajda-line"` — the overline of a sajda ayah, not a diacritic | «خَرُّواْۤ» (19:58) | +| one letter | sometimes **two** ligatures, the second markless; and **not in reading order** | «فَلَا» → `[فلا\|ا]`, «ٱلرَّحِيمِ» → `[لر\|حيم\|ٱ]` | + +The last row is why the check is no longer a left-to-right walk. `align` matches ligature +text to letters by **content**, as a search over which ligature draws which run, so a +ligature emitted last is assigned the letters it actually spells. That is strictly stronger +than the length comparison it replaced: «ٱلرَّحِيمِ» used to *pass* — six letters, six drawn +— and then misassign every mark while the totals balanced. + +**Three entries remain, and all three are the corpus disagreeing with itself.** They are +named rather than absorbed, because a rule for either would be a rule for one word: + +1. **«أَيۡدِيهِمۡ» at 21:28 and 22:76** — the word occurs 26 times. Twenty-four draw + `hamza, fatha, sukun, kasra, kasra, sukun`; these two draw the same list without the + `hamza`. Same spelling, same everything else. +2. **«لِيَسُـُٔواْ» at 17:7** — the print draws a `small waw` and a `maddah` its own + `data-hafs` writes no codepoint for. It is the only word in the corpus where a `small + waw` path appears without a `U+06E5`. + +None of the three is an alignment error — ② already proves every mark sits inside its own +word — and none costs anything downstream, because ② is what decides whether the geometry is +shippable and ② is exact. What ④ bounds is how much of the corpus a *letter*-level highlight +can be offered on, and that bound is now the whole of it. That is the input to §⑧ ①, not its +answer: see §⑦ for what a count still cannot say. ## ⑥ What it would weigh, and why nothing shipped @@ -261,11 +267,22 @@ That order is also the answer the user gave when asked where the marks should ap Whether a mark is on the *right letter*. §⑤'s ligature join narrows this and does not close it. It shows that a ligature drawing -three letters carries the number of marks those three letters call for, 99.90% of the time — -which is what makes a letter-level highlight arithmetically possible at all. But **counts are -necessary and not sufficient**: agreement on three does not establish that the second mark is -over the second letter rather than the third. A word whose marks were internally permuted -would pass ④ exactly as a correct one does. +three letters carries the number of marks those three letters call for, for all but three +words in the corpus — which is what makes a letter-level highlight arithmetically possible +at all. But **counts are necessary and not sufficient**: agreement on three does not +establish that the second mark is over the second letter rather than the third. A word whose +marks were internally permuted would pass ④ exactly as a correct one does. + +The probe prints the evidence for this against itself. Its `codepoint → name` tally is built +*only* from ligatures whose counts agree, and it still carries **611 pairings of `U+06E1` (a +sukun) with a path named `hamza`**. «بِٱلۡأٓخِرَةِ» on p2 is one of them: the run `لأ` is +written `ۡ` then `أ` then `ٓ`, and drawn `hamza`, `sukun`, `fatha`. Three marks wanted, three +marks drawn, ④ passes — and all three pairings are wrong, the third doubly so, since the +print names the madda glyph `fatha`. + +So the tally's head is trustworthy and its tail is not, and no arithmetic distinguishes +them. That is precisely why mark-B puts the boxes on the page for a human before mark-C +ships anything that claims to know which letter a mark is on. Nothing in this repo closes that gap offline, because it is a correspondence between a codepoint in a reconstructed text and an outline on a page, and in the end only a reader's From 996fd3136a7ddce81522e2abdab605667c1f386d Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 13:24:20 -0500 Subject: [PATCH 5/8] The mark that would not say its name, until the others said theirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probe-diacritics grows a fifth question. ④ established that a ligature drawing three letters carries three marks; it never said which drawn path was the tanween and which the sukun, and a tajweed rule that wants to light the tanween needs exactly that. The obvious answer — pair them off left to right — is not an answer. It assumes the print draws marks in the order the text writes them, which is the thing in question, and 1.36% of multi-mark runs say otherwise. The old `pairs` tally did exactly that, and its output was cited in the design doc as proof the gap could not be closed. It was measuring its own assumption. Deleted. ⑤ never looks at position. Each agreeing ligature contributes a bag of codepoint tokens beside a bag of drawn names, and the correspondence falls out of elimination across the corpus: arc consistency over bipartite matchings, where a pairing dies only when no perfect assignment of that run can use it. Set intersection was tried first and is the wrong operator — it presumes the relation is already a function, and drove U+0653 to an empty candidate set. That empty set turned out to be the finding: the print draws a combining madda as `maddah` on nineteen carriers and as `fatha` on a hamza-carrying alef, 277 times of 277, so the codepoint alone is not a function and needs the carrier. `U+0653@hamza` exists for that reason only. Every single-mark run is held out. A one-mark run forces its own pairing, so scoring against it would report 100% by construction — an earlier version of this did, and it was circular. 152,101 runs, 62,931 held out, 2,869 shapes to learn from 34 of 34 tokens pinned in two passes, 0 shapes unsatisfiable 62,931 of 62,931 held-out runs predicted correctly (100.00%) Order is measured only afterwards, once pairing is settled without it: 98.64% drawn as written, 99.56% with R1 (a seated hamza's sign drawn last), stated before it was scored. A second candidate — shadda after its vowel — was put up the same way and refuted by its own score at −40 runs. Dropped. Also fixes the usage line, which named a `--filter @hifth/etl` script that does not exist; and excludes markless ligatures from the run set, where they agreed vacuously and diluted the held-out percentage. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- packages/etl/scripts/probe-diacritics.mjs | 393 ++++++++++++++++++++-- 1 file changed, 358 insertions(+), 35 deletions(-) diff --git a/packages/etl/scripts/probe-diacritics.mjs b/packages/etl/scripts/probe-diacritics.mjs index 0295b80..7de663c 100644 --- a/packages/etl/scripts/probe-diacritics.mjs +++ b/packages/etl/scripts/probe-diacritics.mjs @@ -15,7 +15,7 @@ * out whether the boxes are trustworthy before paying two megabytes to send * them to a phone. * - * ## The four questions, and why these four + * ## The five questions, and why these five * * **① Does the vocabulary hold?** Every `data-diacritic` value in the corpus * must be one `@hifth/core` knows. `readDiacritics` throws otherwise, so this @@ -54,6 +54,27 @@ * ④ is a measurement and does not affect the exit code. It is describing a * property of somebody else's file, not asserting one about ours. * + * **⑤ Which mark is which?** ④ counts. Counting says a ligature drawing three + * letters carries three marks; it does not say which drawn path is the tanween + * and which is the sukun, and a tajweed rule that wants to light the tanween + * needs exactly that. The obvious answer — pair them off left to right — is not + * an answer at all: it *assumes* the print draws marks in the order the text + * writes them, which is the thing in question, and assuming it manufactures + * agreement out of nothing. + * + * So ⑤ never looks at position. Each agreeing ligature contributes a *bag* of + * codepoint tokens beside a *bag* of drawn names, and the correspondence is + * recovered by elimination across the whole corpus: if a run wants + * `{sukun, أ, fatha}` and the print draws `{hamza, sukun, fatha}`, then once + * two are pinned elsewhere the third follows from set arithmetic. The mechanism + * is arc consistency over bipartite matchings — `supported` states why that and + * not plain intersection. + * + * It is checked on data it was not shown. Every run carrying exactly one mark is + * **held out** of the propagation, because a one-mark run forces its own pairing + * and scoring against it would report 100% by construction. Order is measured + * only afterwards, once pairing is settled without it. + * * ## Where the residual went, and why it stops at three * * ④ was chased to 100% on request, and the interesting part is that it got @@ -82,18 +103,26 @@ * * ## What it deliberately does not check * - * Whether a mark is on the *right* letter. ④ can show that a ligature drawing - * three letters carries the three marks those letters call for; it cannot show - * that the second mark is over the second letter and not the third. Counts are - * necessary and not sufficient, and nothing offline closes that gap — it is a - * correspondence between a codepoint in a reconstructed text and an outline on - * a page, and only an eye closes it. That check belongs to the encoding - * inspector (mark-B), where a human sees the boxes on the page beside the three - * other encodings. + * Whether a mark is drawn where a reader would look for it. ⑤ closes *which* + * path is which — the token a path belongs to, and via R1 which box on the page + * carries it. What no count and no set arithmetic can reach is the last step: + * that the path so identified is physically over the letter that wrote it, and + * not floating a letter to its left. Everything here is a correspondence + * between a reconstructed text and an outline on a page, established through + * the corpus's own attributes; whether the ink lands where a reader's eye goes + * is a claim about the picture, and only an eye settles it. That check belongs + * to the encoding inspector (mark-B), where a human sees the boxes on the page + * beside the three other encodings. * * Usage: - * pnpm --filter @hifth/etl probe:diacritics - * pnpm --filter @hifth/etl probe:diacritics --pages 1,2,7 + * pnpm probe:diacritics # all 604 cached pages + * pnpm probe:diacritics --pages 1,2,7 # a fast subset + * + * (from the repo root — the script is registered there, not in this package) + * + * On a subset ⑤ is reporting what those pages alone can settle: the dictionary + * is a corpus-scale result, and a handful of pages will leave tokens open. The + * numbers quoted in `docs/design/sub-word-marks.md` are the full-corpus run. */ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -226,29 +255,58 @@ const TATWEEL = "ـ"; const SAJDA_LINE = "ۤ"; /** - * Which codepoints of a letter the print draws a *named* path for, in order. + * `U+0653` on a `أ` — the one place the print refuses its own `maddah`. + * + * Every other carrier of a combining madda gets a path named `maddah`: the alef + * of «بِمَآ», the yeh of «فِيٓ», the waw of «قَالُوٓاْ», nineteen letters in all, + * 4,682 paths. The hamza-carrying alef gets one **277 times out of 277 and never + * a `maddah`** — the print draws a stroke it names `fatha`, and the outline + * bears that out: measured against an ordinary fatha on the same line it is a + * shortened version of the same curve (median 0.89×, p5 0.72×), not the maddah's + * hooked wave, which is drawn at one constant width throughout the corpus. + * + * Whether that is the madda rendered short or a fatha standing in for it is a + * question about the print's intent, and this file does not have to answer it: + * one codepoint, one path, and ⑤ pins which. It is given a token of its own only + * so that the relation stays a *function*, which is what makes ⑤'s arithmetic + * work — without it `U+0653` maps to two names and arc consistency correctly + * reports a contradiction rather than a dictionary. + */ +const MADDA = "ٓ"; +const MADDA_ON_HAMZA = "أ"; + +/** + * The *tokens* of a letter — one per named path the print is expected to draw, + * in the order the text writes them. + * + * A token is usually just a codepoint, `"U+064E"`. Two carry context, because + * the print composes and the composite has a name of its own: + * + * - `"U+064E+iqlab"` — `DIACRITICS` carries `fatha iqlab`, `kasra iqlab` and + * `damma iqlab` as names in their own right, so where the text writes a vowel + * followed by `ۭ` the print draws a single glyph: «كَافِرِۭ» is two paths on + * `فر`, not three. + * - `"U+0653@hamza"` — see `MADDA_ON_HAMZA` above. * * The hamza or wasla sign comes first where the letter is a hamza form, because - * that is the order the print draws it in: «أَنَّ» is `hamza` then `fatha`. It - * is not universal — «ٱلۡمَلَؤُاْ» draws `damma` before the `hamza` on its `ؤ` — - * and ④ compares counts, so the two disagree without failing. §⑦ of - * `sub-word-marks.md` is about exactly that gap. - * - * The marks follow, minus the two the print draws by other means (the tatweel's - * tooth, the sajda overline) and with one merge: `DIACRITICS` carries `fatha - * iqlab`, `kasra iqlab` and `damma iqlab` as names in their own right, so where - * the text writes a vowel followed by `ۭ` the print draws a single composite - * glyph — «كَافِرِۭ» is two paths on `فر`, not three. Collapsing them is not a - * fudge to raise the number; it is the same fact the vocabulary already states, - * read from the other end. + * that is the order the text writes it in. The print often disagrees — «أَنَّ» + * draws `hamza` then `fatha` but «ٱلۡمَلَؤُاْ» draws `damma` then `hamza` — and + * that disagreement is not swept up here. ④ compares counts and is blind to it; + * ⑤ measures it directly and states the rule it follows. + * + * Excluded are the two codepoints the print draws by other means: the tatweel's + * tooth, folded into a neighbour, and the sajda overline. */ function expected(l) { const out = []; - if (HAMZA_ON[l.letter]) out.push(l.letter); + if (HAMZA_ON[l.letter]) out.push(cp(l.letter)); for (const m of l.marks) { if (m === TATWEEL || m === SAJDA_LINE) continue; - if (IQLAB.test(m) && out.length && VOWEL.test(out[out.length - 1])) continue; - out.push(m); + if (IQLAB.test(m) && out.length && isVowel(out[out.length - 1])) { + out[out.length - 1] += "+iqlab"; + continue; + } + out.push(m === MADDA && l.letter === MADDA_ON_HAMZA ? `${cp(m)}@hamza` : cp(m)); } return out; } @@ -321,6 +379,9 @@ function align(ls, ligs) { const cp = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; +/** Does a token name a vowel? `parseInt` stops at the `+` or `@` of a suffix. */ +const isVowel = (token) => VOWEL.test(String.fromCodePoint(parseInt(token.slice(2), 16))); + const pin = JSON.parse(readFileSync(PIN, "utf8")); const rows = new Map(pin.pages.map((p) => [p.page, p])); const wanted = only ?? pin.pages.map((p) => p.page); @@ -350,9 +411,17 @@ const bucket = { let ligatures = 0; let ligaturesAgree = 0; let textIsImlaey = 0; -const pairs = new Map(); // "U+0650 → kasra" → n, only where a ligature's counts agree const why = new Map(); // a compact signature of a disagreement → [n, example] +/** + * ⑤'s corpus: one entry per ligature whose counts agree, holding the tokens the + * text writes, the names the print drew, and where to look. The two lists are + * the same length and in no stated correspondence — establishing one is ⑤'s + * whole job, and zipping them here by position would answer the question by + * assuming it. + */ +const runs = []; + const blame = (sig, example) => { const e = why.get(sig) ?? [0, example]; why.set(sig, [e[0] + 1, e[1]]); @@ -417,10 +486,12 @@ for (const page of wanted) { continue; } ligaturesAgree += 1; - for (let k = 0; k < want.length; k += 1) { - const p = `${cp(want[k])} ${want[k]} → ${diacriticName(l.marks[k][0])}`; - pairs.set(p, (pairs.get(p) ?? 0) + 1); - } + // ⑤'s raw material: a bag of tokens the text writes beside a bag of + // names the print drew, and no claim about which goes with which. A + // ligature carrying no marks at all agrees vacuously and constrains + // nothing, so it is left out rather than counted as a run — in the + // denominator it would only dilute ⑤'s held-out percentage. + if (want.length) runs.push([want, l.marks.map((m) => diacriticName(m[0])), where]); } bucket[ok ? "joined" : "counts"] += 1; } @@ -544,9 +615,261 @@ for (const [sig, [n, example]] of [...why].sort((a, b) => b[1][0] - a[1][0]).sli console.log(` ${String(n).padStart(7)} ${sig}\n${" ".repeat(15)}e.g. ${example}`); } -console.log("\n codepoint → name, where a ligature's counts agree:"); -for (const [p, n] of [...pairs].sort((a, b) => b[1] - a[1]).slice(0, 20)) { - console.log(` ${String(n).padStart(7)} ${p}`); +// ── ⑤ which mark is which ──────────────────────────────────────────────────── + +/** + * The relation, one entry per token: the set of names it might be drawn as. + * + * Seeded from co-occurrence — every name any run drew beside this token — and + * then narrowed. The seed is deliberately generous: a token starts out able to + * be anything it was ever seen next to, and only elimination takes names away. + */ +const may = new Map(); + +/** + * Every `(token, name)` pairing that *some* one-to-one assignment of this run + * can use, given what the relation currently allows. + * + * This is arc consistency over a bipartite matching, and the distinction from + * plain set intersection matters enough to state: intersection would say "this + * token was seen beside `{a, b}` here and `{b, c}` there, so it must be `b`", + * which is only valid if the relation is a function to begin with. It is not + * known to be one — that is what ⑤ is establishing — and assuming it drove + * `U+0653` to an empty candidate set on the first attempt. Here a pairing + * survives unless *no* perfect assignment of this run's tokens to this run's + * names can use it, which is a claim about the run alone and cannot be wrong. + * + * `reach[i]` is the set of name-subsets consumable by the first `i` tokens; + * `canFinish` asks whether the remaining tokens can consume what is left. A + * pairing is supported when it lies on a path through both. Runs hold at most a + * handful of marks, so the `2^k` masks are cheap, and both halves memoise. + * + * Returns an empty map when the run admits no assignment at all — a + * contradiction, which is reported rather than absorbed. + */ +function supported(want, got) { + const k = want.length; + const all = (1 << k) - 1; + const ok = (i, j) => may.get(want[i]).has(got[j]); + const feas = new Map(); + const canFinish = (i, mask) => { + if (i === k) return mask === all; + const key = i * (all + 1) + mask; + if (feas.has(key)) return feas.get(key); + let v = false; + for (let j = 0; j < k && !v; j += 1) { + if (!(mask & (1 << j)) && ok(i, j)) v = canFinish(i + 1, mask | (1 << j)); + } + feas.set(key, v); + return v; + }; + const reach = Array.from({ length: k + 1 }, () => new Set()); + reach[0].add(0); + const usable = new Map(); + for (let i = 0; i < k; i += 1) { + for (const mask of reach[i]) { + for (let j = 0; j < k; j += 1) { + if (mask & (1 << j)) continue; + if (!ok(i, j) || !canFinish(i + 1, mask | (1 << j))) continue; + reach[i + 1].add(mask | (1 << j)); + if (!usable.has(want[i])) usable.set(want[i], new Set()); + usable.get(want[i]).add(got[j]); + } + } + } + return usable; +} + +/** + * Runs with exactly one mark are **held out**, and that is the load-bearing + * choice in ⑤. + * + * A one-mark run forces its own pairing: one token, one name, nothing to + * decide. Feed it to the propagation and then "check" the dictionary against + * it and the answer is 100% by construction — the check would be reading back + * what it was told. Withheld, the same runs become a genuine test set of + * pairings the propagation never saw and cannot have fitted, and they are the + * overwhelming majority of the corpus. + * + * What the propagation learns from is therefore only the multi-mark runs, where + * every pairing is ambiguous on its own and can only be settled by arithmetic + * across runs. + */ +const shapes = new Map(); +let singles = 0; +for (const [want, got, where] of runs) { + if (want.length < 2) { + singles += 1; + continue; + } + const k = `${want.join(",")}|${got.join(",")}`; + const s = shapes.get(k); + if (s) s.n += 1; + else shapes.set(k, { want, got, where, n: 1 }); +} + +for (const { want, got } of shapes.values()) { + for (const t of want) { + if (!may.has(t)) may.set(t, new Set()); + for (const n of got) may.get(t).add(n); + } +} + +const dead = []; +let passes = 0; +for (;;) { + passes += 1; + let cut = 0; + for (const sh of shapes.values()) { + const usable = supported(sh.want, sh.got); + if (!usable.size) { + if (!sh.dead) { + sh.dead = true; + dead.push(sh); + } + continue; + } + for (const [t, allowed] of usable) { + for (const n of may.get(t)) { + if (!allowed.has(n)) { + may.get(t).delete(n); + cut += 1; + } + } + } + } + if (!cut) break; +} + +const dict = new Map([...may].filter(([, s]) => s.size === 1).map(([t, s]) => [t, [...s][0]])); +const open = [...may].filter(([, s]) => s.size !== 1); + +console.log( + `\n ⑤ which mark is which — ${runs.length} runs whose counts agree, ` + + `${singles} held out\n for the test below, leaving ${shapes.size} distinct token-bag/name-bag ` + + `shapes\n to propagate over; a fixpoint in ${passes} pass(es)`, +); +console.log(`\n the dictionary, from set arithmetic and no assumption about order:`); +for (const [t, s] of [...may].sort((a, b) => a[0].localeCompare(b[0]))) { + const rhs = s.size === 1 ? [...s][0] : `{ ${[...s].join(" | ")} }`; + console.log(` ${t.padEnd(14)} → ${rhs}`); +} +console.log( + ` ${dict.size} of ${may.size} tokens pinned to exactly one name; ` + + `${open.length} still open`, +); +if (dead.length) { + console.log( + ` ${dead.length} shape(s) admit no assignment at all ` + + `(${dead.reduce((a, d) => a + d.n, 0)} runs) — a contradiction, not a gap:`, + ); + for (const d of dead.slice(0, 6)) { + console.log(` ${String(d.n).padStart(7)} [${d.want.join(",")}] vs [${d.got.join(",")}] ${d.where}`); + } +} + +// The held-out test. +let agree = 0; +let differ = 0; +let unseen = 0; +const wrong = new Map(); +for (const [want, got, where] of runs) { + if (want.length !== 1) continue; + const p = dict.get(want[0]); + if (!p) { + unseen += 1; + continue; + } + if (p === got[0]) { + agree += 1; + continue; + } + differ += 1; + const k = `${want[0]} predicted ${p}, drawn ${got[0]}`; + const e = wrong.get(k) ?? { n: 0, where }; + e.n += 1; + wrong.set(k, e); +} +const spct = (n) => (singles ? ((n / singles) * 100).toFixed(2) : "0.00"); +console.log( + `\n the held-out test — ${singles} runs carrying exactly one mark, ` + + "none of which\n the propagation was shown:", +); +console.log(` ${String(agree).padStart(7)} (${spct(agree).padStart(6)}%) the dictionary predicts the drawn name`); +console.log(` ${String(differ).padStart(7)} (${spct(differ).padStart(6)}%) predicts a different name`); +console.log( + ` ${String(unseen).padStart(7)} (${spct(unseen).padStart(6)}%) ` + + "a token that never appears beside another, so nothing was learnt", +); +for (const [k, e] of [...wrong].sort((a, b) => b[1].n - a[1].n).slice(0, 8)) { + console.log(` ${String(e.n).padStart(7)} ${k}\n${" ".repeat(15)}e.g. ${e.where}`); +} + +/** + * R1 — a seated hamza's own sign is drawn *after* every other mark on its + * ligature, though the text writes it first. «يُؤۡمِنُونَ» writes damma, hamza, + * sukun and draws damma, sukun, hamza. + * + * Stated here, before it is scored, so that scoring cannot become fitting. A + * second candidate — that a shadda is drawn after the vowel it shares a letter + * with, as «وَّ» suggests — was put up the same way and **refuted**: it costs + * forty runs, because «نُّؤۡمِنَ» and its family do not swap. It is recorded in + * `docs/design/sub-word-marks.md` §⑦ and deliberately not implemented. + */ +const HAMZAISH = new Set(["hamza", "wasla"]); +const r1 = (names) => [ + ...names.filter((n) => !HAMZAISH.has(n)), + ...names.filter((n) => HAMZAISH.has(n)), +]; + +let multi = 0; +let inOrder = 0; +let byR1 = 0; +let permuted = 0; +let unresolved = 0; +const perms = new Map(); +const same = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]); +for (const [want, got, where] of runs) { + if (want.length < 2) continue; + multi += 1; + const pred = want.map((t) => dict.get(t)); + if (pred.some((p) => !p)) { + unresolved += 1; + continue; + } + if (same(pred, got)) { + inOrder += 1; + byR1 += 1; + continue; + } + const bag = (a) => [...a].sort().join("|"); + if (bag(pred) !== bag(got)) { + unresolved += 1; + continue; + } + permuted += 1; + if (same(r1(pred), got)) byR1 += 1; + else { + const k = `${pred.join(" , ")} drawn ${got.join(" , ")}`; + const e = perms.get(k) ?? { n: 0, where }; + e.n += 1; + perms.set(k, e); + } +} +const mpct = (n) => (multi ? ((n / multi) * 100).toFixed(2) : "0.00"); +console.log( + `\n and only now, order — of ${multi} runs carrying two marks or more, ` + + "pairing\n having been established without ever consulting position:", +); +console.log(` ${String(inOrder).padStart(7)} (${mpct(inOrder).padStart(6)}%) drawn in the order the text writes them`); +console.log(` ${String(permuted).padStart(7)} (${mpct(permuted).padStart(6)}%) the same marks, drawn in another order`); +console.log(` ${String(unresolved).padStart(7)} (${mpct(unresolved).padStart(6)}%) unresolved`); +console.log( + ` ${String(byR1).padStart(7)} (${mpct(byR1).padStart(6)}%) with R1 — the seated hamza drawn last`, +); +console.log(`\n what R1 leaves, most common first:`); +for (const [k, e] of [...perms].sort((a, b) => b[1].n - a[1].n).slice(0, 10)) { + console.log(` ${String(e.n).padStart(7)} ${k}\n${" ".repeat(15)}e.g. ${e.where}`); } console.log(); From d8a457ffffad5eb554327c1dda72f18c44b02551 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 13:24:33 -0500 Subject: [PATCH 6/8] The section that said it could not be done, and the run that did it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sub-word-marks.md §⑦ argued no arithmetic could say which drawn path belongs to which codepoint, and offered as proof that the probe's own tally carried 611 pairings of a sukun with a path named `hamza`. Those 611 were an artifact of how that tally was built — it zipped a run's codepoints against its paths by position, so every word where the print reorders was miscounted by construction. The section is rewritten rather than deleted, because the refutation is worth keeping: a check that shares a mistake with the thing it checks will agree with it. What survives §⑦ is smaller and real. ⑤ closes identity — this path is the sukun and not the hamza — and every step of it is a correspondence between a reconstructed text and the corpus's own attributes. None of it looks at the picture. A print that named its paths correctly and placed one a letter to the left would satisfy ①–⑤ exactly as a correct one does. That is mark-B's check, and it is why mark-B is a separate step rather than a review of mark-C. §⑤ gains the propagation: 34 of 34 tokens pinned, 62,931 held-out runs predicted at 100.00%, and order measured only afterwards — 98.64% as written, 99.56% with R1, with the refuted shadda rule recorded as refuted. The conventions table goes from six to eight. The two new rows are one convention stated as two: a combining madda draws as `maddah` on nineteen carriers and as `fatha` on a hamza-carrying alef, 277 of 277 — the only place in the corpus where a codepoint's drawn name depends on the letter under it. The geometry leans toward it being the madda drawn short (0.89× a same-line fatha, against the maddah's one constant width), and nothing downstream needs the answer. issues.json ① is narrowed, not closed: half of "does a tajweed span land on a mark" is now an offline measurement and should be made before anyone looks at a screen. The half that needs an eye is stated as two questions instead of one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- docs/design/sub-word-marks.md | 165 ++++++++++++++++++++++++++-------- docs/issues.json | 2 +- docs/issues.md | 2 +- docs/map.json | 7 +- 4 files changed, 134 insertions(+), 42 deletions(-) diff --git a/docs/design/sub-word-marks.md b/docs/design/sub-word-marks.md index bef16c4..722e06a 100644 --- a/docs/design/sub-word-marks.md +++ b/docs/design/sub-word-marks.md @@ -6,9 +6,11 @@ > document is the measurement that settles which. **Status:** design of record for the **named-mark layer**. The vocabulary and the extraction -are built and measured over all 604 pages (§③–§⑥); **nothing ships**, on purpose (§⑥). What -is not decided is whether a tajweed span corresponds to a mark a reader can be shown, and -whether the shards are worth their bytes — §⑧ ① and ②. +are built and measured over all 604 pages (§③–§⑥); **nothing ships**, on purpose (§⑥). Which +drawn path answers to which codepoint is settled — thirty-four tokens, each pinned to one +name by elimination, validated on 62,931 runs the propagation never saw (§⑤). What is not +decided is whether a tajweed span corresponds to a mark a reader can be shown, and whether +the shards are worth their bytes — §⑧ ① and ②. ## How to read this, and what it is not @@ -205,9 +207,10 @@ The per-ligature figure is 159,585 of 159,588, but that is conditioned on the pa above, so the table states the unconditional number instead. Quoting the ligature percentage alone would silently condition it on a filter the reader cannot see. -**Six print conventions had to be learned to get there**, each read off a markup dump rather -than assumed. Earlier drafts of the join read 88.79%, then 97.75%, then 99.90%; the number -moved each time a dump explained a family, and never because a rule was added to move it. +**Eight print conventions had to be learned to get there**, each read off a markup dump +rather than assumed. Earlier drafts of the join read 88.79%, then 97.75%, then 99.90%; the +number moved each time a dump explained a family, and never because a rule was added to move +it. | the text writes | the print draws | example | |---|---|---| @@ -217,6 +220,19 @@ moved each time a dump explained a family, and never because a rule was added to | a seated hamza `أ إ ؤ ئ`, and `ٱ` | a base outline **plus** a named `hamza` / `wasla` path — **always**, whether the ligature spells `ا` or `أ` | «أَنزَلَ», «أَنَّ» | | the small high madda `ۤ` U+06E4 | `data-type="sajda-line"` — the overline of a sajda ayah, not a diacritic | «خَرُّواْۤ» (19:58) | | one letter | sometimes **two** ligatures, the second markless; and **not in reading order** | «فَلَا» → `[فلا\|ا]`, «ٱلرَّحِيمِ» → `[لر\|حيم\|ٱ]` | +| a combining madda `ٓ` U+0653 on any of nineteen carriers | a path named `maddah` — 4,682 of them | «بِمَآ», «فِيٓ», «قَالُوٓاْ» | +| the same madda on a hamza-carrying alef `أ` | a stroke named **`fatha`** — **277 times of 277**, never a `maddah` | «ٱلۡأٓخِرِ», «لِأٓدَمَ» | + +The last two rows are one convention stated as two, and it is the only place in the corpus +where a codepoint's drawn name depends on the letter under it. The split is by carrier and +nothing else — `ا` 2,959, `ي` 650, `و` 583, `ى` 392, `ه` 373, `ل` 305 and thirteen more all +take the `maddah`; `أ` takes none. Whether that is the madda drawn short or a fatha standing +in for it is a question about the print's intent, and the geometry leans the first way: the +suspect stroke measures **0.89× an ordinary fatha on the same line** (p5 0.72, p95 1.00), +while the `maddah` is drawn at one constant width corpus-wide. Nothing downstream needs the +answer — one codepoint, one path, and §⑤'s propagation pins which. It is given a token of +its own (`U+0653@hamza`) only so the codepoint→name relation stays a *function*, which is +what makes that propagation's arithmetic work at all. The last row is why the check is no longer a left-to-right walk. `align` matches ligature text to letters by **content**, as a search over which ligature draws which run, so a @@ -237,8 +253,67 @@ named rather than absorbed, because a rule for either would be a rule for one wo None of the three is an alignment error — ② already proves every mark sits inside its own word — and none costs anything downstream, because ② is what decides whether the geometry is shippable and ② is exact. What ④ bounds is how much of the corpus a *letter*-level highlight -can be offered on, and that bound is now the whole of it. That is the input to §⑧ ①, not its -answer: see §⑦ for what a count still cannot say. +can be offered on, and that bound is now the whole of it. + +### Which mark is which, without ever assuming order + +④ counts. Counting says a ligature drawing three letters carries three marks; it does not say +*which* drawn path is the tanween and which is the sukun, and a rule that wants to light the +tanween needs exactly that. The obvious answer — pair them off left to right — is not an +answer: it assumes the print draws marks in the order the text writes them, which is the +thing in question. An earlier draft of this document did zip them positionally and reported +the result as evidence the gap could not be closed; that tally was an artifact of the zip, +and §⑦ records what replaced it. + +**⑤ never looks at position.** Each agreeing ligature contributes a *bag* of codepoint tokens +beside a *bag* of drawn names, and the correspondence is recovered by elimination across the +whole corpus. If a run wants `{sukun, أ, fatha}` and the print draws `{hamza, sukun, fatha}`, +then two being pinned elsewhere forces the third — from set arithmetic, not from where it +sits. The mechanism is **arc consistency over bipartite matchings**: a pairing survives +unless *no* perfect one-to-one assignment of that run's tokens to that run's names can use +it. Plain set intersection is the wrong operator here and was tried first — it presumes the +relation is already a function, and drove `U+0653` to an empty candidate set, which is how +the madda convention above got found. + +Every run carrying **exactly one mark is held out** of the propagation, because a one-mark +run forces its own pairing and scoring against it would report 100% by construction. What is +left to learn from is only the ambiguous runs. All 604 pages, 2026-08-07: + +| | | +|---|---:| +| runs whose counts agree | 152,101 | +| held out for the test — exactly one mark | 62,931 | +| distinct token-bag / name-bag shapes to propagate over | 2,869 | +| passes to a fixpoint | 2 | +| **tokens pinned to exactly one name** | **34 of 34** | +| shapes admitting no assignment at all | 0 | + +And on the held-out runs, which the propagation never saw: + +| | | | +|---|---:|---| +| the dictionary predicts the drawn name | **62,931** | **100.00%** | +| predicts a different name | 0 | 0.00% | +| a token that never appears beside another | 0 | 0.00% | + +The dictionary is printed in full by the probe. It is thirty-four tokens because two carry +context — `U+064E+iqlab`, `U+0653@hamza` — for the composition reasons the conventions table +gives; the other thirty-two are bare codepoints. + +**Order is measured only afterwards**, once pairing has been settled without it. Of the +89,170 runs carrying two marks or more, **98.64% are drawn in the order the text writes +them**. One rule, stated before it was scored, accounts for most of the rest: + +> **R1** — a seated hamza's own sign is drawn *after* every other mark on its ligature, +> though the text writes it first. «يُؤۡمِنُونَ» writes damma, hamza, sukun and draws damma, +> sukun, hamza. + +R1 takes it to **99.56%**, leaving 390 runs (0.44%) — «شَيۡـٔٗا», «سَيِّـَٔاتِكُمۡ», +«تَسۡـَٔلُواْ» and their families, all seated-hamza words where more than the hamza moves. +A second candidate was put up the same way and **refuted by its own score**: that a shadda is +drawn after the vowel it shares a letter with, as «وَّ» suggests. It costs forty runs, +because «نُّؤۡمِنَ» and its family do not swap. It is recorded here and deliberately not +implemented — a rule the corpus contradicts is worse than no rule. ## ⑥ What it would weigh, and why nothing shipped @@ -264,32 +339,37 @@ That order is also the answer the user gave when asked where the marks should ap ## ⑦ What this cannot answer -Whether a mark is on the *right letter*. - -§⑤'s ligature join narrows this and does not close it. It shows that a ligature drawing -three letters carries the number of marks those three letters call for, for all but three -words in the corpus — which is what makes a letter-level highlight arithmetically possible -at all. But **counts are necessary and not sufficient**: agreement on three does not -establish that the second mark is over the second letter rather than the third. A word whose -marks were internally permuted would pass ④ exactly as a correct one does. - -The probe prints the evidence for this against itself. Its `codepoint → name` tally is built -*only* from ligatures whose counts agree, and it still carries **611 pairings of `U+06E1` (a -sukun) with a path named `hamza`**. «بِٱلۡأٓخِرَةِ» on p2 is one of them: the run `لأ` is -written `ۡ` then `أ` then `ٓ`, and drawn `hamza`, `sukun`, `fatha`. Three marks wanted, three -marks drawn, ④ passes — and all three pairings are wrong, the third doubly so, since the -print names the madda glyph `fatha`. - -So the tally's head is trustworthy and its tail is not, and no arithmetic distinguishes -them. That is precisely why mark-B puts the boxes on the page for a human before mark-C -ships anything that claims to know which letter a mark is on. - -Nothing in this repo closes that gap offline, because it is a correspondence between a -codepoint in a reconstructed text and an outline on a page, and in the end only a reader's -eye settles it. Containment proves a mark belongs to its word; the ligature join proves the -counts work out per run; neither says the *k*-th mark is on the *k*-th letter. That is the -inspector's job, and it is why mark-B exists as a separate step rather than a review of -mark-C. +**This section previously said the opposite, and was wrong.** It argued that no arithmetic +could say which drawn path belongs to which codepoint, and offered as proof that the probe's +own tally carried 611 pairings of `U+06E1` (a sukun) with a path named `hamza` — «بِٱلۡأٓخِرَةِ» +on p2 among them. Those 611 were an artifact of *how the tally was built*: it zipped a run's +codepoints against its paths **by position**, so every word where the print reorders — every +seated hamza, 1.36% of multi-mark runs — was miscounted by construction. The tally was +measuring its own assumption. §⑤'s propagation never zips, and the pairing it recovers is +exact and validated on 62,931 held-out runs. The paragraph is retired rather than deleted so +that the refutation is on the record, and because it is a clean example of the failure mode +this repo keeps finding: a check that shares a mistake with the thing it checks will agree +with it. + +What survives is smaller and real: **whether the ink lands where a reader's eye goes.** + +§⑤ closes *identity* — that a given path is the sukun and not the hamza — and via R1 it +predicts which box carries which name for 99.56% of multi-mark runs. Every step of that is a +correspondence between a reconstructed text and the corpus's own attributes. None of it +looks at the picture. A print that named its paths correctly and *placed* one of them a +letter to the left would satisfy ①–⑤ exactly as a correct one does, because nothing here +ever asks where the outline sits relative to the letter that wrote it. + +That is not a gap arithmetic can close from inside the file, and it does not need to be +large to matter: a tanween highlight that lights the letter beside the tanween is worse than +no highlight, because a hafiz would trust it. It is why mark-B puts the boxes on the page for +a human before mark-C ships anything, and why mark-B is a separate step rather than a review +of mark-C. + +The 390 runs R1 leaves are the concrete place to start looking — they are enumerated by +family in the probe's output, they are almost all seated-hamza words, and if the inspector +shows their marks sitting correctly then the residual is an ordering curiosity rather than a +placement defect. --- @@ -307,11 +387,18 @@ document puts 326,515 named marks inside those same words. The question is wheth meet: when `madd_246` opens at a codepoint, is there a `maddah` box there — and if there is, is highlighting *it* a truer rendering of the rule than washing the whole word? -**What would answer it:** the encoding inspector (mark-B). It already reconciles the print, -the ligature corpus, QAC and the tajweed offsets on one screen for one page; adding the mark -boxes puts all four descriptions and the geometry in one place where a human can see whether -a span and a mark coincide. Nothing offline can do this — the correspondence is between a -codepoint in a reconstructed text and an outline on a page, and only an eye closes that gap. +**What §⑤ changed about this.** Half of it is now arithmetic. With the codepoint→name +dictionary pinned, "does `madd_246` open at a codepoint the print draws a `maddah` for" is a +question the corpus answers offline, without an eye and without a guess — and that half +should be measured before anyone looks at a screen, because it is cheap and it bounds what +the looking is for. What §⑤ did **not** close is the second half. + +**What would answer the rest:** the encoding inspector (mark-B). It already reconciles the +print, the ligature corpus, QAC and the tajweed offsets on one screen for one page; adding +the mark boxes puts all four descriptions and the geometry in one place. Two things only an +eye settles there: whether the box the dictionary names sits where a reader looks for that +mark, and whether lighting *it* reads as a truer rendering of the rule than washing the +whole word. The first is §⑦'s remaining gap; the second was never a measurement at all. **What must not happen instead:** deriving the correspondence from the fact that both numbers exist. Reading a mapping off where the offsets happen to land and then declaring diff --git a/docs/issues.json b/docs/issues.json index 3ae9201..d071b99 100644 --- a/docs/issues.json +++ b/docs/issues.json @@ -465,7 +465,7 @@ "severity": "question", "owner": "agent", "blockedBy": ["the encoding inspector drawing the marks"], - "note": "Opened 2026-08-07 with mark-A. Two measurements now exist over the same words and nothing has checked whether they meet. word-indexing.md ⑪ ⑤ lands 59,975 of 60,057 tajweed annotations (99.86%) on the letter their rule names, 83.31% of them inside a single print word; sub-word-marks.md ⑤ puts 326,515 named mark boxes inside those same words, 0 of them outside. The open question is whether a span's [start,end) coincides with a mark a reader can be SHOWN — whether madd_246 opening at a codepoint means there is a maddah box there — and, if so, whether highlighting the mark is a truer rendering of the rule than washing the whole word. What would answer it: the encoding inspector (mark-B), which already reconciles the print, the ligature corpus, QAC and the tajweed offsets on one screen and would gain the mark boxes over the same frame. Nothing offline can: the correspondence is between a codepoint in a text word-indexing.md ⑪ ⑤ RECONSTRUCTS and an outline on a page, and only an eye closes that gap. What must not happen instead is deriving the correspondence from the fact that both numbers exist — reading a mapping off where the offsets happen to land and then declaring that they land there is the circularity ⑪ ⑤ names about its own oracle, and it passes on a broken answer. Note this is a rendering question and not a recitation one: DIACRITICS records what the corpus wrote in an attribute and this repo asserts nothing about how any of the twenty-six are pronounced." + "note": "Opened 2026-08-07 with mark-A. Two measurements now exist over the same words and nothing has checked whether they meet. word-indexing.md ⑪ ⑤ lands 59,975 of 60,057 tajweed annotations (99.86%) on the letter their rule names, 83.31% of them inside a single print word; sub-word-marks.md ⑤ puts 326,515 named mark boxes inside those same words, 0 of them outside. The open question is whether a span's [start,end) coincides with a mark a reader can be SHOWN — whether madd_246 opening at a codepoint means there is a maddah box there — and, if so, whether highlighting the mark is a truer rendering of the rule than washing the whole word. Narrowed 2026-08-07 by sub-word-marks.md ⑤: with the codepoint→name dictionary pinned (34 of 34 tokens, by elimination, validated on 62,931 held-out runs), the first half — does madd_246 open at a codepoint the print draws a maddah for — is now an offline measurement, and should be made before anyone looks at a screen because it is cheap and it bounds what the looking is for. What ⑤ did NOT close, and what still needs the encoding inspector (mark-B), which already reconciles the print, the ligature corpus, QAC and the tajweed offsets on one screen and would gain the mark boxes over the same frame: whether the box the dictionary NAMES sits where a reader looks for that mark, and whether lighting it reads as a truer rendering of the rule than washing the word. The first is a claim about the picture — every step of ⑤ is a correspondence between a reconstructed text and the corpus's own attributes, and none of it looks at where the outline sits relative to the letter that wrote it — and the second was never a measurement at all. What must not happen instead is deriving the correspondence from the fact that both numbers exist — reading a mapping off where the offsets happen to land and then declaring that they land there is the circularity ⑪ ⑤ names about its own oracle, and it passes on a broken answer. Note this is a rendering question and not a recitation one: DIACRITICS records what the corpus wrote in an attribute and this repo asserts nothing about how any of the twenty-six are pronounced." }, { diff --git a/docs/issues.md b/docs/issues.md index 5339882..d087f52 100644 --- a/docs/issues.md +++ b/docs/issues.md @@ -1,5 +1,5 @@ - + # Open items diff --git a/docs/map.json b/docs/map.json index e607dfb..be47aae 100644 --- a/docs/map.json +++ b/docs/map.json @@ -624,7 +624,12 @@ { "file": "packages/etl/scripts/probe-diacritics.mjs", "symbol": "const SLACK", - "note": "The measurement that had to come before anything shipped, over all 604 cached pages: the vocabulary is complete (26 of 26 names drawn), every one of 326,515 marks lands inside the word box the app already ships (0 escapes, 0 unmatched, slack 0.2 — rounding, not registration), and the shard tree would weigh 7.35 MB raw / 2.28 MB gz. Containment is checked against the *committed* shards rather than boxes computed in the same pass, so the two sides cannot share a mistake and agree about it. It ships nothing on purpose: 2.28 MB of assets no caller fetches is the same waste `gate:assets` names for a non-vendored edition. What it cannot answer is whether a mark is on the right *letter* — that needs an eye, and belongs in the encoding inspector." + "note": "The measurement that had to come before anything shipped, over all 604 cached pages: the vocabulary is complete (26 of 26 names drawn), every one of 326,515 marks lands inside the word box the app already ships (0 escapes, 0 unmatched, slack 0.2 — rounding, not registration), and the shard tree would weigh 7.35 MB raw / 2.28 MB gz. Containment is checked against the *committed* shards rather than boxes computed in the same pass, so the two sides cannot share a mistake and agree about it. It ships nothing on purpose: 2.28 MB of assets no caller fetches is the same waste `gate:assets` names for a non-vendored edition. Five questions, and the order between them is the argument: ④ establishes that the counts work out per ligature, and only then does ⑤ establish which path is which. What none of them can answer is whether the ink lands where a reader's eye goes — that needs an eye, and belongs in the encoding inspector." + }, + { + "file": "packages/etl/scripts/probe-diacritics.mjs", + "symbol": "function supported(want, got)", + "note": "⑤'s engine, and the reason the codepoint→name dictionary is evidence rather than an assumption. Arc consistency over bipartite matchings: a run contributes a *bag* of codepoint tokens beside a *bag* of drawn names, and a pairing is deleted only when no perfect one-to-one assignment of that run can use it. Position is never consulted — pairing them off left to right would assume the print draws marks in written order, which is the thing in question, and 1.36% of multi-mark runs say otherwise. Plain set intersection is the wrong operator and was tried first: it presumes the relation is already a function, and drove `U+0653` to an empty candidate set, which is how the `أ`-carrier madda convention was found. Every single-mark run is held out, because a one-mark run forces its own pairing and scoring against it would report 100% by construction. Result: 34 of 34 tokens pinned in two passes, 62,931 of 62,931 held-out runs predicted correctly." } ], "extend": [ From 2d52bfdcc853fa84f417751c4a86a1c43bfa55b7 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 14:07:01 -0500 Subject: [PATCH 7/8] The offset walks down to a rectangle, and half the rules have nowhere to land MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lib/mark-join.mjs` extracts ④ and ⑤'s arithmetic out of `probe-diacritics.mjs` so there is exactly one of it: `letters` folds `\p{Mn}` and `\p{Lm}` onto the letter before, `align` matches ligatures to letter runs by content with backtracking, and `pairMarks` pairs a wanted token to a drawn name through the frozen 34-entry `DRAWN_NAME`. `markPaths` is the one entry point a caller should want — a word in, `[{ at, len, token, name, mark }]` out, with `at` a codepoint index into that word's own `data-hafs`, which is the bridge a tajweed offset needs. It refuses rather than approximates. `null` for a mark count that disagrees, a name the tokens did not ask for, or letters no assignment of ligatures can cover — because a partial answer here would look exactly like an answer, which is the defect ④'s left-to-right walk already shipped once while every length balanced. `mark-join.test.mjs` is 28 tests on words small enough to count by hand and most of them assert one of those refusals. `probe-encodings.mjs --marks` is the caller. It walks all 60,057 tajweed annotations from a Tanzil offset down to a drawn path and counts the outcome into eight named classes rather than a rate: drawn 28535 47.51% a named path — the rectangle to light letter 30943 51.52% a base letter, with nothing above it respelt 497 0.83% the offset does not address `data-hafs` oracle-miss 82 0.14% no position to resolve in the first place no-host / no-word / unjoined / basmala 0 0.00% The 51.52% is an answer. Ten of the eighteen rules name a consonant — qalqalah on ق, lam_shamsiyyah on ل, ghunnah on ن or م — and the print draws a consonant as a letter outline. The eight that name a mark reach one almost always: hamzat_wasl → wasla 98.11%, madd_2 98.58%, iqlab 99.82%. So the finding is a split rather than a rate, and any mark-granular UI has to say which of the two it is doing. The inspector draws the boxes with the selected annotation's own mark lit inside its word. Hollow and hairline: the outlines-yes-ink-no rule holds a level finer, and it is under more pressure there, because a mark's box is small enough that filling it in would read as the mark itself. `--marks` stays opt-in for one reason and it is a size: 326,515 more rectangles takes the report from 5.0 MB to 13.8 MB. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- .../scripts/lib/encoding-inspector.client.mjs | 58 ++- .../etl/scripts/lib/encoding-inspector.css | 9 + packages/etl/scripts/lib/mark-join.mjs | 420 ++++++++++++++++++ packages/etl/scripts/lib/mark-join.test.mjs | 217 +++++++++ packages/etl/scripts/probe-diacritics.mjs | 312 ++++--------- packages/etl/scripts/probe-encodings.mjs | 220 ++++++++- 6 files changed, 1007 insertions(+), 229 deletions(-) create mode 100644 packages/etl/scripts/lib/mark-join.mjs create mode 100644 packages/etl/scripts/lib/mark-join.test.mjs diff --git a/packages/etl/scripts/lib/encoding-inspector.client.mjs b/packages/etl/scripts/lib/encoding-inspector.client.mjs index 8dcd6cf..2ca87fb 100644 --- a/packages/etl/scripts/lib/encoding-inspector.client.mjs +++ b/packages/etl/scripts/lib/encoding-inspector.client.mjs @@ -24,7 +24,7 @@ * on a checkbox, and it means the number under the toggle is measured rather * than remembered. */ -/* global CORRECTIONS, ALL_CORRECTIONS, DRIFT_LIMIT, foldAyah, touchClass, touched, oracleOf, driftOnset, driftShape, oracleLabel, oracleDensity, nameOf, nameWindow, HIFTH_DATA */ +/* global CORRECTIONS, ALL_CORRECTIONS, DRIFT_LIMIT, foldAyah, respellerFor, touchClass, touched, oracleOf, driftOnset, driftShape, oracleLabel, oracleDensity, nameOf, nameWindow, HIFTH_DATA */ const DATA = HIFTH_DATA; const $ = (sel, root = document) => root.querySelector(sel); @@ -412,18 +412,74 @@ function outline(key, f, sel) { ); }); + // ⑥ — the level below a word, when the report was generated with `--marks`. + // + // Drawn last so a mark paints over its own word's box rather than under the + // next one's, and drawn as rectangles for the same reason the words are: the + // rule is outlines, not ink. A mark's box is inside its word's box by + // construction — the word's box is the union over every path in its segment, + // marks included — so an escape here would be an alignment error, and seeing + // one is the point of drawing them at this scale. + const rows = DATA.diacritics ? (f.entry.d ?? null) : null; + let marksDrawn = 0; + let unjoined = 0; + let unaddressable = 0; + if (rows) { + const respell = respellerFor(state.on); + for (const [printStr, list] of Object.entries(rows)) { + const print = Number(printStr); + if (list === null) { + unjoined += 1; + continue; + } + const host = f.hosts[hostOfPrint.get(print)]; + const hafs = f.words[print - 1]?.hafs; + // A respelt word spends a different number of codepoints than the string + // the offsets address, so `at` is not a fold offset there. Its marks are + // still drawn — the geometry is unaffected — and simply never lit. + const addressable = host !== undefined && hafs !== undefined && respell(hafs) === hafs; + if (!addressable) unaddressable += 1; + for (const [at, len, id, x, y, w, h] of list) { + marksDrawn += 1; + const from = addressable ? host.from + at : null; + const isLit = sel && from !== null && from < sel.end && from + len > sel.start; + const span = len > 1 ? `${at}–${at + len - 1}` : String(at); + svg.append( + svgEl( + "rect", + { class: `mk${isLit ? " lit" : ""}`, x, y, width: w, height: h }, + svgEl("title", { + text: + `${DATA.diacritics[id] ?? `#${id}`} · ${hafs ?? ""} codepoint ${span} · ` + + (from === null ? "respelt here, so it has no fold offset" : `fold ${from}`), + }), + ), + ); + } + } + } + const wrap = el("div", { class: "outline-wrap" }, svg); const bits = []; if (!trusted) { bits.push(el("p", { class: "warn", text: `${boxes.length} boxes but ${f.words.length} print words. Two independent descriptions of this ayah disagree on how many words it has, so the outline draws the geometry and deliberately does not label it — a guessed alignment here would be a picture that lies quietly. Worth filing.` })); } bits.push(wrap); + if (rows) { + const parts = [`${marksDrawn} named marks drawn inside these boxes`]; + if (unaddressable) parts.push(`${unaddressable} word(s) respelt by the fold, so their marks carry no offset and never light`); + if (unjoined) parts.push(`${unjoined} word(s) the ligature join could not resolve, so their marks are absent here`); + bits.push(el("p", { class: "note", text: `${parts.join("; ")}. Hover a mark for its name and the codepoint it was drawn for.` })); + } else if (DATA.diacritics === null) { + bits.push(el("p", { class: "note", text: "Generated without --marks, so the level below a word is not in this report. Re-run probe-encodings.mjs with --marks to draw it." })); + } bits.push(el("p", { class: "note legend" }, [ ["even", "word"], ["mark", "pause mark"], ["gap", "dropped by the fold"], ["lit", "touched by the selected annotation"], ["other", "the rest of the page"], + ...(rows ? [["mk", "a named mark"]] : []), // Each swatch and its label are one inline-flex item, so a wrap breaks // between pairs rather than stranding a swatch on the line above its word. ].map(([cls, label]) => el("span", { class: "item" }, [el("span", { class: `sw ${cls}` }), el("span", { text: label })])))); diff --git a/packages/etl/scripts/lib/encoding-inspector.css b/packages/etl/scripts/lib/encoding-inspector.css index 2144ede..ea8db21 100644 --- a/packages/etl/scripts/lib/encoding-inspector.css +++ b/packages/etl/scripts/lib/encoding-inspector.css @@ -135,6 +135,14 @@ svg.outline .box.mark { fill: none; stroke: var(--accent); stroke-dasharray: 1.5 svg.outline .box.lit { fill: var(--span); stroke: var(--accent); stroke-width: 1; } svg.outline .box.untrusted { fill: none; stroke: var(--bad); stroke-width: 0.6; cursor: default; } svg.outline .box:hover { stroke: var(--accent); stroke-width: 1; } +/* ⑥ the mark level. A named path's own rectangle, hollow and hairline — it sits + inside a word box that is already filled, so a fill here would just repaint + the word. `pointer-events` stays on so a mark can be hovered for its name, + and the stroke is thin enough that two adjacent marks do not merge into one + smear at the scale a whole page is drawn at. */ +svg.outline .mk { fill: none; stroke: var(--dim); stroke-width: 0.25; } +svg.outline .mk.lit { stroke: var(--accent); stroke-width: 0.6; } +svg.outline .mk:hover { stroke: var(--accent); stroke-width: 0.8; } .warn { color: var(--bad); font-size: 13px; max-width: 78ch; border-left: 3px solid var(--bad); padding-left: 10px; @@ -147,6 +155,7 @@ svg.outline .box:hover { stroke: var(--accent); stroke-width: 1; } .legend .sw.gap { border-style: dashed; } .legend .sw.lit { background: var(--span); border-color: var(--accent); } .legend .sw.other { background: var(--line); opacity: 0.55; border-color: var(--line); } +.legend .sw.mk { background: none; border-color: var(--dim); height: 6px; width: 8px; } .diff { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 14px 16px; } .verdict { font-size: 14px; } diff --git a/packages/etl/scripts/lib/mark-join.mjs b/packages/etl/scripts/lib/mark-join.mjs new file mode 100644 index 0000000..4d10877 --- /dev/null +++ b/packages/etl/scripts/lib/mark-join.mjs @@ -0,0 +1,420 @@ +/** + * The join from a codepoint to the path the print drew for it. + * + * `lib/diacritics.mjs` reads the marks out of a page: rectangles with names, + * grouped by word and by the ligature inside it. It stops there deliberately — + * its own header says a caller "does its own arithmetic". This file is that + * arithmetic, extracted once so there is exactly one of it. + * + * Three steps, and each was measured before it was written down. + * `probe-diacritics.mjs` is where to read *why* each has the shape it has; its + * ④ and ⑤ are the evidence, and what moved here is only the maths. + * + * 1. **{@link letters}** — a word's `data-hafs` as the letters the print draws + * an outline for, each carrying the codepoints written on it. + * 2. **{@link align}** — which ligature draws which run of those letters. This + * is the only join the corpus offers between a mark and a codepoint, and ④ + * reports it holding for 86,962 of 86,965 lettered words. + * 3. **{@link DRAWN_NAME} + {@link pairMarks}** — which *drawn path* answers to + * which written codepoint. ⑤ recovered the dictionary by elimination and + * validated it on 62,931 runs it was never shown. + * + * ## Why this is a lib and not more of the probe + * + * Two callers, the same reason `tajweed-fold.mjs` gives about itself. The probe + * derives the dictionary from the corpus and checks it; `probe-encodings.mjs` + * *uses* it, to ask whether a tajweed span opens on a codepoint the print drew + * a mark for and to put that mark's rectangle on a screen. If the inspector's + * join and the probe's join were two implementations, a clean screen would stop + * being evidence about the probe — which is the one failure a diagnostic tool + * must not have. + */ +import { diacriticName } from "@hifth/core"; + +/** + * A word's `data-hafs` as the letters the print draws an *outline* for, each + * carrying the codepoints written on it: `بِسْمِ` → `[ب:[ِ], س:[ْ], م:[ِ]]`. + * + * Two Unicode categories are not outlines and fold into the letter before them: + * + * - **`\p{Mn}`**, the combining marks. Obvious, and the reason this exists. + * - **`\p{Lm}`**, the modifier letters — and this one is the whole reason ④'s + * first draft disagreed. Three of them occur in this text: the tatweel + * `U+0640` that seats a hamza in `شَيۡـٔٗا`, and the small waw `U+06E5` and + * small yeh `U+06E6` of `بِهِۦ`. The text calls all three letters. The print + * does not: the tatweel is drawn as a tooth folded into its neighbour's + * ligature, and the two small letters are drawn as `data-diacritic="small + * waw"` and `"small yeh"` — named marks, sitting in `DIACRITICS` beside the + * fatha. Counting them as base letters made the partition off by one for + * every word containing a seated hamza. + * + * Both rules are Unicode's own categories rather than a codepoint list this + * repo maintains, because a list would be a third place with an opinion about + * Arabic marks and would drift from the other two. + * + * `at` is the **codepoint** index into `hafs`, on both the letter and each mark, + * and it is the whole reason a tajweed offset can reach a rectangle: the offsets + * count codepoints, and so does this. It is carried rather than recomputed + * because folding a `\p{Lm}` into its neighbour destroys the correspondence and + * a caller that tried to recover it would be guessing. + * + * The class is `const` and shared with `align`, which has to fold a ligature's + * `data-text` by exactly the same rule for the two to be comparable at all. + * `FOLD` tests one character and `FOLDS` strips a run; they are the same class + * written once, because two copies of it would be the drift this paragraph is + * about. + */ +const FOLD_CLASS = "[\\p{Mn}\\p{Lm}]"; +const FOLD = new RegExp(FOLD_CLASS, "u"); +const FOLDS = new RegExp(FOLD_CLASS, "gu"); + +export function letters(hafs) { + const out = []; + let at = 0; + for (const c of hafs) { + if (FOLD.test(c) && out.length) out[out.length - 1].marks.push({ ch: c, at }); + else out.push({ letter: c, at, marks: [] }); + at += 1; + } + return out; +} + +/** + * A hamza form written as one codepoint, and the carrier it is written on. + * + * Two separate facts live here, and conflating them cost a pass of the corpus. + * + * **The print always draws the sign.** `أ` gets a `hamza` path, `ٱ` a `wasla` + * path, every time, in all 9,168 and 13,476 places they occur. The ligature's + * own spelling does *not* decide it: «أَنَّ» on p119 is drawn `[أ | ن]` and the + * first ligature still carries `hamza` then `fatha`. Making the expectation + * conditional on the ligature spelling the bare carrier — which a first reading + * of «أَيۡدِيهِمۡ» seemed to show — put 151 words on seven pages into the + * residual, and the markup dump said plainly why. + * + * **The spelling still matters for matching.** `align` compares a ligature's + * `data-text` to the word's letters, and the two disagree about the carrier: a + * ligature may spell `ا` where the word writes `أ`. So `base()` folds a hamza + * form to its carrier for that comparison only, and never for what the print + * is expected to draw. + * + * The bare hamza `ء` `U+0621` is deliberately absent: with no carrier to sit + * on it is drawn as `data-type="text"` like any other letter, always. + */ +const HAMZA_ON = { آ: "ا", أ: "ا", إ: "ا", ٱ: "ا", ؤ: "و", ئ: "ي" }; + +/** The letter under a hamza form, for matching a ligature's text to the word's. */ +const base = (c) => HAMZA_ON[c] ?? c; + +/** A short vowel or a tanween — the thing an iqlab meem merges into. */ +const VOWEL = /[ً-ِٗٞ]/; + +/** + * The iqlab meem, which the print never draws on its own beside a vowel — both + * of the forms this text uses. `كَافِرِۭ` writes the final form `U+06ED` and + * `رِكۡزَۢا` the isolated `U+06E2`; the print composes either with the vowel + * before it into one `kasra iqlab` / `fatha iqlab` glyph. + */ +const IQLAB = /[ۭۢ]/; + +/** + * The tatweel. It folds like a mark, because the print does not give it a + * ligature of its own — but unlike the small waw and small yeh it folds beside, + * it is drawn as part of the neighbouring outline (the tooth that seats a hamza in + * `شَيۡـٔٗا`), not as a named path. So it is invisible to the partition on both + * sides: not a letter, and not a mark either. + */ +const TATWEEL = "ـ"; + +/** + * `U+06E4`, the small high madda — which in this print is not a mark at all. + * + * Every word carrying it sits in a sajda ayah (13:15, 17:107, 19:58 …) and the + * print draws it as `data-type="sajda-line"`: the overline stretched above the + * phrase a reader prostrates at, not a diacritic over a letter. It has no + * `data-diacritic`, so `readDiacritics` never sees it, and expecting one for it + * was counting a rubric as a vowel. + */ +const SAJDA_LINE = "ۤ"; + +/** + * `U+0653` on a `أ` — the one place the print refuses its own `maddah`. + * + * Every other carrier of a combining madda gets a path named `maddah`: the alef + * of «بِمَآ», the yeh of «فِيٓ», the waw of «قَالُوٓاْ», nineteen letters in all, + * 4,682 paths. The hamza-carrying alef gets one **277 times out of 277 and never + * a `maddah`** — the print draws a stroke it names `fatha`, and the outline + * bears that out: measured against an ordinary fatha on the same line it is a + * shortened version of the same curve (median 0.89×, p5 0.72×), not the maddah's + * hooked wave, which is drawn at one constant width throughout the corpus. + * + * Whether that is the madda rendered short or a fatha standing in for it is a + * question about the print's intent, and this file does not have to answer it: + * one codepoint, one path, and ⑤ pins which. It is given a token of its own only + * so that the relation stays a *function*, which is what makes ⑤'s arithmetic + * work — without it `U+0653` maps to two names and arc consistency correctly + * reports a contradiction rather than a dictionary. + */ +const MADDA = "ٓ"; +const MADDA_ON_HAMZA = "أ"; + +const cp = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; + +/** Does a token name a vowel? `parseInt` stops at the `+` or `@` of a suffix. */ +const isVowel = (token) => VOWEL.test(String.fromCodePoint(parseInt(token.slice(2), 16))); + +/** + * The *tokens* of a letter — one per named path the print is expected to draw, + * in the order the text writes them, each with the codepoint index it came from. + * + * A token is usually just a codepoint, `"U+064E"`. Two carry context, because + * the print composes and the composite has a name of its own: + * + * - `"U+064E+iqlab"` — `DIACRITICS` carries `fatha iqlab`, `kasra iqlab` and + * `damma iqlab` as names in their own right, so where the text writes a vowel + * followed by `ۭ` the print draws a single glyph: «كَافِرِۭ» is two paths on + * `فر`, not three. Its `[at, at+len)` spans **both** codepoints, because one + * rectangle is the answer for either of them. + * - `"U+0653@hamza"` — see `MADDA_ON_HAMZA` above. + * + * The hamza or wasla sign comes first where the letter is a hamza form, because + * that is the order the text writes it in, and it takes the *letter's* own index + * — the sign is not written separately, so the codepoint that asks for it is the + * carrier. The print often disagrees about the order — «أَنَّ» draws `hamza` then + * `fatha` but «ٱلۡمَلَؤُاْ» draws `damma` then `hamza` — and that disagreement is + * not swept up here: ④ compares counts and is blind to it, ⑤ measures it, and + * {@link pairMarks} never depends on it. + * + * Excluded are the two codepoints the print draws by other means: the tatweel's + * tooth, folded into a neighbour, and the sajda overline. + */ +export function expected(l) { + const out = []; + if (HAMZA_ON[l.letter]) out.push({ token: cp(l.letter), at: l.at, len: 1 }); + for (const m of l.marks) { + if (m.ch === TATWEEL || m.ch === SAJDA_LINE) continue; + const prev = out[out.length - 1]; + if (IQLAB.test(m.ch) && prev && isVowel(prev.token)) { + prev.token += "+iqlab"; + prev.len = m.at - prev.at + 1; + continue; + } + out.push({ + token: m.ch === MADDA && l.letter === MADDA_ON_HAMZA ? `${cp(m.ch)}@hamza` : cp(m.ch), + at: m.at, + len: 1, + }); + } + return out; +} + +/** + * Assign each ligature the letters it draws, or `null` if no assignment exists. + * + * The obvious implementation — walk the ligatures in document order, handing + * each the next `text.length` letters — is what ④'s previous draft did, and it + * is wrong in two ways the markup shows plainly: + * + * **Document order is not reading order.** «ٱلرَّحِيمِ» on p379 is drawn as + * `[لر | حيم | ٱ]`: the alef wasla is a separate ligature emitted *last*. Six + * letters, six drawn, so a length check passes — and then every mark is + * assigned to the wrong letter while the totals still balance. That is the + * failure mode this whole file exists to catch, and counting alone cannot see + * it. + * + * **A letter can be drawn twice.** «فَلَا» is `[فلا | ا]` — four letters drawn + * for a three-letter word, because the print puts the alef's stroke in a second + * ligature. Those continuation runs carry no marks of their own, which is what + * makes them safe to recognise: a repeat that carried marks would be a + * different phenomenon and would still fail here. + * + * So this matches on **content** rather than length, over `base()` so that a + * ligature spelling `ا` matches a word writing `أ`. A ligature may take the + * next letters, or re-draw letters already taken if it has no marks. The search + * is a DFS over (position, set of ligatures used) with memoisation; words have + * a handful of ligatures, so the state space is tiny. + * + * Both sides are reduced the same way, which is the only thing that makes the + * comparison meaningful: `letters` folds a `\p{Lm}` into the letter before it, + * so a ligature's `data-text` has to be folded too. It carries the tatweel — + * «مَـَٔابٗا» is drawn `[مـا | با]`, tatweel and all — and leaving it in made + * ten seated-hamza words on the last two juz look unassignable when they are + * simply spelt with the tooth the print draws them with. + * + * Matching on content is strictly stronger than the length check it replaces — + * some words that used to pass the partition now fail it, and that is the point. + */ +export function align(ls, ligs) { + const target = ls.map((l) => base(l.letter)).join(""); + const texts = ligs.map((l) => [...l.text.replace(FOLDS, "")].map(base)); + const all = (1 << ligs.length) - 1; + const memo = new Map(); + + const go = (pos, used) => { + if (pos === target.length && used === all) return []; + const key = pos * (all + 1) + used; + if (memo.has(key)) return memo.get(key); + let out = null; + for (let i = 0; i < ligs.length && !out; i += 1) { + if (used & (1 << i)) continue; + const t = texts[i]; + const fits = (from) => from >= 0 && t.every((c, j) => target[from + j] === c); + if (pos + t.length <= target.length && fits(pos)) { + const rest = go(pos + t.length, used | (1 << i)); + if (rest) out = [{ lig: i, from: pos, to: pos + t.length }, ...rest]; + } + if (!out && !ligs[i].marks.length && fits(pos - t.length)) { + const rest = go(pos, used | (1 << i)); + if (rest) out = [{ lig: i, from: pos - t.length, to: pos, redraw: true }, ...rest]; + } + } + memo.set(key, out); + return out; + }; + return go(0, 0); +} + +// ----------------------------------------------------- which mark is which -- + +/** + * What the print draws for each codepoint the text writes — thirty-four tokens, + * each pinned to exactly one name. + * + * **This table is a result, not a premise.** It was recovered by + * `probe-diacritics.mjs` ⑤ from 152,101 ligatures whose mark counts agree, by + * arc consistency over bipartite matchings: a pairing is deleted only when no + * perfect one-to-one assignment of a run's token-bag to its name-bag can use it, + * so nothing here rests on the order the print happens to emit its paths in. The + * fixpoint arrives in two passes with 34 of 34 tokens pinned and none left open, + * and it predicts the drawn name on **62,931 of 62,931** single-mark runs that + * were held out of the propagation. `docs/design/sub-word-marks.md` §⑤ is the + * write-up. + * + * It is frozen here rather than re-derived per caller because the derivation + * needs the whole 380 MB cache and two minutes, and a tool that draws one page + * cannot pay that. The guard against the copy going stale is that ⑤ still + * derives it every run and **fails** if the two disagree — so this table cannot + * quietly drift from the corpus it describes, and a corpus that changed its mind + * about a codepoint is a loud error rather than a wrong rectangle. + * + * `U+0653` appearing twice is the one convention in the print where a + * codepoint's drawn name depends on the letter under it; see `MADDA_ON_HAMZA`. + */ +export const DRAWN_NAME = Object.freeze({ + "U+0623": "hamza", // أ alef with hamza above + "U+0624": "hamza", // ؤ waw with hamza above + "U+0625": "hamza", // إ alef with hamza below + "U+0626": "hamza", // ئ yeh with hamza above + "U+064B": "fathatan", + "U+064C": "dammatan", + "U+064D": "kasratan", + "U+064E": "fatha", + "U+064E+iqlab": "fatha iqlab", // the composed glyph, not two paths + "U+064F": "damma", + "U+064F+iqlab": "damma iqlab", + "U+0650": "kasra", + "U+0650+iqlab": "kasra iqlab", + "U+0651": "shadda", + "U+0652": "rounded zero", // the print's sukun-of-silence, drawn as a ring + "U+0653": "maddah", + "U+0653@hamza": "fatha", // the one carrier-dependent name in the table + "U+0654": "hamza", + "U+0655": "hamza", + "U+0656": "successive kasratan", + "U+0657": "successive fathatan", + "U+065E": "successive dammatan", + "U+0670": "superscript alef", + "U+0671": "wasla", + "U+06DC": "small seen", + "U+06E0": "rectangular zero", + "U+06E1": "sukun", + "U+06E2": "small meem", + "U+06E5": "small waw", + "U+06E6": "small yeh", + "U+06E7": "small yeh", + "U+06E8": "small noon", + "U+06EA": "vowel sign", + "U+06EC": "vowel sign", +}); + +/** + * Which drawn path answers to which token, inside one ligature. + * + * `tokens` is {@link expected}'s output for the letters the ligature draws, in + * written order; `marks` is the ligature's paths as `readDiacritics` returns + * them, `[id, x, y, w, h]`, in document order. Returns indices into `marks` + * parallel to `tokens`, or `null` if the two bags do not match — which for a + * count-agreeing ligature cannot happen unless {@link DRAWN_NAME} has gone + * stale, and is exactly what a caller should refuse to draw through. + * + * **Position is not used to pair.** The dictionary names every token, so a + * ligature whose tokens name distinct paths is settled outright and the print's + * emission order is irrelevant — which matters, because ⑤ measured that order + * disagreeing with the text's on 1.36% of multi-mark runs. + * + * A tie is the only place anything is assumed: two tokens in one ligature that + * name the *same* path — a `U+0653@hamza` fatha beside a real fatha, say. The + * tiebreak is **geometry, not document order**: Arabic is set right to left, so + * among paths sharing a name the rightmost is the one the text writes first. + * That is an independent signal rather than a restatement of the question, and + * `probe-diacritics.mjs` ⑤ reports how often the two orders disagree. + */ +export function pairMarks(tokens, marks) { + const byName = new Map(); + marks.forEach((m, i) => { + const n = diacriticName(m[0]); + if (!byName.has(n)) byName.set(n, []); + byName.get(n).push(i); + }); + for (const list of byName.values()) list.sort((a, b) => marks[b][1] - marks[a][1]); + + const cursor = new Map(); + const out = []; + for (const t of tokens) { + const want = DRAWN_NAME[t.token]; + const list = want === undefined ? undefined : byName.get(want); + const k = cursor.get(want) ?? 0; + if (!list || k >= list.length) return null; + cursor.set(want, k + 1); + out.push(list[k]); + } + return out; +} + +/** + * Every codepoint of one word that the print drew a named path for, with the + * path. The whole file in one call, and the only entry point a caller needs. + * + * `word` is one entry from `readDiacritics` — `{ hafs, ligatures, marks }`. + * Returns `[{ at, len, token, name, mark }]` sorted by `at`, where `mark` is the + * `[id, x, y, w, h]` the print drew and `[at, at+len)` is the codepoint range + * into `hafs` that asks for it. `at` counts **codepoints**, which is what the + * tajweed offsets count. + * + * Returns `null` — never a partial answer — when the word does not join: no + * assignment of ligatures to letters exists, a ligature's mark count disagrees, + * or a bag does not pair. Those are ④'s and ⑤'s residual, three words in 86,965 + * and a stale dictionary respectively, and a caller that drew through them would + * be putting a rectangle under a codepoint that did not ask for it — the exact + * off-by-one this repo has already shipped once. + */ +export function markPaths(word) { + const ls = letters(word.hafs); + const plan = align(ls, word.ligatures); + if (!plan) return null; + + const out = []; + for (const step of plan) { + if (step.redraw) continue; // a second stroke of a letter already drawn, markless by construction + const lig = word.ligatures[step.lig]; + const tokens = ls.slice(step.from, step.to).flatMap(expected); + if (tokens.length !== lig.marks.length) return null; + if (!tokens.length) continue; + const pairing = pairMarks(tokens, lig.marks); + if (!pairing) return null; + tokens.forEach((t, i) => { + out.push({ at: t.at, len: t.len, token: t.token, name: DRAWN_NAME[t.token], mark: lig.marks[pairing[i]] }); + }); + } + return out.sort((a, b) => a.at - b.at); +} diff --git a/packages/etl/scripts/lib/mark-join.test.mjs b/packages/etl/scripts/lib/mark-join.test.mjs new file mode 100644 index 0000000..4835d4b --- /dev/null +++ b/packages/etl/scripts/lib/mark-join.test.mjs @@ -0,0 +1,217 @@ +/** + * The join, on words small enough to count by hand. + * + * `probe-diacritics.mjs` ④ and ⑤ are the measurement — 86,962 words joined and + * 34 tokens pinned to one name each — and they need 380 MB of gitignored cache + * to say anything, which is why they are a probe and not a gate. What a test + * *can* hold is the arithmetic and, more importantly, the **refusals**: this + * file's whole value downstream is that a caller can trust a non-null answer, + * and it can only trust one if a partial answer is impossible to get. + * + * So most of what is asserted below is that `markPaths` returns `null` — for a + * mark count that disagrees, for a name the tokens did not ask for, for letters + * no assignment of ligatures can cover. A version of this file that guessed + * would pass a happy-path test and would be exactly the defect this repo has + * already shipped once, where a length check agreed while every mark was + * misassigned (④'s left-to-right walk). + * + * Marks are written as `[id, x, y, w, h]` with ids from `@hifth/core`'s + * `DIACRITICS`, looked up by name rather than pasted as numbers — a reordering + * of that table must not silently rewrite what this file claims. + */ +import { diacriticId } from "@hifth/core"; +import { describe, expect, it } from "vitest"; +import { DRAWN_NAME, align, expected, letters, markPaths, pairMarks } from "./mark-join.mjs"; + +/** A mark at `x`, named. Only the name and the x ever matter here. */ +const m = (name, x = 0) => [diacriticId(name), x, 0, 1, 1]; + +/** A word in the shape `readDiacritics` returns, minus the fields the join ignores. */ +const word = (hafs, ligatures) => ({ hafs, ligatures }); + +/** `{ text, marks }` without the ceremony. */ +const lig = (text, marks = []) => ({ text, marks }); + +describe("letters", () => { + it("folds every combining mark onto the letter before it, carrying its index", () => { + // «بِهِۦ» — beh, kasra, heh, kasra, small yeh. + const ls = letters("بِهِۦ"); + expect(ls.map((l) => l.letter)).toEqual(["ب", "ه"]); + expect(ls[0]).toMatchObject({ at: 0, marks: [{ ch: "ِ", at: 1 }] }); + expect(ls[1].at).toBe(2); + expect(ls[1].marks.map((x) => x.at)).toEqual([3, 4]); + }); + + it("counts in codepoints, so `at` is an index into the string the print wrote", () => { + const hafs = "شَيۡـٔٗا"; + const ls = letters(hafs); + for (const l of ls) { + expect([...hafs][l.at]).toBe(l.letter); + for (const mk of l.marks) expect([...hafs][mk.at]).toBe(mk.ch); + } + }); + + it("a leading combining mark has no letter to fold onto and stands alone", () => { + // Not a word the print writes; the guard exists so a corrupt `data-hafs` + // cannot index past the start of the array. + expect(letters("ِب").map((l) => l.letter)).toEqual(["ِ", "ب"]); + }); +}); + +describe("expected", () => { + const tokensOf = (hafs) => letters(hafs).flatMap(expected).map((t) => t.token); + + it("gives a seated hamza its own path, always", () => { + // «أَنَّ» is drawn `[أ|ن]` and still carries hamza then fatha — the + // conditional version of this cost 151 words. + expect(tokensOf("أَ")).toEqual(["U+0623", "U+064E"]); + expect(tokensOf("ٱل")).toEqual(["U+0671"]); + }); + + it("does not give a bare hamza one — the print draws it as a letter", () => { + expect(tokensOf("ء")).toEqual([]); + }); + + it("folds an iqlab meem into the vowel before it, as one composite glyph", () => { + // «كَافِرِۭ» — the kasra and U+06ED are one path the corpus calls `kasra iqlab`. + expect(tokensOf("رِۭ")).toEqual(["U+0650+iqlab"]); + expect(tokensOf("رَۢ")).toEqual(["U+064E+iqlab"]); + }); + + it("spans the vowel and the meem together, so a rule on either reaches the path", () => { + const t = letters("رِۭ").flatMap(expected); + expect(t).toEqual([{ token: "U+0650+iqlab", at: 1, len: 2 }]); + }); + + it("draws nothing for a tatweel or a sajda overline", () => { + // «مَـَٔابٗا» — the tatweel is a tooth folded into its neighbour and gets no + // path; the two fathas and the hamza-above on it do. U+06E4 is the sajda + // overline, `data-type="sajda-line"`, which is not a named mark. + expect(tokensOf("مَـَٔ")).toEqual(["U+064E", "U+064E", "U+0654"]); + expect(tokensOf("اۤ")).toEqual([]); + }); + + it("names a madda on a seated hamza differently from a madda anywhere else", () => { + // The one place in the dictionary where a codepoint's drawn name depends on + // the letter under it — 277 of 277, measured, not assumed. + expect(tokensOf("أٓ")).toEqual(["U+0623", "U+0653@hamza"]); + expect(tokensOf("مٓ")).toEqual(["U+0653"]); + expect(DRAWN_NAME["U+0653@hamza"]).toBe("fatha"); + expect(DRAWN_NAME["U+0653"]).toBe("maddah"); + }); +}); + +describe("align", () => { + it("matches on content, not on order — ligature order is not reading order", () => { + // «ٱلرَّحِيمِ» is drawn `[لر|حيم|ٱ]`, and a left-to-right walk got this wrong + // while every length balanced. + const ls = letters("ٱلرَّحِيمِ"); + const plan = align(ls, [lig("لر"), lig("حيم"), lig("ٱ")]); + expect(plan).not.toBeNull(); + const drawn = plan.filter((s) => !s.redraw).map((s) => s.lig); + expect(drawn).toEqual([2, 0, 1]); + }); + + it("allows a letter to be redrawn, markless, in a second ligature", () => { + // «فَلَا» → `[فلا|ا]`: the alef is drawn twice and the second stroke carries + // nothing. That second step is `redraw`, and `markPaths` skips it. + const plan = align(letters("فَلَا"), [lig("فلا"), lig("ا")]); + expect(plan).not.toBeNull(); + expect(plan.some((s) => s.redraw)).toBe(true); + }); + + it("returns null when no assignment covers the letters", () => { + expect(align(letters("فَلَا"), [lig("فل")])).toBeNull(); + }); + + it("returns null for a word with no ligatures at all", () => { + expect(align(letters("فَلَا"), [])).toBeNull(); + }); +}); + +describe("pairMarks", () => { + const tokens = (hafs) => letters(hafs).flatMap(expected); + + it("pairs by the name the dictionary predicts, not by position", () => { + // «أَ» wants [hamza, fatha]; the print here draws them fatha-first. + const marks = [m("fatha", 10), m("hamza", 20)]; + expect(pairMarks(tokens("أَ"), marks)).toEqual([1, 0]); + }); + + it("breaks a same-name tie right to left, the direction the script runs", () => { + // Two fathas, drawn in document order left then right; the first token is + // the first letter, which is the rightmost mark. + const marks = [m("fatha", 10), m("fatha", 30)]; + expect(pairMarks(tokens("بَبَ"), marks)).toEqual([1, 0]); + }); + + it("returns null when a wanted name is not drawn at all", () => { + expect(pairMarks(tokens("أَ"), [m("fatha"), m("fatha")])).toBeNull(); + }); + + it("returns null when a name runs out before the tokens do", () => { + expect(pairMarks(tokens("بَبَ"), [m("fatha", 10)])).toBeNull(); + }); + + it("returns null for a token the dictionary does not name", () => { + expect(pairMarks([{ token: "U+FFFF", at: 0, len: 1 }], [m("fatha")])).toBeNull(); + }); +}); + +describe("markPaths", () => { + it("returns one row per drawn path, indexed into the word's own `data-hafs`", () => { + // «بِهِۦ» drawn as one ligature carrying kasra, kasra, small yeh. + const marks = [m("kasra", 30), m("kasra", 20), m("small yeh", 10)]; + const out = markPaths(word("بِهِۦ", [lig("به", marks)])); + expect(out.map((r) => [r.at, r.len, r.name])).toEqual([ + [1, 1, "kasra"], + [3, 1, "kasra"], + [4, 1, "small yeh"], + ]); + expect(out[2].mark).toBe(marks[2]); + }); + + it("sorts by codepoint index, whatever order the print drew them in", () => { + const out = markPaths(word("أَ", [lig("ا", [m("fatha", 10), m("hamza", 20)])])); + expect(out.map((r) => r.at)).toEqual([0, 1]); + }); + + it("skips a redraw, which carries no marks by construction", () => { + const out = markPaths(word("فَلَا", [lig("فلا", [m("fatha", 30), m("fatha", 20)]), lig("ا")])); + expect(out.map((r) => r.at)).toEqual([1, 3]); + }); + + it("returns null — not a partial answer — when a mark count disagrees", () => { + // The three residual words of ④ land here. A row for the marks it *could* + // place would be worse than nothing: it would look like an answer. + expect(markPaths(word("بِهِۦ", [lig("به", [m("kasra"), m("kasra")])]))).toBeNull(); + }); + + it("returns null when the letters cannot be assigned to the ligatures", () => { + expect(markPaths(word("فَلَا", [lig("فل", [m("fatha")])]))).toBeNull(); + }); + + it("returns null when the drawn names cannot satisfy the tokens", () => { + expect(markPaths(word("أَ", [lig("ا", [m("fatha"), m("damma")])]))).toBeNull(); + }); + + it("returns an empty list for a word the print draws with no named path", () => { + expect(markPaths(word("من", [lig("من")]))).toEqual([]); + }); +}); + +describe("DRAWN_NAME", () => { + it("is frozen — ⑤ re-derives it every full run and reports any drift", () => { + expect(Object.isFrozen(DRAWN_NAME)).toBe(true); + }); + + it("names only marks the shipped vocabulary carries", () => { + for (const name of Object.values(DRAWN_NAME)) expect(diacriticId(name)).toBeGreaterThanOrEqual(0); + }); + + it("keys are `U+XXXX`, optionally with one qualifier", () => { + for (const token of Object.keys(DRAWN_NAME)) { + expect(token).toMatch(/^U\+[0-9A-F]{4}(\+iqlab|@hamza)?$/); + } + }); +}); diff --git a/packages/etl/scripts/probe-diacritics.mjs b/packages/etl/scripts/probe-diacritics.mjs index 7de663c..95bba1f 100644 --- a/packages/etl/scripts/probe-diacritics.mjs +++ b/packages/etl/scripts/probe-diacritics.mjs @@ -133,6 +133,7 @@ import { DIACRITICS, diacriticName } from "@hifth/core"; import { candidatePage } from "./lib/candidate-pages.mjs"; import { applierFromPin, readDiacritics } from "./lib/diacritics.mjs"; +import { DRAWN_NAME, align, expected, letters, pairMarks } from "./lib/mark-join.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const REPO = join(HERE, "..", "..", ".."); @@ -157,230 +158,12 @@ const only = (() => { const SLACK = 0.2; /** - * A word's `data-hafs` as the letters the print draws an *outline* for, each - * carrying the codepoints written on it: `بِسْمِ` → `[ب:[ِ], س:[ْ], م:[ِ]]`. - * - * Two Unicode categories are not outlines and fold into the letter before them: - * - * - **`\p{Mn}`**, the combining marks. Obvious, and the reason this exists. - * - **`\p{Lm}`**, the modifier letters — and this one is the whole reason ④'s - * first draft disagreed. Three of them occur in this text: the tatweel - * `U+0640` that seats a hamza in `شَيۡـٔٗا`, and the small waw `U+06E5` and - * small yeh `U+06E6` of `بِهِۦ`. The text calls all three letters. The print - * does not: the tatweel is drawn as a tooth folded into its neighbour's - * ligature, and the two small letters are drawn as `data-diacritic="small - * waw"` and `"small yeh"` — named marks, sitting in `DIACRITICS` beside the - * fatha. Counting them as base letters made the partition off by one for - * every word containing a seated hamza. - * - * Both rules are Unicode's own categories rather than a codepoint list this - * repo maintains, because a list would be a third place with an opinion about - * Arabic marks and would drift from the other two. - * - * The class is `const` and shared with `align`, which has to fold a ligature's - * `data-text` by exactly the same rule for the two to be comparable at all. - * `FOLD` tests one character and `FOLDS` strips a run; they are the same class - * written once, because two copies of it would be the drift this paragraph is - * about. + * The join itself — `letters`, `align`, `expected`, and the conventions each + * encodes — lives in `lib/mark-join.mjs`, because `probe-encodings.mjs` draws + * the marks it resolves and the two must be the same arithmetic. ④ and ⑤ below + * are still where the evidence for every one of those conventions is written + * down; what moved is only the code. */ -const FOLD_CLASS = "[\\p{Mn}\\p{Lm}]"; -const FOLD = new RegExp(FOLD_CLASS, "u"); -const FOLDS = new RegExp(FOLD_CLASS, "gu"); - -function letters(hafs) { - const out = []; - for (const c of hafs) { - if (FOLD.test(c) && out.length) out[out.length - 1].marks.push(c); - else out.push({ letter: c, marks: [] }); - } - return out; -} - -/** - * A hamza form written as one codepoint, and the carrier it is written on. - * - * Two separate facts live here, and conflating them cost a pass of the corpus. - * - * **The print always draws the sign.** `أ` gets a `hamza` path, `ٱ` a `wasla` - * path, every time, in all 9,168 and 13,476 places they occur. The ligature's - * own spelling does *not* decide it: «أَنَّ» on p119 is drawn `[أ | ن]` and the - * first ligature still carries `hamza` then `fatha`. Making the expectation - * conditional on the ligature spelling the bare carrier — which a first reading - * of «أَيۡدِيهِمۡ» seemed to show — put 151 words on seven pages into the - * residual, and the markup dump said plainly why. - * - * **The spelling still matters for matching.** `align` compares a ligature's - * `data-text` to the word's letters, and the two disagree about the carrier: a - * ligature may spell `ا` where the word writes `أ`. So `base()` folds a hamza - * form to its carrier for that comparison only, and never for what the print - * is expected to draw. - * - * The bare hamza `ء` `U+0621` is deliberately absent: with no carrier to sit - * on it is drawn as `data-type="text"` like any other letter, always. - */ -const HAMZA_ON = { آ: "ا", أ: "ا", إ: "ا", ٱ: "ا", ؤ: "و", ئ: "ي" }; - -/** The letter under a hamza form, for matching a ligature's text to the word's. */ -const base = (c) => HAMZA_ON[c] ?? c; - -/** A short vowel or a tanween — the thing an iqlab meem merges into. */ -const VOWEL = /[ً-ِٗٞ]/; - -/** - * The iqlab meem, which the print never draws on its own beside a vowel — both - * of the forms this text uses. `كَافِرِۭ` writes the final form `U+06ED` and - * `رِكۡزَۢا` the isolated `U+06E2`; the print composes either with the vowel - * before it into one `kasra iqlab` / `fatha iqlab` glyph. - */ -const IQLAB = /[ۭۢ]/; - -/** - * The tatweel. It folds like a mark, because the print does not give it a - * ligature of its own — but unlike the small waw and small yeh it folds beside, - * it is drawn as part of the neighbouring outline (the tooth that seats a hamza in - * `شَيۡـٔٗا`), not as a named path. So it is invisible to the partition on both - * sides: not a letter, and not a mark either. - */ -const TATWEEL = "ـ"; - -/** - * `U+06E4`, the small high madda — which in this print is not a mark at all. - * - * Every word carrying it sits in a sajda ayah (13:15, 17:107, 19:58 …) and the - * print draws it as `data-type="sajda-line"`: the overline stretched above the - * phrase a reader prostrates at, not a diacritic over a letter. It has no - * `data-diacritic`, so `readDiacritics` never sees it, and expecting one for it - * was counting a rubric as a vowel. - */ -const SAJDA_LINE = "ۤ"; - -/** - * `U+0653` on a `أ` — the one place the print refuses its own `maddah`. - * - * Every other carrier of a combining madda gets a path named `maddah`: the alef - * of «بِمَآ», the yeh of «فِيٓ», the waw of «قَالُوٓاْ», nineteen letters in all, - * 4,682 paths. The hamza-carrying alef gets one **277 times out of 277 and never - * a `maddah`** — the print draws a stroke it names `fatha`, and the outline - * bears that out: measured against an ordinary fatha on the same line it is a - * shortened version of the same curve (median 0.89×, p5 0.72×), not the maddah's - * hooked wave, which is drawn at one constant width throughout the corpus. - * - * Whether that is the madda rendered short or a fatha standing in for it is a - * question about the print's intent, and this file does not have to answer it: - * one codepoint, one path, and ⑤ pins which. It is given a token of its own only - * so that the relation stays a *function*, which is what makes ⑤'s arithmetic - * work — without it `U+0653` maps to two names and arc consistency correctly - * reports a contradiction rather than a dictionary. - */ -const MADDA = "ٓ"; -const MADDA_ON_HAMZA = "أ"; - -/** - * The *tokens* of a letter — one per named path the print is expected to draw, - * in the order the text writes them. - * - * A token is usually just a codepoint, `"U+064E"`. Two carry context, because - * the print composes and the composite has a name of its own: - * - * - `"U+064E+iqlab"` — `DIACRITICS` carries `fatha iqlab`, `kasra iqlab` and - * `damma iqlab` as names in their own right, so where the text writes a vowel - * followed by `ۭ` the print draws a single glyph: «كَافِرِۭ» is two paths on - * `فر`, not three. - * - `"U+0653@hamza"` — see `MADDA_ON_HAMZA` above. - * - * The hamza or wasla sign comes first where the letter is a hamza form, because - * that is the order the text writes it in. The print often disagrees — «أَنَّ» - * draws `hamza` then `fatha` but «ٱلۡمَلَؤُاْ» draws `damma` then `hamza` — and - * that disagreement is not swept up here. ④ compares counts and is blind to it; - * ⑤ measures it directly and states the rule it follows. - * - * Excluded are the two codepoints the print draws by other means: the tatweel's - * tooth, folded into a neighbour, and the sajda overline. - */ -function expected(l) { - const out = []; - if (HAMZA_ON[l.letter]) out.push(cp(l.letter)); - for (const m of l.marks) { - if (m === TATWEEL || m === SAJDA_LINE) continue; - if (IQLAB.test(m) && out.length && isVowel(out[out.length - 1])) { - out[out.length - 1] += "+iqlab"; - continue; - } - out.push(m === MADDA && l.letter === MADDA_ON_HAMZA ? `${cp(m)}@hamza` : cp(m)); - } - return out; -} - -/** - * Assign each ligature the letters it draws, or `null` if no assignment exists. - * - * The obvious implementation — walk the ligatures in document order, handing - * each the next `text.length` letters — is what ④'s previous draft did, and it - * is wrong in two ways the markup shows plainly: - * - * **Document order is not reading order.** «ٱلرَّحِيمِ» on p379 is drawn as - * `[لر | حيم | ٱ]`: the alef wasla is a separate ligature emitted *last*. Six - * letters, six drawn, so a length check passes — and then every mark is - * assigned to the wrong letter while the totals still balance. That is the - * failure mode this whole file exists to catch, and counting alone cannot see - * it. - * - * **A letter can be drawn twice.** «فَلَا» is `[فلا | ا]` — four letters drawn - * for a three-letter word, because the print puts the alef's stroke in a second - * ligature. Those continuation runs carry no marks of their own, which is what - * makes them safe to recognise: a repeat that carried marks would be a - * different phenomenon and would still fail here. - * - * So this matches on **content** rather than length, over `base()` so that a - * ligature spelling `ا` matches a word writing `أ`. A ligature may take the - * next letters, or re-draw letters already taken if it has no marks. The search - * is a DFS over (position, set of ligatures used) with memoisation; words have - * a handful of ligatures, so the state space is tiny. - * - * Both sides are reduced the same way, which is the only thing that makes the - * comparison meaningful: `letters` folds a `\p{Lm}` into the letter before it, - * so a ligature's `data-text` has to be folded too. It carries the tatweel — - * «مَـَٔابٗا» is drawn `[مـا | با]`, tatweel and all — and leaving it in made - * ten seated-hamza words on the last two juz look unassignable when they are - * simply spelt with the tooth the print draws them with. - * - * Matching on content is strictly stronger than the length check it replaces — - * some words that used to pass the partition now fail it, and that is the point. - */ -function align(ls, ligs) { - const target = ls.map((l) => base(l.letter)).join(""); - const texts = ligs.map((l) => [...l.text.replace(FOLDS, "")].map(base)); - const all = (1 << ligs.length) - 1; - const memo = new Map(); - - const go = (pos, used) => { - if (pos === target.length && used === all) return []; - const key = pos * (all + 1) + used; - if (memo.has(key)) return memo.get(key); - let out = null; - for (let i = 0; i < ligs.length && !out; i += 1) { - if (used & (1 << i)) continue; - const t = texts[i]; - const fits = (from) => from >= 0 && t.every((c, j) => target[from + j] === c); - if (pos + t.length <= target.length && fits(pos)) { - const rest = go(pos + t.length, used | (1 << i)); - if (rest) out = [{ lig: i, from: pos, to: pos + t.length }, ...rest]; - } - if (!out && !ligs[i].marks.length && fits(pos - t.length)) { - const rest = go(pos, used | (1 << i)); - if (rest) out = [{ lig: i, from: pos - t.length, to: pos, redraw: true }, ...rest]; - } - } - memo.set(key, out); - return out; - }; - return go(0, 0); -} - -const cp = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; - -/** Does a token name a vowel? `parseInt` stops at the `+` or `@` of a suffix. */ -const isVowel = (token) => VOWEL.test(String.fromCodePoint(parseInt(token.slice(2), 16))); const pin = JSON.parse(readFileSync(PIN, "utf8")); const rows = new Map(pin.pages.map((p) => [p.page, p])); @@ -427,6 +210,45 @@ const blame = (sig, example) => { why.set(sig, [e[0] + 1, e[1]]); }; +/** + * The one thing `pairMarks` assumes, counted. + * + * The dictionary settles a ligature outright when its tokens name distinct + * paths. Where two tokens name the *same* path — a `U+0653@hamza` fatha beside + * an ordinary one — something has to break the tie, and `lib/mark-join.mjs` + * breaks it with geometry: Arabic is set right to left, so among same-named + * paths the rightmost is the one the text writes first. + * + * That is a claim, so it is measured rather than asserted. `ties` counts the + * ligatures where a tie exists at all — if it is rare, the assumption is cheap + * whatever it is worth — and `tiesDiffer` counts where the geometric order and + * the print's own emission order disagree, which is the only case in which + * choosing between them changes a rectangle. + */ +let ties = 0; +let tiesDiffer = 0; +const tieWhere = []; +const tie = (tokens, marks, where) => { + const names = tokens.map((t) => DRAWN_NAME[t.token]); + if (new Set(names).size === names.length) return; + ties += 1; + const doc = marks.map((_, i) => i); + const geom = pairMarks(tokens, marks); + // `doc` is document order restricted to the tied name; comparing the whole + // pairing against it would count the print's known reordering of the seated + // hamza, which is R1's business and not this one. + const dup = names.filter((n, i) => names.indexOf(n) !== i); + for (const n of new Set(dup)) { + const inDoc = doc.filter((i) => diacriticName(marks[i][0]) === n); + const inGeom = names.map((x, i) => [x, i]).filter(([x]) => x === n).map(([, i]) => geom?.[i]); + if (inDoc.join() !== inGeom.join()) { + tiesDiffer += 1; + if (tieWhere.length < 6) tieWhere.push(`${n} × ${inDoc.length} — ${where}`); + break; + } + } +}; + for (const page of wanted) { const row = rows.get(page); if (!row) { @@ -476,11 +298,11 @@ for (const page of wanted) { // denominator with runs that cannot disagree. if (step.redraw) continue; ligatures += 1; - const want = ls.slice(step.from, step.to).flatMap(expected); - if (want.length !== l.marks.length) { + const tokens = ls.slice(step.from, step.to).flatMap(expected); + if (tokens.length !== l.marks.length) { ok = false; blame( - `ligature “${l.text}” wants ${want.length} mark(s), the print draws ${l.marks.length}`, + `ligature “${l.text}” wants ${tokens.length} mark(s), the print draws ${l.marks.length}`, where, ); continue; @@ -491,7 +313,10 @@ for (const page of wanted) { // ligature carrying no marks at all agrees vacuously and constrains // nothing, so it is left out rather than counted as a run — in the // denominator it would only dilute ⑤'s held-out percentage. - if (want.length) runs.push([want, l.marks.map((m) => diacriticName(m[0])), where]); + if (tokens.length) { + runs.push([tokens.map((t) => t.token), l.marks.map((m) => diacriticName(m[0])), where]); + tie(tokens, l.marks, where); + } } bucket[ok ? "joined" : "counts"] += 1; } @@ -758,6 +583,39 @@ console.log( ` ${dict.size} of ${may.size} tokens pinned to exactly one name; ` + `${open.length} still open`, ); + +/** + * The frozen copy, checked against the run that earned it. + * + * `lib/mark-join.mjs` ships `DRAWN_NAME` so that a tool drawing one page does + * not have to read 380 MB to know what a `U+0651` looks like. A frozen copy of + * a measured result is a liability unless something re-measures it, so this is + * that something: every full run re-derives the dictionary from the corpus and + * says whether the table still describes it. + * + * Only on a full run. A subset settles fewer tokens by design — `--pages 1,2,7` + * has not seen enough of the corpus to pin all thirty-four — so a short run + * reports what it *can* confirm rather than failing for being short. + */ +const full = wanted.length === pin.pages.length; +const drift = []; +for (const [t, n] of dict) if (DRAWN_NAME[t] !== n) drift.push(`${t} → ${n}, the table says ${DRAWN_NAME[t] ?? "nothing"}`); +if (full) for (const t of Object.keys(DRAWN_NAME)) if (!dict.has(t)) drift.push(`${t} is in the table and this run did not pin it`); +if (drift.length) { + console.log(`\n ⚠ lib/mark-join.mjs DRAWN_NAME disagrees with this run on ${drift.length}:`); + for (const d of drift) console.log(` ${d}`); +} else { + console.log( + ` lib/mark-join.mjs DRAWN_NAME agrees on ${dict.size} of them` + + (full ? " and carries no token this run did not pin" : " (subset run — completeness not checked)"), + ); +} + +console.log( + `\n the tie the pairing does assume — ${ties} of ${runs.length} runs draw two ` + + `paths\n of one name, and geometry disagrees with document order on ${tiesDiffer}`, +); +for (const t of tieWhere) console.log(` ${t}`); if (dead.length) { console.log( ` ${dead.length} shape(s) admit no assignment at all ` + diff --git a/packages/etl/scripts/probe-encodings.mjs b/packages/etl/scripts/probe-encodings.mjs index 151afa4..13bc8c0 100644 --- a/packages/etl/scripts/probe-encodings.mjs +++ b/packages/etl/scripts/probe-encodings.mjs @@ -46,10 +46,26 @@ * Not a gate, and never will be: no cache, nothing to read. Named `probe-` for * exactly the reason `probe-tajweed-words.mjs` is. * + * ## `--marks`, and why it is opt-in + * + * The level below a word. With the flag, every page is also read through + * `lib/diacritics.mjs` and joined by `lib/mark-join.mjs`, so each tajweed + * annotation can be asked a question the four encodings alone cannot answer: + * **does the codepoint this rule opens on have a drawn path, and which one.** + * That is the offline half of `sub-word-marks.md` §⑧ ①, and the boxes it finds + * are what the outline draws inside its word rectangles. + * + * It is a flag rather than the default for one reason: it puts 326,515 more + * rectangles in the payload, which roughly doubles a report that is already + * megabytes. The default run is the one a maintainer opens to read four + * encodings; `--marks` is the one they open to look at ink they are not allowed + * to draw. + * * Usage: * node packages/etl/scripts/probe-encodings.mjs # from the cache * node packages/etl/scripts/probe-encodings.mjs --fetch # fill it first * node packages/etl/scripts/probe-encodings.mjs --pages 40 # a fast subset + * node packages/etl/scripts/probe-encodings.mjs --marks # + the mark level * node packages/etl/scripts/probe-encodings.mjs --out /tmp/x.html */ import { createHash } from "node:crypto"; @@ -57,10 +73,21 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import vm from "node:vm"; +import { DIACRITICS } from "@hifth/core"; import { candidatePage, pin } from "./lib/candidate-pages.mjs"; +import { applierFromPin, readDiacritics } from "./lib/diacritics.mjs"; +import { DRAWN_NAME, markPaths } from "./lib/mark-join.mjs"; import { WAQF } from "./lib/mushaf-frame.mjs"; import { EXCEPTIONS, lexicalIndices, openAlignment, qacSkeletons } from "./lib/segmentation.mjs"; -import { ALL_CORRECTIONS, foldAyah, oracleDensity, oracleOf, touchClass } from "./lib/tajweed-fold.mjs"; +import { + ALL_CORRECTIONS, + ORACLE, + foldAyah, + oracleDensity, + oracleOf, + respellerFor, + touchClass, +} from "./lib/tajweed-fold.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const DATA = join(HERE, "..", "data"); @@ -73,12 +100,17 @@ const DEFAULT_OUT = join(HERE, "..", "out", "encoding-inspector.html"); const ASSETS = join(HERE, "..", "..", "..", "apps", "web", "public", "assets"); const WORD_SHARDS = join(ASSETS, "words", "hafs-kfqc"); const MANIFEST = join(ASSETS, "manifest.json"); +// The per-page fit from their frame to ours, four numbers a page, committed. +// `--marks` needs it and nothing else here does; see `lib/diacritics.mjs` on +// why a caller reconstitutes the transform rather than re-fitting one. +const WORD_PIN = join(DATA, "pages", "word-boxes.pin.json"); const argOf = (name, fallback) => { const i = process.argv.indexOf(name); return i === -1 ? fallback : process.argv[i + 1]; }; const fetchMissing = process.argv.includes("--fetch"); +const wantMarks = process.argv.includes("--marks"); const lastPage = Number(argOf("--pages", 604)); const out = argOf("--out", DEFAULT_OUT); @@ -102,11 +134,33 @@ const isMark = (text) => [...text].length > 0 && [...text].every((c) => WAQF.has /** "surah:ayah" → Map(1-based print index → { hafs, waw, mark }). */ const byAyah = new Map(); +/** + * `--marks` only: "surah:ayah" → Map(print index → the word's resolved marks). + * + * Each value is {@link markPaths}'s answer — `[{ at, len, token, name, mark }]` + * with `at` a **codepoint** index into that word's own `data-hafs` — or `null` + * for a word that does not join. The null is kept rather than dropped, because + * "this word has no marks" and "this word could not be resolved" are different + * findings and the second must not be able to hide inside the first. + */ +const marksByAyah = new Map(); +let markRows = null; + async function readPages() { let bytes = 0; + if (wantMarks) { + markRows = new Map(JSON.parse(readFileSync(WORD_PIN, "utf8")).pages.map((p) => [p.page, p])); + } for (let page = 1; page <= lastPage; page += 1) { const { body } = await candidatePage(page, { offline: !fetchMissing }); bytes += body.length; + if (wantMarks && markRows.has(page)) { + for (const w of readDiacritics(body.toString("utf8"), applierFromPin(markRows.get(page)))) { + const key = `${w.surah}:${w.aya}`; + if (!marksByAyah.has(key)) marksByAyah.set(key, new Map()); + marksByAyah.get(key).set(w.idx, markPaths(w)); + } + } for (const m of body.toString("utf8").matchAll(WORD)) { const a = m[1]; const surah = Number(attr(a, "data-surah")); @@ -284,6 +338,22 @@ for (const key of byAyah.keys()) { } else if (EXCEPTIONS[key]) { entry.x = EXCEPTIONS[key]; } + // `d` — the mark level, print index → `[at, len, id, x, y, w, h]` rows, or + // `null` for a word ④ could not join. A word that simply carries no marks is + // omitted rather than stored as `[]`: absent and empty mean the same thing to + // the outline, and 91,451 empty arrays are not free. `null` is stored, + // because "no answer" and "the answer is none" are not the same claim. + if (wantMarks) { + const byIdx = marksByAyah.get(key); + if (byIdx) { + const d = {}; + for (const [i, resolved] of byIdx) { + if (resolved === null) d[i] = null; + else if (resolved.length) d[i] = resolved.map((m) => [m.at, m.len, ...m.mark]); + } + if (Object.keys(d).length) entry.d = d; + } + } ayahs[key] = entry; } @@ -307,6 +377,125 @@ let oracleHit = 0; let oracleSens = 0; let residualAyahs = 0; +// ------------------------------------------------------- `--marks`: the level -- + +/** + * The offline half of `sub-word-marks.md` §⑧ ①, measured per annotation. + * + * ①–⑤ of `probe-diacritics.mjs` end at a word: every mark is named, joined to + * the codepoint the print drew it for, and inside its own word's box. What is + * still unmeasured is whether a *tajweed rule* can reach one — the rule speaks + * in Tanzil offsets, the mark answers to a `data-hafs` codepoint index, and the + * fold that connects them **respells** some words. So each annotation is walked + * all the way down and its outcome recorded, including every way down that does + * not arrive: + * + * `oracle-miss` the offset does not land on the letter its rule names, so + * there is no position to resolve; already counted above. + * `basmala` the letter is in the prefixed basmala — ink from 1:1, with + * no print index in this ayah. + * `no-host` the position falls in a space between words. + * `no-word` the corpus has no word at that print index (page not read). + * `respelt` the fold rewrote this word, so an offset into the string is + * not an offset into `data-hafs`. Counted, never guessed. + * `unjoined` `markPaths` refused the word (④'s residual). + * `letter` the codepoint is a base letter and the print drew no named + * path for it. **This is an answer, not a failure** — qalqalah + * opens on ق, and the box to light is the letter's, which the + * word shards do not carry at letter granularity. + * `drawn` the codepoint has a named path, and this is its rectangle. + * + * The predicted name is `DRAWN_NAME[cp(letter)]` and the observed one is the + * path's own — for a mark these agree by construction, because `pairMarks` + * pairs *by* name, so the comparison is not evidence and is not reported as + * such. What the per-rule tally is actually for is the shape of the answer: + * which rules land on a mark, which land on a letter, and which cannot be + * reached at all. `composite` is the one real disagreement — a vowel that + * carries an iqlab meem is drawn as one glyph, so a rule naming the bare vowel + * gets `fatha iqlab` where the bare-codepoint lookup says `fatha`. + */ +const markOutcome = { + "oracle-miss": 0, + basmala: 0, + "no-host": 0, + "no-word": 0, + respelt: 0, + unjoined: 0, + letter: 0, + drawn: 0, +}; +const markByRule = new Map(); +const markNames = new Map(); +let markComposite = 0; +const respell = respellerFor(on); +const cpOf = (c) => `U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`; +const ORACLE_SETS = new Map(Object.entries(ORACLE).map(([r, e]) => [r, { set: new Set(e.letters), near: e.near ?? 0 }])); + +/** Where in `cps` the rule's letter actually sits, for a hit. `near` is re-walked, not guessed. */ +function letterAt(cps, rule, start) { + const spec = ORACLE_SETS.get(rule); + if (!spec) return -1; + for (let d = 0; d <= spec.near; d += 1) if (spec.set.has(cps[start + d])) return start + d; + return -1; +} + +function markLevel(key, entry, cps, hosts) { + const byIdx = marksByAyah.get(key); + const tally = (rule, k) => { + if (!markByRule.has(rule)) markByRule.set(rule, { n: 0, drawn: 0, letter: 0, unreached: 0 }); + const t = markByRule.get(rule); + t.n += 1; + if (k === "drawn") t.drawn += 1; + else if (k === "letter") t.letter += 1; + else t.unreached += 1; + markOutcome[k] += 1; + }; + + for (const [r, start, end] of entry.a) { + const rule = rules[r]; + const o = oracleOf(cps, { rule, start, end }); + if (!o || !o.hit) { + tally(rule, "oracle-miss"); + continue; + } + const pos = letterAt(cps, rule, start); + const host = hosts.find((h) => pos >= h.from && pos < h.to); + if (pos < 0 || !host) { + tally(rule, "no-host"); + continue; + } + if (host.print === null) { + tally(rule, "basmala"); + continue; + } + const word = byIdx?.get(host.print); + const hafs = entry.w[host.print - 1]; + if (word === undefined || hafs === undefined) { + tally(rule, "no-word"); + continue; + } + if (respell(hafs) !== hafs) { + tally(rule, "respelt"); + continue; + } + if (word === null) { + tally(rule, "unjoined"); + continue; + } + const at = pos - host.from; + const hit = word.find((m) => at >= m.at && at < m.at + m.len); + if (!hit) { + tally(rule, "letter"); + continue; + } + tally(rule, "drawn"); + if (DRAWN_NAME[cpOf(cps[pos])] !== hit.name) markComposite += 1; + const seen = markNames.get(rule) ?? new Map(); + seen.set(hit.name, (seen.get(hit.name) ?? 0) + 1); + markNames.set(rule, seen); + } +} + for (const [key, entry] of Object.entries(ayahs)) { const [surah, ayah] = key.split(":").map(Number); const words = entry.w.map((hafs, i) => ({ hafs, waw: entry.v[i] === "1", mark: entry.m[i] === "1" })); @@ -326,6 +515,7 @@ for (const [key, entry] of Object.entries(ayahs)) { } } if (missed) residualAyahs += 1; + if (wantMarks) markLevel(key, entry, cps, hosts); } // ---------------------------------------------------------------- the report -- @@ -374,6 +564,9 @@ const payload = { megabytes: (bytes / 1024 / 1024).toFixed(0), }, rules, + // The mark vocabulary, so a row's `id` can be named in the browser. Null + // without `--marks`, and the client keys the whole mark level off that. + diacritics: wantMarks ? DIACRITICS : null, surahs: surahs(), exceptions: EXCEPTIONS, pages: Object.fromEntries(Object.entries(ayahs).map(([k, e]) => [k, e.p])), @@ -448,5 +641,30 @@ console.log(`\n── the residual: ${residualAyahs} ayahs, ${oracleN - oracleHi for (const [d, n] of [...drift].sort((a, b) => (a[0] ?? 99) - (b[0] ?? 99))) { console.log(` ${(d === null ? "∅" : String(d)).padStart(3)} ${String(n).padStart(4)} ${pct(n, oracleN - oracleHit)}`); } +if (wantMarks) { + const total = Object.values(markOutcome).reduce((a, b) => a + b, 0); + console.log(`\n── the mark level — ${total} annotations walked from an offset to a drawn path`); + for (const [k, n] of Object.entries(markOutcome)) { + console.log(` ${k.padEnd(12)} ${String(n).padStart(6)} ${pct(n, total)}`); + } + console.log( + ` of the ${markOutcome.drawn} that reach a path, ${markComposite} are drawn as a composite` + + " — a vowel and its iqlab meem in one glyph, which the bare codepoint does not predict", + ); + console.log("\n── per rule: where the rule's own letter is drawn"); + const rows = [...markByRule].sort((a, b) => b[1].n - a[1].n); + for (const [rule, t] of rows) { + const names = [...(markNames.get(rule) ?? new Map())].sort((a, b) => b[1] - a[1]); + const top = names + .slice(0, 3) + .map(([n, c]) => `${n} ×${c}`) + .join(", "); + console.log( + ` ${rule.padEnd(16)} ${String(t.n).padStart(5)} drawn ${pct(t.drawn, t.n).padStart(7)}` + + ` letter ${pct(t.letter, t.n).padStart(7)} unreached ${pct(t.unreached, t.n).padStart(7)}` + + (top ? ` · ${top}${names.length > 3 ? ", …" : ""}` : ""), + ); + } +} console.log(`\n wrote ${out} (${(html.length / 1024 / 1024).toFixed(1)} MB, self-contained, gitignored)`); console.log(" open it with: open " + out + "\n"); From 0f385cfbee3fd0590e03b6991d58e328d121161f Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 14:07:17 -0500 Subject: [PATCH 8/8] The registers learn that half of an open question closed, and the other half needs an eye MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sub-word-marks.md` §⑧ ① keeps its **open** status and gains the measurement that halves it. The offline half is answered: all 60,057 tajweed annotations walked from a Tanzil offset to a drawn path, 47.51% reaching a named rectangle and 51.52% landing on a base letter, with nothing unreachable for a structural reason and the remainder — 497 respelt words, 82 oracle misses — named rather than dropped. The section argues why the 51.52% is an answer instead of a shortfall, because that is the part a reader would otherwise take for a coverage number and try to raise: ten of the eighteen rules name a consonant, and the print draws a consonant as a letter outline. So the finding is a split, and a mark-granular UI has to say which of the two it is doing. That is the constraint mark-C inherits and it was not visible before the walk. What stays open is what arithmetic cannot reach — whether the box the dictionary names sits where a reader *looks* for that mark, and whether lighting it reads truer than washing the word. The inspector now draws that screen, so the blocker moves from "the inspector drawing the marks" to a hafiz looking at them. `encoding-inspector.md` gains `--marks` in §④ ①, §⑥ 4 and §⑧: the level, the size it costs, and the restated ink line. §⑥ 4 is the one that needed rewriting rather than appending — the temptation the rule anticipates gets *stronger* a level down, because a mark's box is small enough that filling it in would look like the mark. `map.json` gains three pointers, hand-edited: `lib/mark-join.mjs` and its test under word-geometry, `markLevel` under encoding-inspector. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- docs/design/encoding-inspector.md | 20 ++++++++++++-- docs/design/sub-word-marks.md | 45 ++++++++++++++++++++++--------- docs/issues.json | 4 +-- docs/issues.md | 4 +-- docs/map.json | 15 +++++++++++ 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/docs/design/encoding-inspector.md b/docs/design/encoding-inspector.md index 54cf764..fbc017a 100644 --- a/docs/design/encoding-inspector.md +++ b/docs/design/encoding-inspector.md @@ -148,7 +148,11 @@ Six views, one ayah picker, three checkboxes. ⑨ ④ is the record of what moved and what did not). A box carries the same even/odd tint its codepoints carry on the ruler at (4), so a span lit in one panel is lit in the other; clicking one selects the annotation over it. If the box count and the `data-hafs` word - count ever disagree, the view draws the geometry and **refuses to number it**; + count ever disagree, the view draws the geometry and **refuses to number it**. With + `--marks`, each word's **named marks** are drawn as hairline rectangles inside it — the + name and the codepoint index on hover, and the selected annotation's own mark lit inside + its lit word. That is the level below a word, and + [`sub-word-marks.md`](sub-word-marks.md) §⑧ ① is where the arithmetic behind it lives; 2. the print's words: `data-hafs`, codepoint count, kind (word / split waw / pause mark), the half-open span each occupies in the fold, and the QAC word it maps to; 3. the print↔QAC map for the ayah: every QAC word, the folded consonant skeleton the aligner @@ -231,7 +235,11 @@ Six things, and the last two are the ones that will eventually tempt somebody. authority rests on being about *identity* rather than appearance. A rectangle from a `gate:words`-checked shard makes no claim about how anything *looks*; it says only where a word the tool is already reasoning about happens to sit. The distinction is the whole - of what ⑨ ④ decided, and it is worth holding: outlines yes, ink no. + of what ⑨ ④ decided, and it is worth holding: outlines yes, ink no. `--marks` goes a + level finer and stays on the same side of that line — a mark's rectangle is geometry from + `lib/diacritics.mjs` with a name from a measured dictionary, and it still draws no stroke + of the print. The temptation the rule anticipates gets stronger here, because a mark's box + is small enough that filling it in would *look* like the mark. It must not. 5. **QAC segment granularity.** A print word maps to a QAC *word*. PREFIX/STEM/SUFFIX is not in the alignment and is not shown, because the alignment does not know it. 6. **Any other print, and any other edition of QAC.** Both are pinned. A different pin is a @@ -327,10 +335,18 @@ count so a future disagreement surfaces as a number rather than as a wrong scree pnpm probe:encodings # from the cache, all 604 pages, ~90s pnpm probe:encodings --fetch # fill the cache first pnpm probe:encodings --pages 30 # a fast subset while changing the client +pnpm probe:encodings --marks # + the level below a word pnpm probe:encodings --out /tmp/x.html open packages/etl/out/encoding-inspector.html ``` +`--marks` is opt-in rather than the default for one reason, and it is a size: it puts 326,515 +more rectangles in the payload and takes the report from 5.0 MB to **13.8 MB**. The extraction +itself is cheap — `readDiacritics` bboxes only the mark paths, so the whole corpus costs +about a second on top of a run that already reads all 604 pages. What is expensive is the +page a browser then has to hold. Most questions this tool is opened for are about the four +encodings and do not need it. + `packages/etl/out/` is gitignored, and that is load-bearing rather than tidy: **there is no Quran text in this repo and there will not be.** The report is full of Arabic; every codepoint of it is derived at runtime from the gitignored cache, and committing one would diff --git a/docs/design/sub-word-marks.md b/docs/design/sub-word-marks.md index 722e06a..87234e4 100644 --- a/docs/design/sub-word-marks.md +++ b/docs/design/sub-word-marks.md @@ -387,18 +387,39 @@ document puts 326,515 named marks inside those same words. The question is wheth meet: when `madd_246` opens at a codepoint, is there a `maddah` box there — and if there is, is highlighting *it* a truer rendering of the rule than washing the whole word? -**What §⑤ changed about this.** Half of it is now arithmetic. With the codepoint→name -dictionary pinned, "does `madd_246` open at a codepoint the print draws a `maddah` for" is a -question the corpus answers offline, without an eye and without a guess — and that half -should be measured before anyone looks at a screen, because it is cheap and it bounds what -the looking is for. What §⑤ did **not** close is the second half. - -**What would answer the rest:** the encoding inspector (mark-B). It already reconciles the -print, the ligature corpus, QAC and the tajweed offsets on one screen for one page; adding -the mark boxes puts all four descriptions and the geometry in one place. Two things only an -eye settles there: whether the box the dictionary names sits where a reader looks for that -mark, and whether lighting *it* reads as a truer rendering of the rule than washing the -whole word. The first is §⑦'s remaining gap; the second was never a measurement at all. +**The offline half, measured.** `pnpm probe:encodings --marks` now walks every one of the +60,057 annotations from its Tanzil offset down to a drawn path, using `lib/mark-join.mjs` — +the same join `probe:diacritics` ④/⑤ measured, extracted so the two cannot drift. Nothing is +guessed on the way: a word the fold *respells* is counted as not-checkable rather than +addressed, because an offset into the respelled string is not an offset into `data-hafs`. + +| of 60,057 annotations, where the rule's own letter is drawn | | | +|---|---:|---:| +| a named path — this is the rectangle to light | **28,535** | **47.51%** | +| a base letter with no named path | 30,943 | 51.52% | +| the word was respelt, so the offset does not address `data-hafs` | 497 | 0.83% | +| the oracle itself misses, so there is no position to resolve | 82 | 0.14% | +| no host, no word, or a word the join refused | **0** | **0.00%** | + +**The 51.52% is an answer, not a shortfall.** Ten of the eighteen rules name a *consonant* — +`qalqalah` opens on ق, `lam_shamsiyyah` on ل, `ghunnah` on ن or م — and the print draws a +consonant as a letter outline, not as a named mark. Those rules land on a letter 99%+ of the +time and there is nothing above them to light. The eight that name a mark reach one almost +always: `hamzat_wasl` → `wasla` 98.11%, `madd_2` → `superscript alef`/`small waw`/`small +yeh` 98.58%, `iqlab` → `small meem` or a `… iqlab` composite 99.82%, and the ikhfa/idghaam +family → the `successive fathatan/kasratan/dammatan` the print writes for them. + +So the shape of the finding is a **split**, not a rate: a mark-granular highlight is +available for the rules about marks and not for the rules about letters, and any UI built on +this has to say which it is doing. That is a design constraint mark-C inherits, and it was +not visible before the walk. + +**What is still not answered, and cannot be by arithmetic:** whether the box the dictionary +names sits where a reader *looks* for that mark, and whether lighting it reads as a truer +rendering of the rule than washing the whole word. The inspector now draws the boxes +(`--marks`, §⑥ of [`encoding-inspector.md`](encoding-inspector.md)) with the selected +annotation's own mark lit inside its word, which is the screen those two questions need. The +first is §⑦'s remaining gap; the second was never a measurement at all. **What must not happen instead:** deriving the correspondence from the fact that both numbers exist. Reading a mapping off where the offsets happen to land and then declaring diff --git a/docs/issues.json b/docs/issues.json index d071b99..774c8dd 100644 --- a/docs/issues.json +++ b/docs/issues.json @@ -464,8 +464,8 @@ "status": "open", "severity": "question", "owner": "agent", - "blockedBy": ["the encoding inspector drawing the marks"], - "note": "Opened 2026-08-07 with mark-A. Two measurements now exist over the same words and nothing has checked whether they meet. word-indexing.md ⑪ ⑤ lands 59,975 of 60,057 tajweed annotations (99.86%) on the letter their rule names, 83.31% of them inside a single print word; sub-word-marks.md ⑤ puts 326,515 named mark boxes inside those same words, 0 of them outside. The open question is whether a span's [start,end) coincides with a mark a reader can be SHOWN — whether madd_246 opening at a codepoint means there is a maddah box there — and, if so, whether highlighting the mark is a truer rendering of the rule than washing the whole word. Narrowed 2026-08-07 by sub-word-marks.md ⑤: with the codepoint→name dictionary pinned (34 of 34 tokens, by elimination, validated on 62,931 held-out runs), the first half — does madd_246 open at a codepoint the print draws a maddah for — is now an offline measurement, and should be made before anyone looks at a screen because it is cheap and it bounds what the looking is for. What ⑤ did NOT close, and what still needs the encoding inspector (mark-B), which already reconciles the print, the ligature corpus, QAC and the tajweed offsets on one screen and would gain the mark boxes over the same frame: whether the box the dictionary NAMES sits where a reader looks for that mark, and whether lighting it reads as a truer rendering of the rule than washing the word. The first is a claim about the picture — every step of ⑤ is a correspondence between a reconstructed text and the corpus's own attributes, and none of it looks at where the outline sits relative to the letter that wrote it — and the second was never a measurement at all. What must not happen instead is deriving the correspondence from the fact that both numbers exist — reading a mapping off where the offsets happen to land and then declaring that they land there is the circularity ⑪ ⑤ names about its own oracle, and it passes on a broken answer. Note this is a rendering question and not a recitation one: DIACRITICS records what the corpus wrote in an attribute and this repo asserts nothing about how any of the twenty-six are pronounced." + "blockedBy": ["a hafiz looking at the drawn marks"], + "note": "Narrowed again 2026-08-07 by mark-B, and the offline half is now ANSWERED — the remaining half needs an eye and nothing else. `pnpm probe:encodings --marks` walks all 60,057 annotations from a Tanzil offset down to a drawn path through lib/mark-join.mjs (the same join probe:diacritics ④/⑤ measured, extracted so the inspector and the probe cannot drift), and the answer is a SPLIT rather than a rate: 47.51% open on a codepoint the print draws a named path for — the rectangle to light — and 51.52% open on a base letter with no path above it, which is the CORRECT answer and not a shortfall, because ten of the eighteen rules name a consonant (qalqalah → ق, lam_shamsiyyah → ل, ghunnah → ن/م) and the print draws a consonant as a letter outline. The eight rules that name a mark reach one almost always: hamzat_wasl → wasla 98.11%, madd_2 98.58%, iqlab 99.82%. Nothing is unreachable for a structural reason — no-host, no-word and unjoined are all 0 — and the remainder is explained rather than dropped: 497 respelt words, whose offsets do not address data-hafs by construction, and 82 oracle misses. So a mark-granular highlight is available for the rules about marks and not for the rules about letters, and any UI built on this has to SAY WHICH IT IS DOING; that is a constraint mark-C inherits and it was not visible before the walk. What is left is what arithmetic cannot reach: whether the box the dictionary names sits where a reader LOOKS for that mark, and whether lighting it reads as a truer rendering of the rule than washing the word. The inspector now draws the boxes with the selected annotation's own mark lit inside its word, so the screen those two questions need exists — sub-word-marks.md §⑧ ① carries the table. Opened 2026-08-07 with mark-A. Two measurements now exist over the same words and nothing has checked whether they meet. word-indexing.md ⑪ ⑤ lands 59,975 of 60,057 tajweed annotations (99.86%) on the letter their rule names, 83.31% of them inside a single print word; sub-word-marks.md ⑤ puts 326,515 named mark boxes inside those same words, 0 of them outside. The open question is whether a span's [start,end) coincides with a mark a reader can be SHOWN — whether madd_246 opening at a codepoint means there is a maddah box there — and, if so, whether highlighting the mark is a truer rendering of the rule than washing the whole word. Narrowed 2026-08-07 by sub-word-marks.md ⑤: with the codepoint→name dictionary pinned (34 of 34 tokens, by elimination, validated on 62,931 held-out runs), the first half — does madd_246 open at a codepoint the print draws a maddah for — is now an offline measurement, and should be made before anyone looks at a screen because it is cheap and it bounds what the looking is for. What ⑤ did NOT close, and what still needs the encoding inspector (mark-B), which already reconciles the print, the ligature corpus, QAC and the tajweed offsets on one screen and would gain the mark boxes over the same frame: whether the box the dictionary NAMES sits where a reader looks for that mark, and whether lighting it reads as a truer rendering of the rule than washing the word. The first is a claim about the picture — every step of ⑤ is a correspondence between a reconstructed text and the corpus's own attributes, and none of it looks at where the outline sits relative to the letter that wrote it — and the second was never a measurement at all. What must not happen instead is deriving the correspondence from the fact that both numbers exist — reading a mapping off where the offsets happen to land and then declaring that they land there is the circularity ⑪ ⑤ names about its own oracle, and it passes on a broken answer. Note this is a rendering question and not a recitation one: DIACRITICS records what the corpus wrote in an attribute and this repo asserts nothing about how any of the twenty-six are pronounced." }, { diff --git a/docs/issues.md b/docs/issues.md index d087f52..fae1c50 100644 --- a/docs/issues.md +++ b/docs/issues.md @@ -1,5 +1,5 @@ - + # Open items @@ -38,7 +38,7 @@ test that would fail if it came back; the gate refuses the word without one. | [Does a real fore-edge stack vary?](design/page-transition.md#-does-a-real-fore-edge-stack-vary--open) | open | question | user | — | | [Whether this document should be generated rather than written](design/etl-pipeline.md#-whether-this-document-should-be-generated-rather-than-written--open) | open | question | agent | — | | [Whether the GPL/App-Store reading in ①–③ is right](design/track-b-native.md#-whether-the-gplapp-store-reading-in--is-right--open) | open | risk | user | a licensing opinion | -| [Does a tajweed span land on a mark a reader can be shown](design/sub-word-marks.md#-does-a-tajweed-span-land-on-a-mark-a-reader-can-be-shown--open) | open | question | agent | the encoding inspector drawing the marks | +| [Does a tajweed span land on a mark a reader can be shown](design/sub-word-marks.md#-does-a-tajweed-span-land-on-a-mark-a-reader-can-be-shown--open) | open | question | agent | a hafiz looking at the drawn marks | | [`PLAN.md` states an unachievable order and one wrong citation](design/track-b-native.md#-planmd-states-an-unachievable-order-and-one-wrong-citation--open) | open | defect | agent | gpl-and-the-app-store | | [Whether Track B should exist at all after ④ and ⑤](design/track-b-native.md#-whether-track-b-should-exist-at-all-after--and---open) | open | question | user | web v1.0 and somebody using it | | [The CI frame budget is a number from an emulator](backlog.md#-the-ci-frame-budget-is-a-number-from-an-emulator--blocked) | blocked | risk | agent | perf-verdict-on-device | diff --git a/docs/map.json b/docs/map.json index be47aae..321a664 100644 --- a/docs/map.json +++ b/docs/map.json @@ -630,6 +630,16 @@ "file": "packages/etl/scripts/probe-diacritics.mjs", "symbol": "function supported(want, got)", "note": "⑤'s engine, and the reason the codepoint→name dictionary is evidence rather than an assumption. Arc consistency over bipartite matchings: a run contributes a *bag* of codepoint tokens beside a *bag* of drawn names, and a pairing is deleted only when no perfect one-to-one assignment of that run can use it. Position is never consulted — pairing them off left to right would assume the print draws marks in written order, which is the thing in question, and 1.36% of multi-mark runs say otherwise. Plain set intersection is the wrong operator and was tried first: it presumes the relation is already a function, and drove `U+0653` to an empty candidate set, which is how the `أ`-carrier madda convention was found. Every single-mark run is held out, because a one-mark run forces its own pairing and scoring against it would report 100% by construction. Result: 34 of 34 tokens pinned in two passes, 62,931 of 62,931 held-out runs predicted correctly." + }, + { + "file": "packages/etl/scripts/lib/mark-join.mjs", + "symbol": "export function markPaths", + "note": "④ and ⑤'s arithmetic extracted so there is exactly one of it, and the only entry point a caller should want: a word as `readDiacritics` returns it → `[{ at, len, token, name, mark }]`, where `at` is a **codepoint** index into that word's own `data-hafs`. That index is the whole point — tajweed offsets count codepoints, so this is the bridge from an offset to a rectangle. Three steps behind it: `letters` folds `\\p{Mn}` *and* `\\p{Lm}` onto the letter before (the tatweel and the small waw/yeh are text letters the print draws as marks, which is what made ④'s first draft off by one for every seated hamza), `align` matches ligatures to letter runs by CONTENT with backtracking (order is not reading order — «ٱلرَّحِيمِ» draws `[لر|حيم|ٱ]`, and a left-to-right walk agreed on every length while misassigning every mark), and `pairMarks` pairs a wanted token to a drawn name through the frozen 34-entry `DRAWN_NAME`, breaking same-name ties right to left. Its refusals are the load-bearing part: it returns `null` — never a partial answer — for a mark count that disagrees, a name the tokens did not ask for, or letters no assignment of ligatures can cover, because a partial answer would look exactly like an answer. Extracted rather than left in the probe for the reason `tajweed-fold.mjs` gives about itself: `probe-encodings.mjs --marks` draws these rectangles, and if its join were a second implementation, a clean screen would stop being evidence about the probe." + }, + { + "file": "packages/etl/scripts/lib/mark-join.test.mjs", + "symbol": "describe(\"markPaths\"", + "note": "28 tests on words small enough to count by hand, and most of them assert that `markPaths` returns `null`. The probe is the measurement — 86,962 words joined, 34 tokens pinned — and it needs 380 MB of gitignored cache to say anything, which is why it is a probe; what a test can hold is the arithmetic and the refusals. Marks are written as `[diacriticId(name), x, y, w, h]` rather than pasted ids, so reordering `DIACRITICS` fails here instead of silently re-labelling what this file claims. One test earned its comment the hard way: `مَـَٔ` yields three tokens, not two — the tatweel and the sajda overline get no path, but the hamza-above on the tatweel does." } ], "extend": [ @@ -722,6 +732,11 @@ "symbol": "describe(\"driftOnset\"", "note": "54 tests, and they are what makes the shared module safe to share. They pin the sign convention both probes now read, the eighteen-entry ORACLE (its size, that every entry carries the tajweed reason for its letter set, that any letter of a set counts, and that `near` is a one-position window rather than a slop allowance — `iqlab`'s meem is written high over a fatha and LOW over a kasra, and missing the low form scored 85.05%), `oracleDensity`'s pricing of a hit, each of the eight corrections' effect in isolation — including a `respellerFor` block that checks each `respell` fires on its own shape and on nothing else, so deleting one fails a test rather than quietly moving an aggregate — and `driftOnset`'s refusals — it does not narrow with a hit AFTER the miss, and it reports `bounded: false` rather than pretending to a left edge it does not have. Vitest globals are imported explicitly here: the eslint test-globals block covers `**/*.test.{ts,tsx}` and this is `.mjs`." }, + { + "file": "packages/etl/scripts/probe-encodings.mjs", + "symbol": "function markLevel(key, entry, cps, hosts)", + "note": "`--marks`, the level below a word, opt-in because it is a size: 326,515 more rectangles takes the report from 5.0 MB to 13.8 MB. It walks every one of the 60,057 tajweed annotations from its codepoint offset to the path the print drew — oracle hit, host word, respell check, then `markPaths` — and counts the outcome into one of eight named classes rather than a rate, because `letter` is an ANSWER and not a shortfall: ten of the eighteen rules name a consonant, and the print draws a consonant as a letter outline, not a mark. The measured split is 47.51% reaching a named path against 51.52% landing on a base letter, with 0 unreachable for any structural reason (`no-host`, `no-word`, `unjoined` all zero) and the remainder explained rather than dropped — 497 respelt words, whose offsets do not address `data-hafs` by construction, and 82 oracle misses. `docs/design/sub-word-marks.md` §⑧ ① carries the table and the argument that this is a split rather than a rate, which is the constraint any mark-granular UI inherits. The predicted-vs-observed name tally it prints is a smoke alarm, not evidence: `pairMarks` pairs BY name, so the two can only disagree where a composite spans more than one codepoint. On the client side the rectangles go in `outline()` beside the word boxes, hollow and hairline — the ink rule holds a level finer, and it is under more pressure there, because a mark's box is small enough that filling it in would read as the mark itself." + }, { "file": ".gitignore", "symbol": "packages/etl/out/",