From ec4d9c56efcfef1dd33b487ff95bdb57df2a7104 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 08:20:16 -0500 Subject: [PATCH 1/2] The print already named its marks, and nothing had asked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `skins.ts` has said since Loop 6a that a tajweed colour cannot land on part of a word, because the print's glyphs are "anonymous outlined ``s". That is true of `assets/pages/**`, which is what the app draws. It is false of the ligature corpus, which names every mark it draws in a `data-diacritic` attribute — and the same comment already named that corpus as its own gate. This is the measurement, and nothing else. - `packages/core/src/diacritics.ts` — the twenty-six names, in frequency order, with the counts beside them. Twenty-six is measured over all 604 pages, not chosen. A shard would say `0`, not `"fatha"`, because a page carries ~540 marks and the names are the larger half of the bytes — which makes the array's *order* load-bearing in a way nothing else in core is. Appending is safe; reordering silently re-labels the whole corpus and moves no geometry, so nothing that measures rectangles would notice. Six ids are pinned in a test for that reason. Dots are excluded on purpose: i'jam is part of the letter's identity, not a mark a reader is told to look at. - `packages/etl/scripts/lib/diacritics.mjs` — a named, exact box per mark, grouped by the word it sits on. It fits nothing: `apply` is handed in and `applierFromPin` rebuilds it from the four numbers `word-boxes.pin.json` already records, so a mark cannot land on a second transform that disagrees with its own word's by a tenth of a unit. It throws on a name core has never heard of rather than inventing an id. - `packages/etl/scripts/probe-diacritics.mjs` — the corpus-wide run. It ships nothing: 2.28 MB of assets no caller fetches is the same waste `gate:assets` names for a non-vendored edition. The invariant that makes any of this verifiable offline is that a word's shipped box is the union over *every* path in its group, marks included. So containment holds by construction, and an escape can only be an alignment error — a mark filed under the wrong word, the failure class behind `gate:edges`. Checked against the *committed* shards rather than boxes computed in the same pass, because two sides of one computation share their mistakes. 26 of 26 names drawn · 326,515 marks · 0 outside their word · 0 unmatched 7.35 MB raw / 2.28 MB gz · smallest mark 1.8 units on our frame The 4,487 words carrying no mark are the 4,486 pause marks `gate:words` already counts, plus one: 20:1's opening, which this print writes with nothing above or below it. An off-by-one would have scattered those empties. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- package.json | 1 + packages/core/src/diacritics.test.ts | 55 ++++++ packages/core/src/diacritics.ts | 112 ++++++++++++ packages/core/src/index.ts | 12 ++ packages/etl/scripts/lib/diacritics.mjs | 139 +++++++++++++++ packages/etl/scripts/probe-diacritics.mjs | 202 ++++++++++++++++++++++ 6 files changed, 521 insertions(+) create mode 100644 packages/core/src/diacritics.test.ts create mode 100644 packages/core/src/diacritics.ts create mode 100644 packages/etl/scripts/lib/diacritics.mjs create mode 100644 packages/etl/scripts/probe-diacritics.mjs diff --git a/package.json b/package.json index c047e3c..4ea1cdf 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "probe:word-registration": "node packages/etl/scripts/probe-word-registration.mjs", "probe:tajweed-words": "node packages/etl/scripts/probe-tajweed-words.mjs", "probe:encodings": "node packages/etl/scripts/probe-encodings.mjs", + "probe:diacritics": "node packages/etl/scripts/probe-diacritics.mjs", "sample:edges": "node packages/etl/scripts/sample-edges.mjs", "guide": "node scripts/build-validation-guide.mjs", "record": "node scripts/record-validation.mjs", diff --git a/packages/core/src/diacritics.test.ts b/packages/core/src/diacritics.test.ts new file mode 100644 index 0000000..a6df7cb --- /dev/null +++ b/packages/core/src/diacritics.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { DIACRITICS, diacriticId, diacriticName, isDiacriticName } from "./diacritics.js"; + +describe("the mark vocabulary", () => { + it("carries every name the print draws, and no duplicates", () => { + // 26 is measured, not chosen: it is the number of distinct data-diacritic + // values across all 604 pages of the ligature corpus. A duplicate would be + // worse than a missing name — two ids for one mark, and half the shards + // would use the other one. + expect(DIACRITICS).toHaveLength(26); + expect(new Set(DIACRITICS).size).toBe(26); + }); + + it("keeps the ids a shard was written against", () => { + // These are the ids in every shipped shard. Reordering DIACRITICS silently + // re-labels the whole corpus — a fatha becomes a kasra on every page — and + // nothing else in the build would notice, because the geometry is + // unchanged and only the integer beside it moved. This test is the notice. + expect(diacriticId("fatha")).toBe(0); + expect(diacriticId("kasra")).toBe(1); + expect(diacriticId("damma")).toBe(2); + expect(diacriticId("sukun")).toBe(3); + expect(diacriticId("shadda")).toBe(4); + expect(diacriticId("small noon")).toBe(25); + }); + + it("says -1 for a name the print has never drawn", () => { + // Not undefined: the ETL is expected to throw on this, and a number that + // indexes nothing is louder downstream than a hole would be. + expect(diacriticId("two dots")).toBe(-1); + expect(diacriticId("")).toBe(-1); + expect(isDiacriticName("two dots")).toBe(false); + expect(isDiacriticName("fatha")).toBe(true); + }); + + it("round-trips a name through its id and back", () => { + for (const name of DIACRITICS) { + expect(diacriticName(diacriticId(name))).toBe(name); + } + }); + + it("says nothing rather than something for an id off the end", () => { + expect(diacriticName(26)).toBeNull(); + expect(diacriticName(-1)).toBeNull(); + }); + + it("excludes the dots on purpose", () => { + // i'jam distinguishes ب from ت from ث; it is part of the letter, not a + // mark a reader is told to notice. The corpus draws 105,269 of them under + // data-dots and this vocabulary deliberately cannot name one. + for (const dots of ["one dot", "two dots", "three dots"]) { + expect(isDiacriticName(dots)).toBe(false); + } + }); +}); diff --git a/packages/core/src/diacritics.ts b/packages/core/src/diacritics.ts new file mode 100644 index 0000000..985a3c0 --- /dev/null +++ b/packages/core/src/diacritics.ts @@ -0,0 +1,112 @@ +/** + * The print's mark vocabulary — the twenty-six things it draws above and below + * a letter, and the integer each one is called by. + * + * Every layer that touches sub-word geometry needs the same list, and the list + * is not a design choice anybody gets to make: it is what the ligature corpus's + * `data-diacritic` attribute actually contains, measured over all 604 pages on + * 2026-08-07. So it is written down once, here, and the ETL imports it rather + * than growing its own copy — the same arrangement `build-tajweed.mjs` already + * has with {@link isTajweedRuleId}, where a name the app cannot paint stops the + * build instead of reaching a shard. + * + * ── Why an integer and not the name ───────────────────────────────────────── + * + * A page carries ~540 marks. Spelling "superscript alef" beside each one would + * cost more than the geometry does; an index costs one or two characters. The + * order below is therefore load-bearing — an id is only meaningful against this + * array — which is why the order is *frequency*, descending, and why the counts + * are written beside it. Frequency order is not cosmetic: it puts the cheap + * one-character ids on the marks that occur most, and it makes a re-measure + * that reorders the list obvious rather than silent. + * + * **Appending is safe. Reordering is not.** A shard written before a reorder + * would read as a different mark on every page. If a future print introduces a + * twenty-seventh mark, add it at the end and leave the rest alone, even though + * that breaks the frequency ordering — the ordering is a nicety and the ids are + * a contract. + * + * ── What is deliberately not here ─────────────────────────────────────────── + * + * **Dots.** The corpus draws i'jam — the dots that distinguish ب ت ث — as a + * separate `data-dots` attribute with three values over 105,269 paths, and they + * are excluded on purpose. A dot is not a mark a reader is ever told to notice; + * it is part of the letter's identity. Including them would add a third of the + * corpus's paths to carry something nobody would ever highlight. + * + * **Waqf marks, the juz star and the sajda ornaments.** The print sets each of + * those as its own *word*, with its own `data-word-index-in-ayah`, and the word + * shards already flag them (`marks`, see `build-words.mjs`'s `isPauseMark`). + * They are pauses and page furniture, not marks on a letter, and they are + * already addressable one level up. + * + * **Any claim about pronunciation.** These are the names the print's own + * markup uses. "damma iqlab" is a damma the corpus tagged as participating in + * iqlab; this module records that the corpus said so and nothing more. Mapping + * marks to tajweed rules is {@link TajweedRuleId}'s job and it is a different, + * harder question — the tajweed shards index into Tanzil Uthmani codepoints, + * not into this print's paths. + */ + +/** + * Every named mark the print draws, most frequent first, with the path count + * measured over the whole mus'haf. The index of a name in this array is its id. + * + * The long tail is real and is kept: `small noon` occurs exactly once in the + * Qur'an (68:1, نٓ), and a vocabulary that dropped the singletons would be a + * vocabulary that quietly cannot describe one page. + */ +export const DIACRITICS = [ + "fatha", // 122,948 + "kasra", // 45,970 + "damma", // 37,320 + "sukun", // 37,148 + "shadda", // 22,678 + "hamza", // 16,385 + "wasla", // 13,483 + "superscript alef", // 9,726 + "maddah", // 5,376 + "rounded zero", // 3,988 + "successive fathatan", // 2,901 + "successive kasratan", // 1,935 + "successive dammatan", // 1,807 + "small waw", // 1,257 + "small yeh", // 995 + "fathatan", // 734 + "kasratan", // 599 + "dammatan", // 578 + "small meem", // 270 + "damma iqlab", // 134 + "fatha iqlab", // 106 + "kasra iqlab", // 99 + "rectangular zero", // 66 + "small seen", // 8 + "vowel sign", // 3 + "small noon", // 1 +] as const; + +/** One of the names the print draws. */ +export type DiacriticName = (typeof DIACRITICS)[number]; + +/** + * The integer a shard calls this mark by, or `-1` if the print has never drawn + * it. + * + * `-1` rather than `undefined` so a caller that forgets to check writes an + * impossible index instead of `undefined` into geometry — the ETL is expected + * to throw on it, and a number that indexes nothing is louder downstream than a + * hole. See `readDiacritics` in the ETL, which does exactly that. + */ +export function diacriticId(name: string): number { + return (DIACRITICS as readonly string[]).indexOf(name); +} + +/** Is this a name the print draws? The ETL's guard against a corpus that grew. */ +export function isDiacriticName(name: string): name is DiacriticName { + return diacriticId(name) >= 0; +} + +/** The name an id stands for, or `null` if it stands for nothing. */ +export function diacriticName(id: number): DiacriticName | null { + return DIACRITICS[id] ?? null; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0cfe468..02bc5e1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -259,6 +259,18 @@ export { type TajweedLookup, } from "./skins.js"; +// The print's own mark vocabulary — what the ligature corpus draws above and +// below a letter, and the integer sub-word geometry calls each one by. Not the +// tajweed rules above: these are marks the print draws, those are rules a +// reader applies, and the two are indexed against different corpora. +export { + DIACRITICS, + diacriticId, + diacriticName, + isDiacriticName, + type DiacriticName, +} from "./diacritics.js"; + // Loop 6a — wayfinding: the jumper's query language, the app-level keyboard // precedence rule, and the edition registry + cross-edition concordance seam. export { diff --git a/packages/etl/scripts/lib/diacritics.mjs b/packages/etl/scripts/lib/diacritics.mjs new file mode 100644 index 0000000..efad0b4 --- /dev/null +++ b/packages/etl/scripts/lib/diacritics.mjs @@ -0,0 +1,139 @@ +/** + * The marks inside a word, on our frame. + * + * `build-words.mjs` ships one rectangle per word. This reads the level below + * it: the fatha, the shadda, the superscript alef — the twenty-six things the + * ligature corpus names in `data-diacritic`, each with its own exact box, each + * attached to the word it sits on. + * + * ## Why this is a lib and not a script + * + * It has two callers with different appetites. `probe-encodings.mjs` wants the + * marks for one page at a time, to draw them and check the tajweed spans + * against them, and ships nothing. `build-words.mjs` will eventually want all + * 604 to write shards. Those are the same extraction and must stay the same + * extraction, for the reason `mushaf-frame.mjs`'s header gives about itself: if + * the inspector's boxes and the shipped boxes came from two implementations, + * looking at the inspector would stop being evidence about what gets shipped, + * which is the only reason to look at it. + * + * ## Nothing here fits anything + * + * `apply` is handed in. The transform from their frame to ours is fitted once + * per page by `build-words.mjs` and recorded in `word-boxes.pin.json`, so a + * caller reconstitutes it from four committed numbers rather than re-deriving + * it from the ornaments. Re-fitting here would create a second transform that + * could disagree with the one the word boxes were built with, and a mark that + * disagrees with its own word by a tenth of a unit is indistinguishable from a + * mark on the wrong letter. + * + * ## The containment invariant + * + * A word's box in the shards is `union(pathBBox(d))` over **every** path in the + * word's segment — the letters and the marks alike. So a mark's box is inside + * its word's box by construction, exactly, before rounding. That is not a + * pleasing coincidence; it is the check that makes this data verifiable + * offline. A mark that escapes its word's box cannot be a geometry error, + * because the geometry is a subset of the geometry the word box was computed + * from — it can only be an *alignment* error, a mark filed under the wrong + * word. Which is the failure mode worth catching, and the one that produced the + * off-by-one behind `gate:edges`. + */ +import { isDiacriticName, diacriticId } from "@hifth/core"; + +import { pathBBox } from "./mushaf-frame.mjs"; + +/** + * Where a word ends. The same boundary `readTheirs` uses, and it has to be: + * this walks the same document with the same idea of where a word's paths stop, + * so the marks it collects for word *n* are the paths that went into word *n*'s + * box. + */ +const BOUNDARY = / tag.match(new RegExp(`\\s${name}="([^"]*)"`))?.[1] ?? null; + +const round = (n) => Math.round(n * 10) / 10; + +/** + * Every named mark on the page, grouped by word, in their document order. + * + * Returns one entry per `` the page carries, in the same + * order `readTheirs().words` returns them, so a caller can zip the two without + * a join. Each entry is `{ surah, aya, idx, marks }` where `idx` is the print's + * `data-word-index-in-ayah` — the same index the word shards' `from` counts + * from, and emphatically not QAC's (see `build-words.mjs` on the 4,499 ayahs + * where the two segmentations disagree). + * + * `marks` is `[id, x, y, w, h]` per mark, `id` indexing `DIACRITICS`, the box + * already through `apply` and rounded to a tenth of a viewBox unit — the same + * precision the word boxes ship at, and the smallest mark in the corpus is + * 1.42 × 1.64 units in their frame (≈1.9 × 2.2 in ours), so a tenth cannot + * round one away. + * + * @param {string} svg a ligature-corpus page, verbatim + * @param {(b: number[]) => number[]} apply their frame → ours + */ +export function readDiacritics(svg, apply) { + const out = []; + for (const m of svg.matchAll(/]*)>/g)) { + const rest = svg.slice(m.index + m[0].length); + const nxt = rest.match(BOUNDARY); + const seg = nxt ? rest.slice(0, nxt.index) : rest; + + const marks = []; + for (const p of seg.matchAll(PATHS)) { + const name = attr(p[0], "data-diacritic"); + if (name === null) continue; + if (!isDiacriticName(name)) { + // A name @hifth/core has never heard of is a corpus that grew, and the + // only safe response is to stop. Emitting it under a made-up id would + // write geometry nothing can name; skipping it would silently drop a + // mark from a page and look like the print simply has fewer. + throw new Error( + `data-diacritic="${name}" on ${attr(p[0], "id")} is not in DIACRITICS. ` + + "Append it to packages/core/src/diacritics.ts — append, never reorder, " + + "because an id is only meaningful against that array's order.", + ); + } + const d = attr(p[0], "d"); + if (d === null) continue; + const [x0, y0, x1, y1] = apply(pathBBox(d)); + marks.push([diacriticId(name), round(x0), round(y0), round(x1 - x0), round(y1 - y0)]); + } + + out.push({ + surah: Number(attr(m[2], "data-surah")), + aya: Number(attr(m[2], "data-aya")), + idx: Number(attr(m[2], "data-word-index-in-ayah")), + marks, + }); + } + return out; +} + +/** + * `ours = s·theirs + t`, rebuilt from the four numbers `word-boxes.pin.json` + * records for a page. + * + * A copy of `build-words.mjs`'s `applier` in shape but not in role: that one + * closes over a fit that just happened, this one over a fit that happened once + * and was written down. Both exist so that nobody hand-multiplies the numbers + * at a call site, which is where a sign or an axis gets swapped. + */ +export const applierFromPin = (row) => (b) => [ + row.sx * b[0] + row.tx, + row.sy * b[1] + row.ty, + row.sx * b[2] + row.tx, + row.sy * b[3] + row.ty, +]; diff --git a/packages/etl/scripts/probe-diacritics.mjs b/packages/etl/scripts/probe-diacritics.mjs new file mode 100644 index 0000000..3aeb375 --- /dev/null +++ b/packages/etl/scripts/probe-diacritics.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +/** + * Measure the sub-word layer before anything ships it. + * + * The user's question was whether Hifth can highlight *part* of a word — a + * tanween, a shadda, a superscript alef — and `packages/core/src/skins.ts` has + * said since Loop 6a that it cannot, because "its glyphs are anonymous outlined + * ``s". That is true of `assets/pages/**`, which is what the app draws. + * It is false of the ligature corpus, which names every mark it draws in a + * `data-diacritic` attribute — and this probe is how that gets established with + * a number rather than asserted from a grep. + * + * It ships nothing. `build-words.mjs` will emit the shards when there is a + * caller for them (mark-C); this runs first because the honest order is to find + * out whether the boxes are trustworthy before paying two megabytes to send + * them to a phone. + * + * ## The three questions, and why these three + * + * **① Does the vocabulary hold?** Every `data-diacritic` value in the corpus + * must be one `@hifth/core` knows. `readDiacritics` throws otherwise, so this + * is really asking whether the twenty-six names in `DIACRITICS` are all of + * them — a claim that can only be made by reading all 604 pages, which is what + * this does. + * + * **② Does every mark stay inside its own word?** This is the load-bearing one. + * A word's shipped box is `union(pathBBox)` over *every* path in the word, + * marks included, so containment holds exactly before rounding. That makes an + * escape impossible for geometric reasons and possible only for alignment + * reasons: a mark filed under the wrong word. It is the same failure mode as + * the off-by-one that made 47.8% of hop edges wrong, and it is worth measuring + * against the *committed* shards rather than against boxes computed in the same + * pass — otherwise both sides share a mistake and agree about it. + * + * **③ What would it weigh?** Because that decides whether mark-C is a decision + * or a formality. Measured as the shard text `build-words.mjs` would write, not + * estimated from a path count. + * + * ## What it deliberately does not check + * + * Whether a mark is on the *right letter*. Nothing in this repo can answer that + * offline — it would need the print's own letter order, which the corpus gives + * as ligature ids this does not read, and ultimately a reader's eye. That check + * belongs to the encoding inspector (mark-B), where a human can see the boxes + * on the page beside the three other encodings. + * + * Usage: + * pnpm --filter @hifth/etl probe:diacritics + * pnpm --filter @hifth/etl probe:diacritics --pages 1,2,7 + */ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { gzipSync } from "node:zlib"; + +import { DIACRITICS, diacriticName } from "@hifth/core"; + +import { candidatePage } from "./lib/candidate-pages.mjs"; +import { applierFromPin, readDiacritics } from "./lib/diacritics.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = join(HERE, "..", "..", ".."); +const PIN = join(HERE, "..", "data", "pages", "word-boxes.pin.json"); +const WORDS = join(REPO, "apps", "web", "public", "assets", "words", "hafs-kfqc"); + +const argv = process.argv.slice(2); +const only = (() => { + const i = argv.indexOf("--pages"); + return i < 0 ? null : argv[i + 1].split(",").map(Number); +})(); + +/** + * How far outside its word's box a mark may sit, in viewBox units. + * + * Rounding only. Both boxes are written to one decimal, so each edge can move + * 0.05 and the two can move in opposite directions — 0.1 is the arithmetic + * bound and 0.2 is that with one decimal place of slack. It is deliberately not + * a tolerance for misregistration: there is nothing to misregister, because + * both boxes come out of the same fit applied to paths from the same file. + */ +const SLACK = 0.2; + +const pin = JSON.parse(readFileSync(PIN, "utf8")); +const rows = new Map(pin.pages.map((p) => [p.page, p])); +const wanted = only ?? pin.pages.map((p) => p.page); + +console.log(`\n probe:diacritics — ${wanted.length} page(s), reading the cache and nothing else\n`); + +const seen = new Map(); +const escapes = []; +let marks = 0; +let words = 0; +let wordsWithMarks = 0; +let unmatched = 0; +let raw = 0; +let gz = 0; +let smallest = Infinity; + +for (const page of wanted) { + const row = rows.get(page); + if (!row) { + console.error(`\n FAIL p${page}: word-boxes.pin.json has no row for it\n`); + process.exit(1); + } + const { body } = await candidatePage(page, { offline: true }); + const shard = JSON.parse(readFileSync(join(WORDS, `${page}.json`), "utf8")); + + const perWord = readDiacritics(body.toString("utf8"), applierFromPin(row)); + + for (const w of perWord) { + words += 1; + if (w.marks.length) wordsWithMarks += 1; + for (const m of w.marks) { + marks += 1; + seen.set(m[0], (seen.get(m[0]) ?? 0) + 1); + smallest = Math.min(smallest, m[3], m[4]); + } + + const key = `${w.surah}:${w.aya}`; + const ayah = shard.words[key]; + if (!ayah) { + // A word in the corpus whose ayah the committed shard does not carry. + // Counted rather than thrown: it would mean the two page reads disagree + // about what is on the page, which is a finding, not a crash. + unmatched += 1; + continue; + } + const at = w.idx - ayah.from; + const box = ayah.boxes[at]; + if (!box) { + unmatched += 1; + continue; + } + for (const [id, x, y, mw, mh] of w.marks) { + const outside = + x < box[0] - SLACK || + y < box[1] - SLACK || + x + mw > box[0] + box[2] + SLACK || + y + mh > box[1] + box[3] + SLACK; + if (outside) { + escapes.push({ page, key, idx: w.idx, name: diacriticName(id), mark: [x, y, mw, mh], box }); + } + } + } + + // Weighed on the real shape rather than the placeholder above: per ayah, a + // `from` and a dense array of per-word mark lists, empty where a word has + // none. The empties are kept because position IS the word index — dropping + // them would need a second index per entry, which costs more than the `[]`. + const dense = {}; + for (const [key, ayah] of Object.entries(shard.words)) { + const list = ayah.boxes.map(() => []); + for (const w of perWord) { + if (`${w.surah}:${w.aya}` !== key) continue; + const at = w.idx - ayah.from; + if (at >= 0 && at < list.length) list[at] = w.marks; + } + if (list.some((l) => l.length)) dense[key] = { from: ayah.from, words: list }; + } + const text = `${JSON.stringify({ page, diacritics: dense })}\n`; + raw += Buffer.byteLength(text); + gz += gzipSync(text).length; +} + +// ── ① the vocabulary ───────────────────────────────────────────────────────── + +const unused = DIACRITICS.map((n, i) => [i, n]).filter(([i]) => !seen.has(i)); +console.log(` ① vocabulary — ${seen.size} of ${DIACRITICS.length} names drawn`); +for (const [id, n] of [...seen].sort((a, b) => b[1] - a[1])) { + console.log(` ${String(id).padStart(2)} ${String(n).padStart(7)} ${diacriticName(id)}`); +} +if (unused.length) { + console.log(` unused on these pages: ${unused.map(([, n]) => n).join(", ")}`); +} + +// ── ② containment ──────────────────────────────────────────────────────────── + +console.log( + `\n ② containment — ${marks} marks on ${wordsWithMarks} of ${words} words, ` + + `${escapes.length} outside their word (slack ${SLACK})`, +); +for (const e of escapes.slice(0, 10)) { + console.log( + ` p${e.page} ${e.key} word ${e.idx} ${e.name}: ` + + `[${e.mark.join(",")}] not inside [${e.box.join(",")}]`, + ); +} +if (unmatched) console.log(` ${unmatched} word(s) had no box in the committed shard`); + +// ── ③ weight ───────────────────────────────────────────────────────────────── + +console.log( + `\n ③ weight — ${(raw / 1024 / 1024).toFixed(2)} MB raw / ` + + `${(gz / 1024 / 1024).toFixed(2)} MB gz across ${wanted.length} shard(s)` + + `\n smallest mark on our frame: ${smallest.toFixed(1)} units\n`, +); + +if (escapes.length || unmatched) { + console.error(" probe:diacritics — the boxes are not yet trustworthy; see above\n"); + process.exit(1); +} +console.log(" probe:diacritics — every mark sits inside its own word\n"); From 1e00f50667752c65a02b8fae889f31feefc53b97 Mon Sep 17 00:00:00 2001 From: Omar Eid Date: Fri, 7 Aug 2026 08:20:31 -0500 Subject: [PATCH 2/2] The registers learn that a word has a level below it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docs/design/sub-word-marks.md` — the design of record for the named-mark layer, and the place §② keeps two different things called a mark apart: the word shards' `marks` are *pause* marks whose integers are word indices, and these are drawn inside a word. That collision is why the asset kind would be `diacritics`, and why `DIACRITICS` deliberately cannot name a waqf sign. Four rows under its open-question section, indexed in `docs/issues.json`: ① does a tajweed span land on a mark a reader can be shown · open Two measurements now exist over the same words and nothing has checked whether they meet. Only the encoding inspector can: the correspondence is between a codepoint in a reconstructed text and an outline on a page. Deriving it from the fact that both numbers exist is the circularity `word-indexing.md` ⑪ ⑤ names about its own oracle. ② are the shards worth 2.28 MB, and against what ceiling · open Waiting on a caller, not on a number. `gate:assets` fails outright on a kind it has never heard of, and ⑪ ① is the standing warning that shipping an answer costs multiples of shipping the question. ③ `skins.ts` still says the print's glyphs are anonymous · confirmed Left as a defect rather than edited in place: it is load-bearing prose about why a shipped feature has the granularity it has, and rewriting it before the replacement exists would claim a capability the app lacks. ④ a mark id is only meaningful against an array's order · open Deferred deliberately — the risk does not exist until something is shipped that an id can be stale in. `docs/map.json` gains four pointers on `word-geometry`, hand-edited. The doc's pointer names a body line rather than a heading, because `code-pointers.mjs` reads a markdown `#` as a comment. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EuhvbUKjGesE3uMhjCzBGt --- docs/design/sub-word-marks.md | 260 ++++++++++++++++++++++++++++++++++ docs/issues.json | 40 ++++++ docs/issues.md | 8 +- docs/map.json | 20 +++ 4 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 docs/design/sub-word-marks.md diff --git a/docs/design/sub-word-marks.md b/docs/design/sub-word-marks.md new file mode 100644 index 0000000..696d9a9 --- /dev/null +++ b/docs/design/sub-word-marks.md @@ -0,0 +1,260 @@ +# Sub-word marks: the level below a word, and the print that already named it + +> Read this before answering "can Hifth highlight the tanween?" with the sentence +> [`skins.ts`](../../packages/core/src/skins.ts) has carried since Loop 6a. That sentence was +> true about the pictures we draw and false about the corpus we measure from, and this +> document is the measurement that settles which. + +**Status:** design of record for the **named-mark layer**. The vocabulary and the extraction +are built and measured over all 604 pages (§③–§⑥); **nothing ships**, on purpose (§⑥). What +is not decided is whether a tajweed span corresponds to a mark a reader can be shown, and +whether the shards are worth their bytes — §⑧ ① and ②. + +## How to read this, and what it is not + +`docs/decisions/loop-*.md` record what a finished loop settled; `docs/design/` is what a +reader consults *before* changing a feature. This file covers +[`packages/core/src/diacritics.ts`](../../packages/core/src/diacritics.ts), +[`packages/etl/scripts/lib/diacritics.mjs`](../../packages/etl/scripts/lib/diacritics.mjs) +and [`probe-diacritics.mjs`](../../packages/etl/scripts/probe-diacritics.mjs). + +Its companions, cited rather than restated: + +- [`word-indexing.md`](word-indexing.md) — the level above. Its §⑪ ⑤ reconstructs the text + the tajweed offsets are counted against, and lands 99.86% of 60,057 annotations on the + letter their rule names. This document is what those annotations would be *painted on*. +- [`encoding-inspector.md`](encoding-inspector.md) — where the boxes get looked at. §⑧ ① + is owed to it and cannot be answered anywhere else. +- [`SOURCES.md`](../../SOURCES.md) — the ligature corpus's provenance. No new upstream is + involved here; this reads bytes already accounted for. +- [`docs/map.json`](../map.json), feature `word-geometry` — the code pointers. + +**This is not a tajweed feature.** A mark the print draws and a rule a reader applies are +different things indexed against different corpora, and §② is about keeping them apart. It +is also not a text: nothing here holds, reconstructs or ships scripture — it holds +rectangles and integers. + +--- + +## ① The claim this contradicts + +`packages/core/src/skins.ts` says the tajweed skin ships at ayah granularity because the +print's "glyphs are anonymous outlined ``s". That is exactly true of +`apps/web/public/assets/pages/**`, which is what the app draws: a page there is outlines, +and an outline does not know it is a fatha. + +It is false of the ligature corpus — the second print of the same mus'haf that +[`word-indexing.md`](word-indexing.md) already reads for word boxes. There every mark is +drawn by a `` inside its word's group, named +by the publisher, with its own outline. The blocker was never that the marks are anonymous; +it is that they are anonymous *in the file we ship*. Those are different problems with +different costs, and the same comment already anticipated this: it names Loop 4b's ligature +corpus as its own gate. + +The honest reading is that §① is a **stale comment**, not a stale design — the geometry it +was waiting for arrived and nothing went back to it. It is filed as such: §⑧ ③. + +## ② Two things called a mark + +The word shards already carry a field called `marks`: + +```json +{"page":3,"words":{"2:6":{"from":1,"boxes":[[x,y,w,h]],"marks":[8]}}} +``` + +Those are **pause marks** — waqf signs, the sajdah, the juz star — and the integers are +*word* indices, because the print numbers a pause mark as a word of its own. +`gate:words` measures 4,486 of them and checks each one hangs no more than 4 units below +its ayah's polygon. + +The marks in this document are a level down: a fatha, a shadda, a superscript alef, drawn +*inside* a word rather than beside it. The collision is why the asset kind is called +`diacritics` and not `marks`, and why `DIACRITICS` deliberately cannot name a waqf sign. +A pause mark is already flagged one level up, where it belongs. + +## ③ The vocabulary, and why an integer + +Twenty-six names. The number is measured, not chosen: it is the count of distinct +`data-diacritic` values across all 604 pages, in frequency order, with the counts written +into `diacritics.ts` beside each name. + +| id | name | drawn | | id | name | drawn | +|---:|---|---:|---|---:|---|---:| +| 0 | fatha | 122,948 | | 13 | small waw | 1,257 | +| 1 | kasra | 45,970 | | 14 | small yeh | 995 | +| 2 | damma | 37,320 | | 15 | fathatan | 734 | +| 3 | sukun | 37,148 | | 16 | kasratan | 599 | +| 4 | shadda | 22,678 | | 17 | dammatan | 578 | +| 5 | hamza | 16,385 | | 18 | small meem | 270 | +| 6 | wasla | 13,483 | | 19 | damma iqlab | 134 | +| 7 | superscript alef | 9,726 | | 20 | fatha iqlab | 106 | +| 8 | maddah | 5,376 | | 21 | kasra iqlab | 99 | +| 9 | rounded zero | 3,988 | | 22 | rectangular zero | 66 | +| 10 | successive fathatan | 2,901 | | 23 | small seen | 8 | +| 11 | successive kasratan | 1,935 | | 24 | vowel sign | 3 | +| 12 | successive dammatan | 1,807 | | 25 | small noon | 1 | + +A shard would say `0`, not `"fatha"`. A page carries about 540 marks and the names are the +larger half of the bytes, so the integer is the difference between an asset that is worth +shipping and one that is not. + +**That makes the array's order load-bearing in a way nothing else in `@hifth/core` is.** +Appending a name is safe. Reordering silently re-labels the entire corpus — every fatha in +604 shards becomes a kasra — and changes no geometry, so no gate that measures rectangles +would notice. `diacritics.test.ts` pins six ids for exactly that reason, and the docblock +says the rule in one line: *appending is safe, reordering is not*. + +**What the vocabulary excludes, and why.** The dots. `data-dots` carries three values across +105,269 paths and none of them is in `DIACRITICS`, because i'jam is what distinguishes one +letter from another — it is part of the letter's identity, not a mark a reader is told to +notice. A test asserts `isDiacriticName("two dots")` is false so the exclusion is a decision +rather than an omission. Pause marks are excluded for §②'s reason. + +**And what a name is not a claim about.** "damma iqlab" records what the corpus wrote in an +attribute. It is not this repo asserting a recitation rule; nothing here says how any of +these twenty-six are pronounced, and §⑧ ① is where that question is allowed to be asked at +all. + +## ④ The containment invariant + +`build-words.mjs` computes a word's box as `union(pathBBox(d))` over **every** path in the +word's group — the letters and the marks alike. So a mark's box is inside its word's box by +construction, exactly, before rounding. + +That is not a pleasing coincidence; it is what makes this data verifiable offline with no +second opinion. A mark that escapes its word cannot be a *geometry* error, because the +geometry is a subset of the geometry the word box was computed from. It can only be an +**alignment** error — a mark filed under the wrong word — which is the same failure class as +the off-by-one that made 47.8% of hop edges point at the wrong ayah, and the one worth +paying a corpus-wide pass to rule out. + +Two things follow, and both are enforced rather than intended: + +- **Nothing here fits anything.** `readDiacritics` takes its transform as an argument; + `applierFromPin` rebuilds it from the four numbers `word-boxes.pin.json` already records + per page. Re-fitting would create a second transform that could disagree with the first, + and a mark that disagrees with its own word by a tenth of a unit is indistinguishable from + a mark on the wrong letter. +- **Containment is checked against the committed shards**, not against boxes computed in the + same pass. Two sides derived from one computation share their mistakes and agree about + them. + +## ⑤ What was measured + +`pnpm probe:diacritics`, all 604 cached pages, 2026-08-07: + +| measure | result | +|---|---| +| distinct `data-diacritic` values, all pages | **26 of 26** in `DIACRITICS`; none unknown | +| marks extracted | **326,515** | +| words carrying at least one | 86,964 of 91,451 | +| marks outside their own word's box | **0** (slack 0.2 — see below) | +| words with no box in the committed shard | **0** | +| smallest mark on our frame | 1.8 viewBox units | + +The slack is 0.1 of arithmetic plus a decimal place of room, and it is **not** a tolerance +for misregistration. Both boxes are written to one decimal, so each edge can move 0.05 and +the two can move in opposite directions; there is nothing else to absorb, because both come +out of one fit applied to paths from one file. + +**The 4,487 words with no mark are 4,486 pause marks and one word.** The one is +p312, 20:1 word 1 — «طه», the muqatta'at that opens its surah, which this print writes with +no vowel, no sukun and no maddah. Every other opening of that kind carries at least one. +That the residual came out as *the pause marks, exactly, plus a letter pair that visibly has +nothing on it* is the strongest evidence here that the extraction is filing marks under the +right words: an off-by-one would have scattered the empties. + +## ⑥ What it would weigh, and why nothing shipped + +Measured as the shard text `build-words.mjs` would actually write — a `from` and a dense +per-word list per ayah, empties kept because position *is* the word index: + +**7.35 MB raw / 2.28 MB gz across 604 shards.** + +Against `gate:assets`'s `MAX_MUSHAF_GZ` of 32 MB with the corpus at 27.87 MB today, that +fits. It is also 2.28 MB nobody has asked for yet, and shipping an asset with no reader is +the precise failure that gate already names for a non-vendored edition: *the download is +paid for and unreachable*. So mark-A ships **nothing** to `apps/web/public/assets`, and the +order is: + +1. **mark-A** *(this)* — the vocabulary, the extraction, the corpus-wide measurement. +2. **mark-B** — draw the marks in the encoding inspector, beside the three encodings it + already reconciles, and answer §⑧ ① with an eye on them. +3. **mark-C** — only then `build-words.mjs` emits `assets/diacritics/**` from the same + per-page fit, `gate:words` gains the containment check, `gate:assets` gains a ceiling. + +That order is also the answer the user gave when asked where the marks should appear first: +**the inspector, then the app**. + +## ⑦ What this cannot answer + +Whether a mark is on the *right letter*. Nothing in this repo can settle that offline — it +would need the print's own letter order, which the corpus expresses as ligature ids this +does not read, and in the end a reader's eye. Containment proves a mark belongs to its word; +it says nothing about where inside the word it belongs. That is the inspector's job, and it +is why mark-B exists as a separate step rather than a review of mark-C. + +--- + +## ⑧ Open questions, and what would answer each + +Every design doc in this repo ends under this heading, and every item is an +`### ⓝ … · **status**` row so `pnpm gate:issues` can read it. The vocabulary is defined once +in [`docs/issues.json`](../issues.json). + +### ① Does a tajweed span land on a mark a reader can be shown · **open** + +[`word-indexing.md`](word-indexing.md) §⑪ ⑤ puts 60,057 tajweed annotations on the letter +their rule names, 99.86% of the time, and 83.31% of them inside a single print word. This +document puts 326,515 named marks inside those same words. The question is whether the two +meet: when `madd_246` opens at a codepoint, is there a `maddah` box there — and if there is, +is highlighting *it* a truer rendering of the rule than washing the whole word? + +**What would answer it:** the encoding inspector (mark-B). It already reconciles the print, +the ligature corpus, QAC and the tajweed offsets on one screen for one page; adding the mark +boxes puts all four descriptions and the geometry in one place where a human can see whether +a span and a mark coincide. Nothing offline can do this — the correspondence is between a +codepoint in a reconstructed text and an outline on a page, and only an eye closes that gap. + +**What must not happen instead:** deriving the correspondence from the fact that both +numbers exist. Reading a mapping off where the offsets happen to land and then declaring +they land there is the circularity §⑪ ⑤ names about its own oracle, and it passes on a +broken answer. + +### ② Are the shards worth 2.28 MB, and against what ceiling · **open** + +§⑥ prices the tree. It fits under `MAX_MUSHAF_GZ` and it is still the second-largest asset +kind this repo would own, behind the pages themselves. + +**What would answer it:** a caller. If mark-B shows that a mark-granular highlight reads +better than a word wash, the bytes buy something and mark-C spends them; if it does not, +this stays a probe and the corpus stays uncosted, which is a legitimate outcome and not a +failure. `word-indexing.md` §⑪ ① is the standing warning about the arithmetic: shipping an +*answer* costs multiples of shipping the *question*, so a `diacritics` ceiling must be set +from a build rather than from the 2.28 MB above, and `gate:assets` reviewed rather than +assumed — a kind it has never heard of fails it outright. + +### ③ `skins.ts` still says the print's glyphs are anonymous · **confirmed** + +`packages/core/src/skins.ts` justifies ayah-granular tajweed painting with the claim that +the print's glyphs are anonymous outlined ``s. §① is why that is a half-truth, and +§⑤ is 326,515 counterexamples in the corpus the same comment names as its own gate. + +Left as a defect rather than edited in place, deliberately: the comment is load-bearing +prose about why a shipped feature has the granularity it has, and rewriting it before the +replacement granularity exists would leave the file claiming a capability the app does not +have. It is closed by mark-C, in the commit that gives the skin something finer to paint — +or, if §⑧ ② answers "not worth it", by a correction that says the marks are named and +measured and still not shipped, which is a different sentence from the one there now. + +### ④ A mark id is only meaningful against an array's order · **open** + +`DIACRITICS`'s order is a wire format with no version in it. Today one test pins six ids and +one docblock states the rule; both are inside the package that would be doing the +reordering. + +**What would answer it:** either a gate that reads the committed shards and asserts the +vocabulary they were written against — cheap once shards exist, meaningless before — or a +decision that the test is enough because the ETL rebuilds every shard from source in one +pass and a stale shard cannot survive a build. Deferred to mark-C on purpose: the risk does +not exist until something is shipped that an id can be stale *in*. diff --git a/docs/issues.json b/docs/issues.json index dd92ecd..3ae9201 100644 --- a/docs/issues.json +++ b/docs/issues.json @@ -458,6 +458,46 @@ "note": "Track B's stated payload was: bundle the corpus offline, native share sheet, Universal Links, haptics, state restoration, iPad two-page spread, App Store review notes. Read one at a time against what the web build now does, most of it has already been delivered or dismissed. Offline-by-default is Loop 6b's pin-a-juz packs, and the whole print is 27.87 MB gz across 1,584 files measured by gate:assets — under every store threshold that would have forced staged delivery, which retires the delivery plan's On-Demand-Resources risk outright. The iPad spread is recorded at PLAN.md:1038 as having become a web concern and desktop.md shipped it. navigator.share already exists; the wrapper buys the fallback, not the feature. Haptics and state restoration are small and are not a reason. What is left is ONE load-bearing promise — Universal Links, i.e. a tapped hop link opening an installed app, which research §6 confirms is a genuine iOS platform limit — and it is on the one platform whose store is blocked. The honest remaining options: drop Track B; ship Android only and accept the asymmetry; or treat the iOS deep-link gap as a documented limitation, which onboarding already respects (research §6 says it must not promise link-into-app on iOS, and it does not). Not decidable from here: what would answer it is web v1.0 shipping and someone using it. Worth recording that the gate did its job — Track B was always gated on v1.0, and in the waiting its reason was half-dismantled by the web build itself rather than by an argument." }, + { + "id": "tajweed-span-to-named-mark", + "source": { "file": "docs/design/sub-word-marks.md", "item": "①" }, + "status": "open", + "severity": "question", + "owner": "agent", + "blockedBy": ["the encoding inspector drawing the marks"], + "note": "Opened 2026-08-07 with mark-A. Two measurements now exist over the same words and nothing has checked whether they meet. word-indexing.md ⑪ ⑤ lands 59,975 of 60,057 tajweed annotations (99.86%) on the letter their rule names, 83.31% of them inside a single print word; sub-word-marks.md ⑤ puts 326,515 named mark boxes inside those same words, 0 of them outside. The open question is whether a span's [start,end) coincides with a mark a reader can be SHOWN — whether madd_246 opening at a codepoint means there is a maddah box there — and, if so, whether highlighting the mark is a truer rendering of the rule than washing the whole word. What would answer it: the encoding inspector (mark-B), which already reconciles the print, the ligature corpus, QAC and the tajweed offsets on one screen and would gain the mark boxes over the same frame. Nothing offline can: the correspondence is between a codepoint in a text word-indexing.md ⑪ ⑤ RECONSTRUCTS and an outline on a page, and only an eye closes that gap. What must not happen instead is deriving the correspondence from the fact that both numbers exist — reading a mapping off where the offsets happen to land and then declaring that they land there is the circularity ⑪ ⑤ names about its own oracle, and it passes on a broken answer. Note this is a rendering question and not a recitation one: DIACRITICS records what the corpus wrote in an attribute and this repo asserts nothing about how any of the twenty-six are pronounced." + }, + + { + "id": "diacritic-shards-need-a-caller", + "source": { "file": "docs/design/sub-word-marks.md", "item": "②" }, + "status": "open", + "severity": "question", + "owner": "agent", + "blockedBy": ["tajweed-span-to-named-mark"], + "note": "Measured with mark-A and deliberately not spent: the mark tree would be 7.35 MB raw / 2.28 MB gz across 604 shards, weighed as the shard text build-words.mjs would actually write rather than estimated from a path count. It fits — gate:assets allows 32 MB gz for the mushaf and the corpus is at 27.87 MB — and it is still 2.28 MB nobody has asked for, which is precisely the waste gate:assets already names for a non-vendored edition: the download is paid for and unreachable. So mark-A ships nothing to apps/web/public/assets and this stays open until mark-B produces a caller. If the inspector shows a mark-granular highlight reads better than a word wash, mark-C spends the bytes; if it does not, this stays a probe and the corpus stays uncosted, which is a legitimate outcome. Two constraints on whoever does spend them: word-indexing.md ⑪ ① is the standing warning that shipping an ANSWER costs multiples of shipping the QUESTION (placement C cost 8.9x its estimate for exactly that reason), so a diacritics ceiling must be set from a build rather than from the 2.28 MB above; and gate:assets fails outright on an asset kind with no CEILINGS entry, so the gate is edited in the same commit or CI goes red." + }, + + { + "id": "skins-says-the-glyphs-are-anonymous", + "source": { "file": "docs/design/sub-word-marks.md", "item": "③" }, + "status": "confirmed", + "severity": "defect", + "owner": "agent", + "blockedBy": ["diacritic-shards-need-a-caller"], + "note": "packages/core/src/skins.ts justifies ayah-granular tajweed painting with the claim that the print's glyphs are anonymous outlined s. That is true of apps/web/public/assets/pages/**, which is what the app draws, and false of the ligature corpus, where every mark is drawn by inside its word's group — 326,515 of them, 26 distinct names, measured over all 604 pages by pnpm probe:diacritics. The comment already names Loop 4b's ligature corpus as its own gate; the geometry arrived and nothing went back to it. Left as a defect rather than edited in place on purpose: it is load-bearing prose about why a shipped feature has the granularity it has, and rewriting it before the replacement granularity exists would leave the file claiming a capability the app does not have. Closed by mark-C, in the commit that gives the skin something finer to paint — or, if diacritic-shards-need-a-caller answers 'not worth it', by a correction saying the marks are named and measured and still not shipped, which is a different sentence from the one there now." + }, + + { + "id": "diacritic-ids-are-an-unversioned-wire-format", + "source": { "file": "docs/design/sub-word-marks.md", "item": "④" }, + "status": "open", + "severity": "risk", + "owner": "agent", + "blockedBy": ["diacritic-shards-need-a-caller"], + "note": "A mark ships as an integer and not a name because a page carries ~540 of them and the names are the larger half of the bytes. That makes DIACRITICS's ORDER a wire format, and it is the only array in @hifth/core that is one. Appending is safe; reordering silently re-labels the whole corpus — every fatha in 604 shards becomes a kasra — while changing no geometry, so no gate that measures rectangles would notice, and readDiacritics would keep passing because it looks names up rather than positions. Today the defence is one docblock stating the rule and six ids pinned in diacritics.test.ts, both inside the package that would be doing the reordering. What would answer it: either a gate reading the committed shards and asserting the vocabulary they were written against — cheap once shards exist, meaningless before — or a decision that the test suffices because the ETL rebuilds every shard from source in one pass and a stale shard cannot survive a build. Deferred to mark-C deliberately: the risk does not exist until something is shipped that an id can be stale in." + }, + { "id": "etl-map-should-generate-itself", "source": { "file": "docs/design/etl-pipeline.md", "item": "①" }, diff --git a/docs/issues.md b/docs/issues.md index 1a5b76b..5339882 100644 --- a/docs/issues.md +++ b/docs/issues.md @@ -1,5 +1,5 @@ - + # Open items @@ -26,15 +26,19 @@ code, never reproduced. `open` — undecided, nothing blocking. `blocked` — th and unavailable. `answered` — decided, nothing owed in code. `fixed` — closed in code *and* in a test that would fail if it came back; the gate refuses the word without one. -## Open — 26 +## Open — 30 | item | status | severity | owner | waiting on | | --- | --- | --- | --- | --- | | [`LICENSES.md` buckets `assets/adj/**` under one of its two upstreams](design/track-b-native.md#-licensesmd-buckets-assetsadj-under-one-of-its-two-upstreams--confirmed) | confirmed | defect | agent | — | +| [`skins.ts` still says the print's glyphs are anonymous](design/sub-word-marks.md#-skinsts-still-says-the-prints-glyphs-are-anonymous--confirmed) | confirmed | defect | agent | diacritic-shards-need-a-caller | | [Arabic number agreement outside `distance`](design/i18n.md#-arabic-number-agreement-outside-distance--open) | open | defect | user | a hafiz | +| [A mark id is only meaningful against an array's order](design/sub-word-marks.md#-a-mark-id-is-only-meaningful-against-an-arrays-order--open) | open | risk | agent | diacritic-shards-need-a-caller | +| [Are the shards worth 2.28 MB, and against what ceiling](design/sub-word-marks.md#-are-the-shards-worth-228-mb-and-against-what-ceiling--open) | open | question | agent | tajweed-span-to-named-mark | | [Does a real fore-edge stack vary?](design/page-transition.md#-does-a-real-fore-edge-stack-vary--open) | open | question | user | — | | [Whether this document should be generated rather than written](design/etl-pipeline.md#-whether-this-document-should-be-generated-rather-than-written--open) | open | question | agent | — | | [Whether the GPL/App-Store reading in ①–③ is right](design/track-b-native.md#-whether-the-gplapp-store-reading-in--is-right--open) | open | risk | user | a licensing opinion | +| [Does a tajweed span land on a mark a reader can be shown](design/sub-word-marks.md#-does-a-tajweed-span-land-on-a-mark-a-reader-can-be-shown--open) | open | question | agent | the encoding inspector drawing the marks | | [`PLAN.md` states an unachievable order and one wrong citation](design/track-b-native.md#-planmd-states-an-unachievable-order-and-one-wrong-citation--open) | open | defect | agent | gpl-and-the-app-store | | [Whether Track B should exist at all after ④ and ⑤](design/track-b-native.md#-whether-track-b-should-exist-at-all-after--and---open) | open | question | user | web v1.0 and somebody using it | | [The CI frame budget is a number from an emulator](backlog.md#-the-ci-frame-budget-is-a-number-from-an-emulator--blocked) | blocked | risk | agent | perf-verdict-on-device | diff --git a/docs/map.json b/docs/map.json index ad4298e..e607dfb 100644 --- a/docs/map.json +++ b/docs/map.json @@ -605,6 +605,26 @@ "file": "scripts/gate-words.mjs", "symbol": "MARK_DROP", "note": "Re-measures every committed box against the polygon it claims — offline, and with its own path parser rather than the ETL's, because a gate that imports the code it is checking only proves the code agrees with itself. Three checks: LEXICAL (a word's centre is inside its own ayah), MARK (a pause mark's bottom edge meets its polygon within `MARK_DROP`, 4 units — measured: 4,453 of 4,486 need nothing and the worst needs 2.9), PAIRING (shard keys and polygon keys match in both directions, 604 pages, zero either way). This is the gate that found the one defect `gate:pages` structurally cannot see: on p577 the ink of 75:5's first word *was* covered by a tappable box — the wrong ayah's. No orphan, so no orphan check could fire. Two prints of the same page can say that; one cannot." + }, + { + "file": "docs/design/sub-word-marks.md", + "symbol": "The blocker was never that the marks are anonymous", + "note": "The design of record for the level below a word, and the answer to \"can Hifth highlight the tanween?\". Read it before touching `skins.ts`'s ayah-granularity comment, which it demotes from a design constraint to a stale sentence (§⑧ ③). Also the place §② keeps two different things called a mark apart: the word shards' `marks` are *pause* marks and their integers are word indices, while these are drawn inside a word — which is why the asset kind would be `diacritics` and why `DIACRITICS` deliberately cannot name a waqf sign." + }, + { + "file": "packages/core/src/diacritics.ts", + "symbol": "export const DIACRITICS", + "note": "The level below a word: the twenty-six names the ligature corpus draws above and below a letter, in the order an id means. A shard says `0`, not `\"fatha\"`, because a page carries ~540 marks and the names are the larger half of the bytes — which makes the array's *order* load-bearing in a way nothing else here is. Appending is safe; reordering silently re-labels the whole corpus and changes no geometry, so nothing downstream would notice. `diacritics.test.ts` pins six ids for that reason. Deliberately excludes the dots (`data-dots`, 105,269 paths, three values): i'jam is part of the letter's identity, not a mark a reader is told to look at. It lives in core and not in the ETL because it is the one place the vocabulary is written down — `build-tajweed.mjs`'s arrangement with `isTajweedRuleId`, for the same reason." + }, + { + "file": "packages/etl/scripts/lib/diacritics.mjs", + "symbol": "export function readDiacritics", + "note": "A named, exact box per mark, on our frame, grouped by the word it sits on. Fits nothing: `apply` is handed in, rebuilt by `applierFromPin` from the four numbers `word-boxes.pin.json` already records, so a mark cannot end up on a second transform that disagrees with its own word's by a tenth of a unit. The invariant that makes this verifiable offline is that a word's shipped box is the union over *every* path in its segment, marks included — so containment holds by construction, and an escape can only be an alignment error, a mark filed under the wrong word. Throws on a `data-diacritic` core has never heard of rather than inventing an id, and its `attr` docblock records the regex trap that cost a corpus-wide run of `[Infinity, …, -Infinity]` boxes: the leading `\\s` is load-bearing, because `d` otherwise matches the tail of `id=\"md-path-…\"`." + }, + { + "file": "packages/etl/scripts/probe-diacritics.mjs", + "symbol": "const SLACK", + "note": "The measurement that had to come before anything shipped, over all 604 cached pages: the vocabulary is complete (26 of 26 names drawn), every one of 326,515 marks lands inside the word box the app already ships (0 escapes, 0 unmatched, slack 0.2 — rounding, not registration), and the shard tree would weigh 7.35 MB raw / 2.28 MB gz. Containment is checked against the *committed* shards rather than boxes computed in the same pass, so the two sides cannot share a mistake and agree about it. It ships nothing on purpose: 2.28 MB of assets no caller fetches is the same waste `gate:assets` names for a non-vendored edition. What it cannot answer is whether a mark is on the right *letter* — that needs an eye, and belongs in the encoding inspector." } ], "extend": [