fix(segment): a table cell is not a sentence — split it, like a paragraph

package.json

@@ -1,6 +1,6 @@
 {
   "name": "klaar",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "KLAAR — an open, machine-readable controlled language for specifications that humans and AI agents both have to execute. Reference linter and conformance suite.",
   "keywords": [
     "controlled-language",

tools/klaar/src/segment.js

@@ -240,15 +240,53 @@ export function segment(source, opts = {}) {
 
     const scope = inlineScopeOverride ?? currentScope;
 
-    // --- table rows: each cell is its own unit -----------------------------
+    // --- table rows: each cell is its own block, split into sentences ------
+    //
+    // A cell used to become ONE unit however much prose it held, and that is
+    // wrong against the statement of CORE.LEN.01 itself: that rule limits a
+    // SENTENCE. A reference-table cell carrying five sentences was reported as
+    // one sentence of 85 words, so splitting on the full stop changed nothing
+    // and the only ways out were cutting past the point of meaning or raising
+    // the baseline. Both are the wrong lever, and both were pulled in anger.
+    //
+    // A cell is treated exactly like a paragraph now: collect, mask, split.
+    // Two things deliberately do NOT change. The unit kind stays 'line',
+    // because a cell is not a free-standing sentence and the shape rules lean
+    // on that distinction. And every sentence of one cell keeps the SAME block
+    // id, so a rule declaring `unit: 'block'` still sees the whole cell —
+    // groupByLine() groups on the block id, not the line.
     if (/^\|.*\|\s*$/.test(trimmed)) {
       if (/^\|[\s:|-]+\|$/.test(trimmed)) continue; // separator row
       const cells = trimmed.slice(1, -1).split('|');
       let col = 2;
       for (const cell of cells) {
         const t = cell.trim();
         if (t && !/^[-:\s]+$/.test(t)) {
-          units.push(makeUnit(t, maskInline(t), lineNo, col + (cell.length - cell.trimStart().length), scope, 'line', lang, currentHeading, ++blockId));
+          const cellCol = col + (cell.length - cell.trimStart().length);
+          const masked = maskInline(t);
+          const sentences = splitSentences(masked);
+          blockId++;
+          if (sentences.length <= 1) {
+            units.push(makeUnit(t, masked, lineNo, cellCol, scope, 'line', lang, currentHeading, blockId));
+          } else {
+            // maskInline preserves length, so an offset into the masked cell is
+            // the same offset into the raw cell — and into the source column.
+            for (const s of sentences) {
+              units.push(
+                makeUnit(
+                  t.slice(s.offset, s.offset + s.text.length),
+                  s.text,
+                  lineNo,
+                  cellCol + s.offset,
+                  scope,
+                  'line',
+                  lang,
+                  currentHeading,
+                  blockId,
+                ),
+              );
+            }
+          }
         }
         col += cell.length + 1;
       }

tools/klaar/test/segment.test.js

@@ -86,6 +86,65 @@ test('table cells become their own units', () => {
   assert.ok(!texts.some((t) => /^[-:| ]+$/.test(t)), 'the separator row must not become a unit');
 });
 
+// A cell that holds prose is prose. CORE.LEN.01 limits a SENTENCE, so a cell carrying five
+// of them must not be measured as one. This is the case that made the rule unfixable: the
+// author splits on the full stop, the count does not move, and the only remaining lever is
+// the baseline. Measured on a real reference table, not invented for the test.
+test('a table cell holding several sentences is split into sentences', () => {
+  const cel =
+    'Hoeveel verdicts er zijn uitgesproken. Bij 0 blijft het rood. Vanaf 1 telt het als late vondst.';
+  const doc = ['| Veld | Betekenis |', '|---|---|', `| afmaakRonde | ${cel} |`].join('\n');
+  const { units } = segment(doc);
+  const uit = units.filter((u) => u.line === 3).map((u) => u.text);
+
+  assert.ok(uit.includes('Hoeveel verdicts er zijn uitgesproken.'), 'de eerste zin staat op zichzelf');
+  assert.ok(uit.includes('Bij 0 blijft het rood.'), 'de tweede zin staat op zichzelf');
+  assert.ok(uit.includes('Vanaf 1 telt het als late vondst.'), 'de derde zin staat op zichzelf');
+  assert.ok(!uit.includes(cel), 'de hele cel mag NIET meer als een enkele eenheid gelden');
+});
+
+test('the sentences of one cell keep one block id, so block rules still see the whole cell', () => {
+  const doc = [
+    '| Veld | Betekenis |',
+    '|---|---|',
+    '| a | Eerste zin hier. Tweede zin hier. |',
+    '| b | Losse cel. |',
+  ].join('\n');
+  const { units } = segment(doc);
+  const rij = units.filter((u) => u.line === 3);
+  const cel = rij.filter((u) => u.text.includes('zin hier'));
+
+  assert.equal(cel.length, 2, 'twee zinnen');
+  assert.equal(cel[0].block, cel[1].block, 'delen een block-id');
+  assert.notEqual(rij[0].block, cel[0].block, 'maar de cel ernaast heeft een eigen block-id');
+});
+
+test('a split cell still points at the right column', () => {
+  // Zonder dit klopt de melding wel, maar wijst hij naar het begin van de cel — en dan zoekt
+  // de schrijver de verkeerde zin. De offset moet meebewegen met de zin binnen de cel.
+  const doc = ['| A | Eerste zin. Tweede zin. |', '|---|---|'].join('\n');
+  const { units } = segment(doc);
+  const zinnen = units.filter((u) => u.text.endsWith('zin.'));
+
+  assert.equal(zinnen.length, 2);
+  const regel = doc.split('\n')[0];
+  for (const z of zinnen) {
+    assert.equal(regel.slice(z.column - 1, z.column - 1 + z.text.length), z.text, `kolom ${z.column} wijst naar "${z.text}"`);
+  }
+});
+
+test('inline code in a split cell does not shift the columns', () => {
+  // maskInline vervangt lengte-behoudend; zou dat ooit veranderen, dan schuiven alle offsets
+  // in een cel met code-spans stil op. Deze test valt dan om in plaats van de meting.
+  const doc = ['| A | Zet `AGENT_POOL_MIN` op nul. Daarna schaalt hij mee. |', '|---|---|'].join('\n');
+  const { units } = segment(doc);
+  const tweede = units.find((u) => u.text.startsWith('Daarna'));
+
+  assert.ok(tweede, 'de tweede zin bestaat');
+  const regel = doc.split('\n')[0];
+  assert.equal(regel.slice(tweede.column - 1, tweede.column - 1 + tweede.text.length), tweede.text);
+});
+
 test('front matter is stripped and line numbers still point at the source', () => {
   const doc = ['---', 'type: offerte', '---', '', 'Het systeem MOET werken.'].join('\n');
   const { units, frontMatter } = segment(doc);