diff --git a/package.json b/package.json index 1406beb..cf2b567 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "klaar", - "version": "0.2.0", + "version": "0.3.0", "description": "KLAAR — an open, machine-readable controlled language for specifications that humans and AI agents both have to execute. Reference linter and conformance suite.", "keywords": [ "controlled-language", diff --git a/tools/klaar/src/segment.js b/tools/klaar/src/segment.js index 415fcb0..94948c9 100644 --- a/tools/klaar/src/segment.js +++ b/tools/klaar/src/segment.js @@ -240,7 +240,21 @@ export function segment(source, opts = {}) { const scope = inlineScopeOverride ?? currentScope; - // --- table rows: each cell is its own unit ----------------------------- + // --- table rows: each cell is its own block, split into sentences ------ + // + // A cell used to become ONE unit however much prose it held, and that is + // wrong against the statement of CORE.LEN.01 itself: that rule limits a + // SENTENCE. A reference-table cell carrying five sentences was reported as + // one sentence of 85 words, so splitting on the full stop changed nothing + // and the only ways out were cutting past the point of meaning or raising + // the baseline. Both are the wrong lever, and both were pulled in anger. + // + // A cell is treated exactly like a paragraph now: collect, mask, split. + // Two things deliberately do NOT change. The unit kind stays 'line', + // because a cell is not a free-standing sentence and the shape rules lean + // on that distinction. And every sentence of one cell keeps the SAME block + // id, so a rule declaring `unit: 'block'` still sees the whole cell — + // groupByLine() groups on the block id, not the line. if (/^\|.*\|\s*$/.test(trimmed)) { if (/^\|[\s:|-]+\|$/.test(trimmed)) continue; // separator row const cells = trimmed.slice(1, -1).split('|'); @@ -248,7 +262,31 @@ export function segment(source, opts = {}) { for (const cell of cells) { const t = cell.trim(); if (t && !/^[-:\s]+$/.test(t)) { - units.push(makeUnit(t, maskInline(t), lineNo, col + (cell.length - cell.trimStart().length), scope, 'line', lang, currentHeading, ++blockId)); + const cellCol = col + (cell.length - cell.trimStart().length); + const masked = maskInline(t); + const sentences = splitSentences(masked); + blockId++; + if (sentences.length <= 1) { + units.push(makeUnit(t, masked, lineNo, cellCol, scope, 'line', lang, currentHeading, blockId)); + } else { + // maskInline preserves length, so an offset into the masked cell is + // the same offset into the raw cell — and into the source column. + for (const s of sentences) { + units.push( + makeUnit( + t.slice(s.offset, s.offset + s.text.length), + s.text, + lineNo, + cellCol + s.offset, + scope, + 'line', + lang, + currentHeading, + blockId, + ), + ); + } + } } col += cell.length + 1; } diff --git a/tools/klaar/test/segment.test.js b/tools/klaar/test/segment.test.js index 0215bb2..3422ea1 100644 --- a/tools/klaar/test/segment.test.js +++ b/tools/klaar/test/segment.test.js @@ -86,6 +86,65 @@ test('table cells become their own units', () => { assert.ok(!texts.some((t) => /^[-:| ]+$/.test(t)), 'the separator row must not become a unit'); }); +// A cell that holds prose is prose. CORE.LEN.01 limits a SENTENCE, so a cell carrying five +// of them must not be measured as one. This is the case that made the rule unfixable: the +// author splits on the full stop, the count does not move, and the only remaining lever is +// the baseline. Measured on a real reference table, not invented for the test. +test('a table cell holding several sentences is split into sentences', () => { + const cel = + 'Hoeveel verdicts er zijn uitgesproken. Bij 0 blijft het rood. Vanaf 1 telt het als late vondst.'; + const doc = ['| Veld | Betekenis |', '|---|---|', `| afmaakRonde | ${cel} |`].join('\n'); + const { units } = segment(doc); + const uit = units.filter((u) => u.line === 3).map((u) => u.text); + + assert.ok(uit.includes('Hoeveel verdicts er zijn uitgesproken.'), 'de eerste zin staat op zichzelf'); + assert.ok(uit.includes('Bij 0 blijft het rood.'), 'de tweede zin staat op zichzelf'); + assert.ok(uit.includes('Vanaf 1 telt het als late vondst.'), 'de derde zin staat op zichzelf'); + assert.ok(!uit.includes(cel), 'de hele cel mag NIET meer als een enkele eenheid gelden'); +}); + +test('the sentences of one cell keep one block id, so block rules still see the whole cell', () => { + const doc = [ + '| Veld | Betekenis |', + '|---|---|', + '| a | Eerste zin hier. Tweede zin hier. |', + '| b | Losse cel. |', + ].join('\n'); + const { units } = segment(doc); + const rij = units.filter((u) => u.line === 3); + const cel = rij.filter((u) => u.text.includes('zin hier')); + + assert.equal(cel.length, 2, 'twee zinnen'); + assert.equal(cel[0].block, cel[1].block, 'delen een block-id'); + assert.notEqual(rij[0].block, cel[0].block, 'maar de cel ernaast heeft een eigen block-id'); +}); + +test('a split cell still points at the right column', () => { + // Zonder dit klopt de melding wel, maar wijst hij naar het begin van de cel — en dan zoekt + // de schrijver de verkeerde zin. De offset moet meebewegen met de zin binnen de cel. + const doc = ['| A | Eerste zin. Tweede zin. |', '|---|---|'].join('\n'); + const { units } = segment(doc); + const zinnen = units.filter((u) => u.text.endsWith('zin.')); + + assert.equal(zinnen.length, 2); + const regel = doc.split('\n')[0]; + for (const z of zinnen) { + assert.equal(regel.slice(z.column - 1, z.column - 1 + z.text.length), z.text, `kolom ${z.column} wijst naar "${z.text}"`); + } +}); + +test('inline code in a split cell does not shift the columns', () => { + // maskInline vervangt lengte-behoudend; zou dat ooit veranderen, dan schuiven alle offsets + // in een cel met code-spans stil op. Deze test valt dan om in plaats van de meting. + const doc = ['| A | Zet `AGENT_POOL_MIN` op nul. Daarna schaalt hij mee. |', '|---|---|'].join('\n'); + const { units } = segment(doc); + const tweede = units.find((u) => u.text.startsWith('Daarna')); + + assert.ok(tweede, 'de tweede zin bestaat'); + const regel = doc.split('\n')[0]; + assert.equal(regel.slice(tweede.column - 1, tweede.column - 1 + tweede.text.length), tweede.text); +}); + test('front matter is stripped and line numbers still point at the source', () => { const doc = ['---', 'type: offerte', '---', '', 'Het systeem MOET werken.'].join('\n'); const { units, frontMatter } = segment(doc);