diff --git a/scripts/check-kko-provenance.mjs b/scripts/check-kko-provenance.mjs index a5098cd..e5c2b58 100644 --- a/scripts/check-kko-provenance.mjs +++ b/scripts/check-kko-provenance.mjs @@ -13,9 +13,14 @@ * ── What it enforces ───────────────────────────────────────────────────────────────── * 1. `ontology/kko/kko-2.10.n3` still hashes to the sha256 recorded in PROVENANCE.md, and is * still the recorded byte length. The doc is the source of the expectation, so the doc and - * the artifact can never silently disagree. + * the artifact can never silently disagree. **SHA-256:** and **Bytes:** are both REQUIRED + * fields: a record that omits or mangles one FAILS this gate. It does not skip the + * comparison and report success — see the note above the byte check for why that distinction + * is the whole point of the script. * 2. PROVENANCE.md pins a COMMIT, not just a branch. `@ master` is a moving reference: re-run * the retrieval later and you may get different bytes while claiming the same provenance. + * The pin is recognised by its label — ``commit `<40-hex>` `` — not by "some 40-hex run + * appears somewhere in the document". * 3. The generated `ts/src/kko-data.ts` is exactly what `scripts/gen-kko.mjs` produces from * that .n3 — i.e. the ontology the engine actually SHIPS derives from the vendored source. * A hand-edit to kko-data.ts, or a re-vendored .n3 without a regeneration, otherwise leaves @@ -50,21 +55,45 @@ if (actual !== declared) ` This digest is pinned by consumers too (prophet-platform apps/owl-reasoner asserts it at import).\n` + ` Re-vendor, then update PROVENANCE.md AND every consumer pin in the same change.`) -const declaredBytes = Number(((doc.match(/\*\*Bytes:\*\*\s*([\d,]+)/) || [])[1] || '0').replace(/,/g, '')) -if (declaredBytes && bytes.length !== declaredBytes) +// The byte count is a REQUIRED field and the comparison below is UNCONDITIONAL. +// This used to read `Number(… || '0')` guarded by `if (declaredBytes && …)`. A missing or +// unparseable **Bytes:** therefore became 0, `if (0 && …)` never fired, and the byte check +// silently evaporated — while the run still printed +// ✓ KKO provenance verified — d907919fb40f20ed… (327,797 bytes) +// quoting the file's OWN length as though it had been checked against the record. A check that +// disappears when its input goes missing is worse than no check at all: it manufactures the +// evidence that it ran. Absent or malformed input must FAIL, exactly the way the missing +// **SHA-256:** case above already fails. (Same disease as the commit-pin note below.) +const declaredBytesText = (doc.match(/\*\*Bytes:\*\*\s*([0-9][0-9,]*)/) || [])[1] +const declaredBytes = declaredBytesText === undefined ? NaN : Number(declaredBytesText.replace(/,/g, '')) +if (!Number.isSafeInteger(declaredBytes)) + die('ontology/kko/PROVENANCE.md does not declare a parseable **Bytes:** — the record must state what the artifact should be') +if (bytes.length !== declaredBytes) die(`vendored KKO TBox is ${bytes.length} bytes, PROVENANCE.md declares ${declaredBytes}`) // ── 2) the source must be PINNED, not a moving branch ref ── -// NB: search for the commit pin only AFTER removing the sha256 from the text. A naive -// /[0-9a-f]{40}/ over the whole document matches a 40-char window INSIDE the 64-char sha256 — -// the check would have been satisfied by the very digest it is supposed to be independent of. +// The pin is located by its LABEL — ``commit `<40-hex>` `` — not by scanning the document for a +// bare 40-hex run. An unanchored scan does not answer "is the source pinned?", it answers "does +// any 40-hex string appear anywhere in this file?", and those come apart: a sha1 of some other +// artifact, another repo's commit sha, a hash sitting inside an unrelated URL, all satisfy it +// while the source line still reads `@ master`. The check would pass by accident and report a pin +// that is not there. +// Two details are load-bearing: +// · the (?![0-9a-f]) tail — without it the pattern accepts the first 40 characters of a 64-char +// sha256 that happens to be labelled `commit`; +// · the digest strip below — so the artifact's own sha256 can never be the thing that supplies +// its own pin, even if this pattern is loosened later. // (A checker that validates itself validates nothing; this repo's estate has been bitten by -// exactly that shape before.) +// exactly that shape before — including by the byte check a few lines above, which until +// 2026-07-30 switched ITSELF off whenever PROVENANCE.md stopped declaring a byte count, and went +// on printing a success line that named the bytes it had not verified. Absent input must fail, +// never skip.) const withoutDigest = doc.split(declared).join('') -const commitPin = (withoutDigest.match(/\b([0-9a-f]{40})\b/) || [])[1] +const commitPin = (withoutDigest.match(/\b[Cc]ommit\s+`?([0-9a-f]{40})(?![0-9a-f])/) || [])[1] if (!commitPin) die('ontology/kko/PROVENANCE.md pins no commit sha. "@ master" is a MOVING reference — the same ' + - 'retrieval later can return different bytes and still claim this provenance. Pin the commit.') + 'retrieval later can return different bytes and still claim this provenance. ' + + 'Pin the commit, and record it in the form: commit `<40-hex-sha>`') for (const needle of ['SocioProphet/kbpedia', 'CC-BY-4.0']) if (!doc.includes(needle)) die(`ontology/kko/PROVENANCE.md does not record ${needle}`) diff --git a/ts/src/kko-provenance-guard.test.ts b/ts/src/kko-provenance-guard.test.ts new file mode 100644 index 0000000..609aa56 --- /dev/null +++ b/ts/src/kko-provenance-guard.test.ts @@ -0,0 +1,123 @@ +/** + * Regression tests for `scripts/check-kko-provenance.mjs` — the gate itself, not the ontology. + * + * Why these exist: the byte-count check in that script used to read + * const declaredBytes = Number((doc.match(/\*\*Bytes:\*\*\s*([\d,]+)/) || [])[1] || '0') + * if (declaredBytes && bytes.length !== declaredBytes) die(...) + * so a PROVENANCE.md with no parseable **Bytes:** produced 0, `if (0 && …)` never fired, and the + * comparison silently evaporated — while the run still printed a success line quoting the file's + * OWN byte count as though it had been verified against the record. The gate reported that it had + * checked something it had not checked. + * + * A test suite for a gate has to prove teeth in BOTH directions: that the real record still + * passes, and that each specific corruption still fails. Asserting only the happy path is how a + * check that cannot fail gets to look healthy forever. + * + * These drive the REAL script as a subprocess rather than re-declaring its regexes here — a test + * that re-implements the thing it is testing agrees with itself by construction and proves + * nothing. The script resolves every path relative to its own file URL, so each case runs in a + * throwaway root: a copy of the script, a copy of the record under test, and symlinks back to the + * real .n3 and ts/ tree. The repo's own ontology/kko/PROVENANCE.md is never written to. + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const repo = path.resolve(here, '..', '..') +const guardName = 'check-kko-provenance.mjs' +const guardPath = path.join(repo, 'scripts', guardName) +const recordPath = path.join(repo, 'ontology', 'kko', 'PROVENANCE.md') +const n3Path = path.join(repo, 'ontology', 'kko', 'kko-2.10.n3') + +/** Run the guard against a rewritten record. Returns its exit code and combined output. */ +const runGuard = (mutate: (doc: string) => string): { code: number; out: string } => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'hg-kko-guard-')) + try { + fs.mkdirSync(path.join(root, 'scripts')) + fs.mkdirSync(path.join(root, 'ontology', 'kko'), { recursive: true }) + fs.copyFileSync(guardPath, path.join(root, 'scripts', guardName)) + fs.symlinkSync(n3Path, path.join(root, 'ontology', 'kko', 'kko-2.10.n3')) + fs.symlinkSync(path.join(repo, 'ts'), path.join(root, 'ts')) + fs.writeFileSync( + path.join(root, 'ontology', 'kko', 'PROVENANCE.md'), + mutate(fs.readFileSync(recordPath, 'utf-8')), + ) + const r = spawnSync(process.execPath, ['--import', 'tsx', path.join(root, 'scripts', guardName)], { + cwd: repo, // so `--import tsx` resolves from the repo's node_modules + encoding: 'utf-8', + }) + return { code: r.status ?? -1, out: `${r.stdout ?? ''}${r.stderr ?? ''}` } + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +} + +/** + * String surgery that must actually bite. If PROVENANCE.md is reworded so a needle stops + * matching, the mutation would no-op and the case would "pass" by testing an unmodified + * document — the same silently-skipped-check failure this file exists to prevent. + */ +const rewrite = (doc: string, from: string, to: string): string => { + assert.ok(doc.includes(from), `test fixture is stale: PROVENANCE.md no longer contains ${JSON.stringify(from)}`) + return doc.split(from).join(to) +} + +const BYTES_LINE = '**Bytes:** 327,797' +const PIN_LINE = '@ commit `3f888b397255b69d1439fd95823e97011ed9440b` (branch `master`)' +const PIN_URL = '/SocioProphet/kbpedia/3f888b397255b69d1439fd95823e97011ed9440b/' +const OTHER_SHA256 = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +const OTHER_SHA1 = 'aaaaaaaabbbbbbbbccccccccddddddddeeeeeeee' + +// ─── control: the gate must still accept the real record ────────────────────────── +// A checker that has been "hardened" into rejecting the genuine artifact is worse than the bug. +test('the committed PROVENANCE.md passes the guard unchanged', () => { + const { code, out } = runGuard((d) => d) + assert.equal(code, 0, out) + assert.match(out, /✓ KKO provenance verified/) +}) + +// ─── the byte count is required, and the comparison is unconditional ────────────── +test('a PROVENANCE.md with no **Bytes:** line FAILS (it must not skip the byte check)', () => { + const { code, out } = runGuard((d) => rewrite(d, `${BYTES_LINE}\n`, '')) + assert.equal(code, 1, `expected the guard to fail, got:\n${out}`) + assert.match(out, /does not declare a parseable \*\*Bytes:\*\*/) + // The old code printed the success line here, quoting the file's own length as verified. + assert.doesNotMatch(out, /provenance verified/) +}) + +test('a malformed **Bytes:** FAILS rather than defaulting to 0', () => { + const { code, out } = runGuard((d) => rewrite(d, BYTES_LINE, '**Bytes:** abc')) + assert.equal(code, 1, `expected the guard to fail, got:\n${out}`) + assert.match(out, /does not declare a parseable \*\*Bytes:\*\*/) + assert.doesNotMatch(out, /provenance verified/) +}) + +test('a **Bytes:** that disagrees with the artifact FAILS', () => { + const { code, out } = runGuard((d) => rewrite(d, BYTES_LINE, '**Bytes:** 999,999')) + assert.equal(code, 1, `expected the guard to fail, got:\n${out}`) + assert.match(out, /is 327797 bytes, PROVENANCE\.md declares 999999/) +}) + +// ─── the commit pin is found by its label, not by "some 40-hex run exists" ──────── +test('an unpinned source FAILS even when an unrelated 40-hex sha appears in the record', () => { + const { code, out } = runGuard((d) => { + let s = rewrite(d, PIN_LINE, '@ `master`') + s = rewrite(s, PIN_URL, '/SocioProphet/kbpedia/master/') + return rewrite(s, '## What it is', `**Consumer pin landed in commit** \`${OTHER_SHA1}\`.\n\n## What it is`) + }) + assert.equal(code, 1, `expected the guard to fail, got:\n${out}`) + assert.match(out, /pins no commit sha/) +}) + +test('a second sha256 recorded alongside the genuine pin does not break the pin check', () => { + const { code, out } = runGuard((d) => + rewrite(d, '## What it is', `**Consumer copy SHA-256:** \`${OTHER_SHA256}\`\n\n## What it is`), + ) + assert.equal(code, 0, out) + assert.match(out, /✓ KKO provenance verified/) +})