From ccad335e54518fd46d475b99dbd64250da574848 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 18:36:43 +0000 Subject: [PATCH 1/2] feat(scripts): derive the `body`-dialect producer table from a scan under a written criterion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The producer table was an enumeration, and every round added to it by hand after somebody noticed a site. Two live producers proved the classifier behind it is permanently blind in two directions rather than merely incomplete: it scores a child list only on an object that also carries a string-literal `type`, and it never reads inside a string or template literal. `scripts/body-dialect-producer-scan.mjs` states the criterion instead — C1 emission, C2 resolution, C3 carrier — and derives the table from it on every run. The criterion, its exclusions and its limits travel with the reading in both output modes. C2's reader set is derived per run rather than listed, so a registration that starts or stops resolving `body` joins or leaves it by itself. Both blind spots are closed and each closure is pinned against the census as a differential control, on the values the two implementations disagree about. An item-carried `body` is filed `unruled:item-carrier` and is never folded into objectui#6771's ruled total: the item face declares `content` and declares no `body` on either published face, so deciding it would extend a ruled family rather than apply one. The instrument reports that question, it does not answer it. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_018HrVaotisyhgmot9o2MLRq --- .changeset/9871-body-dialect-producer-scan.md | 24 + package.json | 1 + scripts/__tests__/body-dialect-census.test.ts | 78 +- .../body-dialect-producer-scan.test.ts | 495 ++++++++++ scripts/body-dialect-producer-scan.mjs | 920 ++++++++++++++++++ 5 files changed, 1491 insertions(+), 27 deletions(-) create mode 100644 .changeset/9871-body-dialect-producer-scan.md create mode 100644 scripts/__tests__/body-dialect-producer-scan.test.ts create mode 100644 scripts/body-dialect-producer-scan.mjs diff --git a/.changeset/9871-body-dialect-producer-scan.md b/.changeset/9871-body-dialect-producer-scan.md new file mode 100644 index 0000000000..7dc8496139 --- /dev/null +++ b/.changeset/9871-body-dialect-producer-scan.md @@ -0,0 +1,24 @@ +--- +--- + +Derive the `body`-dialect producer table from a SCAN under a written criterion, +instead of from an enumeration (objectui#9871). + +`scripts/body-dialect-producer-scan.mjs` states what counts as a producer — C1 +emission, C2 resolution, C3 carrier — and answers it over the tree on every run. +The criterion, the exclusions and the limits are emitted with the reading in +both output modes, so the table cannot be quoted without what made it a table. + +The scan closes the two directions `scripts/body-dialect-census.mjs` declares it +is blind in: it requires no sibling `type` (so a `body` on a non-node item is +visible) and it reads inside string and template literals (so a spelling a tool +writes into the author's document is visible). Both closures are pinned against +the census as a differential control in +`scripts/__tests__/body-dialect-producer-scan.test.ts`, on the values the two +implementations disagree about. + +The `the platform SHIPS the dialect it is being asked to refuse` block in +`scripts/__tests__/body-dialect-census.test.ts` now reads that table rather than +naming sites by hand; its claim and its handoff condition are unchanged. + +Tooling and tests only — no package is released by this change. diff --git a/package.json b/package.json index fdd1707688..79aab57139 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "check:esm-specifiers": "node scripts/check-node-esm-load.mjs --specifiers-only", "check:node-esm-load": "node scripts/check-node-esm-load.mjs", "census:body-dialect": "node scripts/body-dialect-census.mjs", + "census:body-dialect-producers": "node scripts/body-dialect-producer-scan.mjs", "census:tsconfig-test-parity": "node scripts/tsconfig-test-parity-census.mjs", "census:cross-file-line-citations": "node scripts/cross-file-line-citation-census.mjs", "census:page-key-reads": "node scripts/page-key-read-census.mjs", diff --git a/scripts/__tests__/body-dialect-census.test.ts b/scripts/__tests__/body-dialect-census.test.ts index 461df31a8f..aeb76e1cb5 100644 --- a/scripts/__tests__/body-dialect-census.test.ts +++ b/scripts/__tests__/body-dialect-census.test.ts @@ -44,6 +44,7 @@ import { RULED_BUT_NOT_A_READER, BODY_ONLY_UNRULED, } from '../body-dialect-census.mjs'; +import { scan, producersOf } from '../body-dialect-producer-scan.mjs'; const REPO_ROOT = join(__dirname, '..', '..'); const read = (p: string) => readFileSync(join(REPO_ROOT, p), 'utf8'); @@ -313,34 +314,57 @@ describe('the `body` consumers the ruling does not enumerate', () => { // limits and TWO of them hide a producer: it scores a child list only on an // object that also carries a string-LITERAL `type` (a `tabs` ITEM carries // `label`/`value`/`body` and no `type`), and it never reads inside a string - // or a template literal (a VS Code completion snippet is a string). Both - // shapes are live in shipped source today, so the claim in this test's name - // is still TRUE — with a third subject, and step 4 is still not landable. + // or a template literal (a VS Code completion snippet is a string). // - // ⭐ When either assertion below reds, that is the HANDOFF, not a - // regression: re-point this block at whatever still ships the dialect — or, - // once nothing does, say so here in these terms, which is the day step 4 - // becomes landable. ⛔ Do not delete the claim to make the block green. - - // Inserted into the user's own document the moment the completion is - // accepted, which is the most direct form a producer takes: the author does - // not even type the spelling. - const completion = readCode('packages/vscode-extension/src/providers/CompletionProvider.ts'); - expect(completion, 'lit control — the snippet table is being read at all').toContain( - '"className": "$1"' - ); - expect(completion).toContain('"body": {'); - - // A shipped `defaultProps` child list, the same construct as the three - // registrations objectui#7181 moved — spelled on a tab ITEM, where the - // renderer's canonical key is `content`. ⛔ Whether it belongs to - // objectui#6771's ruled family is NOT decided here; that it still ships the - // spelling is what is asserted. - const tabs = readCode('packages/components/src/renderers/layout/tabs.tsx'); - expect(tabs, 'lit control — the defaultProps block is being read at all').toContain( - 'defaultProps:' - ); - expect(tabs).toMatch(/value:\s*'tab1',\s*body:\s*\[/); + // ⭐ objectui#9871 REPLACED the enumeration that used to stand here. Two + // named sites were an enumeration with the census's blind spots written + // into it by hand — the next producer in a third blind direction would have + // needed somebody to notice it and add a line. The subject is now DERIVED: + // `scripts/body-dialect-producer-scan.mjs` states a CRITERION (C1 emission + // · C2 resolution · C3 carrier) and this block reads its table. + // + // ⭐ When the table empties, THAT is the handoff, and the message on the + // assertion says so in these terms — it is the day step 4 becomes landable. + // ⛔ Do not delete the claim to make the block green. + const producerScan = scan(REPO_ROOT); + + // Two lit controls, because an empty table is the assertion's own shape: a + // scan that walked nothing, or derived no reader, would satisfy a bare + // "non-empty" check by accident in neither direction — so both are checked + // before the table is read at all. + expect(producerScan.filesScanned, 'the producer scan walked nothing').toBeGreaterThan(0); + expect( + producerScan.readers.reads.length, + 'the producer scan derived NO reader, so C2 would fail for every site' + ).toBeGreaterThan(0); + + const producers = producersOf(producerScan.hits, producerScan.readers); + expect( + producers.length, + 'the producer table is EMPTY — under objectui#9871\'s criterion nothing in shipped source ' + + 'emits the dialect any more, which is the day objectui#6771 step 4 becomes landable. ' + + 'Re-point this block and say so in those terms; ⛔ do not delete the claim.' + ).toBeGreaterThan(0); + + // ⭐ The two SHAPES the census structurally cannot reach, asserted as + // shapes rather than as paths: a producer arriving in a file nobody has + // named joins `producers` with no list to extend. + expect( + producers.filter((hit: { carrier: string }) => hit.carrier === 'item').length, + 'no item-carried producer — the B1 shape (a `body` on an object with no `type`)' + ).toBeGreaterThan(0); + expect( + producers.filter((hit: { source: string }) => hit.source !== 'code-key').length, + 'no string-carried producer — the B2 shape (the key spelled inside a literal)' + ).toBeGreaterThan(0); + + // ⚠️ And the family question stays OPEN in the instrument rather than being + // settled by it: an item-carried `body` is filed `unruled` and is never + // folded into objectui#6771's ruled total. objectui#9871 handed that + // question back rather than extending a ruled family from a dev seat. + for (const hit of producers.filter((h: { carrier: string }) => h.carrier === 'item')) { + expect(hit.disposition).toBe('unruled:item-carrier'); + } }); }); diff --git a/scripts/__tests__/body-dialect-producer-scan.test.ts b/scripts/__tests__/body-dialect-producer-scan.test.ts new file mode 100644 index 0000000000..9de241aff0 --- /dev/null +++ b/scripts/__tests__/body-dialect-producer-scan.test.ts @@ -0,0 +1,495 @@ +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * Pins for `scripts/body-dialect-producer-scan.mjs` (objectui#9871). + * + * ⭐ The differential control in this file is THE OTHER INSTRUMENT. Most blocks + * plant one corpus and run BOTH `scripts/body-dialect-census.mjs` and the + * producer scan over it, asserting they DISAGREE — because "the scan sees more" + * is the claim, and a scan asserted only against itself cannot fail that way. + * + * ⛔ The value each ablation and each differential is taken on is deliberately + * NOT the headline one. A `body` on an object that already carries a + * string-literal `type` is a value both implementations agree about, so a test + * built on it is green on the census's classifier and on this one alike. The + * values here are the ones they disagree about: a `body` on an object with NO + * `type`, and a `body` spelled INSIDE a string literal. + */ + +import { afterAll, describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { + scan, + deriveReaders, + producersOf, + unchannelledOf, + passesC2, + dispositionOf, + valueKindAt, + finalVerdict, + walkFrames, + enclosingScope, + CRITERION, + NOT_A_PRODUCER, + KNOWN_LIMITS, + EMISSION_CHANNELS, + DECLARATION_SEGMENTS, + DATA_VALUE_KINDS, + EXIT_UNREADABLE, +} from '../body-dialect-producer-scan.mjs'; +import { census } from '../body-dialect-census.mjs'; +import { scanSource, blank } from '../js-comment-mask.mjs'; + +const REPO_ROOT = join(__dirname, '..', '..'); +const read = (p: string) => readFileSync(join(REPO_ROOT, p), 'utf8'); + +const roots: string[] = []; +afterAll(() => { + for (const root of roots) rmSync(root, { recursive: true, force: true }); +}); + +/** + * A planted tree. Every corpus needs a READER too: C2 is applied, not assumed, + * so a corpus with no reader is refused rather than reported as empty — which + * is itself pinned below. + */ +const READER_FILE = ` +import { renderChildren } from '../../lib/utils'; +export const Sidebar = ({ schema }) =>
{renderChildren(schema.body)}
; +export const Tabs = ({ schema }) => schema.items?.map((item) => renderChildren(item.content || (item as any).body)); +`; + +function plant(files: Record) { + const root = mkdtempSync(join(tmpdir(), 'body-dialect-producer-9871-')); + roots.push(root); + const all = { 'packages/reader/src/readers.tsx': READER_FILE, ...files }; + for (const [rel, text] of Object.entries(all)) { + const full = join(root, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, text); + } + return root; +} + +describe('⭐ B1 — the classifier does NOT require a sibling `type`', () => { + /** + * The census scores a child list only on an object that also carries a + * string-literal `type`. A tab ITEM carries `label` / `value` / `body` and no + * `type`, so it is PERMANENTLY invisible to that key — not missed, unreachable. + */ + const ITEM_PRODUCER = ` +import { ComponentRegistry } from '@object-ui/core'; +ComponentRegistry.register('tabs', TabsRenderer, { + defaultProps: { + items: [ + { label: 'Tab 1', value: 'tab1', body: [{ type: 'text', content: 'x' }] }, + ], + }, +}); +`; + + it('DIFFERENTIAL — the census reads 0 on the item and the scan reads it as a producer', () => { + const root = plant({ 'packages/widget/src/tabs.tsx': ITEM_PRODUCER }); + + // ── The census, over the same bytes ────────────────────────────────── + const censusRun = census(root); + // Lit control FIRST: the census walked this tree and resolved the INNER + // node, so its zero below is about its classifier and not about a blind + // walk. Without this the assertion after it passes on an empty scan. + expect(censusRun.filesScanned).toBeGreaterThan(0); + expect(censusRun.hits.filter((h: { type: string }) => h.type === 'text').length).toBe(0); + const censusOnItem = censusRun.hits.filter( + (hit: { file: string; body: boolean }) => hit.file.includes('tabs') && hit.body, + ); + expect(censusOnItem).toEqual([]); + + // ── The producer scan, same bytes, same command ────────────────────── + const run = scan(root); + const producers = producersOf(run.hits, run.readers); + expect(producers.map((p: { carrier: string }) => p.carrier)).toEqual(['item']); + expect(producers[0].line).toBe(6); + expect(producers[0].channel).toBe('default-props'); + }); + + it('`type` is a CARRIER FACT, never an admission test — and any value counts', () => { + // The census requires a string LITERAL `type`. Here the value is an + // identifier, so the census leaves the frame unscored; the carrier is still + // recorded as a node because the key is present. + const text = '{ type: SOME_CONST, body: [] }'; + const { comment, literal } = scanSource(text); + const both = new Uint8Array(comment.length); + for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; + const { hits } = walkFrames(blank(text, both), blank(text, comment), literal); + expect(hits).toHaveLength(1); + expect(hits[0].hasType).toBe(true); + expect(hits[0].nodeType).toBeNull(); + // Lit control — a string-literal `type` still resolves its value. + const lit = '{ type: "badge", body: [] }'; + const f = scanSource(lit); + const b2 = new Uint8Array(f.comment.length); + for (let k = 0; k < b2.length; k++) b2[k] = f.comment[k] || f.literal[k] ? 1 : 0; + expect(walkFrames(blank(lit, b2), blank(lit, f.comment), f.literal).hits[0].nodeType).toBe('badge'); + }); +}); + +describe('⭐ B2 — the scanner READS INSIDE string and template literals', () => { + const SNIPPET_PRODUCER = ` +export class CompletionProvider { + provide() { + const common = [{ name: 'body', desc: 'Child components' }]; + const item = new vscode.CompletionItem('x'); + item.insertText = new vscode.SnippetString('"body": {\\n $0\\n}'); + return [item]; + } +} +`; + + it('DIFFERENTIAL — the census reads 0 inside the literal and the scan reads two producers', () => { + const root = plant({ 'packages/vscode-extension/src/providers/CompletionProvider.ts': SNIPPET_PRODUCER }); + + const censusRun = census(root); + // Lit control — the census walked the tree. + expect(censusRun.filesScanned).toBeGreaterThan(0); + expect( + censusRun.hits.filter((hit: { file: string }) => hit.file.includes('CompletionProvider')), + ).toEqual([]); + + const run = scan(root); + const producers = producersOf(run.hits, run.readers).filter((p: { file: string }) => + p.file.includes('CompletionProvider'), + ); + expect(producers.map((p: { source: string }) => p.source).sort()).toEqual([ + 'key-name-datum', + 'literal-key-syntax', + ]); + for (const producer of producers) { + expect(producer.carrier).toBe('string'); + expect(producer.channel).toBe('snippet'); + // ⛔ A string-carried hit gets NO family disposition: the carrier is + // whatever node the author accepts it onto, which this tree cannot see. + expect(producer.disposition).toBe('carrier-undetermined'); + } + }); + + it('a spelling with NO emission verb in scope is teaching, ⛔ not a producer', () => { + // The same bytes inside a hover card. C1 asks whether the spelling LEAVES + // this tree into somebody's metadata; showing it in documentation does not. + const root = plant({ + 'packages/vscode-extension/src/providers/HoverProvider.ts': ` +export class HoverProvider { + provide() { + return new vscode.MarkdownString('Example:\\n"body": [ ... ]'); + } +} +`, + }); + const run = scan(root); + expect(producersOf(run.hits, run.readers)).toEqual([]); + // Lit control — it is SEEN, and reported; it is only held out of the table. + const unchannelled = unchannelledOf(run.hits, run.readers); + expect(unchannelled.map((h: { file: string }) => h.file.includes('HoverProvider'))).toContain(true); + }); +}); + +describe('C1 — emission, and the three things that are ⛔ not it', () => { + it('a DECLARED value is not emitted data, whatever the key is spelled', () => { + expect(valueKindAt('body: [1]', 5)).toBe('array'); + expect(valueKindAt('body: { a: 1 }', 5)).toBe('object'); + expect(valueKindAt("body: 'x'", 5)).toBe('string'); + expect(valueKindAt('body: z.union([])', 5)).toBe('expression'); + expect(valueKindAt('body: aliasKeyRefusal(x)', 5)).toBe('expression'); + expect(valueKindAt('body: _body,', 5)).toBe('expression'); + expect([...DATA_VALUE_KINDS].sort()).toEqual(['array', 'object', 'string', 'template']); + }); + + it('a zod tombstone is NOT a producer, while real data beside it IS', () => { + const root = plant({ + 'packages/types/src/zod/layout.zod.ts': ` +export const BoxSchema = BaseSchema.extend({ + type: z.literal('box'), + body: aliasKeyRefusal('body', 'children', 'box reads children'), +}); +`, + 'packages/widget/src/w.tsx': ` +ComponentRegistry.register('w', W, { defaultProps: { type: 'w', body: [{ type: 'text' }] } }); +`, + }); + const run = scan(root); + const producers = producersOf(run.hits, run.readers); + // Lit control in the SAME run — the emitting file IS scored, so the zero on + // the tombstone is about the criterion and not about a blind walk. + expect(producers.map((p: { file: string }) => p.file.split('\\').join('/'))).toEqual([ + 'packages/widget/src/w.tsx', + ]); + }); + + it('a registration `inputs` entry DECLARES the key and produces nothing', () => { + const root = plant({ + 'packages/components/src/renderers/overlay/tooltip.tsx': ` +ComponentRegistry.register('tooltip', T, { + inputs: [{ name: 'body', type: 'slot' }], + defaultProps: { type: 'tooltip', body: [{ type: 'text' }] }, +}); +`, + }); + const run = scan(root); + const producers = producersOf(run.hits, run.readers); + // The `inputs` datum is held out; the `defaultProps` data beside it — the + // lit control — is not, so this is the criterion discriminating rather than + // the file being skipped. + expect(producers).toHaveLength(1); + expect(producers[0].channel).toBe('default-props'); + expect(producers[0].carrier).toBe('node'); + expect(DECLARATION_SEGMENTS.some((re: RegExp) => re.test('inputs'))).toBe(true); + // ⛔ `properties` is deliberately NOT a declaration segment: the completion + // provider holds its offerings under exactly that name and then writes them. + expect(DECLARATION_SEGMENTS.some((re: RegExp) => re.test('properties'))).toBe(false); + }); + + it('a channel is recognised by a KEY PATH or an EMISSION VERB, never by a file name', () => { + const ids = EMISSION_CHANNELS.map((c: { id: string }) => c.id); + expect(ids).toEqual(['default-props', 'scaffold', 'snippet']); + for (const channel of EMISSION_CHANNELS as Array<{ kind: string }>) { + expect(['key-path', 'emission-verb']).toContain(channel.kind); + } + }); + + it('the `scaffold` channel fires — measured on a PLANT, because the tree has none today', () => { + // objectui#7181's six producers and objectui#9847's seventh have all + // migrated, so this channel currently matches ZERO live sites. A rule with + // no live instance is a rule nobody has run: it is exercised here instead. + const root = plant({ + 'packages/cli/src/commands/init.ts': ` +const templates = { + simple: { type: 'div', body: [{ type: 'text', content: 'hi' }] }, +}; +export function init() { writeFileSync('app.json', JSON.stringify(templates.simple)); } +`, + }); + const producers = (() => { + const run = scan(root); + return producersOf(run.hits, run.readers); + })(); + expect(producers).toHaveLength(1); + expect(producers[0].channel).toBe('scaffold'); + expect(producers[0].carrier).toBe('node'); + expect(producers[0].nodeType).toBe('div'); + // ⛔ And the verb is real: the same bytes with the write removed produce + // nothing, so `scaffold` is not matching on the word `templates`. + const inert = plant({ 'packages/cli/src/commands/init.ts': ` +const templates = { simple: { type: 'div', body: [{ type: 'text', content: 'hi' }] } }; +export function describeOnly() { return JSON.stringify(templates.simple); } +` }); + const inertRun = scan(inert); + expect(producersOf(inertRun.hits, inertRun.readers)).toEqual([]); + }); +}); + +describe('C2 — applied, ⛔ not merely derived', () => { + it('an HTTP payload `body` fails C2, and the SAME bytes with a node carrier pass', () => { + const root = plant({ + 'packages/data/src/client.ts': ` +export async function send() { + return fetch(url, { method: 'POST', body: { name: 'x' } }); +} +`, + 'packages/widget/src/w.tsx': ` +ComponentRegistry.register('w', W, { defaultProps: { type: 'w', body: [{ type: 'text' }] } }); +`, + }); + const run = scan(root); + const failed = run.hits.filter( + (hit: { file: string }) => hit.file.split('\\').join('/') === 'packages/data/src/client.ts', + ); + expect(failed).toHaveLength(1); + expect(failed[0].carrier).toBe('unknown'); + expect(passesC2(failed[0], run.readers)).toBe(false); + // Lit control — the node-carried twin in the same run passes. + const node = run.hits.find((hit: { carrier: string }) => hit.carrier === 'node'); + expect(node).toBeDefined(); + expect(passesC2(node, run.readers)).toBe(true); + }); + + it('the reader set is DERIVED, and prose ABOUT a read is ⛔ not a read', () => { + const root = plant({ + 'packages/types/src/zod/data-display.zod.ts': ` +export const ListSchema = z.object({}).describe( + 'the renderer draws each entry as item.content || renderChildren(item.body), a read filed under ListItem', +); +`, + }); + const readers = deriveReaders(root); + // The planted reader file is the lit control: the derivation is working, so + // the zero on the docstring is about the literal mask and not about a dead + // instrument. + expect(readers.reads.length).toBeGreaterThan(0); + expect(readers.reads.filter((r: { file: string }) => r.file.includes('data-display'))).toEqual([]); + }); + + it('a corpus with NO reader is REFUSED, not reported as a clean zero', () => { + const root = mkdtempSync(join(tmpdir(), 'body-dialect-producer-noreader-')); + roots.push(root); + const full = join(root, 'packages/widget/src/w.tsx'); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, "const x = { type: 'w', body: [] };\n"); + const run = scan(root); + expect(run.readers.reads).toEqual([]); + const verdict = finalVerdict({ filesScanned: run.filesScanned, readers: run.readers }); + expect(verdict.exit).toBe(EXIT_UNREADABLE); + expect(verdict.refusal?.join(' ')).toContain('Zero readers derived'); + + // Lit control, and the point of the whole card: a real ZERO over a real + // corpus with a real reader is a LEGITIMATE reading and is NOT refused — + // it is the day objectui#6771 step 4 becomes landable. + const clean = scan(plant({})); + expect(producersOf(clean.hits, clean.readers)).toEqual([]); + expect(finalVerdict({ filesScanned: clean.filesScanned, readers: clean.readers }).exit).toBe(0); + }); +}); + +describe('⚠️ C3 — the scan REPORTS the unruled family question and ⛔ does not answer it', () => { + it('an item carrier is filed `unruled` and is NEVER folded into the ruled total', () => { + expect(dispositionOf('node')).toBe('ruled:6771'); + expect(dispositionOf('item')).toBe('unruled:item-carrier'); + expect(dispositionOf('string')).toBe('carrier-undetermined'); + + const root = plant({ + 'packages/widget/src/tabs.tsx': ` +ComponentRegistry.register('tabs', T, { + defaultProps: { + items: [{ label: 'a', value: 'a', body: [{ type: 'text' }] }], + body: [{ type: 'text' }], + }, +}); +`, + }); + const run = scan(root); + const producers = producersOf(run.hits, run.readers); + const byDisposition = producers.map((p: { disposition: string }) => p.disposition).sort(); + // Both carriers present in ONE run, so the split is a discrimination and + // not an artifact of a corpus that only held one kind. + // ⭐ A `defaultProps.body` is NODE-carried even with no sibling `type` — + // the node is the registration, and its `type` is the registered key. So + // one run yields BOTH dispositions, which is what makes the split a + // discrimination rather than an artifact of a one-shaped corpus. + expect(byDisposition).toEqual(['ruled:6771', 'unruled:item-carrier']); + const ruled = producers.find((p: { disposition: string }) => p.disposition === 'ruled:6771'); + expect(ruled.nodeType).toBe('tabs'); + }); + + it('the item-array key set is DERIVED from the reader, not named here', () => { + const readers = deriveReaders(plant({})); + expect(readers.itemArrayKeys).toEqual(['items']); + expect(readers.receivers).toContain('item'); + expect(readers.receivers).toContain('schema'); + }); +}); + +describe('the criterion TRAVELS with the reading', () => { + it('every declaration is emitted, machine-readable and unique', () => { + for (const list of [CRITERION, NOT_A_PRODUCER, KNOWN_LIMITS]) { + const ids = (list as Array<{ id: string; what: string }>).map((l) => l.id); + expect(ids.length).toBeGreaterThan(0); + expect(new Set(ids).size).toBe(ids.length); + for (const entry of list as Array<{ what: string }>) expect(entry.what.length).toBeGreaterThan(0); + } + expect(CRITERION.map((c: { id: string }) => c.id)).toEqual([ + 'C1-emission', + 'C2-resolution', + 'C3-carrier', + ]); + }); + + it('CLI — the payload always carries the criterion, and a refusal reaches the exit code', () => { + const script = join(REPO_ROOT, 'scripts', 'body-dialect-producer-scan.mjs'); + const run = (args: string[]) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' }); + + const ok = run(['--root', plant({ 'packages/w/src/w.tsx': "const x = { type: 'w', body: [] };\n" }), '--json']); + expect(ok.status).toBe(0); + const payload = JSON.parse(ok.stdout) as { + criterion: Array<{ id: string }>; + notAProducer: Array<{ id: string }>; + knownLimits: Array<{ id: string }>; + readers: { reads: unknown[] }; + }; + // ⭐ There is no spelling of this payload that reports a table without + // reporting what made it a table: no flag suppresses any of the three. + expect(payload.criterion.map((c) => c.id)).toContain('C2-resolution'); + expect(payload.notAProducer.map((c) => c.id)).toContain('declaration'); + expect(payload.knownLimits.map((c) => c.id)).toContain('unruled-item-carrier'); + expect(payload.readers.reads.length).toBeGreaterThan(0); + + const blind = run(['--root', join(REPO_ROOT, 'no-such-directory-9871'), '--json']); + expect(blind.status).toBe(EXIT_UNREADABLE); + expect(blind.stdout).toBe(''); + expect(blind.stderr).toContain('Zero files scanned'); + }); + + it('is wired into the root package.json, like every other instrument here', () => { + const pkg = JSON.parse(read('package.json')) as { scripts: Record }; + expect(pkg.scripts['census:body-dialect-producers']).toBe( + 'node scripts/body-dialect-producer-scan.mjs', + ); + }); +}); + +describe('the emission verb is scoped to the enclosing FUNCTION, not to the file', () => { + it('a verb in a different function does not vouch for this one', () => { + const text = ` +function writes() { editor.insertText('x'); } +function declares() { return { name: 'body' }; } +`; + const { comment, literal } = scanSource(text); + const both = new Uint8Array(comment.length); + for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; + const { frames } = walkFrames(blank(text, both), blank(text, comment), literal); + const offset = text.indexOf("name: 'body'"); + const scope = enclosingScope(frames, offset); + expect(scope.length).toBeGreaterThan(0); + const scopeText = text.slice(scope[scope.length - 1].start, scope[scope.length - 1].end); + expect(scopeText).toContain("name: 'body'"); + // The lit control is the other half of the same file, which DOES hold a verb. + expect(scopeText).not.toContain('insertText'); + expect(text).toContain('insertText'); + }); +}); + +describe('the LIVE tree, read through the criterion', () => { + it('the producer table is DERIVED and reaches both shapes the census cannot', () => { + const run = scan(REPO_ROOT); + // Two lit controls: the walk resolved files, and C2 derived readers — so a + // zero below would be a reading and not a blind instrument. + expect(run.filesScanned).toBeGreaterThan(0); + expect(run.readers.reads.length).toBeGreaterThan(0); + + const producers = producersOf(run.hits, run.readers); + const files = new Set(producers.map((p: { file: string }) => p.file.split('\\').join('/'))); + + // ⭐ Asserted as SHAPES, not as a count and not as a list of paths: a + // producer arriving in a file nobody named appears in `producers` without + // anyone extending anything, which is the whole point of this card. + expect(producers.filter((p: { carrier: string }) => p.carrier === 'item').length).toBeGreaterThan(0); + expect(producers.filter((p: { source: string }) => p.source !== 'code-key').length).toBeGreaterThan(0); + expect(files.size).toBeGreaterThan(0); + + // ⭐ And the claim that outlives every one of them: while the table is + // non-empty, objectui#6771 step 4 is not landable. ⛔ When this reds + // because the table emptied, that is the HANDOFF — say so here in those + // terms rather than deleting the assertion. + expect( + producers.length, + 'the producer table is EMPTY — under this criterion nothing ships the dialect, which is ' + + 'the day objectui#6771 step 4 becomes landable. Re-point this block, do not delete it.', + ).toBeGreaterThan(0); + }); +}); diff --git a/scripts/body-dialect-producer-scan.mjs b/scripts/body-dialect-producer-scan.mjs new file mode 100644 index 0000000000..6211cdc04c --- /dev/null +++ b/scripts/body-dialect-producer-scan.mjs @@ -0,0 +1,920 @@ +#!/usr/bin/env node +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * `body` dialect PRODUCER scan (objectui#9871). + * + * ## The question this answers, and the one it refuses + * + * ⛔ NOT "how many producers are left". That question is what this file exists + * to end. objectui#7181 named six producers, objectui#9847 found a seventh the + * table never listed, and objectui#9871's retriage found two more — while + * `pnpm census:body-dialect` read CLEAN (`app-metadata` 1 → 0) across all of + * it. The two latest sites share something sharper than "they were missed": + * ⭐ NO instrument in the tree could have found them. + * + * ⇒ the question is **WHAT MAKES THE TABLE COMPLETE**, and the answer has to be + * a CRITERION a future site can be held against — ⛔ not a list a future site + * has to be remembered onto. + * + * ## ⭐ THE CRITERION — what counts as a producer of the `body` child-list dialect + * + * A site is a PRODUCER iff C1 and C2 both hold. C3 is not a qualifier; it is + * the axis the answer is FILED under, and it is recorded rather than assumed. + * + * C1 — EMISSION. The site causes the literal key `body` to appear in metadata + * that a party OUTSIDE this tree then owns. Three channels are measured + * (`EMISSION_CHANNELS`): `default-props` (a registration's declared + * default, materialised into the author's page when the component is + * dropped), `scaffold` (written into files a tool creates), and + * `snippet` (inserted into the author's already-open document). + * + * C2 — RESOLUTION. Some reader in this tree resolves a `body` off that + * carrier INTO A CHILD LIST. This is what separates the dialect from + * every unrelated `body`: an HTTP payload, `bodyExtra`, the `body` + * typography variant. ⭐ The reader set is DERIVED from source on every + * run (`deriveReaders`) — ⛔ never enumerated here, because an + * enumeration is the defect this file replaces. + * + * C3 — CARRIER. Which object the key is spelled ON, because that decides + * WHICH PUBLISHED CONTRACT the site is judged against: + * `node` — the object is a `BaseSchema` node. `body` is a DECLARED + * twin of `children` there (`packages/types/src/base.ts`). + * This is objectui#6771's ruled family. + * `item` — the object is a member of a registration's declared item + * array (`TabItem`, `ListItem`). ⚠️ Its canonical child-list + * key is `content`, and `body` is declared on NEITHER + * published face. ⛔ UNRULED — see "The open question". + * `string` — the spelling is carried by a string the tool writes into + * the author's document. The carrier is whatever node the + * author accepts it onto, which this instrument does not + * see. Reported as such, ⛔ never guessed. + * + * ## ⛔ What the criterion deliberately EXCLUDES, so a zero is readable + * + * `NOT_A_PRODUCER` below is the machine-readable form and every run emits it. + * The four exclusions, and why each is not appetite: + * + * - a DECLARATION (`inputs: [{ name: 'body' }]`, a published type member) + * TEACHES the spelling and writes nothing. It is a real surface and a real + * problem — it is simply a different one, and folding it in would make the + * producer count unreadable. + * - a READER (`renderChildren(schema.body)`) consumes; it does not emit. The + * readers are C2's instrument, so scoring them as producers would make this + * scan report its own denominator. + * - a `body` that no reader resolves as a child list fails C2 outright. + * - authored CORPUS, teaching snippets and fixtures are the dialect's + * VICTIMS, not its producers. They are bucketed and reported apart. + * + * ## ⭐ The two blindnesses this closes, and they are STRUCTURAL not incidental + * + * `scripts/body-dialect-census.mjs` scores a child list only on an object that + * ALSO carries a string-literal `type`, and never reads inside a string or + * template literal. Both limits are DECLARED there — ⛔ this is not a bug + * report against the census. They are the two directions a classifier keyed on + * "object with a `type`" is PERMANENTLY blind in, and each has a live site: + * + * B1 — no sibling `type` is required here. A frame is scored on the `body` + * key ITSELF, so a non-node item (`{ label, value, body: [...] }`) is + * visible. `walkFrames` records `type` as a CARRIER fact, ⛔ never as an + * admission test — and it accepts a `type` of ANY value, where the + * census requires a string literal. + * B2 — literals are READ, not skipped. `scanSource` (the shared masker, + * `scripts/js-comment-mask.mjs`) is projected TWICE over one scan: + * comments-and-literals blanked for the structural walk, comments-only + * blanked for the key sweep. The `literal` flag is then what TELLS THE + * TWO APART — a key token at a literal-flagged offset is a spelling + * inside a string, which is exactly B2's population. + * + * ⚠️ Both projections come from ONE `scanSource` pass, ⛔ never a private + * comment stripper: a regex has no idea what a string literal is, and this file + * would be the last place that could afford to get literals wrong. + * + * ## ⚠️ The open question this scan REPORTS and ⛔ does not answer + * + * Is an `item`-carried `body` inside objectui#6771's ruled family? + * + * ⛔ UNRULED, and this instrument does not rule it. It files `item` hits in + * their own table and REFUSES to fold them into the ruled total, because every + * measured fact points at a DIFFERENT contract rather than the same one: + * #6771's remedy is "`children` is the one spelling" and its step 3 retires + * `body` from `BaseSchema` — while the item face declares `content` (required + * on `TabItem`, optional on `ListItem`), declares no `body` on either the TS or + * the Zod face, and objectui#9256 filed the item channel separately and pinned + * it LIVE on purpose. Deciding it here would EXTEND a ruled family rather than + * apply one. The measurements are on objectui#9871. + * + * Usage: + * node scripts/body-dialect-producer-scan.mjs + * node scripts/body-dialect-producer-scan.mjs --root . --json + */ + +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join, relative, extname, sep } from 'node:path'; + +import { scanSource, blank } from './js-comment-mask.mjs'; +import { isEntrypoint } from './invoked-as.mjs'; + +/** ⭐ The criterion, machine-readable. Emitted by every run, both modes. */ +export const CRITERION = [ + { + id: 'C1-emission', + what: + 'The site causes the literal key `body` to appear in metadata a party OUTSIDE this tree then ' + + 'owns. Channels measured: ' + '`default-props`, `scaffold`, `snippet`.', + }, + { + id: 'C2-resolution', + what: + 'Some reader in this tree resolves a `body` off that carrier INTO A CHILD LIST. The reader set ' + + 'is DERIVED per run, never enumerated — an enumeration is the defect this scan replaces.', + }, + { + id: 'C3-carrier', + what: + 'Which object the key is spelled ON — `node` (a `BaseSchema` node, objectui#6771\'s ruled ' + + 'family), `item` (a member of a declared item array, whose canonical key is `content`; ' + + 'UNRULED), or `string` (carried into the author\'s document, carrier not visible here). ' + + 'Recorded, never assumed; it decides which published contract judges the site.', + }, +]; + +/** ⛔ What the criterion excludes. A zero is only readable beside this. */ +export const NOT_A_PRODUCER = [ + { + id: 'declaration', + what: + 'An `inputs: [{ name: \'body\' }]` entry or a published type member TEACHES the spelling and ' + + 'writes nothing. A real surface, a DIFFERENT one — folding it in makes the count unreadable.', + }, + { + id: 'reader', + what: + 'A `renderChildren(x.body)` read CONSUMES the key. Readers are C2\'s instrument, so scoring ' + + 'them here would make this scan report its own denominator.', + }, + { + id: 'unresolved-body', + what: + 'A `body` no derived reader resolves as a child list fails C2: an HTTP payload, `bodyExtra`, ' + + 'the `body` typography variant.', + }, + { + id: 'authored-corpus', + what: + 'Fixtures, tests, docs and example apps AUTHOR the dialect — they are its victims, not its ' + + 'producers. Bucketed and reported apart, never summed into the producer table.', + }, +]; + +/** Every way a zero here is NOT "no producer ships the dialect". */ +export const KNOWN_LIMITS = [ + { + id: 'channel-key-path', + what: + 'An emission channel is recognised by the KEY PATH the frame sits under (`EMISSION_CHANNELS`). ' + + 'A producer that emits from a path none of those patterns name is scored `channel: null` and ' + + 'reported under `unclassified` — visible, but NOT in the producer table.', + }, + { + id: 'string-carrier', + what: + 'A `string`-carried hit has no carrier object in this tree: the author\'s acceptance decides ' + + 'it. So it is a producer with NO family disposition, and asking this scan for one is asking ' + + 'the wrong instrument.', + }, + { + id: 'item-array-derivation', + what: + 'The item-array key set is derived from `schema..map(` / `Array.isArray(schema.)` in ' + + 'the files that read a non-`schema` receiver. An item array reached any other way leaves its ' + + 'members scored `unknown` rather than `item`.', + }, + { + id: 'skipped-directory', + what: + '`SKIP_DIRS` are never walked, and only `SCANNED_EXT` are read. Absence under either is an ' + + 'exclusion, not a reading.', + }, + { + id: 'unruled-item-carrier', + what: + '`item` hits are NOT folded into the ruled total. Whether they belong to objectui#6771 is ' + + 'unruled, and this instrument reports the question rather than answering it.', + }, +]; + +const SCANNED_EXT = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']); +const SKIP_DIRS = new Set([ + 'node_modules', '.git', 'dist', 'build', 'coverage', '.next', '.turbo', 'out', + '.pnpm-store', 'playwright-report', 'test-results', '.objectui-tmp', +]); + +/** + * How a key path is recognised as an EMISSION channel (C1). + * + * Matched against the chain of keys that OPENED the enclosing frames, so + * `defaultProps.items` and `templates.simple.body` both resolve without anyone + * naming a file. ⭐ A file allow-list would be the enumeration this scan exists + * to replace, one level up. + */ +export const EMISSION_CHANNELS = [ + { id: 'default-props', kind: 'key-path', segment: /^defaultProps$/ }, + { + // ⭐ Recognised by an EMISSION VERB, ⛔ never by a file name. A file + // allow-list would be the enumeration this scan exists to replace, one + // level up: the next scaffold would have to be remembered onto it. + id: 'scaffold', + kind: 'emission-verb', + verb: /\b(?:writeFileSync|writeFile|outputFileSync|outputFile|createFile)\s*\(/, + }, + { + id: 'snippet', + kind: 'emission-verb', + verb: /\b(?:insertText|SnippetString|TextEdit|WorkspaceEdit|applyEdit)\b/, + }, +]; + +/** + * Key paths that mark a DECLARATION rather than an emission (⛔ not a producer). + * + * ⚠️ `properties` is deliberately ABSENT. The VS Code completion provider holds + * its offerings under exactly that key and then WRITES them into the author's + * document, so excluding the name would have hidden the sharpest producer in + * the tree behind a word. Declaration is decided by `inputs` — the registration + * input list — and by the absence of an emission verb, ⛔ never by a key name + * that merely sounds descriptive. + */ +export const DECLARATION_SEGMENTS = [/^inputs$/]; + +// ── C2: derive the reader set ────────────────────────────────────────────── + +/** + * A read that resolves `body` INTO A CHILD LIST, in the two shapes this tree + * writes. Both are anchored on a child-list construct, ⛔ never on the bare key: + * `x.body` alone is not evidence of anything. + */ +const READ_SHAPES = [ + { + id: 'render-children', + // renderChildren( .body ) — including `(recv as any).body` and `recv?.body` + re: /renderChildren\s*\(\s*\(?\s*([A-Za-z_$][\w$]*)(?:\s+as\s+\w+)?\s*\)?\s*\??\.body\b/g, + }, + { + id: 'alternation', + // .children|content ||/?? .body (either order) + re: /([A-Za-z_$][\w$]*)\s*\??\.(?:children|content)\s*(?:\|\||\?\?)\s*(?:renderChildren\s*\(\s*)?\(?\s*\1(?:\s+as\s+\w+)?\s*\)?\s*\??\.body\b/g, + }, + { + id: 'alternation', + re: /([A-Za-z_$][\w$]*)\s*\??\.body\s*(?:\|\||\?\?)\s*\(?\s*\1(?:\s+as\s+\w+)?\s*\)?\s*\??\.(?:children|content)\b/g, + }, +]; + +/** `schema..map(` / `Array.isArray(schema.)` — where an item receiver comes from. */ +const ITEM_ARRAY_SHAPES = [ + /schema\s*\??\.([A-Za-z_$][\w$]*)\s*\??\.map\s*\(/g, + /Array\.isArray\s*\(\s*schema\s*\??\.([A-Za-z_$][\w$]*)\s*\)/g, +]; + +/** + * Derive C2's reader set from source. + * + * Returns the read sites, the receiver identifiers they read from, and the item + * array keys those non-`schema` receivers are drawn from. ⛔ Nothing here is + * hard-coded: a registration that starts resolving `body` joins the set on the + * next run, and one that stops leaves it. + */ +export function deriveReaders(root) { + const reads = []; + const receivers = new Set(); + const itemArrayKeys = new Set(); + let filesScanned = 0; + for (const file of walk(root)) { + let text; + try { + text = readFileSync(file, 'utf8'); + } catch { + continue; + } + filesScanned++; + if (!text.includes('.body')) continue; + const rel = relative(root, file); + // ⭐ C2 asks what the PLATFORM resolves, so the reader set is taken over + // shipped source only: a read inside a test or a fixture is that test's + // subject, not a reason the dialect renders for an author. + if (bucketOf(rel) !== 'shipped-source') continue; + const { comment, literal } = scanSource(text); + const code = blank(text, comment); + let sawNonSchema = false; + for (const shape of READ_SHAPES) { + shape.re.lastIndex = 0; + let m; + while ((m = shape.re.exec(code)) !== null) { + // ⚠️ A read is CODE. The same sentence inside a `.describe(...)` string + // or a docblock quote is prose about a read — scoring it would let this + // tree's own documentation manufacture C2 evidence for itself. + if (literal[m.index]) continue; + const receiver = m[1]; + reads.push({ file: rel, line: lineOf(text, m.index), receiver, shape: shape.id }); + receivers.add(receiver); + if (receiver !== 'schema') sawNonSchema = true; + } + } + if (sawNonSchema) { + for (const re of ITEM_ARRAY_SHAPES) { + re.lastIndex = 0; + let m; + while ((m = re.exec(code)) !== null) if (!literal[m.index]) itemArrayKeys.add(m[1]); + } + } + } + return { reads, receivers: [...receivers].sort(), itemArrayKeys: [...itemArrayKeys].sort(), filesScanned }; +} + +// ── The scanner: one `scanSource` pass, two projections ──────────────────── + +/** A key token in key position: `body:`, `"body":`, `'body':`. */ +const KEY_TOKEN = /^(?:"([A-Za-z_$][\w$-]*)"|'([A-Za-z_$][\w$-]*)'|([A-Za-z_$][\w$]*))\s*:/; + +/** `name: 'body'` — the key spelled as a DATUM that names a key (B2, second shape). */ +const KEY_NAME_DATUM = /\b(?:name|key|prop|property)\s*:\s*(?:"body"|'body')/g; + +/** + * Walk object/array frames, recording every `body` key with the frame that owns + * it, that frame's complete sibling key set, and the key path above it. + * + * ⭐ B1 lives here: the frame is scored on the `body` key ITSELF. `type` is + * recorded as a carrier FACT and is ⛔ never an admission test, so an object + * with no `type` — a tab item — is seen. `type` of ANY value counts, where the + * census requires a string literal. + * + * @param {string} structural comments AND literals blanked — the brace stack + * @param {string} keyText comments blanked, literals INTACT — the key tokens + * @param {Uint8Array} literal the flag that tells a code key from a spelling + * inside a string + */ +export function walkFrames(structural, keyText, literal) { + const hits = []; + const frames = []; + const stack = []; + const n = structural.length; + let i = 0; + let prevSignificant = ''; + let prevWord = ''; + + const finish = (frame) => { + frame.end = i; + frames.push(frame); + for (const pending of frame.pending) { + hits.push({ + ...pending, + siblingKeys: [...frame.keys], + hasType: frame.keys.has('type'), + nodeType: frame.typeValue ?? null, + }); + } + }; + + while (i < n) { + const c = structural[i]; + const top = stack[stack.length - 1]; + + if (top && top.isObject && /[A-Za-z_$"']/.test(keyText[i]) && !literal[i]) { + const m = KEY_TOKEN.exec(keyText.slice(i, i + 160)); + if (m) { + const key = m[1] ?? m[2] ?? m[3]; + top.keys.add(key); + top.lastKey = key; + if (key === 'type') { + const v = /^\s*(?:"([^"\\]*)"|'([^'\\]*)')/.exec(keyText.slice(i + m[0].length, i + m[0].length + 160)); + if (v) top.typeValue = v[1] ?? v[2]; + } + if (key === 'body') { + top.pending.push({ + offset: i, + carrierPath: stack.map((f) => f.openerKey).filter(Boolean), + source: 'code-key', + valueKind: valueKindAt(keyText, i + m[0].length), + }); + } + i += m[0].length; + prevSignificant = ':'; + prevWord = ''; + continue; + } + } + + if (c === '{' || c === '[') { + // ⭐ A `{` is an object literal OR a FUNCTION BODY, and the difference is + // load-bearing twice over: a block holds statements rather than keys, and + // it is the scope an emission verb has to be found in. A block opener + // follows `)` (a parameter list), `=>`, or one of the statement keywords. + const isBlock = + c === '{' && + (prevSignificant === ')' || + prevSignificant === '>' || + BLOCK_KEYWORDS.has(prevWord)); + const openerKey = top ? (top.isObject ? top.lastKey : top.openerKey) : null; + stack.push({ + isObject: c === '{' && !isBlock, + isBlock, + start: i, + end: n, + keys: new Set(), + pending: [], + openerKey: isBlock ? null : openerKey, + lastKey: null, + typeValue: null, + }); + i++; + prevSignificant = c; + prevWord = ''; + continue; + } + if (c === '}' || c === ']') { + const frame = stack.pop(); + if (frame) finish(frame); + i++; + prevSignificant = c; + prevWord = ''; + continue; + } + if (/[A-Za-z_$]/.test(c)) { + prevWord += c; + prevSignificant = c; + } else if (!/\s/.test(c)) { + prevWord = ''; + prevSignificant = c; + } + i++; + } + // An unbalanced tail still yields its hits — dropping them would turn a + // scanner limitation into a silent zero, which is this card's whole subject. + while (stack.length) finish(stack.pop()); + return { hits, frames }; +} + +/** + * What KIND of thing the `body` key is bound to — ⭐ C1's sharpest edge. + * + * A producer emits DATA: an array, an object, a string, a template. A + * DECLARATION binds an expression: `body: z.union(...)`, `body: + * aliasKeyRefusal(...)`, `body: c.body ?? ''`, `body: _body`, `body: unknown`. + * ⛔ Not a taste call — a validator, a destructured rename and a TS annotation + * put no `body` into anybody's metadata, so they cannot satisfy C1 however the + * key is spelled. + */ +export function valueKindAt(text, offset) { + let i = offset; + while (i < text.length && /\s/.test(text[i])) i++; + const c = text[i]; + if (c === '[') return 'array'; + if (c === '{') return 'object'; + if (c === '"' || c === "'") return 'string'; + if (c === '`') return 'template'; + return 'expression'; +} + +/** The value kinds that can carry emitted metadata. Anything else fails C1. */ +export const DATA_VALUE_KINDS = new Set(['array', 'object', 'string', 'template']); + +/** Statement keywords after which a `{` opens a BLOCK, not an object literal. */ +const BLOCK_KEYWORDS = new Set([ + 'else', 'try', 'finally', 'do', 'catch', 'return', +]); + +/** + * The frames enclosing `offset`, innermost first, stopping AT the nearest + * function body. + * + * ⭐ The stop is the point. An emission verb is evidence about the scope that + * emits, so searching past the enclosing function would let any verb anywhere + * in a file vouch for every `body` in it — which is a file allow-list wearing a + * different hat. + */ +export function enclosingScope(frames, offset) { + const containing = frames + .filter((f) => f.start <= offset && offset <= f.end) + .sort((a, b) => b.start - a.start); + const out = []; + for (const frame of containing) { + out.push(frame); + if (frame.isBlock) break; + } + return out; +} + +/** Bucket a file by WHAT KIND of artifact it is, so victims never sum with producers. */ +export function bucketOf(relPath) { + const p = relPath.split(sep).join('/'); + if (/(^|\/)__tests__\//.test(p) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(p)) return 'test'; + if (/(^|\/)(__fixtures__|fixtures|__mocks__)\//.test(p)) return 'fixture'; + if (/(^|\/)e2e\//.test(p)) return 'e2e'; + if (/(^|\/)examples\//.test(p)) return 'example-app'; + if (/(^|\/)scripts\//.test(p)) return 'tooling'; + if (/(^|\/)(apps|packages)\//.test(p)) return 'shipped-source'; + return 'other'; +} + +function channelOf(carrierPath, scopeText) { + for (const channel of EMISSION_CHANNELS) { + if (channel.kind === 'key-path') { + for (const segment of carrierPath) if (channel.segment.test(segment)) return channel.id; + } else if (channel.kind === 'emission-verb') { + if (channel.verb.test(scopeText)) return channel.id; + } + } + return null; +} + +function isDeclaration(carrierPath) { + return carrierPath.some((segment) => DECLARATION_SEGMENTS.some((re) => re.test(segment))); +} + +/** + * C3 — which object the key is spelled ON. + * + * ⭐ `type` is a CARRIER FACT here and ⛔ never an admission test, which is the + * whole of B1: the census requires a sibling string-literal `type` before it + * will score a child list, and that requirement is what makes a tab item + * permanently invisible to it. A `type` of ANY value counts. + */ +function carrierOf(hit, itemArrayKeys) { + if (hit.source !== 'code-key') return 'string'; + if (hit.hasType) return 'node'; + const owner = hit.carrierPath[hit.carrierPath.length - 1]; + if (owner && itemArrayKeys.includes(owner)) return 'item'; + // ⭐ A `body` directly under `defaultProps` IS node-carried: the node is the + // registration, and its `type` is the registered key — which sits in + // `ComponentRegistry.register('', …)` rather than as a sibling of the + // `body`. Requiring a SIBLING `type` here is B1 all over again one frame up, + // and it is the shape objectui#7181's three `defaultProps` producers had. + if (hit.carrierPath.includes('defaultProps')) return 'node'; + return 'unknown'; +} + +/** `ComponentRegistry.register(''` — where a `defaultProps` node's `type` really lives. */ +const REGISTRATION = /ComponentRegistry\s*\.\s*register\s*\(\s*['"]([^'"]+)['"]/g; + +/** The key of the registration a hit sits inside, or null. */ +export function registrationAt(text, offset) { + REGISTRATION.lastIndex = 0; + let key = null; + let m; + while ((m = REGISTRATION.exec(text)) !== null) { + if (m.index > offset) break; + key = m[1]; + } + return key; +} + +/** ⭐ The disposition axis. `item` is UNRULED and this function says so. */ +export function dispositionOf(carrier) { + if (carrier === 'node') return 'ruled:6771'; + if (carrier === 'item') return 'unruled:item-carrier'; + if (carrier === 'string') return 'carrier-undetermined'; + return 'unknown'; +} + +/** + * Scan one tree for producers under the criterion. + * + * @param {string} root + * @param {{ readers?: ReturnType }} [options] + */ +export function scan(root, options = {}) { + const readers = options.readers ?? deriveReaders(root); + const hits = []; + let filesScanned = 0; + + for (const file of walk(root)) { + let text; + try { + text = readFileSync(file, 'utf8'); + } catch { + continue; + } + filesScanned++; + if (!text.includes('body')) continue; + const rel = relative(root, file); + const bucket = bucketOf(rel); + + const { comment, literal } = scanSource(text); + const both = new Uint8Array(comment.length); + for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; + const structural = blank(text, both); + const keyText = blank(text, comment); + + const { hits: frameHits, frames } = walkFrames(structural, keyText, literal); + + const pathAt = (offset) => + enclosingScope(frames, offset) + .map((f) => f.openerKey) + .filter(Boolean) + .reverse(); + + const record = (hit) => { + const scope = enclosingScope(frames, hit.offset); + // ⭐ The scope an emission verb must be found in is the enclosing + // FUNCTION — and when there is none, it is the MODULE. A module-level + // template constant is written out by a function further down the file, + // and scoping its verb to the object literal it sits in would have made + // every scaffold of that shape invisible. That shape is not hypothetical: + // it is how `objectui init` held its templates. + const outermost = scope[scope.length - 1]; + const scopeText = outermost && outermost.isBlock + ? text.slice(outermost.start, outermost.end) + : text; + const carrierPath = hit.carrierPath ?? pathAt(hit.offset); + const carrier = carrierOf({ ...hit, carrierPath }, readers.itemArrayKeys); + const declaration = isDeclaration(carrierPath); + hits.push({ + file: rel, + line: lineOf(text, hit.offset), + bucket, + source: hit.source, + carrierPath, + siblingKeys: hit.siblingKeys ?? [], + carrier, + nodeType: + hit.nodeType + ?? (carrier === 'node' && carrierPath.includes('defaultProps') + ? registrationAt(text, hit.offset) + : null), + valueKind: hit.valueKind ?? 'string', + disposition: dispositionOf(carrier), + declaration, + channel: + declaration || !DATA_VALUE_KINDS.has(hit.valueKind ?? 'string') + ? null + : channelOf(carrierPath, scopeText), + }); + }; + + for (const hit of frameHits) record(hit); + + // ⭐ B2: the same key, spelled INSIDE a literal. The `literal` flag is the + // whole discriminator — a key token at a flagged offset is a spelling the + // tool may write into somebody's document, not a key in this tree's own + // code. Whether it IS written is C1, and C1 is the emission verb above: + // a hover card that merely SHOWS the spelling has no verb and no channel. + const inLiteralKey = /(?:"body"|'body'|\bbody)\s*:/g; + let m; + while ((m = inLiteralKey.exec(keyText)) !== null) { + if (!literal[m.index]) continue; + record({ offset: m.index, source: 'literal-key-syntax' }); + } + KEY_NAME_DATUM.lastIndex = 0; + while ((m = KEY_NAME_DATUM.exec(keyText)) !== null) { + record({ offset: m.index, source: 'key-name-datum' }); + } + } + + return { readers, filesScanned, hits }; +} + +/** + * ⭐ C2, APPLIED — ⛔ not merely derived. + * + * A `body` passes C2 when the derived reader set actually reads from a carrier + * of its class. The receivers those readers name are what decides it, so this + * is the criterion doing work rather than a sentence in a header: + * + * `node` — a `schema.body` reader exists ⇒ passes. + * `item` — an `item.body` reader exists, drawn from a declared item array + * ⇒ passes. + * `string` — the spelling is written into somebody's document as a key; the + * carrier it lands on is theirs ⇒ passes, with no disposition. + * `unknown` — neither a node nor a declared item member. NO derived reader + * resolves such an object's `body` into a child list, so it fails + * C2. ⭐ This is the entire HTTP-payload / `bodyExtra` population, + * and dropping it is the criterion working, not a blind spot — + * the count is REPORTED so the drop is never silent. + */ +export function passesC2(hit, readers) { + if (hit.carrier === 'node') return readers.receivers.includes('schema'); + if (hit.carrier === 'item') return readers.receivers.some((r) => r !== 'schema'); + if (hit.carrier === 'string') return readers.reads.length > 0; + return false; +} + +/** The producer table: C1 ∧ C2, with victims and declarations held out. */ +export function producersOf(hits, readers) { + return hits.filter( + (hit) => + hit.bucket === 'shipped-source' && + !hit.declaration && + passesC2(hit, readers) && + hit.channel !== null, + ); +} + +/** + * Shipped-source spellings that pass C2 but that no `EMISSION_CHANNELS` pattern + * claims. ⭐ This is where a NEW emission channel shows up as a row instead of + * as a silence — the failure this whole card is about. + */ +export function unchannelledOf(hits, readers) { + return hits.filter( + (hit) => + hit.bucket === 'shipped-source' && + !hit.declaration && + DATA_VALUE_KINDS.has(hit.valueKind) && + passesC2(hit, readers) && + hit.channel === null, + ); +} + +// ── Walk / util ──────────────────────────────────────────────────────────── + +function* walk(dir) { + let entries; + try { + entries = readdirSync(dir); + } catch { + return; + } + for (const entry of entries) { + if (SKIP_DIRS.has(entry)) continue; + const full = join(dir, entry); + let st; + try { + st = statSync(full); + } catch { + continue; + } + if (st.isDirectory()) yield* walk(full); + else if (st.isFile() && SCANNED_EXT.has(extname(entry))) yield full; + } +} + +function lineOf(text, offset) { + let line = 1; + for (let i = 0; i < offset && i < text.length; i++) if (text[i] === '\n') line++; + return line; +} + +/** + * Exit code for a run that REFUSES to report a number, kept distinct from 1 so + * "the instrument refused" and "the instrument crashed" are not one reading. + */ +export const EXIT_UNREADABLE = 2; + +/** + * Whether a run may report a table at all. + * + * ⭐ A zero PRODUCER count is a legitimate reading and is NOT refused — it is + * the day objectui#6771 step 4 becomes landable, and refusing it would delete + * the one answer this scan exists to be able to give. What IS refused is a run + * that could not have found anything: a blind walk, or a C2 instrument that + * derived no readers, in which case every `body` fails C2 for a reason that has + * nothing to do with the corpus. + */ +export function finalVerdict({ filesScanned, readers }) { + if (filesScanned === 0) { + return { + exit: EXIT_UNREADABLE, + refusal: [ + '✗ Zero files scanned. A scan that reads nothing because it is blind is', + ' indistinguishable from a tree that ships no producer. Check `--root`.', + ], + }; + } + if (readers.reads.length === 0) { + return { + exit: EXIT_UNREADABLE, + refusal: [ + '✗ Zero readers derived. C2 asks whether a reader resolves `body` into a', + ' child list; with no reader, every site fails C2 for a reason that has', + ' nothing to do with the corpus, and the producer table would read 0.', + ], + }; + } + return { exit: 0, refusal: null }; +} + +function printList(title, entries) { + console.log(`\n#### ${title}\n`); + for (const entry of entries) console.log(`- \`${entry.id}\` — ${entry.what}`); +} + +function main(argv = process.argv.slice(2)) { + const arg = (name, dflt) => { + const idx = argv.indexOf(name); + return idx >= 0 && argv[idx + 1] ? argv[idx + 1] : dflt; + }; + const root = arg('--root', '.'); + const asJson = argv.includes('--json'); + + const { readers, filesScanned, hits } = scan(root); + const verdict = finalVerdict({ filesScanned, readers }); + if (verdict.refusal) { + for (const line of verdict.refusal) console.error(line); + return verdict.exit; + } + + const producers = producersOf(hits, readers); + const unclassified = unchannelledOf(hits, readers); + const failedC2 = hits.filter( + (hit) => hit.bucket === 'shipped-source' && !hit.declaration && !passesC2(hit, readers), + ); + + if (asJson) { + // ⭐ The criterion travels WITH the reading. There is no spelling of this + // payload that reports a table without reporting what made it a table. + console.log( + JSON.stringify( + { + root, + criterion: CRITERION, + notAProducer: NOT_A_PRODUCER, + knownLimits: KNOWN_LIMITS, + readers, + filesScanned, + producers, + unclassified, + failedC2, + hits, + }, + null, + 2, + ), + ); + return verdict.exit; + } + + console.log(`## \`body\` dialect PRODUCER scan (objectui#9871) — root: ${root}`); + console.log(`\nFiles scanned: ${filesScanned}. Candidate \`body\` spellings: ${hits.length}.`); + + printList('⭐ The criterion — what makes a site a producer', CRITERION); + printList('⛔ Excluded by the criterion — so a zero is readable', NOT_A_PRODUCER); + printList('Known limits — every other way a 0 here is not "nothing ships it"', KNOWN_LIMITS); + + console.log('\n### C2 — the reader set, DERIVED this run (never enumerated)\n'); + console.log(`Receivers: ${readers.receivers.map((r) => `\`${r}\``).join(' ')}`); + console.log(`Item-array keys: ${readers.itemArrayKeys.map((k) => `\`${k}\``).join(' ') || '_none_'}`); + console.log(`\n| file:line | receiver | shape |`); + console.log('|:--|:--|:--|'); + for (const read of readers.reads) { + console.log(`| \`${read.file}:${read.line}\` | \`${read.receiver}\` | ${read.shape} |`); + } + + console.log(`\n### ⭐ The PRODUCER table — derived, ${producers.length} site(s)\n`); + if (producers.length === 0) { + console.log('_None._ ⇒ under this criterion no shipped source emits the dialect.'); + } else { + console.log('| file:line | carrier | `type` | channel | disposition | source |'); + console.log('|:--|:--|:--|:--|:--|:--|'); + for (const hit of producers.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line)) { + console.log( + `| \`${hit.file}:${hit.line}\` | ${hit.carrier} | ${hit.nodeType ? `\`${hit.nodeType}\`` : '—'}` + + ` | ${hit.channel} | ${hit.disposition} | ${hit.source} |`, + ); + } + } + + const byDisposition = new Map(); + for (const hit of producers) byDisposition.set(hit.disposition, (byDisposition.get(hit.disposition) ?? 0) + 1); + console.log('\n| disposition | producers |'); + console.log('|:--|--:|'); + for (const [d, c] of [...byDisposition].sort()) console.log(`| ${d} | ${c} |`); + console.log( + '\n⚠️ `unruled:item-carrier` is ⛔ NOT folded into the ruled total. Whether an item-carried' + + ' `body` belongs to objectui#6771 is unruled; this scan reports the question.', + ); + + console.log( + `\n### C2 held ${failedC2.length} shipped-source \`body\` spelling(s) OUT of the table` + + ' — ⛔ not silently\n', + ); + console.log( + 'Their carrier is neither a node nor a member of a declared item array, so NO derived reader' + + '\nresolves them into a child list: an HTTP payload, a `bodyExtra`, a `body` typography' + + '\nvariant. ⭐ The count is printed because a criterion that drops a population silently is' + + '\nthe defect this scan replaces. `--json` carries every row under `failedC2`.', + ); + + if (unclassified.length > 0) { + console.log(`\n### ⚠️ Passes C2, but NO recognised emission channel (${unclassified.length})\n`); + console.log('These carry the dialect on a real carrier and no `EMISSION_CHANNELS` pattern claims'); + console.log('them, so they are visible and ⛔ NOT in the table above. ⭐ A producer emitting by a'); + console.log('route nobody has named yet appears HERE — that is what makes this list load-bearing.\n'); + console.log('| file:line | carrier | path |'); + console.log('|:--|:--|:--|'); + for (const hit of unclassified.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line)) { + console.log(`| \`${hit.file}:${hit.line}\` | ${hit.carrier} | ${hit.carrierPath.join('.') || '_root_'} |`); + } + } + + return verdict.exit; +} + +// The ONE entry-guard predicate, never a hand-typed `process.argv[1]` compare. +if (isEntrypoint(import.meta.url)) process.exit(main()); From 0bbb39c89e970b81e1607fa7e66447ff66e08554 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 19:24:56 +0000 Subject: [PATCH 2/2] perf(scripts): make the producer scan single-pass, and time-box the two tree-wide blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-pointed `the platform SHIPS the dialect it is being asked to refuse` block timed out on CI shard 3 at the default 15s. The gap was cost, not correctness: the block runs two tree-wide instruments, and the scan half was doing avoidable work. Three cuts, none of which narrows the corpus — every file is still walked and still read: - a cheap NECESSARY CONDITION (`CANDIDATE_FILE`) gates the expensive per-character analysis. 1,852 files contain the substring `body`; 500 can hold a hit under the criterion, so 61% of the bytes never reach `scanSource`. The superset argument is written beside the regex, one line per hit shape, and pinned per shape with a negative control. - ONE directory traversal, shared by the reader derivation and the producer pass, instead of two. - ONE masked projection instead of two. The brace stack and the key tokens differ only at literal-flagged offsets, which the `literal` flag already marks, so `walkFrames` reads the structural character as a space wherever the flag is set and the second `blank()` pass over every candidate file goes away. Verified unchanged, not assumed: the whole-tree hit set is identical across the projection change — total 1092, unique 1083, the same 9 duplicated keys — measured against a reconstruction of the previous implementation in one command. Measured, load-normalised against the untouched census in the same command: the scan went from 2.05x the census to 1.06x, and the block from 10.1s to 7.9s. The remaining cost is inherent to a tree-wide character scan, so the two blocks that run one get an EXPLICIT per-test timeout naming the measured duration and the reason. No global `testTimeout` was touched, and no block was skipped, quarantined or narrowed. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_018HrVaotisyhgmot9o2MLRq --- scripts/__tests__/body-dialect-census.test.ts | 9 +- .../body-dialect-producer-scan.test.ts | 97 +++++++++++++++++-- scripts/body-dialect-producer-scan.mjs | 76 ++++++++++++--- 3 files changed, 155 insertions(+), 27 deletions(-) diff --git a/scripts/__tests__/body-dialect-census.test.ts b/scripts/__tests__/body-dialect-census.test.ts index aeb76e1cb5..13bff3ef3c 100644 --- a/scripts/__tests__/body-dialect-census.test.ts +++ b/scripts/__tests__/body-dialect-census.test.ts @@ -365,7 +365,14 @@ describe('the `body` consumers the ruling does not enumerate', () => { for (const hit of producers.filter((h: { carrier: string }) => h.carrier === 'item')) { expect(hit.disposition).toBe('unruled:item-carrier'); } - }); + // ⏱ Explicit, because this block runs TWO tree-wide instruments (the census + // over 7,855 files and the producer scan over 5,136) and the default 15s is + // not enough on a loaded shard: measured 7.9s here after objectui#9871 made + // the scan single-pass, and the CI runner took the SAME block past 15s when + // it measured 10.1s locally — so that runner is at least 1.9x slower. 60s is + // ~3x the observed CI-scale cost: ordinary contention cannot flake it, and a + // scan that stops terminating still fails. ⛔ Not a global `testTimeout` bump. + }, 60_000); }); /** diff --git a/scripts/__tests__/body-dialect-producer-scan.test.ts b/scripts/__tests__/body-dialect-producer-scan.test.ts index 9de241aff0..bc4ebea62c 100644 --- a/scripts/__tests__/body-dialect-producer-scan.test.ts +++ b/scripts/__tests__/body-dialect-producer-scan.test.ts @@ -43,6 +43,7 @@ import { NOT_A_PRODUCER, KNOWN_LIMITS, EMISSION_CHANNELS, + CANDIDATE_FILE, DECLARATION_SEGMENTS, DATA_VALUE_KINDS, EXIT_UNREADABLE, @@ -127,18 +128,14 @@ ComponentRegistry.register('tabs', TabsRenderer, { // recorded as a node because the key is present. const text = '{ type: SOME_CONST, body: [] }'; const { comment, literal } = scanSource(text); - const both = new Uint8Array(comment.length); - for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; - const { hits } = walkFrames(blank(text, both), blank(text, comment), literal); + const { hits } = walkFrames(blank(text, comment), literal); expect(hits).toHaveLength(1); expect(hits[0].hasType).toBe(true); expect(hits[0].nodeType).toBeNull(); // Lit control — a string-literal `type` still resolves its value. const lit = '{ type: "badge", body: [] }'; const f = scanSource(lit); - const b2 = new Uint8Array(f.comment.length); - for (let k = 0; k < b2.length; k++) b2[k] = f.comment[k] || f.literal[k] ? 1 : 0; - expect(walkFrames(blank(lit, b2), blank(lit, f.comment), f.literal).hits[0].nodeType).toBe('badge'); + expect(walkFrames(blank(lit, f.comment), f.literal).hits[0].nodeType).toBe('badge'); }); }); @@ -443,6 +440,84 @@ describe('the criterion TRAVELS with the reading', () => { }); }); +describe('⭐ the cheap pre-filter is a SUPERSET, so skipping cost never skips a hit', () => { + /** + * ⚠️ A pre-filter that is not a superset is a SILENT NARROWING — a scanner + * that quietly stops reading files is the defect this whole card exists to + * end. So the claim is pinned per hit SHAPE rather than asserted once, and + * the whole-tree hit-set equality is on objectui#9871's PR. + */ + it('fires for every shape the expensive path can find', () => { + for (const [shape, source] of [ + ['code-key, bare', "const x = { type: 'w', body: [] };"], + ['code-key, quoted (JSON)', '{ "type": "w", "body": [] }'], + ['code-key, spaced', 'const x = { body : [] };'], + ['literal-key-syntax', `item.insertText = new vscode.SnippetString('"body": {}');`], + ['key-name-datum', "const common = [{ name: 'body', desc: 'Child components' }];"], + ['key-name-datum, double-quoted', 'const common = [{ name: "body" }];'], + ] as const) { + expect(CANDIDATE_FILE.test(source), `pre-filter drops ${shape}`).toBe(true); + } + }); + + it('⚠️ BOTH alternatives are load-bearing — the datum has a COMMA, not a colon', () => { + // The first alternative alone would drop the sharpest producer in the tree: + // `{ name: 'body', desc: … }` has no colon after the quoted key. + const datum = "const common = [{ name: 'body', desc: 'Child components' }];"; + expect(/body\s*["']?\s*:/.test(datum), 'the colon alternative alone matches').toBe(false); + expect(/["']body["']/.test(datum)).toBe(true); + expect(CANDIDATE_FILE.test(datum)).toBe(true); + }); + + it('does NOT fire on the `body` population that carries no key', () => { + for (const source of [ + 'const n = schema.bodyExtra;', + '// the response body is parsed downstream', + 'function nobody() { return 1; }', + 'const b = res.body;', + ]) { + expect(CANDIDATE_FILE.test(source), `pre-filter fires on ${source}`).toBe(false); + } + }); + + it('⭐ OVER-accepts by design — a superset may say yes too often, NEVER too rarely', () => { + // `const variant = 'body'` is a quoted `body` that is not a key, and the + // pre-filter admits it. That is CORRECT: the cheap side may only ever be + // wider than the expensive side. The expensive path then finds nothing + // there, which costs a little time and ⛔ never a hit. + expect(CANDIDATE_FILE.test("const variant = 'body';")).toBe(true); + const root = plant({ 'packages/noise/src/t.ts': "export const variant = 'body';\n" }); + const run = scan(root); + expect(run.hits.filter((h: { file: string }) => h.file.includes('noise'))).toEqual([]); + // Lit control — the same run DOES resolve the planted reader file, so the + // zero above is about this file and not about a walk that read nothing. + expect(run.filesScanned).toBeGreaterThan(0); + expect(run.readers.reads.length).toBeGreaterThan(0); + }); + + it('DIFFERENTIAL — the same planted corpus yields the same hits with the filter in place', () => { + // Lit control on the cheap side: the corpus DOES produce a hit, so the + // equality below is between two non-empty readings and not two blanks. + const root = plant({ + 'packages/widget/src/w.tsx': ` +ComponentRegistry.register('w', W, { defaultProps: { type: 'w', body: [{ type: 'text' }] } }); +`, + 'packages/noise/src/http.ts': ` +export const send = () => fetch(url, { method: 'POST' }); +// the response body is parsed downstream, and schema.bodyExtra is a different key +`, + }); + const run = scan(root); + expect(run.hits.length).toBeGreaterThan(0); + // Every hit came from a file the pre-filter admits — the property the + // superset argument rests on, checked against the real hits. + for (const hit of run.hits) { + const text = readFileSync(join(root, hit.file), 'utf8'); + expect(CANDIDATE_FILE.test(text), `${hit.file} produced a hit the pre-filter would drop`).toBe(true); + } + }); +}); + describe('the emission verb is scoped to the enclosing FUNCTION, not to the file', () => { it('a verb in a different function does not vouch for this one', () => { const text = ` @@ -450,9 +525,7 @@ function writes() { editor.insertText('x'); } function declares() { return { name: 'body' }; } `; const { comment, literal } = scanSource(text); - const both = new Uint8Array(comment.length); - for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; - const { frames } = walkFrames(blank(text, both), blank(text, comment), literal); + const { frames } = walkFrames(blank(text, comment), literal); const offset = text.indexOf("name: 'body'"); const scope = enclosingScope(frames, offset); expect(scope.length).toBeGreaterThan(0); @@ -491,5 +564,9 @@ describe('the LIVE tree, read through the criterion', () => { 'the producer table is EMPTY — under this criterion nothing ships the dialect, which is ' + 'the day objectui#6771 step 4 becomes landable. Re-point this block, do not delete it.', ).toBeGreaterThan(0); - }); + // ⏱ Explicit — one tree-wide scan over 5,136 files, measured 4.0s here; the + // CI shard is at least 1.9x slower (objectui#9871's timeout reading), and + // the default 15s leaves too little room on a loaded shard. Same 60s and the + // same reason as the census block. ⛔ Not a global `testTimeout` bump. + }, 60_000); }); diff --git a/scripts/body-dialect-producer-scan.mjs b/scripts/body-dialect-producer-scan.mjs index 6211cdc04c..035c746541 100644 --- a/scripts/body-dialect-producer-scan.mjs +++ b/scripts/body-dialect-producer-scan.mjs @@ -253,6 +253,36 @@ export const EMISSION_CHANNELS = [ */ export const DECLARATION_SEGMENTS = [/^inputs$/]; +/** + * ⭐ The cheap NECESSARY CONDITION for a file to hold a hit at all. + * + * ⛔ This is NOT a narrowing of the corpus, and the difference matters: every + * file is still walked and still read. What it skips is the EXPENSIVE + * per-character analysis, and only for a file in which no hit shape could + * match — so the hit set is unchanged BY CONSTRUCTION. + * + * The superset argument, one line per shape this scanner can find, because a + * pre-filter that is not a superset is a silent narrowing and that is the + * defect this whole card exists to end: + * + * - `code-key` / `literal-key-syntax` match `("body"|'body'|body)` followed + * by optional space and `:` ⇒ the raw text holds `body`, an optional + * closing quote, optional space, `:` ⇒ the first alternative below. + * - `key-name-datum` matches `name|key|prop|property` `:` then `"body"` or + * `'body'` ⇒ the raw text holds a QUOTED `body` ⇒ the second. + * + * ⚠️ Both alternatives are load-bearing: a `{ name: 'body', desc: … }` datum + * has a COMMA after the quote, not a colon, so the first alternative alone + * would drop the sharpest producer in the tree. Measured, ⛔ not reasoned + * about — the tests pin one case per shape plus a negative control, and the + * whole-tree hit-set equality is recorded on objectui#9871's PR. + * + * Measured on this tree: 1852 files contain the substring `body`, 500 satisfy + * this predicate — 61% of the bytes never reach `scanSource`, and the hit-set + * digest is identical either way. + */ +export const CANDIDATE_FILE = /body\s*["']?\s*:|["']body["']/; + // ── C2: derive the reader set ────────────────────────────────────────────── /** @@ -291,12 +321,12 @@ const ITEM_ARRAY_SHAPES = [ * hard-coded: a registration that starts resolving `body` joins the set on the * next run, and one that stops leaves it. */ -export function deriveReaders(root) { +export function deriveReaders(root, files = null) { const reads = []; const receivers = new Set(); const itemArrayKeys = new Set(); let filesScanned = 0; - for (const file of walk(root)) { + for (const file of files ?? walk(root)) { let text; try { text = readFileSync(file, 'utf8'); @@ -355,16 +385,24 @@ const KEY_NAME_DATUM = /\b(?:name|key|prop|property)\s*:\s*(?:"body"|'body')/g; * with no `type` — a tab item — is seen. `type` of ANY value counts, where the * census requires a string literal. * - * @param {string} structural comments AND literals blanked — the brace stack - * @param {string} keyText comments blanked, literals INTACT — the key tokens - * @param {Uint8Array} literal the flag that tells a code key from a spelling - * inside a string + * ⭐ ONE projection, not two. The brace stack needs the source with literals + * blanked and the key tokens need it with literals INTACT — but those two + * strings differ ONLY at literal-flagged offsets, which is precisely what + * `literal` already marks. So the structural character is read as a space + * wherever the flag is set, and the second `blank()` pass over every candidate + * file (~0.85s of pure `split('')`/`join('')` across this tree) goes away. + * Byte-identical by construction; the hit-set digest is pinned equal on + * objectui#9871's PR across the change. + * + * @param {string} keyText comments blanked, literals INTACT + * @param {Uint8Array} literal the flag that both blanks the brace stack and + * tells a code key from a spelling inside a string */ -export function walkFrames(structural, keyText, literal) { +export function walkFrames(keyText, literal) { const hits = []; const frames = []; const stack = []; - const n = structural.length; + const n = keyText.length; let i = 0; let prevSignificant = ''; let prevWord = ''; @@ -383,7 +421,9 @@ export function walkFrames(structural, keyText, literal) { }; while (i < n) { - const c = structural[i]; + // A brace inside a string literal is not structure — the flag blanks it, + // exactly as a second masked projection used to. + const c = literal[i] ? ' ' : keyText[i]; const top = stack[stack.length - 1]; if (top && top.isObject && /[A-Za-z_$"']/.test(keyText[i]) && !literal[i]) { @@ -590,11 +630,15 @@ export function dispositionOf(carrier) { * @param {{ readers?: ReturnType }} [options] */ export function scan(root, options = {}) { - const readers = options.readers ?? deriveReaders(root); + // ONE directory traversal, shared by both phases. The reader derivation and + // the producer pass ask different questions of the same tree; walking it + // twice bought nothing but five thousand redundant `statSync` calls. + const files = [...walk(root)]; + const readers = options.readers ?? deriveReaders(root, files); const hits = []; let filesScanned = 0; - for (const file of walk(root)) { + for (const file of files) { let text; try { text = readFileSync(file, 'utf8'); @@ -602,17 +646,17 @@ export function scan(root, options = {}) { continue; } filesScanned++; - if (!text.includes('body')) continue; + // ⭐ The cheap necessary condition — see `CANDIDATE_FILE`. Every file is + // still walked and read; only the per-character analysis is skipped, and + // only where no hit shape could match. + if (!CANDIDATE_FILE.test(text)) continue; const rel = relative(root, file); const bucket = bucketOf(rel); const { comment, literal } = scanSource(text); - const both = new Uint8Array(comment.length); - for (let k = 0; k < both.length; k++) both[k] = comment[k] || literal[k] ? 1 : 0; - const structural = blank(text, both); const keyText = blank(text, comment); - const { hits: frameHits, frames } = walkFrames(structural, keyText, literal); + const { hits: frameHits, frames } = walkFrames(keyText, literal); const pathAt = (offset) => enclosingScope(frames, offset)