From 81aa51fcadb147f71f7eeff3c5ec4573218dd12c Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Tue, 18 Aug 2026 18:22:51 +0200 Subject: [PATCH] Make the ladder's non-vacuity control assert something, and emit its maps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects, shipped together because the second is what keeps the first honest across repos. **The control asserted nothing.** `expect(imports.length + invocations.length).toBeGreaterThanOrEqual(0)` is true of the empty map it existed to exclude. The suite is layered precisely so an app landing on its rung *because nothing was analysed* cannot pass for the wrong reason, and this was one of the two controls holding that up. It now asserts the analysis attributed something: every ladder app imports express — that is why it is in all five — so an inventory without it means the scan came back empty and every assertion in the file is vacuous. Verified by flipping the expected package: five failures, not zero. **The maps had no generator.** A consumer checks these five documents in and grades them, and until now the harness built each app in a temp directory, asserted, and deleted it — so those copies could only be reproduced by hand, and nothing detected them going stale. A stale fixture is the quiet case: it still parses and still grades, it just answers for an app this extractor now reads differently. The emitter lives beside the app definitions, which is the drift it exists to prevent, and runs only when handed an output directory: PS_LADDER_EMIT_DIR= npx vitest run tests/map/ladder-emit It strips the per-machine measurements and writes the consumer's exact on-disk form, byte for byte — confirmed against the checked-in copies — because a regeneration that diffs on noise stops being run. It also writes a manifest pairing each rung with its advisory and package, so a consumer can key expectations off the fixture rather than restating them. Three assertions run in a normal suite without writing anything: the volatile fields are gone, both evidence blocks survive (a map missing one grades LOWER rather than failing, so a silently truncating emitter would produce fixtures that pass as conservative verdicts), and the serialization is exact. --- tests/map/ladder-emit.test.ts | 115 ++++++++++++++++++++++++++++++++++ tests/map/ladder.test.ts | 10 ++- 2 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 tests/map/ladder-emit.test.ts diff --git a/tests/map/ladder-emit.test.ts b/tests/map/ladder-emit.test.ts new file mode 100644 index 0000000..086e1c5 --- /dev/null +++ b/tests/map/ladder-emit.test.ts @@ -0,0 +1,115 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { buildInputMap } from '../../src/map/index.js'; +import { LADDER_CASES, type LadderCase } from './ladder-cases.js'; + +// Regeneration entry point for the ladder maps a CONSUMER checks in and grades. +// +// The platform stores these five documents as fixtures and asserts what each one grades to. They are +// generated artifacts, and until this existed they had no generator: the ladder suite builds each app in a +// temp directory, asserts, and deletes it. So the consumer's copies could only be reproduced by hand, and +// nothing detected them going stale — a fixture older than this extractor still parses and still grades, it +// just answers for an app the extractor now reads differently. +// +// Why a spec rather than a script in `scripts/`: `src/map/*` imports with ESM `.js` specifiers that point at +// `.ts` sources, so plain `node` cannot load it and a script would have to run against a build. Living here +// also keeps the emitter beside the app definitions it emits, which is the drift this is meant to prevent. +// +// Run: +// PS_LADDER_EMIT_DIR=/path/to/back/tests/Fixtures/ReachabilityLadder npx vitest run tests/map/ladder-emit +// +// Skipped otherwise, so a normal suite run neither writes files nor needs a directory. + +const OUT = process.env.PS_LADDER_EMIT_DIR; + +/** + * Per-machine measurements. They carry no contract, and leaving them in would make every regeneration a + * diff on noise — which is how a regeneration step stops being run. + */ +const VOLATILE = ['analysisMs', 'rssBytes', 'peakRssBytes'] as const; + +/** The consumer's on-disk form: two-space JSON with a trailing newline. Byte-identical or it diffs. */ +function serialize(document: unknown): string { + return JSON.stringify(document, null, 2) + '\n'; +} + +async function mapFor(c: LadderCase): Promise> { + const dir = mkdtempSync(join(tmpdir(), 'ps-ladder-emit-')); + try { + for (const [rel, body] of Object.entries(c.files)) { + const path = join(dir, rel); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, body); + } + writeFileSync(join(dir, 'package.json'), JSON.stringify(c.packageJson)); + + const { map, error } = await buildInputMap(dir); + expect(error, `${c.id} must produce a map`).toBeUndefined(); + const document = map as Record; + for (const field of VOLATILE) delete document.coverage[field]; + + return document; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The rung/advisory/package pairing, so a consumer can key its expectations off this rather than restate it. */ +function manifest(): Record { + return { + note: 'Generated from the ladder fixture apps. Do not edit; regenerate.', + cases: LADDER_CASES.map((c) => ({ id: c.id, rung: c.rung, cve: c.cve, package: c.pkg, name: c.name })), + }; +} + +describe.skipIf(!OUT)('emitting the ladder maps for a consumer', () => { + it('writes one document per rung, plus the rung/advisory manifest', async () => { + const written: string[] = []; + for (const c of LADDER_CASES) { + const file = c.id.replace('ladder/', '') + '.json'; + writeFileSync(join(OUT!, file), serialize(await mapFor(c))); + written.push(file); + } + writeFileSync(join(OUT!, 'manifest.json'), serialize(manifest())); + + expect(written).toHaveLength(LADDER_CASES.length); + console.log(`[ladder] wrote ${written.join(', ')} and manifest.json to ${OUT}`); + }); +}); + +describe('the emitter agrees with what it emits', () => { + // The emitter's value is that its output can be committed elsewhere and trusted. Two properties make that + // true, and both are cheap to assert without writing anything: the volatile fields really are gone (a + // machine-specific number in a committed fixture makes every regeneration a diff), and the document still + // carries the blocks the consumer's controls check — a map missing them grades LOWER rather than failing, + // so an emitter that quietly dropped one would produce fixtures that pass as conservative verdicts. + it('emits documents with no per-machine fields and both evidence blocks intact', async () => { + const document = await mapFor(LADDER_CASES[0]); + + for (const field of VOLATILE) expect(document.coverage).not.toHaveProperty(field); + expect(document.version).toBe(3); + expect(Array.isArray(document.imports)).toBe(true); + expect(Array.isArray(document.apiInvocations)).toBe(true); + expect(document.coverage.importsComplete).toBeDefined(); + expect(document.coverage.filesParsed).toBeGreaterThan(0); + }); + + it('serializes exactly as the consumer stores it', () => { + // Formatting is part of the contract here: a different indent or a missing trailing newline rewrites + // every line of every fixture and buries the one change that mattered. + expect(serialize({ a: 1 })).toBe('{\n "a": 1\n}\n'); + }); + + it('names every case in the manifest, with its rung and advisory', () => { + const cases = manifest().cases as Array>; + + expect(cases).toHaveLength(LADDER_CASES.length); + for (const entry of cases) { + expect(entry.id).toMatch(/^ladder\//); + expect(entry.cve).toMatch(/^CVE-/); + expect(entry.package).not.toBe(''); + } + }); +}); diff --git a/tests/map/ladder.test.ts b/tests/map/ladder.test.ts index 2805979..f6fd371 100644 --- a/tests/map/ladder.test.ts +++ b/tests/map/ladder.test.ts @@ -59,11 +59,17 @@ describe('every ladder case was actually analysed', () => { }); it.each(LADDER_CASES.map((c) => [c.id, c] as const))('%s declares its dependency', (_id, c) => { - const declared = (map: InputMap) => ((map.imports ?? []) as Array<{ package: string }>).map((i) => i.package); + const declared = ((mapFor(c).imports ?? []) as Array<{ package: string }>).map((i) => i.package); + // Declared in package.json regardless of whether the map can attribute a usage. For the // `unknown` case this is the whole point: the dependency is present and the usage is invisible. expect(c.packageJson.dependencies).toHaveProperty(c.pkg); - expect(declared(mapFor(c)).length + (mapFor(c).apiInvocations ?? []).length).toBeGreaterThanOrEqual(0); + // And the analysis attributed SOMETHING for this app. Every ladder app imports express — it is the + // shared control dependency for exactly this reason — so an inventory without it means the scan came + // back empty, and every assertion in this file would then pass over nothing. (The version this + // replaced summed two lengths and asserted `>= 0`, which is true of the empty map it existed to + // exclude.) + expect(declared, 'an empty inventory makes every assertion below vacuous').toContain('express'); }); });