From a4f81cfb5ad719304f46e87231eaa8cd470a8b39 Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Fri, 28 Aug 2026 17:38:23 -0400 Subject: [PATCH 1/2] T-ENG01: test the scripts, and validate what they export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The project shipped OSCAL and STIX for other tools to consume and never read either back. It turns out 13 of the 25 frameworks were emitting OSCAL catalogs that no OSCAL tool would load — 750 control ids that are not valid OSCAL tokens — and nothing had ever noticed, because nothing looked. Test suites (3 tests → 50): stats.test.mjs invariants on the numbers every headline renders from generate.test.mjs determinism, round-trip, entry and schema-v2 integrity reports.test.mjs the three report CLIs, through their CLIs ingest.test.mjs the framework ingest path — mostly that it rejects exports.test.mjs OSCAL and STIX validated against schema Export validation uses ajv against three schemas in data/schemas/. They are deliberate subsets, written and owned here, and the filenames and the README say so: passing proves the export has not lost its shape, not that it conforms to OSCAL or STIX. data/schemas/README.md records why fetching or vendoring the upstream schemas costs more than it returns, and how to run the real validators when that is the question. Fixes found by the new tests: - `oscalToken()` in compliance-report.js coerces control ids into valid OSCAL tokens. Real ids routinely are not: `Art. 5`, `SR 1.1`, `4.1`, `V8 Data Protection`. The original is preserved verbatim in the control title and a `source-control-id` prop, and the same rule runs in both the catalog and the component-definition renderer so the two still join. Non-token ids across all 25 frameworks: 750 → 0. - ingest-framework.mjs now rejects an empty control set. It validated cleanly before, so a registry with no controls could land in data/frameworks/ and count as a framework covering nothing. Not fixed: 576 mapping rows still hold prose in `control_id` because the Markdown columns are swapped — the root cause of the invalid ids. That is a parser fix touching 576 rows across 25 frameworks and it deserves its own review, so it is issue #35. exports.test.mjs pins the per-framework counts as a regression fence, with a second test that fails if a baseline entry outlives the framework it names. CI gains a `Unit tests` job that runs `npm ci --ignore-scripts`, the suite, and then `git diff --exit-code` — the suite re-runs the generator to prove determinism, so it has to leave the tree clean. `npm run ci` runs the suite too, between build and reports. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/validate.yml | 33 +++ data/framework-schema.json | 1 + data/schemas/README.md | 61 +++++ data/schemas/oscal-catalog.subset.json | 74 ++++++ .../oscal-component-definition.subset.json | 128 ++++++++++ data/schemas/stix-bundle.subset.json | 91 +++++++ package-lock.json | 78 ++++++ package.json | 7 +- scripts/compliance-report.js | 41 +++- scripts/exports.test.mjs | 232 ++++++++++++++++++ scripts/generate.test.mjs | 127 ++++++++++ scripts/ingest-framework.mjs | 5 + scripts/ingest.test.mjs | 109 ++++++++ scripts/reports.test.mjs | 160 ++++++++++++ scripts/stats.test.mjs | 114 +++++++++ 15 files changed, 1256 insertions(+), 5 deletions(-) create mode 100644 data/schemas/README.md create mode 100644 data/schemas/oscal-catalog.subset.json create mode 100644 data/schemas/oscal-component-definition.subset.json create mode 100644 data/schemas/stix-bundle.subset.json create mode 100644 scripts/exports.test.mjs create mode 100644 scripts/generate.test.mjs create mode 100644 scripts/ingest.test.mjs create mode 100644 scripts/reports.test.mjs create mode 100644 scripts/stats.test.mjs diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 130b743..fca25d8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,6 +14,8 @@ on: - 'README.md' - 'scripts/validate.js' - 'scripts/generate.js' + - 'scripts/*.test.mjs' + - 'package.json' - '.github/workflows/validate.yml' pull_request: branches: [main] @@ -28,6 +30,8 @@ on: - 'README.md' - 'scripts/validate.js' - 'scripts/generate.js' + - 'scripts/*.test.mjs' + - 'package.json' - '.github/workflows/validate.yml' # Both jobs only read the tree. Without this they inherit the repository @@ -112,3 +116,32 @@ jobs: # here rather than shipping. - name: Assert generated output is current run: git diff --exit-code -- data/entries docs/data.js docs/incidents.js + + unit-tests: + name: Unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Use Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + + # ajv is the only runtime dependency the suite needs, and it is a + # devDependency. --ignore-scripts because nothing here needs a postinstall. + - name: Install dependencies + run: npm ci --ignore-scripts + + # Covers stats.js, generate.js, the three report CLIs, the framework + # ingest path, and schema validation of the OSCAL and STIX exports. + - name: Run the test suite + run: npm run test:scripts + + # The suite re-runs the generator to prove it is deterministic. If that + # left anything behind, the tree must still be clean. + - name: Assert the tests left no changes behind + run: git diff --exit-code diff --git a/data/framework-schema.json b/data/framework-schema.json index 5b6dc37..64d71ca 100644 --- a/data/framework-schema.json +++ b/data/framework-schema.json @@ -73,6 +73,7 @@ "controls": { "type": "array", "description": "Complete inventory of controls/clauses/requirements", + "minItems": 1, "items": { "type": "object", "required": [ diff --git a/data/schemas/README.md b/data/schemas/README.md new file mode 100644 index 0000000..98234ab --- /dev/null +++ b/data/schemas/README.md @@ -0,0 +1,61 @@ + + +# Export schemas + +The crosswalk emits three machine-readable formats that other tools consume: + +| Format | Emitted by | Schema here | +|---|---|---| +| OSCAL 1.1.2 Component Definition | `compliance-report.js --format oscal` | `oscal-component-definition.subset.json` | +| OSCAL 1.1.2 Catalog | `compliance-report.js --format oscal-catalog` | `oscal-catalog.subset.json` | +| STIX 2.1 Bundle | `incidents-report.js --format stix` | `stix-bundle.subset.json` | + +`scripts/exports.test.mjs` validates every emitted document against these on +each `node --test` run, and `npm run ci` includes it. + +## These are subsets, and the filename says so + +They are **not** the NIST and OASIS schemas. They encode the structural rules +this project's output has to satisfy — required members, id formats, the +`spec_version` and `oscal-version` constants, UUID and timestamp shapes, +`additionalProperties` where the spec closes an object — and they are written +here, in this repository, by this project. + +Passing them proves the export has not silently lost its shape. It does not +prove full OSCAL or STIX conformance, and no sentence in this repository should +claim it does. + +## Why not the upstream schemas + +Two options were considered and both cost more than they return here: + +- **Fetch at CI time.** Makes every build depend on a third-party host being up + and on a schema that can change without warning. A red build that means + "NIST changed a description" is a build people learn to ignore. +- **Vendor the upstream files.** The OSCAL complete schema is over a megabyte of + JSON this project does not control and cannot meaningfully review on update. + +A small schema that is read, understood and owned catches the regressions that +actually happen — a renamed member, a dropped `spec_version`, a malformed +UUID — and it does so without pretending to an authority it does not have. + +For genuine conformance, run the emitted file through the upstream validators: + +```bash +# OSCAL — https://github.com/usnistgov/OSCAL +node scripts/compliance-report.js --framework "NIST AI RMF 1.0" --format oscal --stdout > oscal.json + +# STIX 2.1 — https://github.com/oasis-open/cti-stix-validator +node scripts/incidents-report.js --format stix --stdout > stix.json +``` + +--- + +*Part of the [OWASP GenAI Crosswalk](https://github.com/GenAI-Security-Project/crosswalk) — +maintained by the [OWASP GenAI Data Security Initiative](https://genai.owasp.org)* diff --git a/data/schemas/oscal-catalog.subset.json b/data/schemas/oscal-catalog.subset.json new file mode 100644 index 0000000..511d762 --- /dev/null +++ b/data/schemas/oscal-catalog.subset.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/oscal-catalog.subset.json", + "title": "OSCAL 1.1.2 Catalog — structural subset", + "description": "A SUBSET of the NIST OSCAL 1.1.2 catalog model, written and owned by this project. It encodes the structure compliance-report.js --format oscal-catalog must emit. Passing proves the export has not lost its shape; it does not prove OSCAL conformance. See data/schemas/README.md.", + "type": "object", + "required": ["catalog"], + "additionalProperties": false, + "properties": { + "catalog": { + "type": "object", + "required": ["uuid", "metadata"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "metadata": { "$ref": "#/definitions/metadata" }, + "groups": { + "type": "array", + "items": { "$ref": "#/definitions/group" } + }, + "controls": { + "type": "array", + "items": { "$ref": "#/definitions/control" } + }, + "back-matter": { "type": "object" } + } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + }, + "metadata": { + "type": "object", + "required": ["title", "last-modified", "version", "oscal-version"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "last-modified": { "type": "string", "format": "date-time" }, + "version": { "type": "string", "minLength": 1 }, + "oscal-version": { "const": "1.1.2" }, + "roles": { "type": "array" }, + "parties": { "type": "array" }, + "responsible-parties": { "type": "array" } + } + }, + "group": { + "type": "object", + "required": ["id", "title"], + "properties": { + "id": { "$ref": "#/definitions/tokenId" }, + "title": { "type": "string", "minLength": 1 }, + "controls": { "type": "array", "items": { "$ref": "#/definitions/control" } }, + "groups": { "type": "array", "items": { "$ref": "#/definitions/group" } } + } + }, + "control": { + "type": "object", + "required": ["id", "title"], + "properties": { + "id": { "$ref": "#/definitions/tokenId" }, + "title": { "type": "string", "minLength": 1 }, + "props": { "type": "array", "items": { "type": "object", "required": ["name", "value"] } }, + "links": { "type": "array", "items": { "type": "object", "required": ["href"] } }, + "parts": { "type": "array" }, + "controls": { "type": "array", "items": { "$ref": "#/definitions/control" } } + } + }, + "tokenId": { + "type": "string", + "description": "OSCAL identifiers are NCName-like tokens: they must not start with a digit and must not contain whitespace. Control ids taken straight from a framework often do both, so this is the check most likely to catch a real regression.", + "pattern": "^[A-Za-z_][A-Za-z0-9._~:-]*$" + } + } +} diff --git a/data/schemas/oscal-component-definition.subset.json b/data/schemas/oscal-component-definition.subset.json new file mode 100644 index 0000000..62ddea5 --- /dev/null +++ b/data/schemas/oscal-component-definition.subset.json @@ -0,0 +1,128 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/oscal-component-definition.subset.json", + "title": "OSCAL 1.1.2 Component Definition — structural subset", + "description": "A SUBSET of the NIST OSCAL 1.1.2 component-definition model, written and owned by this project. It encodes the structure compliance-report.js --format oscal must emit. Passing proves the export has not lost its shape; it does not prove OSCAL conformance. See data/schemas/README.md.", + "type": "object", + "required": ["component-definition"], + "additionalProperties": false, + "properties": { + "component-definition": { + "type": "object", + "required": ["uuid", "metadata", "components"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "metadata": { "$ref": "#/definitions/metadata" }, + "components": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/component" } + }, + "back-matter": { "type": "object" } + } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + }, + "metadata": { + "type": "object", + "required": ["title", "last-modified", "version", "oscal-version"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "last-modified": { "type": "string", "format": "date-time" }, + "version": { "type": "string", "minLength": 1 }, + "oscal-version": { + "type": "string", + "description": "Pinned. A silent bump here changes what downstream tooling will accept.", + "const": "1.1.2" + }, + "roles": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "title"], + "properties": { "id": { "type": "string" }, "title": { "type": "string" } } + } + }, + "parties": { + "type": "array", + "items": { + "type": "object", + "required": ["uuid", "type", "name"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "type": { "enum": ["person", "organization"] }, + "name": { "type": "string", "minLength": 1 } + } + } + }, + "responsible-parties": { "type": "array" } + } + }, + "component": { + "type": "object", + "required": ["uuid", "type", "title", "description"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "type": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "control-implementations": { + "type": "array", + "items": { + "type": "object", + "required": ["uuid", "source", "description", "implemented-requirements"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "source": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "implemented-requirements": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["uuid", "control-id", "description"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "control-id": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "props": { "$ref": "#/definitions/props" }, + "links": { "$ref": "#/definitions/links" } + } + } + } + } + } + } + } + }, + "props": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "value"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "string" }, + "ns": { "type": "string" }, + "class": { "type": "string" } + } + } + }, + "links": { + "type": "array", + "items": { + "type": "object", + "required": ["href"], + "properties": { + "href": { "type": "string", "minLength": 1 }, + "rel": { "type": "string" }, + "text": { "type": "string" } + } + } + } + } +} diff --git a/data/schemas/stix-bundle.subset.json b/data/schemas/stix-bundle.subset.json new file mode 100644 index 0000000..841f6af --- /dev/null +++ b/data/schemas/stix-bundle.subset.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/stix-bundle.subset.json", + "title": "STIX 2.1 Bundle — structural subset", + "description": "A SUBSET of the OASIS STIX 2.1 specification, written and owned by this project. It encodes the structure incidents-report.js --format stix must emit. Passing proves the export has not lost its shape; it does not prove STIX conformance. See data/schemas/README.md.", + "type": "object", + "required": ["type", "id", "objects"], + "additionalProperties": false, + "properties": { + "type": { "const": "bundle" }, + "id": { "$ref": "#/definitions/stixId" }, + "objects": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/sdo" } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "stixId": { + "type": "string", + "description": "STIX id: --. The double hyphen is required by the spec.", + "pattern": "^[a-z][a-z0-9-]*--[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "timestamp": { "type": "string", "format": "date-time" }, + "externalReference": { + "type": "object", + "required": ["source_name"], + "properties": { + "source_name": { "type": "string", "minLength": 1 }, + "external_id": { "type": "string" }, + "url": { "type": "string", "format": "uri" }, + "description": { "type": "string" } + } + }, + "commonSdo": { + "type": "object", + "required": ["type", "spec_version", "id", "created", "modified"], + "properties": { + "type": { "type": "string", "minLength": 1 }, + "spec_version": { "const": "2.1" }, + "id": { "$ref": "#/definitions/stixId" }, + "created": { "$ref": "#/definitions/timestamp" }, + "modified": { "$ref": "#/definitions/timestamp" }, + "external_references": { + "type": "array", + "items": { "$ref": "#/definitions/externalReference" } + } + } + }, + "sdo": { + "allOf": [ + { "$ref": "#/definitions/commonSdo" }, + { + "if": { "properties": { "type": { "const": "attack-pattern" } } }, + "then": { "required": ["name"], "properties": { "name": { "type": "string", "minLength": 1 } } } + }, + { + "if": { "properties": { "type": { "const": "report" } } }, + "then": { + "required": ["name", "published", "object_refs"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "published": { "$ref": "#/definitions/timestamp" }, + "report_types": { "type": "array", "items": { "type": "string" } }, + "object_refs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/stixId" } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "relationship" } } }, + "then": { + "required": ["relationship_type", "source_ref", "target_ref"], + "properties": { + "relationship_type": { "type": "string", "minLength": 1 }, + "source_ref": { "$ref": "#/definitions/stixId" }, + "target_ref": { "$ref": "#/definitions/stixId" } + } + } + } + ] + } + } +} diff --git a/package-lock.json b/package-lock.json index 5463c5a..7f978b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "license": "CC-BY-SA-4.0", "devDependencies": { "@types/node": "^25.5.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "typescript": "^5.4.0" }, "engines": { @@ -25,6 +27,82 @@ "undici-types": "~7.18.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index 5a9a80a..8ef538b 100644 --- a/package.json +++ b/package.json @@ -39,10 +39,11 @@ "state-report": "node scripts/state-report.js", "build": "npm run generate && npm run validate && npm run stats:check", "build:reports": "npm run compliance && npm run incidents && npm run registry-coverage && npm run density && npm run inventory && npm run audit:incidents && npm run validate:aivss && npm run freshness", - "ci": "npm run build && npm run build:reports", + "ci": "npm run build && npm run test:scripts && npm run build:reports", "compile": "tsc", "prepublishOnly": "npm run compile", - "test:scripts": "node --test \"scripts/**/*.test.mjs\"" + "test:scripts": "node --test \"scripts/**/*.test.mjs\"", + "test": "node --test \"scripts/**/*.test.mjs\"" }, "license": "CC-BY-SA-4.0", "repository": { @@ -74,6 +75,8 @@ ], "devDependencies": { "@types/node": "^25.5.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "typescript": "^5.4.0" } } diff --git a/scripts/compliance-report.js b/scripts/compliance-report.js index e3382e5..395d7f3 100644 --- a/scripts/compliance-report.js +++ b/scripts/compliance-report.js @@ -737,6 +737,39 @@ function renderSummaryMarkdown(frameworks, allEntries, opts) { const crypto = require('crypto'); +/** + * Coerce a framework control identifier into a valid OSCAL token. + * + * OSCAL ids are NCName-like: they may not begin with a digit and may not + * contain whitespace. Real control ids routinely do both — `Art. 5` (DORA), + * `SR 1.1` (ISA 62443), `4.1` (ISO 42001), `V8 Data Protection` (ASVS) — so + * emitting them verbatim produced catalogs that no OSCAL tool would load. + * Thirteen of the twenty-five frameworks were affected and nothing caught it, + * because nothing read the export back. + * + * The rule is deliberately boring and deterministic, so the same control gets + * the same id in the catalog and in the component definition, and so the id is + * stable across runs: + * + * whitespace and invalid characters → `-`, collapsed + * leading digit → `c-` prefix + * longer than 60 characters → truncated, plus a hash of the original + * + * The original is never lost: it is preserved verbatim in the control `title` + * and in a `source-control-id` prop. + */ +function oscalToken(raw) { + const original = String(raw == null ? '' : raw).trim(); + let t = original.replace(/[^A-Za-z0-9._~:-]+/g, '-').replace(/-{2,}/g, '-').replace(/^-+|-+$/g, ''); + if (!t) t = 'control'; + if (!/^[A-Za-z_]/.test(t)) t = 'c-' + t; + if (t.length > 60) { + const h = crypto.createHash('sha1').update(original).digest('hex').slice(0, 6); + t = t.slice(0, 52).replace(/-+$/, '') + '-' + h; + } + return t; +} + function renderOSCAL(fw, allEntries) { const r = buildFrameworkReport(fw, allEntries); const now = new Date().toISOString(); @@ -771,9 +804,10 @@ function renderOSCAL(fw, allEntries) { 'implemented-requirements': [...r.controls.values()].flatMap(ctrl => ctrl.entries.map(entry => ({ uuid: crypto.randomUUID(), - 'control-id': ctrl.control_id, + 'control-id': oscalToken(ctrl.control_id), description: `${entry.id}: ${entry.name} — ${ctrl.control_name}${ctrl.notes.length ? ' | ' + ctrl.notes.join('; ') : ''}`, props: [ + { name: 'source-control-id', value: String(ctrl.control_id) }, { name: 'implementation-status', value: 'planned', ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, { name: 'owasp-entry', value: entry.id, ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, { name: 'severity', value: entry.severity, ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, @@ -844,15 +878,16 @@ function renderOSCALCatalog(fw, allEntries) { links: registryFw ? [{ href: registryFw.url, rel: 'canonical' }] : [], }, groups: [...groups.entries()].map(([groupName, ctrls]) => ({ - id: groupName.toLowerCase().replace(/[^a-z0-9]+/g, '-'), + id: oscalToken(groupName.toLowerCase()), title: groupName, controls: ctrls.map(ctrl => { const cid = registryFw ? ctrl.control_id : ctrl.control_id; const mappedData = r.controls.get(cid); return { - id: cid, + id: oscalToken(cid), title: registryFw ? ctrl.title : (ctrl.control_name || ctrl.title), props: [ + { name: 'source-control-id', value: String(cid) }, ...(ctrl.description ? [{ name: 'description', value: ctrl.description }] : []), ...(mappedData ? [ { name: 'owasp-coverage', value: mappedData.entries.map(e => e.id).join(','), ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, diff --git a/scripts/exports.test.mjs b/scripts/exports.test.mjs new file mode 100644 index 0000000..91200e6 --- /dev/null +++ b/scripts/exports.test.mjs @@ -0,0 +1,232 @@ +/** + * exports.test.mjs — validate the machine-readable exports against schema. + * + * The crosswalk emits OSCAL and STIX for other tools to consume. Those + * consumers fail on a missing member or a malformed id, and until now nothing + * checked either: the exports were generated, written, and never read back. + * + * The schemas in data/schemas/ are deliberate subsets, written and owned here. + * Passing proves the export has not lost its shape. It does not prove OSCAL or + * STIX conformance — see data/schemas/README.md for why, and for how to run the + * upstream validators when that is what you need. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const ajv = new Ajv({ allErrors: true, strict: false }); +addFormats(ajv); + +const schema = (name) => + ajv.compile(JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'schemas', name), 'utf8'))); + +/** Run a repo script and parse its stdout as JSON. */ +function emit(script, args) { + const out = execFileSync(process.execPath, [path.join(ROOT, 'scripts', script), ...args], { + cwd: ROOT, + maxBuffer: 256 * 1024 * 1024, + encoding: 'utf8', + }); + return JSON.parse(out); +} + +const report = (validate) => + (validate.errors || []) + .slice(0, 10) + .map((e) => `${e.instancePath || '/'} ${e.message}`) + .join('\n'); + +// ── OSCAL component definition ─────────────────────────────────────────────── + +test('OSCAL component definition matches the structural subset', () => { + const validate = schema('oscal-component-definition.subset.json'); + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal', '--stdout', + ]); + assert.ok(validate(doc), report(validate)); +}); + +test('OSCAL component definition carries at least one implemented requirement', () => { + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal', '--stdout', + ]); + const impls = doc['component-definition'].components.flatMap( + (c) => c['control-implementations'] || [], + ); + const reqs = impls.flatMap((i) => i['implemented-requirements'] || []); + assert.ok(reqs.length > 0, 'export contains no implemented-requirements — the mapping was dropped'); +}); + +// ── OSCAL catalog ──────────────────────────────────────────────────────────── + +test('OSCAL catalog matches the structural subset', () => { + const validate = schema('oscal-catalog.subset.json'); + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal-catalog', '--stdout', + ]); + assert.ok(validate(doc), report(validate)); +}); + +test('OSCAL catalog ids are OSCAL tokens, not raw control strings', () => { + // The failure this catches: a framework whose control ids start with a digit + // or contain a space is emitted verbatim and produces a catalog no OSCAL tool + // will load. It is invisible in the JSON unless something checks. + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal-catalog', '--stdout', + ]); + const ids = []; + const walk = (c) => { + ids.push(c.id); + (c.controls || []).forEach(walk); + }; + for (const g of doc.catalog.groups || []) { + ids.push(g.id); + (g.controls || []).forEach(walk); + } + (doc.catalog.controls || []).forEach(walk); + + assert.ok(ids.length > 0, 'catalog has no controls'); + const bad = ids.filter((id) => !/^[A-Za-z_][A-Za-z0-9._~:-]*$/.test(id)); + assert.deepEqual(bad, [], `non-token ids: ${bad.slice(0, 5).join(', ')}`); +}); + +// ── STIX 2.1 ───────────────────────────────────────────────────────────────── + +test('STIX bundle matches the structural subset', () => { + const validate = schema('stix-bundle.subset.json'); + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + assert.ok(validate(doc), report(validate)); +}); + +test('every STIX relationship points at an object in the same bundle', () => { + // A dangling source_ref or target_ref is accepted by a shape check and + // rejected by every real STIX consumer, so it needs its own assertion. + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const ids = new Set(doc.objects.map((o) => o.id)); + const dangling = []; + for (const o of doc.objects) { + if (o.type === 'relationship') { + if (!ids.has(o.source_ref)) dangling.push(`${o.id} source_ref ${o.source_ref}`); + if (!ids.has(o.target_ref)) dangling.push(`${o.id} target_ref ${o.target_ref}`); + } + for (const ref of o.object_refs || []) { + if (!ids.has(ref)) dangling.push(`${o.id} object_ref ${ref}`); + } + } + assert.deepEqual(dangling.slice(0, 5), [], `${dangling.length} dangling reference(s)`); +}); + +test('STIX ids are unique across the bundle', () => { + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const seen = new Set(); + const dupes = []; + for (const o of doc.objects) { + if (seen.has(o.id)) dupes.push(o.id); + seen.add(o.id); + } + assert.deepEqual(dupes, [], `duplicate STIX ids: ${dupes.slice(0, 3).join(', ')}`); +}); + +test('STIX bundle covers every incident in the database', () => { + const db = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'incidents.json'), 'utf8')); + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const reports = doc.objects.filter((o) => o.type === 'report'); + assert.equal( + reports.length, + db.incidents.length, + 'incident count and STIX report count disagree — the export is dropping records', + ); +}); + +// ── regression fence for the swapped control_id / control_name rows ────────── + +/** + * A known, tracked defect: in some mapping files the control identifier and the + * requirement prose are in the wrong columns, so `control_id` holds a sentence. + * The OSCAL exports no longer *break* on it — `oscalToken()` coerces the id and + * preserves the original in a `source-control-id` prop — but the underlying data + * is still wrong, and a reader following an id back to the framework cannot. + * + * This is the count as of 2026-08-28, per framework registry. It is a ceiling, + * not a target: the test fails if any framework gains prose ids, or if a + * framework not listed here starts producing them. When the parser is fixed, + * these numbers come down and this table shrinks with them. + * + * Tracked as issue #35. + */ +const PROSE_ID_BASELINE = Object.freeze({ + 'AIUC-1': 3, + 'CIS Controls v8.1': 31, + 'CWE/CVE': 25, + 'EU AI Act': 118, + 'ISO/IEC 42001:2023': 3, + 'NIST SP 800-218A': 40, + 'NIST SP 800-82 Rev 3': 29, + 'OWASP AI Testing Guide': 16, + 'OWASP NHI Top 10': 103, + 'PCI DSS v4.0': 40, + 'SOC 2': 168, +}); + +/** A control id is "prose" when it reads as a sentence rather than an identifier. */ +const isProse = (raw) => String(raw).trim().split(/\s+/).length >= 5; + +test('prose-shaped control ids do not spread beyond the known set', () => { + const fwDir = path.join(ROOT, 'data', 'frameworks'); + const registries = fs.readdirSync(fwDir).filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(fwDir, f), 'utf8'))); + + const counted = {}; + for (const reg of registries) { + // `--framework` is a partial match, so one call can emit several documents. + const raw = execFileSync(process.execPath, [ + path.join(ROOT, 'scripts', 'compliance-report.js'), + '--framework', reg.name, '--format', 'oscal-catalog', '--stdout', + ], { cwd: ROOT, maxBuffer: 256 * 1024 * 1024, encoding: 'utf8' }); + + for (const chunk of raw.split(/\n(?=\{\n)/)) { + const doc = JSON.parse(chunk); + const name = doc.catalog.metadata.title.replace(/ — Control Catalog$/, ''); + const sources = []; + const walk = (c) => { + const p = (c.props || []).find((x) => x.name === 'source-control-id'); + if (p) sources.push(p.value); + (c.controls || []).forEach(walk); + }; + for (const g of doc.catalog.groups || []) (g.controls || []).forEach(walk); + (doc.catalog.controls || []).forEach(walk); + + const n = sources.filter(isProse).length; + if (n) counted[name] = n; + } + } + + const regressions = []; + for (const [fw, n] of Object.entries(counted)) { + const allowed = PROSE_ID_BASELINE[fw]; + if (allowed === undefined) regressions.push(`${fw} newly produces ${n} prose id(s)`); + else if (n > allowed) regressions.push(`${fw}: ${n} prose ids, baseline ${allowed}`); + } + assert.deepEqual(regressions, [], regressions.join('; ')); +}); + +test('the prose-id baseline does not silently overstate the problem', () => { + // The mirror of the test above. If a framework is fixed, its baseline entry + // must be removed rather than left as dead permission — otherwise the fence + // slowly stops fencing anything. + const fwDir = path.join(ROOT, 'data', 'frameworks'); + const names = new Set( + fs.readdirSync(fwDir).filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(fwDir, f), 'utf8')).name), + ); + const unknown = Object.keys(PROSE_ID_BASELINE).filter((f) => !names.has(f)); + assert.deepEqual(unknown, [], `baseline names frameworks that no longer exist: ${unknown.join(', ')}`); +}); diff --git a/scripts/generate.test.mjs b/scripts/generate.test.mjs new file mode 100644 index 0000000..8c31dce --- /dev/null +++ b/scripts/generate.test.mjs @@ -0,0 +1,127 @@ +/** + * generate.test.mjs — the generator must be deterministic, and its output must + * be the thing that is committed. + * + * generate.js produces every file in data/entries/ and the three webapp data + * bundles. It once sat broken on main for several commits with all CI green, + * because no job ran it. CI now does; this covers the properties CI's + * `git diff --exit-code` cannot express on its own. + * + * These tests re-run the generator. Its output is deterministic, so a clean + * tree stays clean — and if it does not, that is the failure being reported. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const ENTRIES = path.join(ROOT, 'data', 'entries'); +const BUNDLES = ['data.js', 'incidents.js', 'backlinks.js', 'frameworks-registry.js'] + .map((f) => path.join(ROOT, 'docs', f)); + +const runGenerate = () => + execFileSync(process.execPath, [path.join(ROOT, 'scripts', 'generate.js')], { + cwd: ROOT, maxBuffer: 256 * 1024 * 1024, encoding: 'utf8', + }); + +/** Hash every generated artefact, EOL-normalised so Windows checkouts agree. */ +function fingerprint() { + const h = crypto.createHash('sha256'); + const files = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')).sort() + .map((f) => path.join(ENTRIES, f)) + .concat(BUNDLES.filter((f) => fs.existsSync(f))); + for (const f of files) { + h.update(path.basename(f)); + h.update(fs.readFileSync(f, 'utf8').replace(/\r\n/g, '\n')); + } + return h.digest('hex'); +} + +test('generating twice produces byte-identical output', () => { + // Any timestamp, random id or Object key-order dependence in the generator + // shows up here as a diff on every run — which is how a repository ends up + // with permanently dirty generated files that everyone learns to ignore. + runGenerate(); + const first = fingerprint(); + runGenerate(); + assert.equal(fingerprint(), first, 'generate.js is not deterministic'); +}); + +test('committed entries match a fresh generation', () => { + const before = fingerprint(); + runGenerate(); + assert.equal(fingerprint(), before, + 'data/entries or docs/*.js differ from what generate.js produces — regenerate and commit'); +}); + +test('every entry declares an id, name, source list and severity', () => { + const files = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')); + assert.ok(files.length > 0); + for (const f of files) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + assert.match(e.id, /^(LLM|ASI|DSGAI|AST)\d{2}$/, `${f} has id "${e.id}"`); + assert.equal(`${e.id}.json`, f, `${f} does not match its own id`); + assert.ok(e.name && e.name.length > 1, `${f} has no name`); + assert.ok(e.source_list, `${f} has no source_list`); + assert.match(e.severity, /^(Critical|High|Medium|Low)$/, `${f} severity "${e.severity}"`); + } +}); + +test('no mapping is missing a framework or a control id', () => { + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + (e.mappings || []).forEach((m, i) => { + assert.ok(m.framework, `${f} mapping[${i}] has no framework`); + assert.ok(m.control_id, `${f} mapping[${i}] has no control_id`); + }); + } +}); + +test('no mapping claims a confidence without a named reviewer', () => { + // The single rule the whole schema-v2 migration exists to enforce. It is + // checked by validate.js too; duplicating it here means `node --test` alone + // is enough to catch an entry file edited by hand. + const offenders = []; + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + for (const m of e.mappings || []) { + if (m.confidence && m.confidence !== 'unreviewed' && !(m.reviewed_by || []).length) { + offenders.push(`${e.id}/${m.framework}:${m.control_id}`); + } + } + } + assert.deepEqual(offenders.slice(0, 5), [], `${offenders.length} unreviewed row(s) claim confidence`); +}); + +test('DRAFT never survives into a stored enum field', () => { + // The Markdown templates carry the literal word DRAFT in the relationship, + // rationale and confidence columns. generate.js is supposed to resolve those + // to `unreviewed` rather than store them, so a stored "DRAFT" means the + // template leaked into the data layer. + const leaked = []; + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + for (const m of e.mappings || []) { + for (const k of ['relationship', 'rationale_type', 'confidence']) { + if (typeof m[k] === 'string' && /^draft$/i.test(m[k].trim())) { + leaked.push(`${e.id}/${m.framework}:${m.control_id}.${k}`); + } + } + } + } + assert.deepEqual(leaked.slice(0, 5), [], `${leaked.length} field(s) stored the literal "DRAFT"`); +}); + +test('webapp bundles stay in step with the entry files', () => { + const src = fs.readFileSync(path.join(ROOT, 'docs', 'data.js'), 'utf8'); + const start = src.indexOf('['); + const end = src.lastIndexOf(']'); + const bundled = JSON.parse(src.slice(start, end + 1)); + const onDisk = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')).length; + assert.equal(bundled.length, onDisk, 'docs/data.js and data/entries hold different entry counts'); +}); diff --git a/scripts/ingest-framework.mjs b/scripts/ingest-framework.mjs index 4d1899a..3abe4e4 100644 --- a/scripts/ingest-framework.mjs +++ b/scripts/ingest-framework.mjs @@ -71,6 +71,11 @@ function validateFramework(fw, schema) { // controls if (!Array.isArray(fw.controls)) { errors.push('controls must be an array'); + } else if (fw.controls.length === 0) { + // A registry with no controls validates cleanly, lands in + // data/frameworks/, and then counts as a framework with an empty + // inventory — inflating the registry count while covering nothing. + errors.push('controls is empty — a registry with no controls is not a framework'); } else { const ids = new Set(); fw.controls.forEach((c, i) => { diff --git a/scripts/ingest.test.mjs b/scripts/ingest.test.mjs new file mode 100644 index 0000000..c100e82 --- /dev/null +++ b/scripts/ingest.test.mjs @@ -0,0 +1,109 @@ +/** + * ingest.test.mjs — the framework ingest path, exercised through its CLI. + * + * ingest-framework.mjs is how a new framework registry enters the project. It + * is the one script whose job is to accept outside data, so the property that + * matters is that it rejects: a malformed source must fail loudly rather than + * land a half-formed registry in data/frameworks/. + * + * Every test uses --validate, so nothing here writes to data/frameworks/. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const SCRIPT = path.join(ROOT, 'scripts', 'ingest-framework.mjs'); + +function run(args) { + return execFileSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, maxBuffer: 32 * 1024 * 1024, encoding: 'utf8', + }); +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'crosswalk-ingest-')); +const write = (name, body) => { + const p = path.join(tmp, name); + fs.writeFileSync(p, body, 'utf8'); + return p; +}; + +test.after(() => fs.rmSync(tmp, { recursive: true, force: true })); + +const VALID = { + id: 'test-framework', + name: 'Test Framework', + short_name: 'TF', + version: '1.0', + url: 'https://example.org/test-framework', + license: 'CC BY 4.0', + publisher: 'Test Publisher', + category: 'ai-governance', + last_synced: '2026-08-28', + source_sha: null, + controls: [ + { control_id: 'TF-1', title: 'First control', description: 'Does a thing', kind: 'control' }, + { control_id: 'TF-2', title: 'Second control', description: 'Does another', kind: 'control' }, + ], +}; + +test('--list names every framework already registered', () => { + const out = run(['--list']); + const names = fs.readdirSync(path.join(ROOT, 'data', 'frameworks')) + .filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'frameworks', f), 'utf8'))); + const missing = names.filter((r) => !out.includes(r.id) && !out.includes(r.name)); + assert.deepEqual(missing.map((r) => r.id), [], 'registered frameworks missing from --list'); +}); + +test('a well-formed JSON source validates', () => { + const p = write('valid.json', JSON.stringify(VALID, null, 2)); + const out = run([p, '--validate']); + assert.match(out, /valid|ok|pass/i, `unexpected output:\n${out}`); +}); + +test('a source with no controls is rejected', () => { + const p = write('empty.json', JSON.stringify({ ...VALID, controls: [] }, null, 2)); + assert.throws(() => run([p, '--validate']), 'an empty control set was accepted'); +}); + +test('a source missing a required top-level field is rejected', () => { + const { name, ...noName } = VALID; + const p = write('no-name.json', JSON.stringify(noName, null, 2)); + assert.throws(() => run([p, '--validate']), 'a registry with no name was accepted'); +}); + +test('a control with no control_id is rejected', () => { + const p = write('bad-control.json', JSON.stringify({ + ...VALID, + controls: [{ title: 'Nameless', description: 'x', kind: 'control' }], + }, null, 2)); + assert.throws(() => run([p, '--validate']), 'a control with no id was accepted'); +}); + +test('a CSV source is parsed into controls', () => { + const p = write('tf.csv', [ + 'control_id,title,description,parent,function', + 'TF-1,First control,Does a thing,,Govern', + 'TF-2,"Second, with a comma","Description, quoted",TF-1,Govern', + ].join('\n')); + const out = run([p, '--validate']); + assert.match(out, /valid|ok|pass|2/i, `unexpected output:\n${out}`); +}); + +test('a non-existent source fails instead of writing an empty registry', () => { + assert.throws(() => run([path.join(tmp, 'does-not-exist.json'), '--validate'])); +}); + +test('--validate leaves data/frameworks untouched', () => { + const dir = path.join(ROOT, 'data', 'frameworks'); + const before = fs.readdirSync(dir).sort().join(','); + const p = write('valid2.json', JSON.stringify(VALID, null, 2)); + try { run([p, '--validate']); } catch { /* the assertion below is the point */ } + assert.equal(fs.readdirSync(dir).sort().join(','), before, '--validate wrote to data/frameworks'); +}); diff --git a/scripts/reports.test.mjs b/scripts/reports.test.mjs new file mode 100644 index 0000000..8426dc0 --- /dev/null +++ b/scripts/reports.test.mjs @@ -0,0 +1,160 @@ +/** + * reports.test.mjs — the report generators, exercised through their CLIs. + * + * compliance-report.js, incidents-report.js and state-report.js are scripts, + * not modules: they parse argv and write files. Testing them through the CLI + * checks what people and CI actually run, and needs no refactor to get there. + * + * Everything here uses --stdout, so no test writes into reports/. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +function run(script, args) { + return execFileSync(process.execPath, [path.join(ROOT, 'scripts', script), ...args], { + cwd: ROOT, + maxBuffer: 256 * 1024 * 1024, + encoding: 'utf8', + }); +} + +const db = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'incidents.json'), 'utf8')); +const stats = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'stats.json'), 'utf8')); + +/** Minimal RFC 4180 row splitter — enough to check quoting is honoured. */ +function csvRows(text) { + const rows = []; + let row = [], cell = '', quoted = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (quoted) { + if (c === '"') { + if (text[i + 1] === '"') { cell += '"'; i++; } else quoted = false; + } else cell += c; + } else if (c === '"') quoted = true; + else if (c === ',') { row.push(cell); cell = ''; } + else if (c === '\n') { row.push(cell); rows.push(row); row = []; cell = ''; } + else if (c !== '\r') cell += c; + } + if (cell.length || row.length) { row.push(cell); rows.push(row); } + return rows.filter((r) => r.length > 1 || r[0] !== ''); +} + +// ── state-report.js ────────────────────────────────────────────────────────── + +test('state-report --json agrees with data/stats.json', () => { + // Two independent counters over the same data. When they disagree, one of + // them is being used to publish a number that is not true. + const r = JSON.parse(run('state-report.js', ['--json'])); + assert.equal(r.totals.incidents, stats.incidents.total); + assert.equal(r.totals.entries, stats.entries.total); + assert.equal(r.totals.mappings, stats.mappings.total); + assert.equal(r.totals.frameworks, stats.frameworks.registries); +}); + +test('state-report incident breakdowns sum to the incident total', () => { + const r = JSON.parse(run('state-report.js', ['--json'])); + const sum = (o) => Object.values(o).reduce((a, b) => a + b, 0); + assert.equal(sum(r.incidents.by_category), r.totals.incidents); + assert.equal(sum(r.incidents.by_severity), r.totals.incidents); +}); + +// ── incidents-report.js ────────────────────────────────────────────────────── + +test('incidents CSV has one data row per incident and a stable header', () => { + const rows = csvRows(run('incidents-report.js', ['--format', 'csv', '--stdout'])); + assert.ok(rows.length > 1, 'CSV has no data rows'); + assert.equal(rows[0][0].toLowerCase(), 'id', `unexpected first column: ${rows[0][0]}`); + assert.equal(rows.length - 1, db.incidents.length); +}); + +test('incidents CSV survives the commas and quotes in real descriptions', () => { + // Incident text is full of commas, quoted phrases and em dashes. An unquoted + // field shifts every later column and the file still opens in Excel, wrong. + const text = run('incidents-report.js', ['--format', 'csv', '--stdout']); + const rows = csvRows(text); + const width = rows[0].length; + const ragged = rows.map((r, i) => [i, r.length]).filter(([, n]) => n !== width); + assert.deepEqual(ragged.slice(0, 3), [], `${ragged.length} row(s) have the wrong column count`); +}); + +test('incidents --entry filters to incidents that name that entry', () => { + const out = JSON.parse(run('incidents-report.js', ['--entry', 'LLM01', '--format', 'json', '--stdout'])); + const list = Array.isArray(out) ? out : out.incidents; + assert.ok(list.length > 0, 'no incidents returned for LLM01'); + for (const inc of list) { + assert.ok(inc.owasp_entries.includes('LLM01'), `${inc.id} does not name LLM01`); + } +}); + +test('incidents --severity filters to that severity only', () => { + const out = JSON.parse(run('incidents-report.js', ['--severity', 'Critical', '--format', 'json', '--stdout'])); + const list = Array.isArray(out) ? out : out.incidents; + assert.ok(list.length > 0); + for (const inc of list) assert.equal(inc.severity, 'Critical'); +}); + +// ── compliance-report.js ───────────────────────────────────────────────────── + +test('compliance --list-frameworks lists every mapped framework', () => { + const out = run('compliance-report.js', ['--list-frameworks']); + const entriesDir = path.join(ROOT, 'data', 'entries'); + const mapped = new Set(); + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) mapped.add(m.framework); + } + const missing = [...mapped].filter((f) => !out.includes(f)); + assert.deepEqual(missing, [], `not listed: ${missing.join(', ')}`); +}); + +test('compliance JSON summary agrees with the coverage array it ships', () => { + // The summary block is what the markdown report and the webapp quote. It is + // computed separately from the per-entry array, so the two can drift. + const doc = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const s = doc.summary; + assert.equal(s.total_entries, doc.coverage.length); + assert.equal(s.total_entries, stats.entries.total, 'report covers a different entry set than stats.json'); + assert.equal(s.covered_entries, doc.coverage.filter((e) => e.mapped).length); + assert.equal(s.uncovered_entries, doc.coverage.filter((e) => !e.mapped).length); + assert.equal(s.covered_entries + s.uncovered_entries, s.total_entries); + assert.equal(s.unique_controls, doc.controls.length); +}); + +test('compliance coverage_rate is the rate it claims to be', () => { + const doc = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const s = doc.summary; + const expected = Math.round((s.covered_entries / s.total_entries) * 1000) / 10; + assert.equal(s.coverage_rate, expected); +}); + +test('compliance --severity narrows the report rather than widening it', () => { + const all = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const crit = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--severity', 'Critical', '--format', 'json', '--stdout', + ])); + assert.ok(crit.coverage.length > 0, 'severity filter returned nothing at all'); + assert.ok(crit.coverage.length <= all.coverage.length, + 'a severity filter returned more entries than no filter'); + for (const e of crit.coverage) assert.equal(e.severity, 'Critical'); +}); + +test('an unknown framework fails loudly instead of emitting an empty report', () => { + assert.throws( + () => run('compliance-report.js', ['--framework', 'Not A Real Framework', '--format', 'json', '--stdout']), + 'a typo in --framework should not produce a clean empty report', + ); +}); diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs new file mode 100644 index 0000000..6ea5e6d --- /dev/null +++ b/scripts/stats.test.mjs @@ -0,0 +1,114 @@ +/** + * stats.test.mjs — invariants for the numbers every headline claim is built on. + * + * README badges, the webapp, and the compliance reports all render from + * data/stats.json. A wrong number here is a wrong number everywhere, and it + * looks authoritative because it was generated. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { computeStats } = require(path.join(ROOT, 'scripts', 'stats.js')); + +const stats = computeStats(); + +test('every count is a finite non-negative integer', () => { + const walk = (o, at) => { + for (const [k, v] of Object.entries(o)) { + if (typeof v === 'number') { + assert.ok(Number.isInteger(v) && v >= 0, `${at}.${k} is ${v}`); + } else if (v && typeof v === 'object' && !Array.isArray(v)) { + walk(v, `${at}.${k}`); + } + } + }; + walk(stats, 'stats'); +}); + +test('per-source-list entry counts sum to the total', () => { + const sum = Object.values(stats.entries.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.entries.total); +}); + +test('per-source-list mapping counts sum to the total', () => { + const sum = Object.values(stats.mappings.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.mappings.total); +}); + +test('per-source-list mapping files sum to the total', () => { + const sum = Object.values(stats.mapping_files.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.mapping_files.total); +}); + +test('frameworks mapped never exceeds registries present', () => { + // These two numbers are deliberately separate and were conflated for a long + // time. `mapped` counts frameworks some entry actually maps a control to; + // `registries` counts inventories on disk. mapped > registries would mean a + // mapping names a framework with no registry behind it. + assert.ok( + stats.frameworks.mapped <= stats.frameworks.registries, + `${stats.frameworks.mapped} mapped > ${stats.frameworks.registries} registries`, + ); +}); + +test('unmapped registries and mapped frameworks account for every registry', () => { + assert.equal( + stats.frameworks.mapped + stats.frameworks.unmapped_registries.length, + stats.frameworks.registries, + ); +}); + +test('draft_only names only frameworks that are actually mapped', () => { + const entriesDir = path.join(ROOT, 'data', 'entries'); + const mapped = new Set(); + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) mapped.add(m.framework); + } + for (const f of stats.frameworks.draft_only) { + assert.ok(mapped.has(f), `draft_only names "${f}", which no entry maps`); + } +}); + +test('draft rows never exceed total mappings', () => { + assert.ok( + stats.frameworks.draft_rows <= stats.mappings.total, + `${stats.frameworks.draft_rows} draft rows > ${stats.mappings.total} mappings`, + ); +}); + +test('a draft_only framework really has no authored row', () => { + // The claim the README renders is "these frameworks carry candidate DRAFT + // rows only". If one of them has an authored row, the caveat understates the + // project and the reader is misled in the safe direction — still wrong. + const entriesDir = path.join(ROOT, 'data', 'entries'); + const authored = {}; + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) { + if (!/^DRAFT\b/.test(m.notes || '')) authored[m.framework] = (authored[m.framework] || 0) + 1; + } + } + for (const f of stats.frameworks.draft_only) { + assert.equal(authored[f], undefined, `"${f}" is listed draft-only but has authored rows`); + } +}); + +test('stats.json on disk matches a fresh computation', () => { + // stats:check enforces this in CI too; having it here means `node --test` + // alone catches a hand-edited stats.json. + const onDisk = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'stats.json'), 'utf8')); + assert.deepEqual(onDisk, stats); +}); + +test('source list count matches the ids and labels it publishes', () => { + assert.equal(stats.source_lists.count, stats.source_lists.ids.length); + assert.equal(stats.source_lists.count, stats.source_lists.labels.length); +}); From 957b59e54640b1e5358f5ae16346961d3b319860 Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Fri, 28 Aug 2026 17:40:10 -0400 Subject: [PATCH 2/2] CI: node --test could not find the suite on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `node --test "scripts/**/*.test.mjs"` fails in CI: bash does not expand `**` without globstar, and Node 20 does not glob the pattern itself. The suite had never run in CI before this PR, so nothing caught it. Unquoted `scripts/*.test.mjs` works both ways — a POSIX shell expands it to a file list, and on Windows, where cmd does not glob, Node expands it. Co-Authored-By: Claude Opus 5 (1M context) --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8ef538b..32a707c 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "ci": "npm run build && npm run test:scripts && npm run build:reports", "compile": "tsc", "prepublishOnly": "npm run compile", - "test:scripts": "node --test \"scripts/**/*.test.mjs\"", - "test": "node --test \"scripts/**/*.test.mjs\"" + "test:scripts": "node --test scripts/*.test.mjs", + "test": "node --test scripts/*.test.mjs" }, "license": "CC-BY-SA-4.0", "repository": {