diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 130b743..fca25d8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -14,6 +14,8 @@ on: - 'README.md' - 'scripts/validate.js' - 'scripts/generate.js' + - 'scripts/*.test.mjs' + - 'package.json' - '.github/workflows/validate.yml' pull_request: branches: [main] @@ -28,6 +30,8 @@ on: - 'README.md' - 'scripts/validate.js' - 'scripts/generate.js' + - 'scripts/*.test.mjs' + - 'package.json' - '.github/workflows/validate.yml' # Both jobs only read the tree. Without this they inherit the repository @@ -112,3 +116,32 @@ jobs: # here rather than shipping. - name: Assert generated output is current run: git diff --exit-code -- data/entries docs/data.js docs/incidents.js + + unit-tests: + name: Unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Use Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + + # ajv is the only runtime dependency the suite needs, and it is a + # devDependency. --ignore-scripts because nothing here needs a postinstall. + - name: Install dependencies + run: npm ci --ignore-scripts + + # Covers stats.js, generate.js, the three report CLIs, the framework + # ingest path, and schema validation of the OSCAL and STIX exports. + - name: Run the test suite + run: npm run test:scripts + + # The suite re-runs the generator to prove it is deterministic. If that + # left anything behind, the tree must still be clean. + - name: Assert the tests left no changes behind + run: git diff --exit-code diff --git a/data/framework-schema.json b/data/framework-schema.json index 5b6dc37..64d71ca 100644 --- a/data/framework-schema.json +++ b/data/framework-schema.json @@ -73,6 +73,7 @@ "controls": { "type": "array", "description": "Complete inventory of controls/clauses/requirements", + "minItems": 1, "items": { "type": "object", "required": [ diff --git a/data/schemas/README.md b/data/schemas/README.md new file mode 100644 index 0000000..98234ab --- /dev/null +++ b/data/schemas/README.md @@ -0,0 +1,61 @@ + + +# Export schemas + +The crosswalk emits three machine-readable formats that other tools consume: + +| Format | Emitted by | Schema here | +|---|---|---| +| OSCAL 1.1.2 Component Definition | `compliance-report.js --format oscal` | `oscal-component-definition.subset.json` | +| OSCAL 1.1.2 Catalog | `compliance-report.js --format oscal-catalog` | `oscal-catalog.subset.json` | +| STIX 2.1 Bundle | `incidents-report.js --format stix` | `stix-bundle.subset.json` | + +`scripts/exports.test.mjs` validates every emitted document against these on +each `node --test` run, and `npm run ci` includes it. + +## These are subsets, and the filename says so + +They are **not** the NIST and OASIS schemas. They encode the structural rules +this project's output has to satisfy — required members, id formats, the +`spec_version` and `oscal-version` constants, UUID and timestamp shapes, +`additionalProperties` where the spec closes an object — and they are written +here, in this repository, by this project. + +Passing them proves the export has not silently lost its shape. It does not +prove full OSCAL or STIX conformance, and no sentence in this repository should +claim it does. + +## Why not the upstream schemas + +Two options were considered and both cost more than they return here: + +- **Fetch at CI time.** Makes every build depend on a third-party host being up + and on a schema that can change without warning. A red build that means + "NIST changed a description" is a build people learn to ignore. +- **Vendor the upstream files.** The OSCAL complete schema is over a megabyte of + JSON this project does not control and cannot meaningfully review on update. + +A small schema that is read, understood and owned catches the regressions that +actually happen — a renamed member, a dropped `spec_version`, a malformed +UUID — and it does so without pretending to an authority it does not have. + +For genuine conformance, run the emitted file through the upstream validators: + +```bash +# OSCAL — https://github.com/usnistgov/OSCAL +node scripts/compliance-report.js --framework "NIST AI RMF 1.0" --format oscal --stdout > oscal.json + +# STIX 2.1 — https://github.com/oasis-open/cti-stix-validator +node scripts/incidents-report.js --format stix --stdout > stix.json +``` + +--- + +*Part of the [OWASP GenAI Crosswalk](https://github.com/GenAI-Security-Project/crosswalk) — +maintained by the [OWASP GenAI Data Security Initiative](https://genai.owasp.org)* diff --git a/data/schemas/oscal-catalog.subset.json b/data/schemas/oscal-catalog.subset.json new file mode 100644 index 0000000..511d762 --- /dev/null +++ b/data/schemas/oscal-catalog.subset.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/oscal-catalog.subset.json", + "title": "OSCAL 1.1.2 Catalog — structural subset", + "description": "A SUBSET of the NIST OSCAL 1.1.2 catalog model, written and owned by this project. It encodes the structure compliance-report.js --format oscal-catalog must emit. Passing proves the export has not lost its shape; it does not prove OSCAL conformance. See data/schemas/README.md.", + "type": "object", + "required": ["catalog"], + "additionalProperties": false, + "properties": { + "catalog": { + "type": "object", + "required": ["uuid", "metadata"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "metadata": { "$ref": "#/definitions/metadata" }, + "groups": { + "type": "array", + "items": { "$ref": "#/definitions/group" } + }, + "controls": { + "type": "array", + "items": { "$ref": "#/definitions/control" } + }, + "back-matter": { "type": "object" } + } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + }, + "metadata": { + "type": "object", + "required": ["title", "last-modified", "version", "oscal-version"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "last-modified": { "type": "string", "format": "date-time" }, + "version": { "type": "string", "minLength": 1 }, + "oscal-version": { "const": "1.1.2" }, + "roles": { "type": "array" }, + "parties": { "type": "array" }, + "responsible-parties": { "type": "array" } + } + }, + "group": { + "type": "object", + "required": ["id", "title"], + "properties": { + "id": { "$ref": "#/definitions/tokenId" }, + "title": { "type": "string", "minLength": 1 }, + "controls": { "type": "array", "items": { "$ref": "#/definitions/control" } }, + "groups": { "type": "array", "items": { "$ref": "#/definitions/group" } } + } + }, + "control": { + "type": "object", + "required": ["id", "title"], + "properties": { + "id": { "$ref": "#/definitions/tokenId" }, + "title": { "type": "string", "minLength": 1 }, + "props": { "type": "array", "items": { "type": "object", "required": ["name", "value"] } }, + "links": { "type": "array", "items": { "type": "object", "required": ["href"] } }, + "parts": { "type": "array" }, + "controls": { "type": "array", "items": { "$ref": "#/definitions/control" } } + } + }, + "tokenId": { + "type": "string", + "description": "OSCAL identifiers are NCName-like tokens: they must not start with a digit and must not contain whitespace. Control ids taken straight from a framework often do both, so this is the check most likely to catch a real regression.", + "pattern": "^[A-Za-z_][A-Za-z0-9._~:-]*$" + } + } +} diff --git a/data/schemas/oscal-component-definition.subset.json b/data/schemas/oscal-component-definition.subset.json new file mode 100644 index 0000000..62ddea5 --- /dev/null +++ b/data/schemas/oscal-component-definition.subset.json @@ -0,0 +1,128 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/oscal-component-definition.subset.json", + "title": "OSCAL 1.1.2 Component Definition — structural subset", + "description": "A SUBSET of the NIST OSCAL 1.1.2 component-definition model, written and owned by this project. It encodes the structure compliance-report.js --format oscal must emit. Passing proves the export has not lost its shape; it does not prove OSCAL conformance. See data/schemas/README.md.", + "type": "object", + "required": ["component-definition"], + "additionalProperties": false, + "properties": { + "component-definition": { + "type": "object", + "required": ["uuid", "metadata", "components"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "metadata": { "$ref": "#/definitions/metadata" }, + "components": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/component" } + }, + "back-matter": { "type": "object" } + } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + }, + "metadata": { + "type": "object", + "required": ["title", "last-modified", "version", "oscal-version"], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "last-modified": { "type": "string", "format": "date-time" }, + "version": { "type": "string", "minLength": 1 }, + "oscal-version": { + "type": "string", + "description": "Pinned. A silent bump here changes what downstream tooling will accept.", + "const": "1.1.2" + }, + "roles": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "title"], + "properties": { "id": { "type": "string" }, "title": { "type": "string" } } + } + }, + "parties": { + "type": "array", + "items": { + "type": "object", + "required": ["uuid", "type", "name"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "type": { "enum": ["person", "organization"] }, + "name": { "type": "string", "minLength": 1 } + } + } + }, + "responsible-parties": { "type": "array" } + } + }, + "component": { + "type": "object", + "required": ["uuid", "type", "title", "description"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "type": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "control-implementations": { + "type": "array", + "items": { + "type": "object", + "required": ["uuid", "source", "description", "implemented-requirements"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "source": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "implemented-requirements": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["uuid", "control-id", "description"], + "properties": { + "uuid": { "$ref": "#/definitions/uuid" }, + "control-id": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "props": { "$ref": "#/definitions/props" }, + "links": { "$ref": "#/definitions/links" } + } + } + } + } + } + } + } + }, + "props": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "value"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "string" }, + "ns": { "type": "string" }, + "class": { "type": "string" } + } + } + }, + "links": { + "type": "array", + "items": { + "type": "object", + "required": ["href"], + "properties": { + "href": { "type": "string", "minLength": 1 }, + "rel": { "type": "string" }, + "text": { "type": "string" } + } + } + } + } +} diff --git a/data/schemas/stix-bundle.subset.json b/data/schemas/stix-bundle.subset.json new file mode 100644 index 0000000..841f6af --- /dev/null +++ b/data/schemas/stix-bundle.subset.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/GenAI-Security-Project/crosswalk/blob/main/data/schemas/stix-bundle.subset.json", + "title": "STIX 2.1 Bundle — structural subset", + "description": "A SUBSET of the OASIS STIX 2.1 specification, written and owned by this project. It encodes the structure incidents-report.js --format stix must emit. Passing proves the export has not lost its shape; it does not prove STIX conformance. See data/schemas/README.md.", + "type": "object", + "required": ["type", "id", "objects"], + "additionalProperties": false, + "properties": { + "type": { "const": "bundle" }, + "id": { "$ref": "#/definitions/stixId" }, + "objects": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/sdo" } + } + }, + "definitions": { + "uuid": { + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "stixId": { + "type": "string", + "description": "STIX id: --. The double hyphen is required by the spec.", + "pattern": "^[a-z][a-z0-9-]*--[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "timestamp": { "type": "string", "format": "date-time" }, + "externalReference": { + "type": "object", + "required": ["source_name"], + "properties": { + "source_name": { "type": "string", "minLength": 1 }, + "external_id": { "type": "string" }, + "url": { "type": "string", "format": "uri" }, + "description": { "type": "string" } + } + }, + "commonSdo": { + "type": "object", + "required": ["type", "spec_version", "id", "created", "modified"], + "properties": { + "type": { "type": "string", "minLength": 1 }, + "spec_version": { "const": "2.1" }, + "id": { "$ref": "#/definitions/stixId" }, + "created": { "$ref": "#/definitions/timestamp" }, + "modified": { "$ref": "#/definitions/timestamp" }, + "external_references": { + "type": "array", + "items": { "$ref": "#/definitions/externalReference" } + } + } + }, + "sdo": { + "allOf": [ + { "$ref": "#/definitions/commonSdo" }, + { + "if": { "properties": { "type": { "const": "attack-pattern" } } }, + "then": { "required": ["name"], "properties": { "name": { "type": "string", "minLength": 1 } } } + }, + { + "if": { "properties": { "type": { "const": "report" } } }, + "then": { + "required": ["name", "published", "object_refs"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "published": { "$ref": "#/definitions/timestamp" }, + "report_types": { "type": "array", "items": { "type": "string" } }, + "object_refs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/stixId" } + } + } + } + }, + { + "if": { "properties": { "type": { "const": "relationship" } } }, + "then": { + "required": ["relationship_type", "source_ref", "target_ref"], + "properties": { + "relationship_type": { "type": "string", "minLength": 1 }, + "source_ref": { "$ref": "#/definitions/stixId" }, + "target_ref": { "$ref": "#/definitions/stixId" } + } + } + } + ] + } + } +} diff --git a/package-lock.json b/package-lock.json index 5463c5a..7f978b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "license": "CC-BY-SA-4.0", "devDependencies": { "@types/node": "^25.5.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "typescript": "^5.4.0" }, "engines": { @@ -25,6 +27,82 @@ "undici-types": "~7.18.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index 5a9a80a..32a707c 100644 --- a/package.json +++ b/package.json @@ -39,10 +39,11 @@ "state-report": "node scripts/state-report.js", "build": "npm run generate && npm run validate && npm run stats:check", "build:reports": "npm run compliance && npm run incidents && npm run registry-coverage && npm run density && npm run inventory && npm run audit:incidents && npm run validate:aivss && npm run freshness", - "ci": "npm run build && npm run build:reports", + "ci": "npm run build && npm run test:scripts && npm run build:reports", "compile": "tsc", "prepublishOnly": "npm run compile", - "test:scripts": "node --test \"scripts/**/*.test.mjs\"" + "test:scripts": "node --test scripts/*.test.mjs", + "test": "node --test scripts/*.test.mjs" }, "license": "CC-BY-SA-4.0", "repository": { @@ -74,6 +75,8 @@ ], "devDependencies": { "@types/node": "^25.5.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "typescript": "^5.4.0" } } diff --git a/scripts/compliance-report.js b/scripts/compliance-report.js index e3382e5..395d7f3 100644 --- a/scripts/compliance-report.js +++ b/scripts/compliance-report.js @@ -737,6 +737,39 @@ function renderSummaryMarkdown(frameworks, allEntries, opts) { const crypto = require('crypto'); +/** + * Coerce a framework control identifier into a valid OSCAL token. + * + * OSCAL ids are NCName-like: they may not begin with a digit and may not + * contain whitespace. Real control ids routinely do both — `Art. 5` (DORA), + * `SR 1.1` (ISA 62443), `4.1` (ISO 42001), `V8 Data Protection` (ASVS) — so + * emitting them verbatim produced catalogs that no OSCAL tool would load. + * Thirteen of the twenty-five frameworks were affected and nothing caught it, + * because nothing read the export back. + * + * The rule is deliberately boring and deterministic, so the same control gets + * the same id in the catalog and in the component definition, and so the id is + * stable across runs: + * + * whitespace and invalid characters → `-`, collapsed + * leading digit → `c-` prefix + * longer than 60 characters → truncated, plus a hash of the original + * + * The original is never lost: it is preserved verbatim in the control `title` + * and in a `source-control-id` prop. + */ +function oscalToken(raw) { + const original = String(raw == null ? '' : raw).trim(); + let t = original.replace(/[^A-Za-z0-9._~:-]+/g, '-').replace(/-{2,}/g, '-').replace(/^-+|-+$/g, ''); + if (!t) t = 'control'; + if (!/^[A-Za-z_]/.test(t)) t = 'c-' + t; + if (t.length > 60) { + const h = crypto.createHash('sha1').update(original).digest('hex').slice(0, 6); + t = t.slice(0, 52).replace(/-+$/, '') + '-' + h; + } + return t; +} + function renderOSCAL(fw, allEntries) { const r = buildFrameworkReport(fw, allEntries); const now = new Date().toISOString(); @@ -771,9 +804,10 @@ function renderOSCAL(fw, allEntries) { 'implemented-requirements': [...r.controls.values()].flatMap(ctrl => ctrl.entries.map(entry => ({ uuid: crypto.randomUUID(), - 'control-id': ctrl.control_id, + 'control-id': oscalToken(ctrl.control_id), description: `${entry.id}: ${entry.name} — ${ctrl.control_name}${ctrl.notes.length ? ' | ' + ctrl.notes.join('; ') : ''}`, props: [ + { name: 'source-control-id', value: String(ctrl.control_id) }, { name: 'implementation-status', value: 'planned', ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, { name: 'owasp-entry', value: entry.id, ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, { name: 'severity', value: entry.severity, ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, @@ -844,15 +878,16 @@ function renderOSCALCatalog(fw, allEntries) { links: registryFw ? [{ href: registryFw.url, rel: 'canonical' }] : [], }, groups: [...groups.entries()].map(([groupName, ctrls]) => ({ - id: groupName.toLowerCase().replace(/[^a-z0-9]+/g, '-'), + id: oscalToken(groupName.toLowerCase()), title: groupName, controls: ctrls.map(ctrl => { const cid = registryFw ? ctrl.control_id : ctrl.control_id; const mappedData = r.controls.get(cid); return { - id: cid, + id: oscalToken(cid), title: registryFw ? ctrl.title : (ctrl.control_name || ctrl.title), props: [ + { name: 'source-control-id', value: String(cid) }, ...(ctrl.description ? [{ name: 'description', value: ctrl.description }] : []), ...(mappedData ? [ { name: 'owasp-coverage', value: mappedData.entries.map(e => e.id).join(','), ns: 'https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/crosswalk' }, diff --git a/scripts/exports.test.mjs b/scripts/exports.test.mjs new file mode 100644 index 0000000..91200e6 --- /dev/null +++ b/scripts/exports.test.mjs @@ -0,0 +1,232 @@ +/** + * exports.test.mjs — validate the machine-readable exports against schema. + * + * The crosswalk emits OSCAL and STIX for other tools to consume. Those + * consumers fail on a missing member or a malformed id, and until now nothing + * checked either: the exports were generated, written, and never read back. + * + * The schemas in data/schemas/ are deliberate subsets, written and owned here. + * Passing proves the export has not lost its shape. It does not prove OSCAL or + * STIX conformance — see data/schemas/README.md for why, and for how to run the + * upstream validators when that is what you need. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const ajv = new Ajv({ allErrors: true, strict: false }); +addFormats(ajv); + +const schema = (name) => + ajv.compile(JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'schemas', name), 'utf8'))); + +/** Run a repo script and parse its stdout as JSON. */ +function emit(script, args) { + const out = execFileSync(process.execPath, [path.join(ROOT, 'scripts', script), ...args], { + cwd: ROOT, + maxBuffer: 256 * 1024 * 1024, + encoding: 'utf8', + }); + return JSON.parse(out); +} + +const report = (validate) => + (validate.errors || []) + .slice(0, 10) + .map((e) => `${e.instancePath || '/'} ${e.message}`) + .join('\n'); + +// ── OSCAL component definition ─────────────────────────────────────────────── + +test('OSCAL component definition matches the structural subset', () => { + const validate = schema('oscal-component-definition.subset.json'); + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal', '--stdout', + ]); + assert.ok(validate(doc), report(validate)); +}); + +test('OSCAL component definition carries at least one implemented requirement', () => { + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal', '--stdout', + ]); + const impls = doc['component-definition'].components.flatMap( + (c) => c['control-implementations'] || [], + ); + const reqs = impls.flatMap((i) => i['implemented-requirements'] || []); + assert.ok(reqs.length > 0, 'export contains no implemented-requirements — the mapping was dropped'); +}); + +// ── OSCAL catalog ──────────────────────────────────────────────────────────── + +test('OSCAL catalog matches the structural subset', () => { + const validate = schema('oscal-catalog.subset.json'); + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal-catalog', '--stdout', + ]); + assert.ok(validate(doc), report(validate)); +}); + +test('OSCAL catalog ids are OSCAL tokens, not raw control strings', () => { + // The failure this catches: a framework whose control ids start with a digit + // or contain a space is emitted verbatim and produces a catalog no OSCAL tool + // will load. It is invisible in the JSON unless something checks. + const doc = emit('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'oscal-catalog', '--stdout', + ]); + const ids = []; + const walk = (c) => { + ids.push(c.id); + (c.controls || []).forEach(walk); + }; + for (const g of doc.catalog.groups || []) { + ids.push(g.id); + (g.controls || []).forEach(walk); + } + (doc.catalog.controls || []).forEach(walk); + + assert.ok(ids.length > 0, 'catalog has no controls'); + const bad = ids.filter((id) => !/^[A-Za-z_][A-Za-z0-9._~:-]*$/.test(id)); + assert.deepEqual(bad, [], `non-token ids: ${bad.slice(0, 5).join(', ')}`); +}); + +// ── STIX 2.1 ───────────────────────────────────────────────────────────────── + +test('STIX bundle matches the structural subset', () => { + const validate = schema('stix-bundle.subset.json'); + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + assert.ok(validate(doc), report(validate)); +}); + +test('every STIX relationship points at an object in the same bundle', () => { + // A dangling source_ref or target_ref is accepted by a shape check and + // rejected by every real STIX consumer, so it needs its own assertion. + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const ids = new Set(doc.objects.map((o) => o.id)); + const dangling = []; + for (const o of doc.objects) { + if (o.type === 'relationship') { + if (!ids.has(o.source_ref)) dangling.push(`${o.id} source_ref ${o.source_ref}`); + if (!ids.has(o.target_ref)) dangling.push(`${o.id} target_ref ${o.target_ref}`); + } + for (const ref of o.object_refs || []) { + if (!ids.has(ref)) dangling.push(`${o.id} object_ref ${ref}`); + } + } + assert.deepEqual(dangling.slice(0, 5), [], `${dangling.length} dangling reference(s)`); +}); + +test('STIX ids are unique across the bundle', () => { + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const seen = new Set(); + const dupes = []; + for (const o of doc.objects) { + if (seen.has(o.id)) dupes.push(o.id); + seen.add(o.id); + } + assert.deepEqual(dupes, [], `duplicate STIX ids: ${dupes.slice(0, 3).join(', ')}`); +}); + +test('STIX bundle covers every incident in the database', () => { + const db = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'incidents.json'), 'utf8')); + const doc = emit('incidents-report.js', ['--format', 'stix', '--stdout']); + const reports = doc.objects.filter((o) => o.type === 'report'); + assert.equal( + reports.length, + db.incidents.length, + 'incident count and STIX report count disagree — the export is dropping records', + ); +}); + +// ── regression fence for the swapped control_id / control_name rows ────────── + +/** + * A known, tracked defect: in some mapping files the control identifier and the + * requirement prose are in the wrong columns, so `control_id` holds a sentence. + * The OSCAL exports no longer *break* on it — `oscalToken()` coerces the id and + * preserves the original in a `source-control-id` prop — but the underlying data + * is still wrong, and a reader following an id back to the framework cannot. + * + * This is the count as of 2026-08-28, per framework registry. It is a ceiling, + * not a target: the test fails if any framework gains prose ids, or if a + * framework not listed here starts producing them. When the parser is fixed, + * these numbers come down and this table shrinks with them. + * + * Tracked as issue #35. + */ +const PROSE_ID_BASELINE = Object.freeze({ + 'AIUC-1': 3, + 'CIS Controls v8.1': 31, + 'CWE/CVE': 25, + 'EU AI Act': 118, + 'ISO/IEC 42001:2023': 3, + 'NIST SP 800-218A': 40, + 'NIST SP 800-82 Rev 3': 29, + 'OWASP AI Testing Guide': 16, + 'OWASP NHI Top 10': 103, + 'PCI DSS v4.0': 40, + 'SOC 2': 168, +}); + +/** A control id is "prose" when it reads as a sentence rather than an identifier. */ +const isProse = (raw) => String(raw).trim().split(/\s+/).length >= 5; + +test('prose-shaped control ids do not spread beyond the known set', () => { + const fwDir = path.join(ROOT, 'data', 'frameworks'); + const registries = fs.readdirSync(fwDir).filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(fwDir, f), 'utf8'))); + + const counted = {}; + for (const reg of registries) { + // `--framework` is a partial match, so one call can emit several documents. + const raw = execFileSync(process.execPath, [ + path.join(ROOT, 'scripts', 'compliance-report.js'), + '--framework', reg.name, '--format', 'oscal-catalog', '--stdout', + ], { cwd: ROOT, maxBuffer: 256 * 1024 * 1024, encoding: 'utf8' }); + + for (const chunk of raw.split(/\n(?=\{\n)/)) { + const doc = JSON.parse(chunk); + const name = doc.catalog.metadata.title.replace(/ — Control Catalog$/, ''); + const sources = []; + const walk = (c) => { + const p = (c.props || []).find((x) => x.name === 'source-control-id'); + if (p) sources.push(p.value); + (c.controls || []).forEach(walk); + }; + for (const g of doc.catalog.groups || []) (g.controls || []).forEach(walk); + (doc.catalog.controls || []).forEach(walk); + + const n = sources.filter(isProse).length; + if (n) counted[name] = n; + } + } + + const regressions = []; + for (const [fw, n] of Object.entries(counted)) { + const allowed = PROSE_ID_BASELINE[fw]; + if (allowed === undefined) regressions.push(`${fw} newly produces ${n} prose id(s)`); + else if (n > allowed) regressions.push(`${fw}: ${n} prose ids, baseline ${allowed}`); + } + assert.deepEqual(regressions, [], regressions.join('; ')); +}); + +test('the prose-id baseline does not silently overstate the problem', () => { + // The mirror of the test above. If a framework is fixed, its baseline entry + // must be removed rather than left as dead permission — otherwise the fence + // slowly stops fencing anything. + const fwDir = path.join(ROOT, 'data', 'frameworks'); + const names = new Set( + fs.readdirSync(fwDir).filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(fwDir, f), 'utf8')).name), + ); + const unknown = Object.keys(PROSE_ID_BASELINE).filter((f) => !names.has(f)); + assert.deepEqual(unknown, [], `baseline names frameworks that no longer exist: ${unknown.join(', ')}`); +}); diff --git a/scripts/generate.test.mjs b/scripts/generate.test.mjs new file mode 100644 index 0000000..8c31dce --- /dev/null +++ b/scripts/generate.test.mjs @@ -0,0 +1,127 @@ +/** + * generate.test.mjs — the generator must be deterministic, and its output must + * be the thing that is committed. + * + * generate.js produces every file in data/entries/ and the three webapp data + * bundles. It once sat broken on main for several commits with all CI green, + * because no job ran it. CI now does; this covers the properties CI's + * `git diff --exit-code` cannot express on its own. + * + * These tests re-run the generator. Its output is deterministic, so a clean + * tree stays clean — and if it does not, that is the failure being reported. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const ENTRIES = path.join(ROOT, 'data', 'entries'); +const BUNDLES = ['data.js', 'incidents.js', 'backlinks.js', 'frameworks-registry.js'] + .map((f) => path.join(ROOT, 'docs', f)); + +const runGenerate = () => + execFileSync(process.execPath, [path.join(ROOT, 'scripts', 'generate.js')], { + cwd: ROOT, maxBuffer: 256 * 1024 * 1024, encoding: 'utf8', + }); + +/** Hash every generated artefact, EOL-normalised so Windows checkouts agree. */ +function fingerprint() { + const h = crypto.createHash('sha256'); + const files = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')).sort() + .map((f) => path.join(ENTRIES, f)) + .concat(BUNDLES.filter((f) => fs.existsSync(f))); + for (const f of files) { + h.update(path.basename(f)); + h.update(fs.readFileSync(f, 'utf8').replace(/\r\n/g, '\n')); + } + return h.digest('hex'); +} + +test('generating twice produces byte-identical output', () => { + // Any timestamp, random id or Object key-order dependence in the generator + // shows up here as a diff on every run — which is how a repository ends up + // with permanently dirty generated files that everyone learns to ignore. + runGenerate(); + const first = fingerprint(); + runGenerate(); + assert.equal(fingerprint(), first, 'generate.js is not deterministic'); +}); + +test('committed entries match a fresh generation', () => { + const before = fingerprint(); + runGenerate(); + assert.equal(fingerprint(), before, + 'data/entries or docs/*.js differ from what generate.js produces — regenerate and commit'); +}); + +test('every entry declares an id, name, source list and severity', () => { + const files = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')); + assert.ok(files.length > 0); + for (const f of files) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + assert.match(e.id, /^(LLM|ASI|DSGAI|AST)\d{2}$/, `${f} has id "${e.id}"`); + assert.equal(`${e.id}.json`, f, `${f} does not match its own id`); + assert.ok(e.name && e.name.length > 1, `${f} has no name`); + assert.ok(e.source_list, `${f} has no source_list`); + assert.match(e.severity, /^(Critical|High|Medium|Low)$/, `${f} severity "${e.severity}"`); + } +}); + +test('no mapping is missing a framework or a control id', () => { + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + (e.mappings || []).forEach((m, i) => { + assert.ok(m.framework, `${f} mapping[${i}] has no framework`); + assert.ok(m.control_id, `${f} mapping[${i}] has no control_id`); + }); + } +}); + +test('no mapping claims a confidence without a named reviewer', () => { + // The single rule the whole schema-v2 migration exists to enforce. It is + // checked by validate.js too; duplicating it here means `node --test` alone + // is enough to catch an entry file edited by hand. + const offenders = []; + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + for (const m of e.mappings || []) { + if (m.confidence && m.confidence !== 'unreviewed' && !(m.reviewed_by || []).length) { + offenders.push(`${e.id}/${m.framework}:${m.control_id}`); + } + } + } + assert.deepEqual(offenders.slice(0, 5), [], `${offenders.length} unreviewed row(s) claim confidence`); +}); + +test('DRAFT never survives into a stored enum field', () => { + // The Markdown templates carry the literal word DRAFT in the relationship, + // rationale and confidence columns. generate.js is supposed to resolve those + // to `unreviewed` rather than store them, so a stored "DRAFT" means the + // template leaked into the data layer. + const leaked = []; + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + for (const m of e.mappings || []) { + for (const k of ['relationship', 'rationale_type', 'confidence']) { + if (typeof m[k] === 'string' && /^draft$/i.test(m[k].trim())) { + leaked.push(`${e.id}/${m.framework}:${m.control_id}.${k}`); + } + } + } + } + assert.deepEqual(leaked.slice(0, 5), [], `${leaked.length} field(s) stored the literal "DRAFT"`); +}); + +test('webapp bundles stay in step with the entry files', () => { + const src = fs.readFileSync(path.join(ROOT, 'docs', 'data.js'), 'utf8'); + const start = src.indexOf('['); + const end = src.lastIndexOf(']'); + const bundled = JSON.parse(src.slice(start, end + 1)); + const onDisk = fs.readdirSync(ENTRIES).filter((f) => f.endsWith('.json')).length; + assert.equal(bundled.length, onDisk, 'docs/data.js and data/entries hold different entry counts'); +}); diff --git a/scripts/ingest-framework.mjs b/scripts/ingest-framework.mjs index 4d1899a..3abe4e4 100644 --- a/scripts/ingest-framework.mjs +++ b/scripts/ingest-framework.mjs @@ -71,6 +71,11 @@ function validateFramework(fw, schema) { // controls if (!Array.isArray(fw.controls)) { errors.push('controls must be an array'); + } else if (fw.controls.length === 0) { + // A registry with no controls validates cleanly, lands in + // data/frameworks/, and then counts as a framework with an empty + // inventory — inflating the registry count while covering nothing. + errors.push('controls is empty — a registry with no controls is not a framework'); } else { const ids = new Set(); fw.controls.forEach((c, i) => { diff --git a/scripts/ingest.test.mjs b/scripts/ingest.test.mjs new file mode 100644 index 0000000..c100e82 --- /dev/null +++ b/scripts/ingest.test.mjs @@ -0,0 +1,109 @@ +/** + * ingest.test.mjs — the framework ingest path, exercised through its CLI. + * + * ingest-framework.mjs is how a new framework registry enters the project. It + * is the one script whose job is to accept outside data, so the property that + * matters is that it rejects: a malformed source must fail loudly rather than + * land a half-formed registry in data/frameworks/. + * + * Every test uses --validate, so nothing here writes to data/frameworks/. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const SCRIPT = path.join(ROOT, 'scripts', 'ingest-framework.mjs'); + +function run(args) { + return execFileSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, maxBuffer: 32 * 1024 * 1024, encoding: 'utf8', + }); +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'crosswalk-ingest-')); +const write = (name, body) => { + const p = path.join(tmp, name); + fs.writeFileSync(p, body, 'utf8'); + return p; +}; + +test.after(() => fs.rmSync(tmp, { recursive: true, force: true })); + +const VALID = { + id: 'test-framework', + name: 'Test Framework', + short_name: 'TF', + version: '1.0', + url: 'https://example.org/test-framework', + license: 'CC BY 4.0', + publisher: 'Test Publisher', + category: 'ai-governance', + last_synced: '2026-08-28', + source_sha: null, + controls: [ + { control_id: 'TF-1', title: 'First control', description: 'Does a thing', kind: 'control' }, + { control_id: 'TF-2', title: 'Second control', description: 'Does another', kind: 'control' }, + ], +}; + +test('--list names every framework already registered', () => { + const out = run(['--list']); + const names = fs.readdirSync(path.join(ROOT, 'data', 'frameworks')) + .filter((f) => f.endsWith('.json')) + .map((f) => JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'frameworks', f), 'utf8'))); + const missing = names.filter((r) => !out.includes(r.id) && !out.includes(r.name)); + assert.deepEqual(missing.map((r) => r.id), [], 'registered frameworks missing from --list'); +}); + +test('a well-formed JSON source validates', () => { + const p = write('valid.json', JSON.stringify(VALID, null, 2)); + const out = run([p, '--validate']); + assert.match(out, /valid|ok|pass/i, `unexpected output:\n${out}`); +}); + +test('a source with no controls is rejected', () => { + const p = write('empty.json', JSON.stringify({ ...VALID, controls: [] }, null, 2)); + assert.throws(() => run([p, '--validate']), 'an empty control set was accepted'); +}); + +test('a source missing a required top-level field is rejected', () => { + const { name, ...noName } = VALID; + const p = write('no-name.json', JSON.stringify(noName, null, 2)); + assert.throws(() => run([p, '--validate']), 'a registry with no name was accepted'); +}); + +test('a control with no control_id is rejected', () => { + const p = write('bad-control.json', JSON.stringify({ + ...VALID, + controls: [{ title: 'Nameless', description: 'x', kind: 'control' }], + }, null, 2)); + assert.throws(() => run([p, '--validate']), 'a control with no id was accepted'); +}); + +test('a CSV source is parsed into controls', () => { + const p = write('tf.csv', [ + 'control_id,title,description,parent,function', + 'TF-1,First control,Does a thing,,Govern', + 'TF-2,"Second, with a comma","Description, quoted",TF-1,Govern', + ].join('\n')); + const out = run([p, '--validate']); + assert.match(out, /valid|ok|pass|2/i, `unexpected output:\n${out}`); +}); + +test('a non-existent source fails instead of writing an empty registry', () => { + assert.throws(() => run([path.join(tmp, 'does-not-exist.json'), '--validate'])); +}); + +test('--validate leaves data/frameworks untouched', () => { + const dir = path.join(ROOT, 'data', 'frameworks'); + const before = fs.readdirSync(dir).sort().join(','); + const p = write('valid2.json', JSON.stringify(VALID, null, 2)); + try { run([p, '--validate']); } catch { /* the assertion below is the point */ } + assert.equal(fs.readdirSync(dir).sort().join(','), before, '--validate wrote to data/frameworks'); +}); diff --git a/scripts/reports.test.mjs b/scripts/reports.test.mjs new file mode 100644 index 0000000..8426dc0 --- /dev/null +++ b/scripts/reports.test.mjs @@ -0,0 +1,160 @@ +/** + * reports.test.mjs — the report generators, exercised through their CLIs. + * + * compliance-report.js, incidents-report.js and state-report.js are scripts, + * not modules: they parse argv and write files. Testing them through the CLI + * checks what people and CI actually run, and needs no refactor to get there. + * + * Everything here uses --stdout, so no test writes into reports/. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +function run(script, args) { + return execFileSync(process.execPath, [path.join(ROOT, 'scripts', script), ...args], { + cwd: ROOT, + maxBuffer: 256 * 1024 * 1024, + encoding: 'utf8', + }); +} + +const db = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'incidents.json'), 'utf8')); +const stats = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'stats.json'), 'utf8')); + +/** Minimal RFC 4180 row splitter — enough to check quoting is honoured. */ +function csvRows(text) { + const rows = []; + let row = [], cell = '', quoted = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (quoted) { + if (c === '"') { + if (text[i + 1] === '"') { cell += '"'; i++; } else quoted = false; + } else cell += c; + } else if (c === '"') quoted = true; + else if (c === ',') { row.push(cell); cell = ''; } + else if (c === '\n') { row.push(cell); rows.push(row); row = []; cell = ''; } + else if (c !== '\r') cell += c; + } + if (cell.length || row.length) { row.push(cell); rows.push(row); } + return rows.filter((r) => r.length > 1 || r[0] !== ''); +} + +// ── state-report.js ────────────────────────────────────────────────────────── + +test('state-report --json agrees with data/stats.json', () => { + // Two independent counters over the same data. When they disagree, one of + // them is being used to publish a number that is not true. + const r = JSON.parse(run('state-report.js', ['--json'])); + assert.equal(r.totals.incidents, stats.incidents.total); + assert.equal(r.totals.entries, stats.entries.total); + assert.equal(r.totals.mappings, stats.mappings.total); + assert.equal(r.totals.frameworks, stats.frameworks.registries); +}); + +test('state-report incident breakdowns sum to the incident total', () => { + const r = JSON.parse(run('state-report.js', ['--json'])); + const sum = (o) => Object.values(o).reduce((a, b) => a + b, 0); + assert.equal(sum(r.incidents.by_category), r.totals.incidents); + assert.equal(sum(r.incidents.by_severity), r.totals.incidents); +}); + +// ── incidents-report.js ────────────────────────────────────────────────────── + +test('incidents CSV has one data row per incident and a stable header', () => { + const rows = csvRows(run('incidents-report.js', ['--format', 'csv', '--stdout'])); + assert.ok(rows.length > 1, 'CSV has no data rows'); + assert.equal(rows[0][0].toLowerCase(), 'id', `unexpected first column: ${rows[0][0]}`); + assert.equal(rows.length - 1, db.incidents.length); +}); + +test('incidents CSV survives the commas and quotes in real descriptions', () => { + // Incident text is full of commas, quoted phrases and em dashes. An unquoted + // field shifts every later column and the file still opens in Excel, wrong. + const text = run('incidents-report.js', ['--format', 'csv', '--stdout']); + const rows = csvRows(text); + const width = rows[0].length; + const ragged = rows.map((r, i) => [i, r.length]).filter(([, n]) => n !== width); + assert.deepEqual(ragged.slice(0, 3), [], `${ragged.length} row(s) have the wrong column count`); +}); + +test('incidents --entry filters to incidents that name that entry', () => { + const out = JSON.parse(run('incidents-report.js', ['--entry', 'LLM01', '--format', 'json', '--stdout'])); + const list = Array.isArray(out) ? out : out.incidents; + assert.ok(list.length > 0, 'no incidents returned for LLM01'); + for (const inc of list) { + assert.ok(inc.owasp_entries.includes('LLM01'), `${inc.id} does not name LLM01`); + } +}); + +test('incidents --severity filters to that severity only', () => { + const out = JSON.parse(run('incidents-report.js', ['--severity', 'Critical', '--format', 'json', '--stdout'])); + const list = Array.isArray(out) ? out : out.incidents; + assert.ok(list.length > 0); + for (const inc of list) assert.equal(inc.severity, 'Critical'); +}); + +// ── compliance-report.js ───────────────────────────────────────────────────── + +test('compliance --list-frameworks lists every mapped framework', () => { + const out = run('compliance-report.js', ['--list-frameworks']); + const entriesDir = path.join(ROOT, 'data', 'entries'); + const mapped = new Set(); + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) mapped.add(m.framework); + } + const missing = [...mapped].filter((f) => !out.includes(f)); + assert.deepEqual(missing, [], `not listed: ${missing.join(', ')}`); +}); + +test('compliance JSON summary agrees with the coverage array it ships', () => { + // The summary block is what the markdown report and the webapp quote. It is + // computed separately from the per-entry array, so the two can drift. + const doc = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const s = doc.summary; + assert.equal(s.total_entries, doc.coverage.length); + assert.equal(s.total_entries, stats.entries.total, 'report covers a different entry set than stats.json'); + assert.equal(s.covered_entries, doc.coverage.filter((e) => e.mapped).length); + assert.equal(s.uncovered_entries, doc.coverage.filter((e) => !e.mapped).length); + assert.equal(s.covered_entries + s.uncovered_entries, s.total_entries); + assert.equal(s.unique_controls, doc.controls.length); +}); + +test('compliance coverage_rate is the rate it claims to be', () => { + const doc = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const s = doc.summary; + const expected = Math.round((s.covered_entries / s.total_entries) * 1000) / 10; + assert.equal(s.coverage_rate, expected); +}); + +test('compliance --severity narrows the report rather than widening it', () => { + const all = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--format', 'json', '--stdout', + ])); + const crit = JSON.parse(run('compliance-report.js', [ + '--framework', 'NIST AI RMF 1.0', '--severity', 'Critical', '--format', 'json', '--stdout', + ])); + assert.ok(crit.coverage.length > 0, 'severity filter returned nothing at all'); + assert.ok(crit.coverage.length <= all.coverage.length, + 'a severity filter returned more entries than no filter'); + for (const e of crit.coverage) assert.equal(e.severity, 'Critical'); +}); + +test('an unknown framework fails loudly instead of emitting an empty report', () => { + assert.throws( + () => run('compliance-report.js', ['--framework', 'Not A Real Framework', '--format', 'json', '--stdout']), + 'a typo in --framework should not produce a clean empty report', + ); +}); diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs new file mode 100644 index 0000000..6ea5e6d --- /dev/null +++ b/scripts/stats.test.mjs @@ -0,0 +1,114 @@ +/** + * stats.test.mjs — invariants for the numbers every headline claim is built on. + * + * README badges, the webapp, and the compliance reports all render from + * data/stats.json. A wrong number here is a wrong number everywhere, and it + * looks authoritative because it was generated. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { computeStats } = require(path.join(ROOT, 'scripts', 'stats.js')); + +const stats = computeStats(); + +test('every count is a finite non-negative integer', () => { + const walk = (o, at) => { + for (const [k, v] of Object.entries(o)) { + if (typeof v === 'number') { + assert.ok(Number.isInteger(v) && v >= 0, `${at}.${k} is ${v}`); + } else if (v && typeof v === 'object' && !Array.isArray(v)) { + walk(v, `${at}.${k}`); + } + } + }; + walk(stats, 'stats'); +}); + +test('per-source-list entry counts sum to the total', () => { + const sum = Object.values(stats.entries.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.entries.total); +}); + +test('per-source-list mapping counts sum to the total', () => { + const sum = Object.values(stats.mappings.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.mappings.total); +}); + +test('per-source-list mapping files sum to the total', () => { + const sum = Object.values(stats.mapping_files.by_list).reduce((a, b) => a + b, 0); + assert.equal(sum, stats.mapping_files.total); +}); + +test('frameworks mapped never exceeds registries present', () => { + // These two numbers are deliberately separate and were conflated for a long + // time. `mapped` counts frameworks some entry actually maps a control to; + // `registries` counts inventories on disk. mapped > registries would mean a + // mapping names a framework with no registry behind it. + assert.ok( + stats.frameworks.mapped <= stats.frameworks.registries, + `${stats.frameworks.mapped} mapped > ${stats.frameworks.registries} registries`, + ); +}); + +test('unmapped registries and mapped frameworks account for every registry', () => { + assert.equal( + stats.frameworks.mapped + stats.frameworks.unmapped_registries.length, + stats.frameworks.registries, + ); +}); + +test('draft_only names only frameworks that are actually mapped', () => { + const entriesDir = path.join(ROOT, 'data', 'entries'); + const mapped = new Set(); + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) mapped.add(m.framework); + } + for (const f of stats.frameworks.draft_only) { + assert.ok(mapped.has(f), `draft_only names "${f}", which no entry maps`); + } +}); + +test('draft rows never exceed total mappings', () => { + assert.ok( + stats.frameworks.draft_rows <= stats.mappings.total, + `${stats.frameworks.draft_rows} draft rows > ${stats.mappings.total} mappings`, + ); +}); + +test('a draft_only framework really has no authored row', () => { + // The claim the README renders is "these frameworks carry candidate DRAFT + // rows only". If one of them has an authored row, the caveat understates the + // project and the reader is misled in the safe direction — still wrong. + const entriesDir = path.join(ROOT, 'data', 'entries'); + const authored = {}; + for (const f of fs.readdirSync(entriesDir).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(entriesDir, f), 'utf8')); + for (const m of e.mappings || []) { + if (!/^DRAFT\b/.test(m.notes || '')) authored[m.framework] = (authored[m.framework] || 0) + 1; + } + } + for (const f of stats.frameworks.draft_only) { + assert.equal(authored[f], undefined, `"${f}" is listed draft-only but has authored rows`); + } +}); + +test('stats.json on disk matches a fresh computation', () => { + // stats:check enforces this in CI too; having it here means `node --test` + // alone catches a hand-edited stats.json. + const onDisk = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'stats.json'), 'utf8')); + assert.deepEqual(onDisk, stats); +}); + +test('source list count matches the ids and labels it publishes', () => { + assert.equal(stats.source_lists.count, stats.source_lists.ids.length); + assert.equal(stats.source_lists.count, stats.source_lists.labels.length); +});