From b76a85ebf3fa32ad3c7521245e3d40ff95ac790e Mon Sep 17 00:00:00 2001 From: Robert DeLanghe <1240090+bdelanghe@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:35:09 -0400 Subject: [PATCH] feat(axe): add axe-core a11y gate (serious/critical, config-driven) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gates/axe-gate.mjs runs axe-core over each built page in a real browser and fails closed on any violation at or above a configurable impact threshold (default: serious). Everything is config-driven (dist dir, page list, ruleset tags, threshold, runner) with neutral defaults — nothing site-specific is hardcoded. It serves dist over an ephemeral origin so absolute asset paths resolve (running file:// would strip styles and fabricate layout-dependent findings), then drives one of two interchangeable engines: - playwright (default, CI): @axe-core/playwright + bundled Chromium - tezcatl (local, macOS WebKit): injects axe.min.js and reads results back The emitted machine-readable report groups violations by impact per page and exposes an `axe: { serious, critical }` envelope — exactly what conformance-report's `a11y.axe-serious-critical` criterion consumes, so a clean run is what lets a site honestly assert it and a regression turns CI red. Pure classification/threshold/report logic is exported and unit-tested deterministically; a fixtures/axe/ known-bad + known-good pair is exercised end-to-end when a browser engine is on PATH (skipped, like the cosign step, when none is). axe-core added as a dependency; @axe-core/playwright + playwright are consumer-supplied (dynamically imported) to keep `npm ci` light. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_011uU1XvPggEPNMiXBUrV8hy --- README.md | 13 +- fixtures/axe/bad.html | 17 +++ fixtures/axe/good.html | 16 ++ gates/axe-gate.mjs | 325 +++++++++++++++++++++++++++++++++++++++++ package-lock.json | 11 ++ package.json | 2 + test/run.mjs | 58 ++++++++ 7 files changed, 438 insertions(+), 4 deletions(-) create mode 100644 fixtures/axe/bad.html create mode 100644 fixtures/axe/good.html create mode 100644 gates/axe-gate.mjs diff --git a/README.md b/README.md index 84553ec..246b796 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ hardcodes `robertdelanghe.dev`, `bounded.tools`, an account, or an email. ``` integrity/ verify-site · verify (sigstore) · gen-sitemanifest · gen-provenance · structure-audit · http-probe -gates/ sbom (gen + completeness) · shacl-runner · seo-gate · readability-gate · commonmark-runner · semantic (lone) +gates/ sbom (gen + completeness) · shacl-runner · seo-gate · axe-gate (axe-core a11y) · readability-gate · commonmark-runner · semantic (lone) gates/conformance/ conformance-report — lone's conformance() projection (Node port of jsr:@bounded-systems/lone@0.4) + a generic HTML renderer generators/ gen-cid (IPFS UnixFS) · gen-identity (did:web + VC) · openapi (static-API helper core) emitters/ reprDigest (RFC 9530) · securityTxt (RFC 9116) · webManifest · markdown-sibling headers @@ -66,6 +66,7 @@ in-process verifier). The Deno semantic runner pins its imports in | `sbom/check-sbom.mjs` | `ROOT=. DIST=dist node …/check-sbom.mjs` | Same `$ROOT`/`$DIST`. Fails closed unless pinned-set ⊆ SBOM ⊆ pinned-set and (optionally) the in-toto attestation reconciles. | | `shacl-runner.mjs` | `node …/shacl-runner.mjs ` | **The SHACL shapes file stays in the site** (its structured-data contract) + the built-HTML dir. Optional `$SHACL_CONTEXT` (custom offline JSON-LD context; default schema.org). Fails unless every JSON-LD block `conforms: true`. | | `seo-gate.mjs` | `node …/seo-gate.mjs [distDir]` | `$DIST`. Optional `$SEO_ERROR_PAGE`, `$SEO_DEPLOY_SIDECARS`. Enforces canonical/title/description uniqueness + self-consistency, robots.txt (RFC 9309), sitemap, internal links. | +| `axe-gate.mjs` | `node …/axe-gate.mjs [distDir]` | `$DIST`. Optional `$AXE_PAGES` (comma list, default: every `*.html` in dist), `$AXE_TAGS` (default `wcag2a,wcag2aa,wcag21a,wcag21aa,wcag22aa`), `$AXE_IMPACT_THRESHOLD` (`minor`/`moderate`/`serious`/`critical`, default `serious`), `$AXE_RUNNER` (`playwright` (CI, needs `playwright` + `@axe-core/playwright` + `npx playwright install chromium`) \| `tezcatl` (macOS WebKit, local)), `$AXE_REPORT` (write the JSON report). Serves dist over an ephemeral origin (so assets resolve), runs **axe-core** per page, and **fails closed** on any violation at/above the threshold. The emitted report's `axe: { serious, critical }` envelope is exactly what `conformance-report`'s `a11y.axe-serious-critical` criterion consumes — a clean run is what lets a site honestly assert it. | | `readability-gate.mjs` | `node …/readability-gate.mjs [--strict]` | **The corpus is an input** the site assembles from its copy: a JSON array of `{id,text}` or an `{id:text}` map. Optional `$READABILITY_THRESHOLDS`, `$READABILITY_MIN_WORDS`, `$READABILITY_KNOWN_ACRONYMS`. WARN-only unless `--strict`. | | `commonmark-runner.mjs` | `node …/commonmark-runner.mjs [fixtures.json]` | **The site's markdown renderer module** (export `renderMarkdown`, or set `$COMMONMARK_RENDER_EXPORT`). Default fixtures pin a safe CommonMark subset + 4 hostile-HTML escapes; a site with a different renderer supplies its own `fixtures.json`. | | `semantic/gate.ts` | `deno run --allow-read --allow-net …/gate.ts` | Built HTML in `$SEMANTIC_DIR` (default `dist/blog`); `$SEMANTIC_SELECTOR` (subject node, default `article`). Imports `jsr:@bounded-systems/lone`; any error-severity finding fails CI. | @@ -109,14 +110,18 @@ deno run -A jsr:@bounded-systems/verify https://your-site ## Test ``` -npm install && npm test # 11 cases against fixtures/, in isolation +npm install && npm test # 13 cases against fixtures/, in isolation ``` The suite verifies the generic logic end-to-end: gen-sbom against a sample lockfile; shacl-runner against sample shapes+HTML → `conforms: true`; structure-audit / seo / readability / commonmark against sample inputs; gen-sitemanifest + gen-cid + verify-site -round-trip on a sample build; gen-identity; and the emitter/openapi/schema helpers. -(The Deno semantic runner is exercised by the consuming site, as it needs Deno + JSR.) +round-trip on a sample build; gen-identity; the emitter/openapi/schema helpers; the +conformance projection; and the **axe-gate** (its classification/threshold/report logic +deterministically, plus a real end-to-end pass on the known-bad + known-good +`fixtures/axe/` snippets when a browser engine — tezcatl or Playwright/Chromium — is on +PATH; skipped, like the cosign step, when none is). (The Deno semantic runner is +exercised by the consuming site, as it needs Deno + JSR.) ## Provenance / determinism diff --git a/fixtures/axe/bad.html b/fixtures/axe/bad.html new file mode 100644 index 0000000..af0bf9c --- /dev/null +++ b/fixtures/axe/bad.html @@ -0,0 +1,17 @@ + + + + + +axe-gate known-bad fixture + + +
+

Known-bad page

+ + +
+ + diff --git a/fixtures/axe/good.html b/fixtures/axe/good.html new file mode 100644 index 0000000..8bec3ed --- /dev/null +++ b/fixtures/axe/good.html @@ -0,0 +1,16 @@ + + + + + +axe-gate known-good fixture + + +
+

Known-good page

+Project logo +Go somewhere +
+ + diff --git a/gates/axe-gate.mjs b/gates/axe-gate.mjs new file mode 100644 index 0000000..a7d4f61 --- /dev/null +++ b/gates/axe-gate.mjs @@ -0,0 +1,325 @@ +#!/usr/bin/env node +// axe accessibility gate — turns "we ran axe once" into a CONTINUOUSLY-ENFORCED +// member of the conformance contract. It loads each BUILT page in a real browser, +// runs axe-core with the WCAG 2.x A/AA ruleset, and FAILS CLOSED (exit 1) on any +// violation at or above a configurable impact threshold (default: serious). The +// machine-readable result it emits is exactly the shape lone's conformance() model +// consumes for `a11y.axe-serious-critical` (`{ serious, critical }`), so a clean run +// is what lets a site honestly assert that criterion — and a regression turns CI red. +// +// node gates/axe-gate.mjs [distDir] # build gate (exit 1 on any blocking violation) +// +// Pure data in → typed report out. The browser is the ONLY impurity: axe needs real +// layout/computed-style (e.g. colour-contrast, target-size), so a DOM shim is not +// enough. Two interchangeable runners drive a real engine: +// - playwright (default) — `@axe-core/playwright` + Playwright's bundled Chromium. +// The CI runner: hermetic, headless, cross-platform. +// - tezcatl — macOS-native headless WebKit. Injects axe.min.js into the +// served page and reads the result back. The LOCAL runner (no Chromium download). +// Both serve dist/ over an ephemeral localhost HTTP origin first, so absolute asset +// paths (`/assets/…css`, fonts) resolve — running file:// would strip the styles and +// fabricate layout-dependent violations. +// +// Everything is config-driven; NOTHING about any one site is hard-coded: +// argv[2] / $DIST built output dir (default: "dist") +// $AXE_PAGES comma list of page paths under dist to scan +// (default: every *.html discovered in dist) +// $AXE_TAGS comma list of axe ruleset tags +// (default: wcag2a,wcag2aa,wcag21a,wcag21aa,wcag22aa) +// $AXE_IMPACT_THRESHOLD lowest impact that BLOCKS: minor|moderate|serious|critical +// (default: serious) +// $AXE_RUNNER playwright | tezcatl (default: playwright) +// $AXE_REPORT path to write the JSON report (default: none → stdout only) +// $AXE_TEZCATL_WAIT ms to let axe settle, tezcatl runner (default: 3000) +// +// The pure evaluation/report functions are exported for unit testing without a browser. +import { readFile, readdir, access, mkdtemp, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { join, relative, resolve, extname } from "node:path"; +import { tmpdir } from "node:os"; +import { createRequire } from "node:module"; +import { spawn } from "node:child_process"; + +// ── Pure core (browser-free; unit-testable) ────────────────────────────────── + +/** Impact levels, weakest → strongest. A violation BLOCKS when its impact ranks at + * or above the configured threshold. axe may report `impact: null`; such findings + * rank below `minor` and so never block (but are still counted/reported). */ +export const IMPACT_ORDER = ["minor", "moderate", "serious", "critical"]; +export const DEFAULT_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa"]; +export const DEFAULT_THRESHOLD = "serious"; + +export const impactRank = (impact) => IMPACT_ORDER.indexOf(impact); // -1 when null/unknown +export const blocksAt = (impact, threshold) => { + const t = impactRank(threshold); + return t >= 0 && impactRank(impact) >= t; +}; + +/** Normalise one axe violation to the compact, stable shape we report/persist. */ +export function normalizeViolation(v) { + const nodes = Array.isArray(v.nodes) ? v.nodes : []; + const targets = nodes + .map((n) => (Array.isArray(n.target) ? n.target.join(" ") : String(n.target ?? ""))) + .filter(Boolean); + return { + id: v.id, + impact: v.impact ?? null, + help: v.help ?? "", + helpUrl: v.helpUrl ?? "", + nodes: nodes.length, + targets: targets.slice(0, 8), // cap; full detail lives in axe's own helpUrl + }; +} + +/** Empty {critical,serious,moderate,minor,unknown} counter. */ +const emptyCounts = () => ({ critical: 0, serious: 0, moderate: 0, minor: 0, unknown: 0 }); + +/** Evaluate one page's raw axe violations against the threshold. */ +export function evaluatePage(page, rawViolations, threshold = DEFAULT_THRESHOLD) { + const violations = (rawViolations ?? []).map(normalizeViolation); + const counts = emptyCounts(); + let blocking = 0; + for (const v of violations) { + counts[v.impact && v.impact in counts ? v.impact : "unknown"]++; + if (blocksAt(v.impact, threshold)) blocking++; + } + // Group by impact for the machine-readable report (serious/critical first). + const byImpact = {}; + for (const lvl of [...IMPACT_ORDER].reverse()) { + const inLvl = violations.filter((v) => v.impact === lvl); + if (inLvl.length) byImpact[lvl] = inLvl; + } + const unknown = violations.filter((v) => impactRank(v.impact) < 0); + if (unknown.length) byImpact.unknown = unknown; + return { page, counts, blocking, violations: byImpact }; +} + +/** Fold per-page evaluations into the whole-run report consumed by conformance(). */ +export function summarize(pageResults, { threshold = DEFAULT_THRESHOLD, tags = DEFAULT_TAGS, runner = "playwright" } = {}) { + const totals = emptyCounts(); + let blocking = 0; + for (const p of pageResults) { + for (const k of Object.keys(totals)) totals[k] += p.counts[k]; + blocking += p.blocking; + } + return { + tool: "axe-core", + runner, + standard: "WCAG 2.x A/AA (axe ruleset)", + tags, + impactThreshold: threshold, + generatedAt: new Date().toISOString(), + pages: pageResults, + totals, + // The exact envelope lone's `a11y.axe-serious-critical` evaluator reads. + axe: { serious: totals.serious, critical: totals.critical }, + blocking, // count of violations at/above threshold across all pages + passed: blocking === 0, + }; +} + +// ── dist discovery + static origin (shared by both runners) ────────────────── + +async function walkHtml(dir, base = dir) { + const out = []; + for (const e of await readdir(dir, { withFileTypes: true })) { + const abs = join(dir, e.name); + if (e.isDirectory()) out.push(...await walkHtml(abs, base)); + else if (e.name.endsWith(".html")) out.push(relative(base, abs).replace(/\\/g, "/")); + } + return out; +} + +const MIME = { + ".html": "text/html; charset=utf-8", ".css": "text/css", ".js": "application/javascript", + ".mjs": "application/javascript", ".json": "application/json", ".svg": "image/svg+xml", + ".png": "image/png", ".jpg": "image/jpeg", ".webp": "image/webp", ".ico": "image/x-icon", + ".woff": "font/woff", ".woff2": "font/woff2", ".ttf": "font/ttf", ".xml": "application/xml", + ".txt": "text/plain", ".webmanifest": "application/manifest+json", ".pdf": "application/pdf", +}; + +/** + * Serve `root` over an ephemeral localhost origin. When `inject` is set, HTML + * responses get axe-core + a runner appended before , and `/__axe-core.js` + * serves the axe source — used by the tezcatl runner, which cannot inject async JS + * itself. Returns { origin, close }. + */ +async function startServer(root, { inject = false, tags = DEFAULT_TAGS } = {}) { + let axeSrc = ""; + if (inject) { + const require = createRequire(import.meta.url); + axeSrc = await readFile(require.resolve("axe-core/axe.min.js"), "utf8"); + } + const runnerScript = + ``; + + const server = createServer(async (req, res) => { + try { + let urlPath = decodeURIComponent((req.url || "/").split("?")[0]); + if (inject && urlPath === "/__axe-core.js") { + res.writeHead(200, { "content-type": "application/javascript" }); + return res.end(axeSrc); + } + let file = join(root, urlPath); + if (urlPath.endsWith("/")) file = join(file, "index.html"); + let buf; + try { buf = await readFile(file); } + catch { try { buf = await readFile(file + ".html"); file += ".html"; } catch { res.writeHead(404); return res.end("not found"); } } + const ext = extname(file).toLowerCase(); + if (inject && ext === ".html") { + let html = buf.toString("utf8"); + html = html.includes("") ? html.replace("", runnerScript + "") : html + runnerScript; + buf = Buffer.from(html, "utf8"); + } + res.writeHead(200, { "content-type": MIME[ext] || "application/octet-stream" }); + res.end(buf); + } catch (e) { res.writeHead(500); res.end(String(e)); } + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address(); + return { origin: `http://127.0.0.1:${port}`, close: () => new Promise((r) => server.close(r)) }; +} + +// ── Runners: page → raw axe violations[] ───────────────────────────────────── + +async function collectWithPlaywright(pages, { dist, tags }) { + let chromium, AxeBuilder; + try { + ({ chromium } = await import("playwright")); + ({ default: AxeBuilder } = await import("@axe-core/playwright")); + } catch (e) { + throw new Error( + "playwright runner needs `playwright` + `@axe-core/playwright` installed " + + "(and `npx playwright install --with-deps chromium`). " + e.message, + ); + } + const srv = await startServer(dist, { inject: false }); + const browser = await chromium.launch(); + const out = new Map(); + try { + const ctx = await browser.newContext(); + for (const page of pages) { + const pg = await ctx.newPage(); + await pg.goto(`${srv.origin}/${page}`, { waitUntil: "load" }); + const results = await new AxeBuilder({ page: pg }).withTags(tags).analyze(); + out.set(page, results.violations); + await pg.close(); + } + } finally { + await browser.close(); + await srv.close(); + } + return out; +} + +// Run tezcatl async (NOT execFileSync) — the static origin lives on this same event +// loop, so a blocking child would deadlock its own server. Resolves to trimmed stdout. +function tezcatl(args) { + return new Promise((res, rej) => { + const ch = spawn("tezcatl", args, { stdio: ["ignore", "pipe", "pipe"] }); + let out = "", err = ""; + ch.stdout.on("data", (d) => (out += d)); + ch.stderr.on("data", (d) => (err += d)); + ch.on("error", (e) => rej(new Error(`tezcatl not runnable (on PATH?): ${e.message}`))); + ch.on("close", (code) => (code === 0 ? res(out.trim()) : rej(new Error(`tezcatl exit ${code}: ${err.trim() || out.trim()}`)))); + }); +} + +async function collectWithTezcatl(pages, { dist, tags }) { + const waitMs = Number(process.env.AXE_TEZCATL_WAIT || 3000); + const readResults = `--eval=(function(){var e=document.getElementById('__axe_results');return e?e.textContent:'';})()`; + const srv = await startServer(dist, { inject: true, tags }); + const out = new Map(); + try { + for (const page of pages) { + const url = `${srv.origin}/${page}`; + let text = ""; + for (const attempt of [waitMs, waitMs * 2]) { // one retry with a longer settle window + const raw = await tezcatl([url, `--wait=${attempt}`, readResults]); + if (raw && raw !== "NORESULT") { text = raw; break; } + } + if (!text) throw new Error(`tezcatl: no axe result for ${page} (raise $AXE_TEZCATL_WAIT?)`); + const parsed = JSON.parse(text); + if (parsed.error) throw new Error(`axe failed on ${page}: ${parsed.error}`); + out.set(page, parsed.violations || []); + } + } finally { + await srv.close(); + } + return out; +} + +const RUNNERS = { playwright: collectWithPlaywright, tezcatl: collectWithTezcatl }; + +/** + * Run the configured runner over `pages` of `dist` and return the summarized report. + * Exposed for programmatic use (and the kit's own test) in addition to the CLI. + */ +export async function runAxeGate({ dist, pages, tags = DEFAULT_TAGS, threshold = DEFAULT_THRESHOLD, runner = "playwright" }) { + const collect = RUNNERS[runner]; + if (!collect) throw new Error(`unknown runner "${runner}" (expected: ${Object.keys(RUNNERS).join(", ")})`); + const raw = await collect(pages, { dist, tags }); + const pageResults = pages.map((p) => evaluatePage(p, raw.get(p) || [], threshold)); + return summarize(pageResults, { threshold, tags, runner }); +} + +// ── CLI ────────────────────────────────────────────────────────────────────── + +async function main() { + const dist = resolve(process.argv[2] && !process.argv[2].startsWith("--") ? process.argv[2] : process.env.DIST || "dist"); + const exists = async (p) => { try { await access(p); return true; } catch { return false; } }; + if (!(await exists(dist))) { console.error(`✗ axe-gate: ${dist} not found — build first.`); process.exit(2); } + + const tags = (process.env.AXE_TAGS || DEFAULT_TAGS.join(",")).split(",").map((s) => s.trim()).filter(Boolean); + const threshold = (process.env.AXE_IMPACT_THRESHOLD || DEFAULT_THRESHOLD).trim(); + if (!IMPACT_ORDER.includes(threshold)) { + console.error(`✗ axe-gate: $AXE_IMPACT_THRESHOLD must be one of ${IMPACT_ORDER.join("|")} (got "${threshold}")`); + process.exit(2); + } + const runner = (process.env.AXE_RUNNER || "playwright").trim(); + let pages = (process.env.AXE_PAGES || "").split(",").map((s) => s.trim().replace(/^\//, "")).filter(Boolean); + if (pages.length === 0) pages = (await walkHtml(dist)).sort(); + if (pages.length === 0) { console.error(`✗ axe-gate: no HTML pages found under ${dist}`); process.exit(2); } + + console.log(`axe-gate: ${runner} runner · ${pages.length} page(s) · tags [${tags.join(", ")}] · block ≥ ${threshold}`); + const report = await runAxeGate({ dist, pages, tags, threshold, runner }); + + if (process.env.AXE_REPORT) { + await writeFile(resolve(process.env.AXE_REPORT), JSON.stringify(report, null, 2) + "\n"); + console.log(` ↳ wrote ${process.env.AXE_REPORT}`); + } + + for (const p of report.pages) { + const tally = IMPACT_ORDER.map((l) => `${p.counts[l]} ${l}`).join(", "); + const mark = p.blocking ? "✗" : "✓"; + console.log(` ${mark} ${p.page} — ${tally}${p.counts.unknown ? `, ${p.counts.unknown} unknown` : ""}`); + if (p.blocking) { + for (const lvl of ["critical", "serious", "moderate", "minor"]) { + for (const v of p.violations[lvl] || []) { + if (!blocksAt(lvl, threshold)) continue; + console.error(` [${lvl}] ${v.id} — ${v.help} (${v.nodes} node(s)) ${v.helpUrl}`); + for (const t of v.targets) console.error(` · ${t}`); + } + } + } + } + + console.log(""); + if (!report.passed) { + console.error(`✗ axe-gate: ${report.blocking} violation(s) at or above "${threshold}" across ${report.pages.length} page(s) (${report.totals.critical} critical, ${report.totals.serious} serious).`); + process.exit(1); + } + console.log(`✓ axe-gate: ${report.pages.length} page(s) clean — 0 violations at or above "${threshold}" (axe ${tags.includes("wcag22aa") ? "WCAG 2.2 A/AA" : "WCAG A/AA"}).`); +} + +// Only run the CLI when invoked directly (not when imported by a test). +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((e) => { console.error("✗ axe-gate: error —", e.stack || e.message); process.exit(1); }); +} diff --git a/package-lock.json b/package-lock.json index 7563d07..bc6a367 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@mozilla/readability": "^0.5.0", "@zazuko/env-node": "^2.1.5", + "axe-core": "^4.10.0", "jsonld": "^8.3.2", "linkedom": "^0.18.0", "n3": "^1.17.3", @@ -18,6 +19,7 @@ "sigstore": "^5.0.0" }, "bin": { + "ck-axe-gate": "gates/axe-gate.mjs", "ck-check-sbom": "gates/sbom/check-sbom.mjs", "ck-commonmark-runner": "gates/commonmark-runner.mjs", "ck-gen-cid": "generators/gen-cid.mjs", @@ -893,6 +895,15 @@ "node": ">= 20" } }, + "node_modules/axe-core": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.12.1.tgz", + "integrity": "sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==", + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", diff --git a/package.json b/package.json index c82b91f..bf12cdc 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "ck-check-sbom": "./gates/sbom/check-sbom.mjs", "ck-shacl-runner": "./gates/shacl-runner.mjs", "ck-seo-gate": "./gates/seo-gate.mjs", + "ck-axe-gate": "./gates/axe-gate.mjs", "ck-readability-gate": "./gates/readability-gate.mjs", "ck-commonmark-runner": "./gates/commonmark-runner.mjs", "ck-gen-cid": "./generators/gen-cid.mjs", @@ -29,6 +30,7 @@ "dependencies": { "@mozilla/readability": "^0.5.0", "@zazuko/env-node": "^2.1.5", + "axe-core": "^4.10.0", "jsonld": "^8.3.2", "linkedom": "^0.18.0", "n3": "^1.17.3", diff --git a/test/run.mjs b/test/run.mjs index 4687a62..931900e 100755 --- a/test/run.mjs +++ b/test/run.mjs @@ -238,6 +238,64 @@ await test("gates/conformance-report: build + render the conformance projection" `partial=${partial.summary.met}met/${partial.summary.unmet}unmet/${partial.summary.notAssessed}n-a · full claim=compact`); }); +// 13. axe-gate: pure classification/threshold/report logic, then a best-effort +// end-to-end run on the known-bad + known-good fixtures (skipped if no browser +// runner is on PATH, like the cosign step above). +await test("gates/axe-gate: classify + threshold + report, e2e on fixtures", async () => { + const { evaluatePage, summarize, blocksAt, normalizeViolation, runAxeGate } = + await import(join(KIT, "gates", "axe-gate.mjs")); + + // (a) threshold semantics: block at/above the configured impact; null never blocks. + if (!blocksAt("critical", "serious") || !blocksAt("serious", "serious")) throw new Error("serious/critical must block at serious"); + if (blocksAt("moderate", "serious") || blocksAt("minor", "serious")) throw new Error("moderate/minor must not block at serious"); + if (blocksAt(null, "serious")) throw new Error("null impact must never block"); + if (!blocksAt("moderate", "moderate")) throw new Error("moderate must block at moderate"); + + // (b) pure evaluation over synthetic axe violations (shaped like axe output). + const synthetic = [ + { id: "image-alt", impact: "critical", help: "Images must have alternate text", helpUrl: "h", nodes: [{ target: ["img"] }] }, + { id: "link-name", impact: "serious", help: "Links must have discernible text", helpUrl: "h", nodes: [{ target: ["a"] }] }, + { id: "landmark", impact: "moderate", help: "x", helpUrl: "h", nodes: [{ target: ["div"] }] }, + ]; + const ev = evaluatePage("bad.html", synthetic, "serious"); + if (ev.blocking !== 2) throw new Error(`expected 2 blocking (critical+serious), got ${ev.blocking}`); + if (ev.counts.critical !== 1 || ev.counts.serious !== 1 || ev.counts.moderate !== 1) throw new Error("impact counts wrong"); + if (!ev.violations.critical || !ev.violations.serious) throw new Error("byImpact grouping missing serious/critical"); + if (normalizeViolation(synthetic[0]).targets[0] !== "img") throw new Error("target normalisation wrong"); + + const rep = summarize([ev, evaluatePage("good.html", [], "serious")], { threshold: "serious", runner: "synthetic" }); + if (rep.axe.critical !== 1 || rep.axe.serious !== 1) throw new Error("report axe envelope must total serious/critical"); + if (rep.passed !== false || rep.blocking !== 2) throw new Error("report with serious/critical must not pass"); + const cleanRep = summarize([evaluatePage("good.html", [], "serious")], { threshold: "serious" }); + if (cleanRep.passed !== true || cleanRep.axe.serious !== 0 || cleanRep.axe.critical !== 0) throw new Error("clean report must pass with axe {0,0}"); + + // (c) end-to-end against the fixtures, with whatever real engine is present. + // tezcatl (macOS WebKit) is preferred locally; Playwright/Chromium is the CI path. + // If neither engine can actually launch (e.g. Chromium not downloaded), SKIP — the + // pure logic above is the deterministic, always-on assertion (cf. the cosign skip). + const hasTezcatl = spawnSync("tezcatl", ["--version"], { stdio: "ignore" }).status === 0; + let hasPlaywright = false; + try { await import("@axe-core/playwright"); await import("playwright"); hasPlaywright = true; } catch { /* optional dep */ } + const runner = hasTezcatl ? "tezcatl" : hasPlaywright ? "playwright" : null; + const fixDir = join(FIX, "axe"); + try { + if (!runner) throw new Error("no browser runner on PATH"); + const badRun = await runAxeGate({ dist: fixDir, pages: ["bad.html"], threshold: "serious", runner }); + if (badRun.passed !== false || badRun.blocking < 1) throw new Error(`known-bad fixture must fail the gate (${runner})`); + if (badRun.axe.serious + badRun.axe.critical < 1) throw new Error("known-bad fixture must surface a serious/critical violation"); + const goodRun = await runAxeGate({ dist: fixDir, pages: ["good.html"], threshold: "serious", runner }); + if (goodRun.passed !== true) throw new Error(`known-good fixture must pass the gate (${runner})`); + ok("gates/axe-gate: classify + threshold + report, e2e on fixtures", + `pure logic asserted · e2e (${runner}): bad=${badRun.axe.critical}c/${badRun.axe.serious}s blocking, good=clean`); + } catch (e) { + // A real assertion failure (the fixtures are wrong) must surface; only a + // missing/unlaunchable engine is a tolerated skip. + if (/must (fail|pass|surface)|grouping|counts|envelope/.test(e.message)) throw e; + ok("gates/axe-gate: classify + threshold + report, e2e on fixtures", + `pure logic asserted · e2e SKIPPED (${e.message.split("\n")[0]})`); + } +}); + await rm(work, { recursive: true, force: true }); console.log(`\n${failed ? "✗" : "✓"} conformance-kit tests: ${passed} passed, ${failed} failed`); process.exit(failed ? 1 : 0);