Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions tests/lib/adapters/cloud-audit.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
import { join } from 'node:path';

/**
* Optional ingest of the GraphDone-Cloud live-audit into the unified harness, so
* one `test:unified --profile full` can also reflect the live Cloud audit when
* both repos are checked out side by side. The Cloud audit (audit-kit AuditRun)
* writes findings.json = { target, stamp, findings:[{dim,name,status,detail}], ... }
* with status in pass|warn|fail|info. Skips cleanly when no findings file exists.
*/
const STATUS_MAP = { pass: 'passed', fail: 'failed', warn: 'warned', info: 'skipped' };

/** Newest live-audit-artifacts/<stamp>/findings.json under a Cloud repo dir, or null. */
export function findLatestCloudFindings(cloudRepoDir) {
const root = join(cloudRepoDir, 'live-audit-artifacts');
if (!existsSync(root)) return null;
const stamps = readdirSync(root)
.map((name) => join(root, name))
.filter((p) => { try { return statSync(p).isDirectory() && existsSync(join(p, 'findings.json')); } catch { return false; } })
.sort();
return stamps.length ? join(stamps[stamps.length - 1], 'findings.json') : null;
}

export function cloudAuditSequence({ id = 'cloud-audit', title = 'Cloud live audit', findingsPath } = {}) {
if (!findingsPath || !existsSync(findingsPath)) {
return { id, title, kind: 'audit', status: 'skipped', durationMs: 0,
counts: { passed: 0, failed: 0, warned: 0, skipped: 1 }, cases: [],
notes: 'no Cloud findings.json found (run GraphDone-Cloud live-audit, or check out the sibling repo)' };
}
let data;
try { data = JSON.parse(readFileSync(findingsPath, 'utf8')); } catch (e) {
return { id, title, kind: 'audit', status: 'failed', durationMs: 0,
counts: { passed: 0, failed: 0, warned: 0, skipped: 0 }, cases: [{ title: 'parse findings.json', status: 'failed', error: String(e) }] };
}
const counts = { passed: 0, failed: 0, warned: 0, skipped: 0 };
const cases = [];
for (const f of data.findings || []) {
const status = STATUS_MAP[f.status] || 'skipped';
counts[status]++;
if (status === 'failed' || status === 'warn') cases.push({ title: `[${f.dim}] ${f.name}`, status, error: f.detail || undefined });
}
const status = counts.failed > 0 ? 'failed' : counts.warned > 0 ? 'warn' : counts.passed > 0 ? 'passed' : 'skipped';
return { id, title: `${title} (${data.target || ''})`, kind: 'audit', status, durationMs: 0, counts, cases,
notes: `ingested ${findingsPath}` };
}
37 changes: 37 additions & 0 deletions tests/lib/adapters/cloud-audit.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { writeFileSync, mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { cloudAuditSequence } from './cloud-audit.mjs';

test('maps pass/warn/fail/info to unified counts + status', () => {
const dir = mkdtempSync(join(tmpdir(), 'ca-'));
const p = join(dir, 'findings.json');
writeFileSync(p, JSON.stringify({ target: 'https://x', findings: [
{ dim: 'A', name: 'ok', status: 'pass' },
{ dim: 'A', name: 'meh', status: 'warn', detail: 'subscription' },
{ dim: 'B', name: 'note', status: 'info' },
] }));
const s = cloudAuditSequence({ findingsPath: p });
assert.equal(s.counts.passed, 1);
assert.equal(s.counts.warned, 1);
assert.equal(s.counts.skipped, 1);
assert.equal(s.counts.failed, 0);
assert.equal(s.status, 'warn'); // warn dominates pass
assert.ok(!('warn' in s.counts) || typeof s.counts.warn !== 'number'); // no stray key
});

test('a failure makes the sequence failed', () => {
const dir = mkdtempSync(join(tmpdir(), 'ca-'));
const p = join(dir, 'findings.json');
writeFileSync(p, JSON.stringify({ findings: [{ dim: 'X', name: 'broke', status: 'fail', detail: 'boom' }] }));
const s = cloudAuditSequence({ findingsPath: p });
assert.equal(s.status, 'failed');
assert.equal(s.cases.length, 1);
});

test('missing file → skipped, never throws', () => {
const s = cloudAuditSequence({ findingsPath: '/no/such/findings.json' });
assert.equal(s.status, 'skipped');
});
7 changes: 7 additions & 0 deletions tests/run-unified.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { mkdirSync, copyFileSync, rmSync } from 'node:fs';
import { join, extname, resolve } from 'node:path';
import { runVitestSequence } from './lib/adapters/vitest.mjs';
import { runPlaywrightSequence } from './lib/adapters/playwright.mjs';
import { cloudAuditSequence, findLatestCloudFindings } from './lib/adapters/cloud-audit.mjs';
import { buildReport } from './lib/reporting/aggregate.mjs';
import { writeJsonReport } from './lib/reporting/json.mjs';
import { renderHtml } from './lib/reporting/html.mjs';
Expand All @@ -36,6 +37,12 @@ async function runOne(id) {
process.stdout.write(`\n▶ ${id} — ${def.title}\n`);
if (def.adapter === 'vitest') return runVitestSequence({ id, title: def.title, cwd: def.cwd });
if (def.adapter === 'playwright') return runPlaywrightSequence({ id, title: def.title, args: def.args, target });
if (def.adapter === 'cloud-audit') {
const cloudDir = def.cloudRepoDir || resolve(process.cwd(), '..', 'GraphDone-Cloud');
const explicit = flag('--cloud-findings', null);
const findingsPath = typeof explicit === 'string' ? explicit : findLatestCloudFindings(cloudDir);
return cloudAuditSequence({ id, title: def.title, findingsPath });
}
return { id, title: def.title, status: 'skipped', counts: { passed: 0, failed: 0, warned: 0, skipped: 1 }, cases: [] };
}

Expand Down
7 changes: 5 additions & 2 deletions tests/sequences/unified.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@ export const SEQUENCES = {
'matrix': { adapter: 'playwright', title: 'Feature × resolution matrix', args: ['--project=matrix'], blocking: false },
'vlm': { adapter: 'playwright', title: 'Local-VLM visual eval (skips w/o endpoints)', args: ['--project=vlm'], blocking: false },
'perf-scale': { adapter: 'playwright', title: 'Large-scale perf sweep', args: ['--project=perf-scale'], blocking: false },
// Cross-repo: ingest the newest GraphDone-Cloud live-audit findings.json (sibling
// repo) into the unified report. Skips cleanly when absent.
'cloud-audit': { adapter: 'cloud-audit', title: 'Cloud live audit (ingested)', blocking: false },
};

export const PROFILES = {
smoke: ['unit-web', 'smoke', 'e2e-auth'],
pr: ['unit-web', 'smoke', 'e2e-auth', 'e2e-graph', 'mobile', 'perf-budgets'],
full: ['unit-web', 'smoke', 'e2e-auth', 'e2e-graph', 'mobile', 'perf-budgets', 'diagnostics', 'showcase', 'matrix', 'vlm', 'perf-scale'],
report: ['diagnostics', 'showcase', 'matrix', 'vlm', 'perf-scale'],
full: ['unit-web', 'smoke', 'e2e-auth', 'e2e-graph', 'mobile', 'perf-budgets', 'diagnostics', 'showcase', 'matrix', 'vlm', 'perf-scale', 'cloud-audit'],
report: ['diagnostics', 'showcase', 'matrix', 'vlm', 'perf-scale', 'cloud-audit'],
};
Loading