From 9619e9df50774a41e56673b42d40482572abde6e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 12:35:22 +0000 Subject: [PATCH 1/7] test: align scripts and artifacts with testing strategy Co-authored-by: Alexander Nazarov --- .github/workflows/ci.yml | 5 +- package.json | 11 +- playwright.config.ts | 40 +++- scripts/test-suite.mjs | 402 +++++++++++++++++++++++++++++++++++++++ tests/echo.spec.ts | 2 +- tests/react-demo.spec.ts | 24 ++- tests/screenshot.spec.ts | 2 +- tests/server-api.spec.ts | 2 +- tests/video.spec.ts | 2 +- tests/zombie.spec.ts | 2 +- 10 files changed, 476 insertions(+), 16 deletions(-) create mode 100644 scripts/test-suite.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e8d07d..fa3afd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,10 @@ jobs: run: pnpm exec playwright install --with-deps chromium - name: Run tests - run: pnpm coverage:ci + run: JABTERM_SKIP_BUILD=1 pnpm coverage:ci + + - name: Run integration tests (docs assets) + run: JABTERM_SKIP_BUILD=1 pnpm run test:integration - name: Upload coverage artifacts if: always() diff --git a/package.json b/package.json index 77342fb..8e8bb30 100644 --- a/package.json +++ b/package.json @@ -25,9 +25,14 @@ "scripts": { "build": "tsc -p tsconfig.server.json && tsc -p tsconfig.react.json", "prepack": "pnpm build", - "test": "pnpm exec playwright test", - "test:client": "pnpm exec vitest run", - "test:server:coverage": "rm -rf .cache/v8-coverage && NODE_V8_COVERAGE=.cache/v8-coverage pnpm test", + "test": "node scripts/test-suite.mjs all", + "test:unit": "node scripts/test-suite.mjs unit", + "test:e2e": "node scripts/test-suite.mjs e2e", + "test:scenario": "node scripts/test-suite.mjs scenario", + "test:smoke": "node scripts/test-suite.mjs smoke", + "test:integration": "node scripts/test-suite.mjs integration", + "test:client": "pnpm run test:unit", + "test:server:coverage": "rm -rf .cache/v8-coverage && NODE_V8_COVERAGE=.cache/v8-coverage pnpm run test:scenario && NODE_V8_COVERAGE=.cache/v8-coverage pnpm run test:e2e", "test:client:coverage": "rm -rf .cache/coverage/client && pnpm exec vitest run --coverage", "coverage:server:report": "rm -rf .cache/coverage/server && pnpm exec c8 report --temp-directory .cache/v8-coverage --reporter=text-summary --reporter=json-summary --report-dir .cache/coverage/server --all --src src/server --exclude \"**/playwright.config.ts\" --exclude \"tests/**\" --exclude \"src/react/**\" && node scripts/normalize-coverage-summary.mjs .cache/coverage/server/coverage-summary.json", "coverage:merge": "node scripts/merge-coverage-summaries.mjs .cache/coverage/server/coverage-summary.json .cache/coverage/client/coverage-summary.json coverage/coverage-summary.json", diff --git a/playwright.config.ts b/playwright.config.ts index a3c097b..419cc69 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,20 +1,52 @@ import { defineConfig } from "@playwright/test"; +import path from "node:path"; const PORT = parseInt(process.env.JABTERM_PORT || "3223", 10); +const ARTIFACTS_DIR = process.env.TEST_ARTIFACTS_DIR; +const IS_SMOKE = process.env.JABTERM_SMOKE === "1"; + +const outputDir = ARTIFACTS_DIR + ? path.join(ARTIFACTS_DIR, "test-results") + : ".cache/test-results"; +const reportDir = ARTIFACTS_DIR + ? path.join(ARTIFACTS_DIR, "report") + : ".cache/report"; + +const smokePerTestTimeoutMs = parseInt( + process.env.SMOKE_PER_TEST_TIMEOUT_MS || "30000", + 10, +); +const smokeTotalTimeoutMs = parseInt( + process.env.SMOKE_TOTAL_TIMEOUT_MS || "180000", + 10, +); export default defineConfig({ testDir: "./tests", // Keep Playwright focused on E2E specs. Vitest unit tests live under `tests/unit/**` // and use the `.test.*` suffix, which Playwright would otherwise pick up by default. testMatch: ["**/*.spec.ts", "**/*.spec.tsx"], - outputDir: ".cache/test-results", + outputDir, + ...(IS_SMOKE ? { timeout: smokePerTestTimeoutMs } : {}), + ...(IS_SMOKE ? { globalTimeout: smokeTotalTimeoutMs } : {}), fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 1 : 0, workers: 1, - reporter: [ - ["html", { outputFolder: ".cache/report", open: "never" }], - ], + reporter: IS_SMOKE + ? [ + [ + "json", + { + outputFile: + process.env.PW_JSON_OUTPUT_FILE || + (ARTIFACTS_DIR + ? path.join(ARTIFACTS_DIR, "playwright-report.json") + : ".cache/playwright-report.json"), + }, + ], + ] + : [["html", { outputFolder: reportDir, open: "never" }]], use: { baseURL: `http://127.0.0.1:${PORT + 1}`, }, diff --git a/scripts/test-suite.mjs b/scripts/test-suite.mjs new file mode 100644 index 0000000..43fb49a --- /dev/null +++ b/scripts/test-suite.mjs @@ -0,0 +1,402 @@ +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { spawn } from "node:child_process"; + +const ROOT = process.cwd(); + +function artifactsDirFor(suite) { + switch (suite) { + case "unit": + return path.join(ROOT, ".cache/tests/test-unit__vitest"); + case "scenario": + return path.join(ROOT, ".cache/tests/test-scenario__playwright"); + case "e2e": + return path.join(ROOT, ".cache/tests/test-e2e__playwright"); + case "integration": + return path.join(ROOT, ".cache/tests/test-integration__playwright"); + case "smoke": + return path.join(ROOT, ".cache/tests/test-smoke__scenario"); + case "all": + return path.join(ROOT, ".cache/tests/test__all"); + default: + throw new Error(`Unknown suite: ${suite}`); + } +} + +async function rmrf(p) { + await fsp.rm(p, { recursive: true, force: true }); +} + +async function mkdirp(p) { + await fsp.mkdir(p, { recursive: true }); +} + +function spawnLogged(cmd, args, { env, cwd, logStream, passthrough }) { + return new Promise((resolve) => { + const child = spawn(cmd, args, { + cwd, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + + const onOut = (chunk) => { + logStream.write(chunk); + if (passthrough) process.stdout.write(chunk); + }; + const onErr = (chunk) => { + logStream.write(chunk); + if (passthrough) process.stderr.write(chunk); + }; + + child.stdout.on("data", onOut); + child.stderr.on("data", onErr); + + child.on("close", (code, signal) => resolve({ code, signal, child })); + }); +} + +function spawnLoggedWithTimeout(cmd, args, { env, cwd, logStream, timeoutMs }) { + return new Promise((resolve) => { + const child = spawn(cmd, args, { + cwd, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + + child.stdout.on("data", (chunk) => logStream.write(chunk)); + child.stderr.on("data", (chunk) => logStream.write(chunk)); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { + child.kill("SIGTERM"); + } catch { + /* ignore */ + } + setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + /* ignore */ + } + }, 1500).unref?.(); + }, timeoutMs); + timer.unref?.(); + + child.on("close", (code, signal) => { + clearTimeout(timer); + resolve({ code, signal, timedOut }); + }); + }); +} + +function envWithArtifacts(artifactsDir, extraEnv = {}) { + return { + ...process.env, + ...extraEnv, + TEST_ARTIFACTS_DIR: artifactsDir, + }; +} + +async function runUnit({ artifactsDir, logStream }) { + const env = envWithArtifacts(artifactsDir); + return await spawnLogged("pnpm", ["exec", "vitest", "run"], { + cwd: ROOT, + env, + logStream, + passthrough: true, + }); +} + +async function runPlaywright({ artifactsDir, logStream, tag, smoke }) { + const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; + + const env = envWithArtifacts(artifactsDir, smoke ? { JABTERM_SMOKE: "1" } : {}); + + if (!skipBuild) { + const build = await spawnLogged("pnpm", ["run", "build"], { + cwd: ROOT, + env, + logStream, + passthrough: true, + }); + if (build.code !== 0) return { code: build.code, signal: build.signal }; + } + + const args = ["exec", "playwright", "test", "--grep", tag]; + if (smoke) args.push("--max-failures=1", "--quiet"); + + return await spawnLogged("pnpm", args, { + cwd: ROOT, + env, + logStream, + passthrough: !smoke, + }); +} + +function truncateOneLine(s, max = 160) { + const one = String(s || "").replace(/\s+/g, " ").trim(); + if (one.length <= max) return one; + return `${one.slice(0, max - 1)}…`; +} + +function collectTestsFromSuites(suites, out) { + for (const s of suites || []) { + if (s.tests) { + for (const t of s.tests) out.push(t); + } + if (s.suites) collectTestsFromSuites(s.suites, out); + } +} + +function getFinalStatus(test) { + const results = test?.results || []; + for (const r of results) { + if (r.status === "failed" || r.status === "timedOut") return "failed"; + } + for (const r of results) { + if (r.status === "passed") return "passed"; + } + for (const r of results) { + if (r.status === "skipped") return "skipped"; + } + return "unknown"; +} + +async function runSmoke({ artifactsDir, logStream }) { + const startedAt = Date.now(); + const perTestTimeoutMs = parseInt( + process.env.SMOKE_PER_TEST_TIMEOUT_MS || "30000", + 10, + ); + const totalTimeoutMs = parseInt( + process.env.SMOKE_TOTAL_TIMEOUT_MS || "180000", + 10, + ); + const warnAfterMs = parseInt(process.env.SMOKE_WARN_AFTER_MS || "60000", 10); + + const jsonReport = path.join(artifactsDir, "playwright-report.json"); + const env = envWithArtifacts(artifactsDir, { + JABTERM_SMOKE: "1", + SMOKE_PER_TEST_TIMEOUT_MS: String(perTestTimeoutMs), + SMOKE_TOTAL_TIMEOUT_MS: String(totalTimeoutMs), + PW_JSON_OUTPUT_FILE: jsonReport, + }); + + const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; + if (!skipBuild) { + const build = await spawnLogged("pnpm", ["run", "build"], { + cwd: ROOT, + env, + logStream, + passthrough: false, + }); + if (build.code !== 0) { + console.log(`SMOKE FAIL: build (exit ${build.code ?? "?"})`); + console.log(`Artifacts: ${artifactsDir}`); + process.exit(build.code || 1); + } + } + + const warn = setTimeout(() => { + /* marker only; we keep success output to one line */ + }, warnAfterMs); + warn.unref?.(); + + const pw = await spawnLoggedWithTimeout( + "pnpm", + [ + "exec", + "playwright", + "test", + "--grep", + "@scenario", + "--max-failures=1", + "--quiet", + ], + { cwd: ROOT, env, logStream, timeoutMs: totalTimeoutMs }, + ); + clearTimeout(warn); + + const elapsedS = (Date.now() - startedAt) / 1000; + const slow = elapsedS * 1000 > warnAfterMs; + + if (pw.timedOut) { + console.log(`SMOKE FAIL: scenario (timeout after ${Math.round(elapsedS)}s)`); + console.log(`Artifacts: ${artifactsDir}`); + process.exit(1); + } + + if (pw.code === 0) { + let passed = 0; + let total = 0; + try { + const raw = await fsp.readFile(jsonReport, "utf8"); + const report = JSON.parse(raw); + const tests = []; + collectTestsFromSuites(report?.suites, tests); + total = tests.length; + for (const t of tests) { + if (getFinalStatus(t) === "passed") passed++; + } + } catch { + // best-effort; keep smoke output stable even if JSON isn't available + } + + const ratio = total > 0 ? `${passed}/${total}` : "all"; + const warnSuffix = slow ? " (warn: exceeded SMOKE_WARN_AFTER_MS)" : ""; + console.log(`SMOKE: ${ratio} passed in ${elapsedS.toFixed(1)}s${warnSuffix}`); + process.exit(0); + } + + // Failure + let failName = "scenario"; + let reason = `exit ${pw.code ?? "?"}`; + try { + const raw = await fsp.readFile(jsonReport, "utf8"); + const report = JSON.parse(raw); + const tests = []; + collectTestsFromSuites(report?.suites, tests); + const firstFail = tests.find((t) => getFinalStatus(t) === "failed"); + if (firstFail) { + failName = firstFail.title ? String(firstFail.title) : failName; + const lastResult = (firstFail.results || []).at(-1); + const err = lastResult?.error?.message || lastResult?.error?.value; + if (err) reason = truncateOneLine(err); + } + } catch { + /* ignore */ + } + + console.log(`SMOKE FAIL: ${truncateOneLine(failName, 80)} (${reason})`); + console.log(`Artifacts: ${artifactsDir}`); + process.exit(pw.code || 1); +} + +async function main() { + const suite = process.argv[2]; + if (!suite) { + throw new Error( + "Usage: node scripts/test-suite.mjs ", + ); + } + + const artifactsDir = artifactsDirFor(suite); + await rmrf(artifactsDir); + await mkdirp(artifactsDir); + + const logPath = path.join(artifactsDir, "run.log"); + const logStream = fs.createWriteStream(logPath, { flags: "a" }); + + try { + if (suite === "smoke") { + await runSmoke({ artifactsDir, logStream }); + return; + } + + if (suite === "unit") { + const res = await runUnit({ artifactsDir, logStream }); + process.exit(res.code || 0); + } + + if (suite === "scenario") { + const res = await runPlaywright({ + artifactsDir, + logStream, + tag: "@scenario", + smoke: false, + }); + process.exit(res.code || 0); + } + + if (suite === "e2e") { + const res = await runPlaywright({ + artifactsDir, + logStream, + tag: "@e2e", + smoke: false, + }); + process.exit(res.code || 0); + } + + if (suite === "integration") { + const res = await runPlaywright({ + artifactsDir, + logStream, + tag: "@integration", + smoke: false, + }); + process.exit(res.code || 0); + } + + if (suite === "all") { + // Unit first for fast feedback. + const unit = await spawnLogged("pnpm", ["exec", "vitest", "run"], { + cwd: ROOT, + env: envWithArtifacts(path.join(artifactsDir, "unit")), + logStream, + passthrough: true, + }); + if (unit.code !== 0) process.exit(unit.code || 1); + + const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; + if (!skipBuild) { + const build = await spawnLogged("pnpm", ["run", "build"], { + cwd: ROOT, + env: envWithArtifacts(artifactsDir), + logStream, + passthrough: true, + }); + if (build.code !== 0) process.exit(build.code || 1); + } + + const pwEnvBase = { + ...process.env, + JABTERM_SKIP_BUILD: "1", + }; + + const scenarioDir = path.join(artifactsDir, "scenario"); + await rmrf(scenarioDir); + await mkdirp(scenarioDir); + const scenario = await spawnLogged( + "pnpm", + ["exec", "playwright", "test", "--grep", "@scenario"], + { + cwd: ROOT, + env: envWithArtifacts(scenarioDir, pwEnvBase), + logStream, + passthrough: true, + }, + ); + if (scenario.code !== 0) process.exit(scenario.code || 1); + + const e2eDir = path.join(artifactsDir, "e2e"); + await rmrf(e2eDir); + await mkdirp(e2eDir); + const e2e = await spawnLogged( + "pnpm", + ["exec", "playwright", "test", "--grep", "@e2e"], + { + cwd: ROOT, + env: envWithArtifacts(e2eDir, pwEnvBase), + logStream, + passthrough: true, + }, + ); + if (e2e.code !== 0) process.exit(e2e.code || 1); + + process.exit(0); + } + + process.exit(1); + } finally { + await new Promise((resolve) => logStream.end(resolve)); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tests/echo.spec.ts b/tests/echo.spec.ts index 78d0505..2dfc8dc 100644 --- a/tests/echo.spec.ts +++ b/tests/echo.spec.ts @@ -18,7 +18,7 @@ const WS_URL = defaultWsUrl(); test.describe.configure({ mode: "serial" }); -test.describe("Terminal — echo round-trip (WS protocol)", () => { +test.describe("@e2e Terminal — echo round-trip (WS protocol)", () => { test("echo command returns output", async () => { let ws!: WsClient; try { diff --git a/tests/react-demo.spec.ts b/tests/react-demo.spec.ts index 295a8ea..835c5eb 100644 --- a/tests/react-demo.spec.ts +++ b/tests/react-demo.spec.ts @@ -7,9 +7,16 @@ import { test, expect } from "@playwright/test"; -test.describe("React demo page", () => { +test.describe("@scenario React demo page", () => { test("mount/unmount and layout resize work", async ({ page }) => { - await page.goto("/"); + const consoleErrors: string[] = []; + page.on("pageerror", (err) => consoleErrors.push(err.message)); + page.on("console", (msg) => { + if (msg.type() === "error") consoleErrors.push(msg.text()); + }); + + const resp = await page.goto("/"); + expect(resp?.ok()).toBe(true); const term1 = page.locator('[data-testid="jabterm-1"] .xterm-screen'); await expect(term1).toBeVisible({ timeout: 15_000 }); @@ -63,10 +70,19 @@ test.describe("React demo page", () => { expect(after).not.toBeNull(); expect(Math.abs(after!.width - before!.width)).toBeGreaterThan(20); + + expect(consoleErrors).toEqual([]); }); test("shows close message when shell exits", async ({ page }) => { - await page.goto("/"); + const consoleErrors: string[] = []; + page.on("pageerror", (err) => consoleErrors.push(err.message)); + page.on("console", (msg) => { + if (msg.type() === "error") consoleErrors.push(msg.text()); + }); + + const resp = await page.goto("/"); + expect(resp?.ok()).toBe(true); const term1 = page.locator('[data-testid="jabterm-1"] .xterm-screen'); await expect(term1).toBeVisible({ timeout: 15_000 }); @@ -83,6 +99,8 @@ test.describe("React demo page", () => { await expect(state1).toHaveAttribute("data-jabterm-state", "closed", { timeout: 15_000, }); + + expect(consoleErrors).toEqual([]); }); }); diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 31a4723..634879b 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -16,7 +16,7 @@ test.beforeAll(() => { fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true }); }); -test.describe("Terminal — README screenshots", () => { +test.describe("@integration Terminal — README screenshots", () => { test("capture echo terminal screenshot", async ({ page }) => { await page.goto("/"); diff --git a/tests/server-api.spec.ts b/tests/server-api.spec.ts index 6462f2e..174949e 100644 --- a/tests/server-api.spec.ts +++ b/tests/server-api.spec.ts @@ -57,7 +57,7 @@ async function waitForMatch( test.describe.configure({ mode: "serial" }); -test.describe("Server API — createJabtermServer", () => { +test.describe("@e2e Server API — createJabtermServer", () => { test("supports ephemeral port and deterministic shutdown", async () => { const server = createJabtermServer({ host: "127.0.0.1", diff --git a/tests/video.spec.ts b/tests/video.spec.ts index 73cec56..b4a678d 100644 --- a/tests/video.spec.ts +++ b/tests/video.spec.ts @@ -80,7 +80,7 @@ test.use({ }, }); -test.describe("Terminal - demo video", () => { +test.describe("@integration Terminal - demo video", () => { test("records terminal usage flow", async ({ page }) => { test.setTimeout(120_000); diff --git a/tests/zombie.spec.ts b/tests/zombie.spec.ts index 4c6be53..53b730b 100644 --- a/tests/zombie.spec.ts +++ b/tests/zombie.spec.ts @@ -14,7 +14,7 @@ const CLEANUP_DELAY_MS = 1500; test.describe.configure({ mode: "serial" }); -test.describe("Terminal — zombie process prevention", () => { +test.describe("@e2e Terminal — zombie process prevention", () => { test("pty process is killed when websocket closes", async () => { const ws = new WsClient(WS_URL); From 39e5869b2f99c213e27eed351c9967c87f7e1448 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 12:42:03 +0000 Subject: [PATCH 2/7] test: remove integration category; keep docs as scenario Co-authored-by: Alexander Nazarov --- .github/workflows/ci.yml | 4 +-- scripts/test-suite.mjs | 66 ++++++++++++++++++++++++++++++++-------- tests/screenshot.spec.ts | 2 +- tests/video.spec.ts | 2 +- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa3afd5..818f402 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,8 @@ jobs: - name: Run tests run: JABTERM_SKIP_BUILD=1 pnpm coverage:ci - - name: Run integration tests (docs assets) - run: JABTERM_SKIP_BUILD=1 pnpm run test:integration + - name: Run docs scenario tests (docs assets) + run: JABTERM_SCENARIO_ONLY_DOCS=1 JABTERM_SKIP_BUILD=1 pnpm run test:scenario - name: Upload coverage artifacts if: always() diff --git a/scripts/test-suite.mjs b/scripts/test-suite.mjs index 43fb49a..1771e73 100644 --- a/scripts/test-suite.mjs +++ b/scripts/test-suite.mjs @@ -14,7 +14,7 @@ function artifactsDirFor(suite) { case "e2e": return path.join(ROOT, ".cache/tests/test-e2e__playwright"); case "integration": - return path.join(ROOT, ".cache/tests/test-integration__playwright"); + return path.join(ROOT, ".cache/tests/test-integration__none"); case "smoke": return path.join(ROOT, ".cache/tests/test-smoke__scenario"); case "all": @@ -110,7 +110,13 @@ async function runUnit({ artifactsDir, logStream }) { }); } -async function runPlaywright({ artifactsDir, logStream, tag, smoke }) { +async function runPlaywright({ + artifactsDir, + logStream, + grep, + grepInvert = [], + smoke, +}) { const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; const env = envWithArtifacts(artifactsDir, smoke ? { JABTERM_SMOKE: "1" } : {}); @@ -125,7 +131,8 @@ async function runPlaywright({ artifactsDir, logStream, tag, smoke }) { if (build.code !== 0) return { code: build.code, signal: build.signal }; } - const args = ["exec", "playwright", "test", "--grep", tag]; + const args = ["exec", "playwright", "test", "--grep", grep]; + for (const inv of grepInvert) args.push("--grep-invert", inv); if (smoke) args.push("--max-failures=1", "--quiet"); return await spawnLogged("pnpm", args, { @@ -165,6 +172,21 @@ function getFinalStatus(test) { return "unknown"; } +function scenarioFiltersFromEnv() { + const onlyDocs = process.env.JABTERM_SCENARIO_ONLY_DOCS === "1"; + const includeDocs = process.env.JABTERM_SCENARIO_INCLUDE_DOCS === "1"; + const includeSlow = process.env.JABTERM_SCENARIO_INCLUDE_SLOW === "1"; + + if (onlyDocs) { + return { grep: "@docs", grepInvert: [] }; + } + + const grepInvert = []; + if (!includeDocs) grepInvert.push("@docs"); + if (!includeSlow) grepInvert.push("@slow"); + return { grep: "@scenario", grepInvert }; +} + async function runSmoke({ artifactsDir, logStream }) { const startedAt = Date.now(); const perTestTimeoutMs = parseInt( @@ -213,6 +235,10 @@ async function runSmoke({ artifactsDir, logStream }) { "test", "--grep", "@scenario", + "--grep-invert", + "@docs", + "--grep-invert", + "@slow", "--max-failures=1", "--quiet", ], @@ -302,10 +328,12 @@ async function main() { } if (suite === "scenario") { + const { grep, grepInvert } = scenarioFiltersFromEnv(); const res = await runPlaywright({ artifactsDir, logStream, - tag: "@scenario", + grep, + grepInvert, smoke: false, }); process.exit(res.code || 0); @@ -315,20 +343,22 @@ async function main() { const res = await runPlaywright({ artifactsDir, logStream, - tag: "@e2e", + grep: "@e2e", smoke: false, }); process.exit(res.code || 0); } if (suite === "integration") { - const res = await runPlaywright({ - artifactsDir, - logStream, - tag: "@integration", - smoke: false, - }); - process.exit(res.code || 0); + logStream.write( + Buffer.from( + "This repo intentionally has no integration tests. (contract script exists)\n", + ), + ); + process.stdout.write( + "No integration tests in this repo. (intentional)\n", + ); + process.exit(0); } if (suite === "all") { @@ -362,7 +392,17 @@ async function main() { await mkdirp(scenarioDir); const scenario = await spawnLogged( "pnpm", - ["exec", "playwright", "test", "--grep", "@scenario"], + [ + "exec", + "playwright", + "test", + "--grep", + "@scenario", + "--grep-invert", + "@docs", + "--grep-invert", + "@slow", + ], { cwd: ROOT, env: envWithArtifacts(scenarioDir, pwEnvBase), diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 634879b..1951f97 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -16,7 +16,7 @@ test.beforeAll(() => { fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true }); }); -test.describe("@integration Terminal — README screenshots", () => { +test.describe("@scenario @docs @slow Terminal — README screenshots", () => { test("capture echo terminal screenshot", async ({ page }) => { await page.goto("/"); diff --git a/tests/video.spec.ts b/tests/video.spec.ts index b4a678d..4a04fa7 100644 --- a/tests/video.spec.ts +++ b/tests/video.spec.ts @@ -80,7 +80,7 @@ test.use({ }, }); -test.describe("@integration Terminal - demo video", () => { +test.describe("@scenario @docs @slow Terminal - demo video", () => { test("records terminal usage flow", async ({ page }) => { test.setTimeout(120_000); From 45b6ace151115cc77ab27af36dd7dcf0dc857c7a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 13:07:55 +0000 Subject: [PATCH 3/7] test: add human mode runner flag and breath helper Co-authored-by: Alexander Nazarov --- playwright.config.ts | 25 +++++++- scripts/test-suite.mjs | 124 +++++++++++++++++++++++++-------------- tests/helpers/human.ts | 9 +++ tests/react-demo.spec.ts | 3 +- 4 files changed, 114 insertions(+), 47 deletions(-) create mode 100644 tests/helpers/human.ts diff --git a/playwright.config.ts b/playwright.config.ts index 419cc69..fbdd8a3 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,6 +4,7 @@ import path from "node:path"; const PORT = parseInt(process.env.JABTERM_PORT || "3223", 10); const ARTIFACTS_DIR = process.env.TEST_ARTIFACTS_DIR; const IS_SMOKE = process.env.JABTERM_SMOKE === "1"; +const HUMAN_SLOWMO_MS = parseInt(process.env.JABTERM_HUMAN_SLOWMO_MS || "0", 10); const outputDir = ARTIFACTS_DIR ? path.join(ARTIFACTS_DIR, "test-results") @@ -47,9 +48,27 @@ export default defineConfig({ ], ] : [["html", { outputFolder: reportDir, open: "never" }]], - use: { - baseURL: `http://127.0.0.1:${PORT + 1}`, - }, + projects: [ + { + name: "default", + use: { + baseURL: `http://127.0.0.1:${PORT + 1}`, + }, + }, + { + name: "human", + use: { + baseURL: `http://127.0.0.1:${PORT + 1}`, + headless: false, + video: "on", + trace: "on", + screenshot: "on", + ...(HUMAN_SLOWMO_MS > 0 + ? { launchOptions: { slowMo: HUMAN_SLOWMO_MS } } + : {}), + }, + }, + ], webServer: [ { command: `node bin/jabterm-server.mjs --port ${PORT}`, diff --git a/scripts/test-suite.mjs b/scripts/test-suite.mjs index 1771e73..f9d62b4 100644 --- a/scripts/test-suite.mjs +++ b/scripts/test-suite.mjs @@ -5,6 +5,24 @@ import { spawn } from "node:child_process"; const ROOT = process.cwd(); +function getPackageManager() { + try { + const pkgPath = path.join(ROOT, "package.json"); + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + const pm = String(pkg?.packageManager || ""); + if (pm.startsWith("pnpm@")) return "pnpm"; + if (pm.startsWith("npm@")) return "npm"; + } catch { + /* ignore */ + } + + if (fs.existsSync(path.join(ROOT, "pnpm-lock.yaml"))) return "pnpm"; + if (fs.existsSync(path.join(ROOT, "package-lock.json"))) return "npm"; + return "npm"; +} + +const PM = getPackageManager(); + function artifactsDirFor(suite) { switch (suite) { case "unit": @@ -102,7 +120,7 @@ function envWithArtifacts(artifactsDir, extraEnv = {}) { async function runUnit({ artifactsDir, logStream }) { const env = envWithArtifacts(artifactsDir); - return await spawnLogged("pnpm", ["exec", "vitest", "run"], { + return await spawnLogged(PM, ["exec", "vitest", "run"], { cwd: ROOT, env, logStream, @@ -116,13 +134,20 @@ async function runPlaywright({ grep, grepInvert = [], smoke, + human, }) { const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; - const env = envWithArtifacts(artifactsDir, smoke ? { JABTERM_SMOKE: "1" } : {}); + const env = envWithArtifacts( + artifactsDir, + { + ...(smoke ? { JABTERM_SMOKE: "1" } : {}), + ...(human ? { JABTERM_HUMAN: "1" } : {}), + }, + ); if (!skipBuild) { - const build = await spawnLogged("pnpm", ["run", "build"], { + const build = await spawnLogged(PM, ["run", "build"], { cwd: ROOT, env, logStream, @@ -133,9 +158,10 @@ async function runPlaywright({ const args = ["exec", "playwright", "test", "--grep", grep]; for (const inv of grepInvert) args.push("--grep-invert", inv); + args.push("--project", human ? "human" : "default"); if (smoke) args.push("--max-failures=1", "--quiet"); - return await spawnLogged("pnpm", args, { + return await spawnLogged(PM, args, { cwd: ROOT, env, logStream, @@ -209,15 +235,16 @@ async function runSmoke({ artifactsDir, logStream }) { const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; if (!skipBuild) { - const build = await spawnLogged("pnpm", ["run", "build"], { + const build = await spawnLogged(PM, ["run", "build"], { cwd: ROOT, env, logStream, passthrough: false, }); if (build.code !== 0) { - console.log(`SMOKE FAIL: build (exit ${build.code ?? "?"})`); - console.log(`Artifacts: ${artifactsDir}`); + console.log( + `SMOKE FAIL: build (exit ${build.code ?? "?"}) | Artifacts: ${artifactsDir}`, + ); process.exit(build.code || 1); } } @@ -227,31 +254,37 @@ async function runSmoke({ artifactsDir, logStream }) { }, warnAfterMs); warn.unref?.(); - const pw = await spawnLoggedWithTimeout( - "pnpm", - [ - "exec", - "playwright", - "test", - "--grep", - "@scenario", - "--grep-invert", - "@docs", - "--grep-invert", - "@slow", - "--max-failures=1", - "--quiet", - ], - { cwd: ROOT, env, logStream, timeoutMs: totalTimeoutMs }, - ); + const smokeArgs = [ + "exec", + "playwright", + "test", + "--grep", + "@scenario", + "--grep-invert", + "@docs", + "--grep-invert", + "@slow", + "--project", + process.env.JABTERM_HUMAN === "1" ? "human" : "default", + "--max-failures=1", + "--quiet", + ]; + + const pw = await spawnLoggedWithTimeout(PM, smokeArgs, { + cwd: ROOT, + env, + logStream, + timeoutMs: totalTimeoutMs, + }); clearTimeout(warn); const elapsedS = (Date.now() - startedAt) / 1000; const slow = elapsedS * 1000 > warnAfterMs; if (pw.timedOut) { - console.log(`SMOKE FAIL: scenario (timeout after ${Math.round(elapsedS)}s)`); - console.log(`Artifacts: ${artifactsDir}`); + console.log( + `SMOKE FAIL: scenario (timeout after ${Math.round(elapsedS)}s) | Artifacts: ${artifactsDir}`, + ); process.exit(1); } @@ -296,19 +329,23 @@ async function runSmoke({ artifactsDir, logStream }) { /* ignore */ } - console.log(`SMOKE FAIL: ${truncateOneLine(failName, 80)} (${reason})`); - console.log(`Artifacts: ${artifactsDir}`); + console.log( + `SMOKE FAIL: ${truncateOneLine(failName, 80)} (${reason}) | Artifacts: ${artifactsDir}`, + ); process.exit(pw.code || 1); } async function main() { - const suite = process.argv[2]; + const args = process.argv.slice(2); + const suite = args[0]; if (!suite) { throw new Error( - "Usage: node scripts/test-suite.mjs ", + "Usage: node scripts/test-suite.mjs [--human]", ); } + const human = args.includes("--human"); + const artifactsDir = artifactsDirFor(suite); await rmrf(artifactsDir); await mkdirp(artifactsDir); @@ -318,6 +355,7 @@ async function main() { try { if (suite === "smoke") { + if (human) process.env.JABTERM_HUMAN = "1"; await runSmoke({ artifactsDir, logStream }); return; } @@ -335,6 +373,7 @@ async function main() { grep, grepInvert, smoke: false, + human, }); process.exit(res.code || 0); } @@ -345,6 +384,7 @@ async function main() { logStream, grep: "@e2e", smoke: false, + human, }); process.exit(res.code || 0); } @@ -363,7 +403,7 @@ async function main() { if (suite === "all") { // Unit first for fast feedback. - const unit = await spawnLogged("pnpm", ["exec", "vitest", "run"], { + const unit = await spawnLogged(PM, ["exec", "vitest", "run"], { cwd: ROOT, env: envWithArtifacts(path.join(artifactsDir, "unit")), logStream, @@ -373,7 +413,7 @@ async function main() { const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; if (!skipBuild) { - const build = await spawnLogged("pnpm", ["run", "build"], { + const build = await spawnLogged(PM, ["run", "build"], { cwd: ROOT, env: envWithArtifacts(artifactsDir), logStream, @@ -391,7 +431,7 @@ async function main() { await rmrf(scenarioDir); await mkdirp(scenarioDir); const scenario = await spawnLogged( - "pnpm", + PM, [ "exec", "playwright", @@ -402,6 +442,8 @@ async function main() { "@docs", "--grep-invert", "@slow", + "--project", + "default", ], { cwd: ROOT, @@ -415,16 +457,12 @@ async function main() { const e2eDir = path.join(artifactsDir, "e2e"); await rmrf(e2eDir); await mkdirp(e2eDir); - const e2e = await spawnLogged( - "pnpm", - ["exec", "playwright", "test", "--grep", "@e2e"], - { - cwd: ROOT, - env: envWithArtifacts(e2eDir, pwEnvBase), - logStream, - passthrough: true, - }, - ); + const e2e = await spawnLogged(PM, ["exec", "playwright", "test", "--grep", "@e2e", "--project", "default"], { + cwd: ROOT, + env: envWithArtifacts(e2eDir, pwEnvBase), + logStream, + passthrough: true, + }); if (e2e.code !== 0) process.exit(e2e.code || 1); process.exit(0); diff --git a/tests/helpers/human.ts b/tests/helpers/human.ts new file mode 100644 index 0000000..8c35f52 --- /dev/null +++ b/tests/helpers/human.ts @@ -0,0 +1,9 @@ +export function isHumanMode(): boolean { + return process.env.JABTERM_HUMAN === "1"; +} + +export async function breath(ms = 350): Promise { + if (!isHumanMode()) return; + await new Promise((resolve) => setTimeout(resolve, ms)); +} + diff --git a/tests/react-demo.spec.ts b/tests/react-demo.spec.ts index 835c5eb..4e94cc8 100644 --- a/tests/react-demo.spec.ts +++ b/tests/react-demo.spec.ts @@ -6,6 +6,7 @@ */ import { test, expect } from "@playwright/test"; +import { breath } from "./helpers/human.js"; test.describe("@scenario React demo page", () => { test("mount/unmount and layout resize work", async ({ page }) => { @@ -88,7 +89,7 @@ test.describe("@scenario React demo page", () => { await expect(term1).toBeVisible({ timeout: 15_000 }); await term1.click(); - await page.waitForTimeout(500); + await breath(500); await page.keyboard.type("exit", { delay: 10 }); await page.keyboard.press("Enter"); From f7374bc92c4dc8016f933ed9c93b5227c19f44e0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 13:20:38 +0000 Subject: [PATCH 4/7] docs: add AGENTS.md link to testing strategy Co-authored-by: Alexander Nazarov --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..97fc531 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,6 @@ +# Agents + +If you are an AI agent (or automation) working on this repository, follow the test contract described in: + +- [`agents/testing-strategy.md`](agents/testing-strategy.md) + From 528ccade452c65c887beb719592725181a4c7392 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 14:15:27 +0000 Subject: [PATCH 5/7] chore: add universal ai-tests runner package Co-authored-by: Alexander Nazarov --- packages/ai-tests/README.md | 69 +++++ packages/ai-tests/bin/ai-tests.mjs | 436 +++++++++++++++++++++++++++++ packages/ai-tests/package.json | 21 ++ packages/ai-tests/src/human.mjs | 9 + 4 files changed, 535 insertions(+) create mode 100644 packages/ai-tests/README.md create mode 100755 packages/ai-tests/bin/ai-tests.mjs create mode 100644 packages/ai-tests/package.json create mode 100644 packages/ai-tests/src/human.mjs diff --git a/packages/ai-tests/README.md b/packages/ai-tests/README.md new file mode 100644 index 0000000..97e2e55 --- /dev/null +++ b/packages/ai-tests/README.md @@ -0,0 +1,69 @@ +# ai-tests + +Small, policy-light test runner intended to be copied between repos (or published). + +## CLI + +```bash +ai-tests [--smoke] [--human] [--pkg ...] [--project ] [--allow-missing-project] +``` + +Suites: + +- `unit`: run unit tests (Vitest if installed; otherwise `node --test`) +- `scenario`: run Playwright project `scenario` (recommended naming: `*.scenario.e2e.ts`) +- `e2e`: run Playwright project `e2e` (recommended naming: `*.e2e.ts`, ignoring scenario) +- `integration`: run Playwright project `integration` (optional) +- `all`: `unit` + `scenario` + `e2e` (never runs `integration`) + +Smoke rules (`--smoke`) apply to Playwright suites only: + +- per-test timeout via `SMOKE_PER_TEST_TIMEOUT_MS` (default 30000) +- total timeout via `SMOKE_TOTAL_TIMEOUT_MS` (default 180000) +- stop on first failure +- dot reporter output goes to `run.log` +- terminal output: **one line only** on success / failure + +Human execution (`--human`) is orthogonal: + +- Playwright: `--headed`, `--workers=1`, `--trace=on` +- Exposes `AI_TEST_HUMAN=1` for test utilities (see `ai-tests/human`) + +## Artifacts + +Each run writes to: + +`.cache/tests//` (cleaned before each run) + +With: + +- `run.log`: full raw output +- `pw-output/`: Playwright output directory (traces/videos/screenshots if enabled by config) + +## Playwright config convention (recommended) + +Define projects so the runner doesn’t need tags or discovery logic: + +```ts +import { defineConfig } from "@playwright/test"; + +const E2E = /.*\.e2e\.(ts|js)x?$/; +const SCENARIO = /.*\.scenario\.e2e\.(ts|js)x?$/; + +export default defineConfig({ + testDir: "tests", + projects: [ + { name: "e2e", testMatch: E2E, testIgnore: [SCENARIO] }, + { name: "scenario", testMatch: SCENARIO }, + ], +}); +``` + +## Test helper: `breath()` + +```js +import { breath } from "ai-tests/human"; + +await breath(300); +``` + diff --git a/packages/ai-tests/bin/ai-tests.mjs b/packages/ai-tests/bin/ai-tests.mjs new file mode 100755 index 0000000..8ab271c --- /dev/null +++ b/packages/ai-tests/bin/ai-tests.mjs @@ -0,0 +1,436 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import yargs from "yargs/yargs"; +import { hideBin } from "yargs/helpers"; + +const ROOT = process.cwd(); + +function getPackageManager(rootDir) { + try { + const pkg = JSON.parse( + fs.readFileSync(path.join(rootDir, "package.json"), "utf8"), + ); + const pm = String(pkg?.packageManager || ""); + if (pm.startsWith("pnpm@")) return "pnpm"; + if (pm.startsWith("npm@")) return "npm"; + } catch { + /* ignore */ + } + if (fs.existsSync(path.join(rootDir, "pnpm-lock.yaml"))) return "pnpm"; + if (fs.existsSync(path.join(rootDir, "package-lock.json"))) return "npm"; + return "npm"; +} + +const PM = getPackageManager(ROOT); + +function pmExecArgs(bin, args) { + if (PM === "npm") return ["exec", "--", bin, ...args]; + return ["exec", bin, ...args]; +} + +async function rmrf(p) { + await fsp.rm(p, { recursive: true, force: true }); +} + +async function mkdirp(p) { + await fsp.mkdir(p, { recursive: true }); +} + +function spawnTee(cmd, args, { cwd, env, logStream, passthrough }) { + return new Promise((resolve) => { + const child = spawn(cmd, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + + child.stdout.on("data", (chunk) => { + logStream.write(chunk); + if (passthrough) process.stdout.write(chunk); + }); + child.stderr.on("data", (chunk) => { + logStream.write(chunk); + if (passthrough) process.stderr.write(chunk); + }); + + child.on("close", (code, signal) => + resolve({ code: code ?? 0, signal }), + ); + }); +} + +function spawnCapture(cmd, args, { cwd, env }) { + return new Promise((resolve) => { + const child = spawn(cmd, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + let err = ""; + child.stdout.on("data", (c) => (out += c.toString("utf8"))); + child.stderr.on("data", (c) => (err += c.toString("utf8"))); + child.on("close", (code) => resolve({ code: code ?? 0, out, err })); + }); +} + +function sanitizePathSegment(s) { + return String(s) + .trim() + .replace(/^@/, "") + .replace(/[\/@]/g, "__") + .replace(/[^a-zA-Z0-9_.-]+/g, "_") + .slice(0, 120); +} + +function runId({ suite, smoke, human }) { + const parts = [`test-${suite}`]; + if (smoke) parts.push("smoke"); + if (human) parts.push("human"); + return parts.join("__"); +} + +function artifactsDirFor({ runId: rid, pkgName }) { + const base = path.join(ROOT, ".cache", "tests", rid); + return pkgName ? path.join(base, sanitizePathSegment(pkgName)) : base; +} + +function readPkgName(dir) { + try { + const pkg = JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")); + return pkg?.name || path.basename(dir); + } catch { + return path.basename(dir); + } +} + +function pkgHasDep(dir, depName) { + try { + const pkg = JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")); + return Boolean(pkg?.dependencies?.[depName] || pkg?.devDependencies?.[depName]); + } catch { + return false; + } +} + +async function listWorkspacePackages() { + const isPnpmWs = + PM === "pnpm" && fs.existsSync(path.join(ROOT, "pnpm-workspace.yaml")); + if (!isPnpmWs) return [{ name: readPkgName(ROOT), dir: ROOT }]; + + const res = await spawnCapture( + PM, + ["-r", "ls", "--depth", "-1", "--json"], + { cwd: ROOT, env: process.env }, + ); + if (res.code !== 0) return [{ name: readPkgName(ROOT), dir: ROOT }]; + + try { + const parsed = JSON.parse(res.out); + const pkgs = Array.isArray(parsed) ? parsed : []; + const out = []; + for (const p of pkgs) { + const dir = p?.path; + if (!dir || typeof dir !== "string") continue; + if (!fs.existsSync(path.join(dir, "package.json"))) continue; + out.push({ name: p?.name || readPkgName(dir), dir }); + } + return out.length ? out : [{ name: readPkgName(ROOT), dir: ROOT }]; + } catch { + return [{ name: readPkgName(ROOT), dir: ROOT }]; + } +} + +function parsePlaywrightCounts(logText) { + const totalMatch = logText.match(/Running\s+(\d+)\s+tests?\b/); + const total = totalMatch ? parseInt(totalMatch[1], 10) : 0; + + // Prefer the final "X passed" summary when available. + const passedMatches = Array.from(logText.matchAll(/^\s*(\d+)\s+passed\b/mg)); + const passed = passedMatches.length + ? parseInt(passedMatches.at(-1)[1], 10) + : 0; + + return { total, passed }; +} + +function parseFirstFailureHeadline(logText) { + const m = logText.match(/^\s*\d+\)\s+(.+)$/m); + if (!m) return null; + return m[1].replace(/\s+/g, " ").trim().slice(0, 180); +} + +function truncateOneLine(s, max = 160) { + const one = String(s || "").replace(/\s+/g, " ").trim(); + if (one.length <= max) return one; + return `${one.slice(0, max - 1)}…`; +} + +async function runUnit({ pkg, artifactsDir, logStream, passthrough }) { + const env = { ...process.env, TEST_ARTIFACTS_DIR: artifactsDir }; + + if (pkgHasDep(pkg.dir, "vitest")) { + return await spawnTee(PM, pmExecArgs("vitest", ["run"]), { + cwd: pkg.dir, + env, + logStream, + passthrough, + }); + } + + return await spawnTee(process.execPath, ["--test"], { + cwd: pkg.dir, + env, + logStream, + passthrough, + }); +} + +function shouldTreatMissingProjectAsSuccess(output, project) { + const text = String(output || ""); + return ( + text.includes(`Project(s) "${project}" not found`) || + text.includes(`Project(s) '${project}' not found`) || + text.includes("Unknown project") || + text.includes("No projects matched") + ); +} + +async function runPlaywright({ + pkg, + project, + smoke, + human, + artifactsDir, + logStream, + passthrough, + globalTimeoutMs, + allowMissingProject, +}) { + const env = { + ...process.env, + TEST_ARTIFACTS_DIR: artifactsDir, + ...(human ? { AI_TEST_HUMAN: "1" } : {}), + }; + + const pwOut = path.join(artifactsDir, "pw-output"); + const args = [ + "test", + "--project", + project, + "--output", + pwOut, + "--pass-with-no-tests", + ...(human ? ["--headed", "--workers=1", "--trace=on"] : []), + ...(smoke + ? [ + "--max-failures=1", + "--workers=1", + "--reporter=dot", + "--timeout", + String( + parseInt(process.env.SMOKE_PER_TEST_TIMEOUT_MS || "30000", 10), + ), + "--global-timeout", + String(globalTimeoutMs), + ] + : []), + ]; + + const res = await spawnTee(PM, pmExecArgs("playwright", args), { + cwd: pkg.dir, + env, + logStream, + passthrough, + }); + + if (res.code !== 0 && allowMissingProject) { + const logPath = path.join(artifactsDir, "run.log"); + const logTxt = await fsp.readFile(logPath, "utf8").catch(() => ""); + if (shouldTreatMissingProjectAsSuccess(logTxt, project)) return { code: 0 }; + } + + return res; +} + +const argv = await yargs(hideBin(process.argv)) + .scriptName("ai-tests") + .command( + "$0 ", + "Run tests", + (y) => + y + .positional("suite", { + choices: ["unit", "e2e", "scenario", "integration", "all"], + type: "string", + }) + .option("smoke", { + type: "boolean", + default: false, + describe: "Enable smoke rules (Playwright suites only)", + }) + .option("human", { + type: "boolean", + default: false, + describe: "Human execution mode (headed + trace + paced via breath())", + }) + .option("pkg", { + type: "array", + describe: "Workspace package name(s) to run (default: all packages)", + }) + .option("project", { + type: "string", + describe: "Override Playwright project name", + }) + .option("allow-missing-project", { + type: "boolean", + default: false, + describe: "Treat missing Playwright project as success", + }), + ) + .help() + .strict() + .parseAsync(); + +const SUITE = argv.suite; +const SMOKE = Boolean(argv.smoke); +const HUMAN = Boolean(argv.human); +const ONLY_PKGS = (argv.pkg || []).map(String); +const PROJECT_OVERRIDE = argv.project ? String(argv.project) : null; +const ALLOW_MISSING_PROJECT = Boolean(argv["allow-missing-project"]); + +const totalTimeoutMs = parseInt(process.env.SMOKE_TOTAL_TIMEOUT_MS || "180000", 10); +const warnAfterMs = parseInt(process.env.SMOKE_WARN_AFTER_MS || "60000", 10); + +const rid = runId({ suite: SUITE, smoke: SMOKE, human: HUMAN }); +const startedAt = Date.now(); + +const pkgsAll = await listWorkspacePackages(); +const pkgs = ONLY_PKGS.length + ? pkgsAll.filter((p) => ONLY_PKGS.includes(p.name)) + : pkgsAll; + +if (pkgs.length === 0) { + console.error(`No matching packages for --pkg=${ONLY_PKGS.join(", ")}`); + process.exit(2); +} + +let aggPassed = 0; +let aggTotal = 0; + +for (const pkg of pkgs) { + const multi = pkgs.length > 1; + const artifactsDir = artifactsDirFor({ runId: rid, pkgName: multi ? pkg.name : null }); + await rmrf(artifactsDir); + await mkdirp(artifactsDir); + + const logPath = path.join(artifactsDir, "run.log"); + const logStream = fs.createWriteStream(logPath, { flags: "w" }); + + const elapsedMs = Date.now() - startedAt; + const remainingMs = Math.max(1000, totalTimeoutMs - elapsedMs); + + try { + if (SUITE === "unit") { + const res = await runUnit({ pkg, artifactsDir, logStream, passthrough: true }); + if (res.code !== 0) process.exit(res.code || 1); + continue; + } + + if (SUITE === "all") { + const unitDir = path.join(artifactsDir, "unit"); + await rmrf(unitDir); + await mkdirp(unitDir); + const u = await runUnit({ + pkg, + artifactsDir: unitDir, + logStream, + passthrough: true, + }); + if (u.code !== 0) process.exit(u.code || 1); + + if (!pkgHasDep(pkg.dir, "@playwright/test")) continue; + + const sDir = path.join(artifactsDir, "scenario"); + await rmrf(sDir); + await mkdirp(sDir); + const s = await runPlaywright({ + pkg, + project: PROJECT_OVERRIDE || "scenario", + smoke: false, + human: false, + artifactsDir: sDir, + logStream, + passthrough: true, + globalTimeoutMs: remainingMs, + allowMissingProject: ALLOW_MISSING_PROJECT, + }); + if (s.code !== 0) process.exit(s.code || 1); + + const eDir = path.join(artifactsDir, "e2e"); + await rmrf(eDir); + await mkdirp(eDir); + const e = await runPlaywright({ + pkg, + project: PROJECT_OVERRIDE || "e2e", + smoke: false, + human: false, + artifactsDir: eDir, + logStream, + passthrough: true, + globalTimeoutMs: remainingMs, + allowMissingProject: ALLOW_MISSING_PROJECT, + }); + if (e.code !== 0) process.exit(e.code || 1); + + continue; + } + + // For Playwright suites: skip packages without Playwright. + if (!pkgHasDep(pkg.dir, "@playwright/test")) continue; + + const project = + PROJECT_OVERRIDE || + (SUITE === "scenario" + ? "scenario" + : SUITE === "e2e" + ? "e2e" + : "integration"); + + const res = await runPlaywright({ + pkg, + project, + smoke: SMOKE, + human: HUMAN, + artifactsDir, + logStream, + passthrough: !SMOKE, + globalTimeoutMs: remainingMs, + allowMissingProject: ALLOW_MISSING_PROJECT, + }); + + if (!SMOKE) { + if (res.code !== 0) process.exit(res.code || 1); + continue; + } + + const logTxt = await fsp.readFile(logPath, "utf8").catch(() => ""); + const { passed, total } = parsePlaywrightCounts(logTxt); + aggPassed += passed; + aggTotal += total; + + if (res.code !== 0) { + const headline = parseFirstFailureHeadline(logTxt); + const what = headline ? `${pkg.name}: ${headline}` : `${pkg.name}: exit ${res.code}`; + console.log( + `SMOKE FAIL: ${truncateOneLine(what)} | Artifacts: ${artifactsDir}`, + ); + process.exit(res.code || 1); + } + } finally { + await new Promise((resolve) => logStream.end(resolve)); + } +} + +if (SMOKE) { + const elapsedS = (Date.now() - startedAt) / 1000; + const ratio = aggTotal > 0 ? `${aggPassed}/${aggTotal}` : "all"; + const warnSuffix = elapsedS * 1000 > warnAfterMs ? " (warn: slow smoke)" : ""; + console.log(`SMOKE: ${ratio} passed in ${elapsedS.toFixed(1)}s${warnSuffix}`); +} + diff --git a/packages/ai-tests/package.json b/packages/ai-tests/package.json new file mode 100644 index 0000000..5d58109 --- /dev/null +++ b/packages/ai-tests/package.json @@ -0,0 +1,21 @@ +{ + "name": "ai-tests", + "version": "0.0.0", + "description": "Universal test runner (unit/e2e/scenario/smoke/human) with stable artifacts.", + "type": "module", + "license": "MIT", + "bin": { + "ai-tests": "./bin/ai-tests.mjs" + }, + "exports": { + "./human": "./src/human.mjs" + }, + "files": [ + "bin", + "src" + ], + "dependencies": { + "yargs": "^18.0.0" + } +} + diff --git a/packages/ai-tests/src/human.mjs b/packages/ai-tests/src/human.mjs new file mode 100644 index 0000000..2f155fc --- /dev/null +++ b/packages/ai-tests/src/human.mjs @@ -0,0 +1,9 @@ +export function isHumanMode() { + return process.env.AI_TEST_HUMAN === "1"; +} + +export async function breath(ms = 250) { + if (!isHumanMode()) return; + await new Promise((resolve) => setTimeout(resolve, ms)); +} + From 878707280854134e62f1b41a2301f94635e75f03 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 14:15:32 +0000 Subject: [PATCH 6/7] test: migrate repo to ai-tests runner and file-based projects Co-authored-by: Alexander Nazarov --- .github/workflows/ci.yml | 2 +- package.json | 16 +- playwright.config.ts | 72 +-- pnpm-lock.yaml | 71 +++ scripts/test-suite.mjs | 480 ------------------ tests/{echo.spec.ts => echo.e2e.ts} | 3 +- tests/helpers/human.ts | 9 - ...emo.spec.ts => react-demo.scenario.e2e.ts} | 7 +- ...eenshot.spec.ts => screenshot.docs.e2e.ts} | 11 +- .../{server-api.spec.ts => server-api.e2e.ts} | 2 +- tests/{video.spec.ts => video.docs.e2e.ts} | 3 +- tests/{zombie.spec.ts => zombie.e2e.ts} | 3 +- 12 files changed, 111 insertions(+), 568 deletions(-) delete mode 100644 scripts/test-suite.mjs rename tests/{echo.spec.ts => echo.e2e.ts} (95%) delete mode 100644 tests/helpers/human.ts rename tests/{react-demo.spec.ts => react-demo.scenario.e2e.ts} (96%) rename tests/{screenshot.spec.ts => screenshot.docs.e2e.ts} (96%) rename tests/{server-api.spec.ts => server-api.e2e.ts} (99%) rename tests/{video.spec.ts => video.docs.e2e.ts} (98%) rename tests/{zombie.spec.ts => zombie.e2e.ts} (98%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818f402..31e3edd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: run: JABTERM_SKIP_BUILD=1 pnpm coverage:ci - name: Run docs scenario tests (docs assets) - run: JABTERM_SCENARIO_ONLY_DOCS=1 JABTERM_SKIP_BUILD=1 pnpm run test:scenario + run: pnpm run test:docs - name: Upload coverage artifacts if: always() diff --git a/package.json b/package.json index 8e8bb30..12db0e7 100644 --- a/package.json +++ b/package.json @@ -25,14 +25,15 @@ "scripts": { "build": "tsc -p tsconfig.server.json && tsc -p tsconfig.react.json", "prepack": "pnpm build", - "test": "node scripts/test-suite.mjs all", - "test:unit": "node scripts/test-suite.mjs unit", - "test:e2e": "node scripts/test-suite.mjs e2e", - "test:scenario": "node scripts/test-suite.mjs scenario", - "test:smoke": "node scripts/test-suite.mjs smoke", - "test:integration": "node scripts/test-suite.mjs integration", + "test": "pnpm run build && ai-tests all", + "test:unit": "ai-tests unit", + "test:e2e": "pnpm run build && ai-tests e2e", + "test:scenario": "pnpm run build && ai-tests scenario", + "test:smoke": "pnpm run build && ai-tests scenario --smoke", + "test:integration": "node -e \"console.log('No integration tests in this repo. (intentional)')\"", + "test:docs": "pnpm run build && ai-tests e2e --project docs", "test:client": "pnpm run test:unit", - "test:server:coverage": "rm -rf .cache/v8-coverage && NODE_V8_COVERAGE=.cache/v8-coverage pnpm run test:scenario && NODE_V8_COVERAGE=.cache/v8-coverage pnpm run test:e2e", + "test:server:coverage": "rm -rf .cache/v8-coverage && pnpm run build && NODE_V8_COVERAGE=.cache/v8-coverage ai-tests scenario && NODE_V8_COVERAGE=.cache/v8-coverage ai-tests e2e", "test:client:coverage": "rm -rf .cache/coverage/client && pnpm exec vitest run --coverage", "coverage:server:report": "rm -rf .cache/coverage/server && pnpm exec c8 report --temp-directory .cache/v8-coverage --reporter=text-summary --reporter=json-summary --report-dir .cache/coverage/server --all --src src/server --exclude \"**/playwright.config.ts\" --exclude \"tests/**\" --exclude \"src/react/**\" && node scripts/normalize-coverage-summary.mjs .cache/coverage/server/coverage-summary.json", "coverage:merge": "node scripts/merge-coverage-summaries.mjs .cache/coverage/server/coverage-summary.json .cache/coverage/client/coverage-summary.json coverage/coverage-summary.json", @@ -64,6 +65,7 @@ "@testing-library/jest-dom": "^6.9.0", "@testing-library/react": "^16.3.0", "@vitest/coverage-v8": "^3.2.4", + "ai-tests": "file:packages/ai-tests", "c8": "^10.1.3", "jsdom": "^26.1.0", "react": "^19.2.0", diff --git a/playwright.config.ts b/playwright.config.ts index fbdd8a3..d11e3d3 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,73 +1,25 @@ import { defineConfig } from "@playwright/test"; -import path from "node:path"; const PORT = parseInt(process.env.JABTERM_PORT || "3223", 10); -const ARTIFACTS_DIR = process.env.TEST_ARTIFACTS_DIR; -const IS_SMOKE = process.env.JABTERM_SMOKE === "1"; -const HUMAN_SLOWMO_MS = parseInt(process.env.JABTERM_HUMAN_SLOWMO_MS || "0", 10); +const DEMO_PORT = PORT + 1; -const outputDir = ARTIFACTS_DIR - ? path.join(ARTIFACTS_DIR, "test-results") - : ".cache/test-results"; -const reportDir = ARTIFACTS_DIR - ? path.join(ARTIFACTS_DIR, "report") - : ".cache/report"; - -const smokePerTestTimeoutMs = parseInt( - process.env.SMOKE_PER_TEST_TIMEOUT_MS || "30000", - 10, -); -const smokeTotalTimeoutMs = parseInt( - process.env.SMOKE_TOTAL_TIMEOUT_MS || "180000", - 10, -); +const E2E = /.*\.e2e\.(ts|js)x?$/; +const SCENARIO = /.*\.scenario\.e2e\.(ts|js)x?$/; +const DOCS = /.*\.docs\.e2e\.(ts|js)x?$/; export default defineConfig({ testDir: "./tests", - // Keep Playwright focused on E2E specs. Vitest unit tests live under `tests/unit/**` - // and use the `.test.*` suffix, which Playwright would otherwise pick up by default. - testMatch: ["**/*.spec.ts", "**/*.spec.tsx"], - outputDir, - ...(IS_SMOKE ? { timeout: smokePerTestTimeoutMs } : {}), - ...(IS_SMOKE ? { globalTimeout: smokeTotalTimeoutMs } : {}), fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 1 : 0, workers: 1, - reporter: IS_SMOKE - ? [ - [ - "json", - { - outputFile: - process.env.PW_JSON_OUTPUT_FILE || - (ARTIFACTS_DIR - ? path.join(ARTIFACTS_DIR, "playwright-report.json") - : ".cache/playwright-report.json"), - }, - ], - ] - : [["html", { outputFolder: reportDir, open: "never" }]], + use: { + baseURL: `http://127.0.0.1:${DEMO_PORT}`, + }, projects: [ - { - name: "default", - use: { - baseURL: `http://127.0.0.1:${PORT + 1}`, - }, - }, - { - name: "human", - use: { - baseURL: `http://127.0.0.1:${PORT + 1}`, - headless: false, - video: "on", - trace: "on", - screenshot: "on", - ...(HUMAN_SLOWMO_MS > 0 - ? { launchOptions: { slowMo: HUMAN_SLOWMO_MS } } - : {}), - }, - }, + { name: "e2e", testMatch: E2E, testIgnore: [SCENARIO, DOCS] }, + { name: "scenario", testMatch: SCENARIO, testIgnore: [DOCS] }, + { name: "docs", testMatch: DOCS }, ], webServer: [ { @@ -86,13 +38,13 @@ export default defineConfig({ }, { command: `node tests/serve-demo.mjs`, - port: PORT + 1, + port: DEMO_PORT, reuseExistingServer: !process.env.CI, timeout: 10_000, stdout: "pipe", stderr: "pipe", env: { - DEMO_PORT: String(PORT + 1), + DEMO_PORT: String(DEMO_PORT), JABTERM_WS_PORT: String(PORT), ...(process.env.NODE_V8_COVERAGE ? { NODE_V8_COVERAGE: process.env.NODE_V8_COVERAGE } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 58e9154..e7f51c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: '@vitest/coverage-v8': specifier: ^3.2.4 version: 3.2.4(vitest@3.2.4(@types/node@22.19.11)(jsdom@26.1.0)) + ai-tests: + specifier: file:packages/ai-tests + version: file:packages/ai-tests c8: specifier: ^10.1.3 version: 10.1.3 @@ -544,6 +547,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + ai-tests@file:packages/ai-tests: + resolution: {directory: packages/ai-tests, type: directory} + hasBin: true + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -610,6 +617,10 @@ packages: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -667,6 +678,9 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} @@ -727,6 +741,10 @@ packages: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} + get-east-asian-width@1.5.0: + resolution: {integrity: sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==} + engines: {node: '>=18'} + glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me @@ -997,6 +1015,10 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -1184,6 +1206,10 @@ packages: resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} engines: {node: '>=12'} + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + ws@8.19.0: resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} engines: {node: '>=10.0.0'} @@ -1211,10 +1237,18 @@ packages: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yargs@18.0.0: + resolution: {integrity: sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -1590,6 +1624,10 @@ snapshots: agent-base@7.1.4: {} + ai-tests@file:packages/ai-tests: + dependencies: + yargs: 18.0.0 + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -1654,6 +1692,12 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -1698,6 +1742,8 @@ snapshots: eastasianwidth@0.2.0: {} + emoji-regex@10.6.0: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} @@ -1765,6 +1811,8 @@ snapshots: get-caller-file@2.0.5: {} + get-east-asian-width@1.5.0: {} + glob@10.5.0: dependencies: foreground-child: 3.3.1 @@ -2052,6 +2100,12 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.1.2 + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.5.0 + strip-ansi: 7.1.2 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -2232,6 +2286,12 @@ snapshots: string-width: 5.1.2 strip-ansi: 7.1.2 + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.1.2 + ws@8.19.0: {} xml-name-validator@5.0.0: {} @@ -2242,6 +2302,8 @@ snapshots: yargs-parser@21.1.1: {} + yargs-parser@22.0.0: {} + yargs@17.7.2: dependencies: cliui: 8.0.1 @@ -2252,4 +2314,13 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yargs@18.0.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 7.2.0 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yocto-queue@0.1.0: {} diff --git a/scripts/test-suite.mjs b/scripts/test-suite.mjs deleted file mode 100644 index f9d62b4..0000000 --- a/scripts/test-suite.mjs +++ /dev/null @@ -1,480 +0,0 @@ -import fs from "node:fs"; -import fsp from "node:fs/promises"; -import path from "node:path"; -import { spawn } from "node:child_process"; - -const ROOT = process.cwd(); - -function getPackageManager() { - try { - const pkgPath = path.join(ROOT, "package.json"); - const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); - const pm = String(pkg?.packageManager || ""); - if (pm.startsWith("pnpm@")) return "pnpm"; - if (pm.startsWith("npm@")) return "npm"; - } catch { - /* ignore */ - } - - if (fs.existsSync(path.join(ROOT, "pnpm-lock.yaml"))) return "pnpm"; - if (fs.existsSync(path.join(ROOT, "package-lock.json"))) return "npm"; - return "npm"; -} - -const PM = getPackageManager(); - -function artifactsDirFor(suite) { - switch (suite) { - case "unit": - return path.join(ROOT, ".cache/tests/test-unit__vitest"); - case "scenario": - return path.join(ROOT, ".cache/tests/test-scenario__playwright"); - case "e2e": - return path.join(ROOT, ".cache/tests/test-e2e__playwright"); - case "integration": - return path.join(ROOT, ".cache/tests/test-integration__none"); - case "smoke": - return path.join(ROOT, ".cache/tests/test-smoke__scenario"); - case "all": - return path.join(ROOT, ".cache/tests/test__all"); - default: - throw new Error(`Unknown suite: ${suite}`); - } -} - -async function rmrf(p) { - await fsp.rm(p, { recursive: true, force: true }); -} - -async function mkdirp(p) { - await fsp.mkdir(p, { recursive: true }); -} - -function spawnLogged(cmd, args, { env, cwd, logStream, passthrough }) { - return new Promise((resolve) => { - const child = spawn(cmd, args, { - cwd, - env, - stdio: ["ignore", "pipe", "pipe"], - }); - - const onOut = (chunk) => { - logStream.write(chunk); - if (passthrough) process.stdout.write(chunk); - }; - const onErr = (chunk) => { - logStream.write(chunk); - if (passthrough) process.stderr.write(chunk); - }; - - child.stdout.on("data", onOut); - child.stderr.on("data", onErr); - - child.on("close", (code, signal) => resolve({ code, signal, child })); - }); -} - -function spawnLoggedWithTimeout(cmd, args, { env, cwd, logStream, timeoutMs }) { - return new Promise((resolve) => { - const child = spawn(cmd, args, { - cwd, - env, - stdio: ["ignore", "pipe", "pipe"], - }); - - child.stdout.on("data", (chunk) => logStream.write(chunk)); - child.stderr.on("data", (chunk) => logStream.write(chunk)); - - let timedOut = false; - const timer = setTimeout(() => { - timedOut = true; - try { - child.kill("SIGTERM"); - } catch { - /* ignore */ - } - setTimeout(() => { - try { - child.kill("SIGKILL"); - } catch { - /* ignore */ - } - }, 1500).unref?.(); - }, timeoutMs); - timer.unref?.(); - - child.on("close", (code, signal) => { - clearTimeout(timer); - resolve({ code, signal, timedOut }); - }); - }); -} - -function envWithArtifacts(artifactsDir, extraEnv = {}) { - return { - ...process.env, - ...extraEnv, - TEST_ARTIFACTS_DIR: artifactsDir, - }; -} - -async function runUnit({ artifactsDir, logStream }) { - const env = envWithArtifacts(artifactsDir); - return await spawnLogged(PM, ["exec", "vitest", "run"], { - cwd: ROOT, - env, - logStream, - passthrough: true, - }); -} - -async function runPlaywright({ - artifactsDir, - logStream, - grep, - grepInvert = [], - smoke, - human, -}) { - const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; - - const env = envWithArtifacts( - artifactsDir, - { - ...(smoke ? { JABTERM_SMOKE: "1" } : {}), - ...(human ? { JABTERM_HUMAN: "1" } : {}), - }, - ); - - if (!skipBuild) { - const build = await spawnLogged(PM, ["run", "build"], { - cwd: ROOT, - env, - logStream, - passthrough: true, - }); - if (build.code !== 0) return { code: build.code, signal: build.signal }; - } - - const args = ["exec", "playwright", "test", "--grep", grep]; - for (const inv of grepInvert) args.push("--grep-invert", inv); - args.push("--project", human ? "human" : "default"); - if (smoke) args.push("--max-failures=1", "--quiet"); - - return await spawnLogged(PM, args, { - cwd: ROOT, - env, - logStream, - passthrough: !smoke, - }); -} - -function truncateOneLine(s, max = 160) { - const one = String(s || "").replace(/\s+/g, " ").trim(); - if (one.length <= max) return one; - return `${one.slice(0, max - 1)}…`; -} - -function collectTestsFromSuites(suites, out) { - for (const s of suites || []) { - if (s.tests) { - for (const t of s.tests) out.push(t); - } - if (s.suites) collectTestsFromSuites(s.suites, out); - } -} - -function getFinalStatus(test) { - const results = test?.results || []; - for (const r of results) { - if (r.status === "failed" || r.status === "timedOut") return "failed"; - } - for (const r of results) { - if (r.status === "passed") return "passed"; - } - for (const r of results) { - if (r.status === "skipped") return "skipped"; - } - return "unknown"; -} - -function scenarioFiltersFromEnv() { - const onlyDocs = process.env.JABTERM_SCENARIO_ONLY_DOCS === "1"; - const includeDocs = process.env.JABTERM_SCENARIO_INCLUDE_DOCS === "1"; - const includeSlow = process.env.JABTERM_SCENARIO_INCLUDE_SLOW === "1"; - - if (onlyDocs) { - return { grep: "@docs", grepInvert: [] }; - } - - const grepInvert = []; - if (!includeDocs) grepInvert.push("@docs"); - if (!includeSlow) grepInvert.push("@slow"); - return { grep: "@scenario", grepInvert }; -} - -async function runSmoke({ artifactsDir, logStream }) { - const startedAt = Date.now(); - const perTestTimeoutMs = parseInt( - process.env.SMOKE_PER_TEST_TIMEOUT_MS || "30000", - 10, - ); - const totalTimeoutMs = parseInt( - process.env.SMOKE_TOTAL_TIMEOUT_MS || "180000", - 10, - ); - const warnAfterMs = parseInt(process.env.SMOKE_WARN_AFTER_MS || "60000", 10); - - const jsonReport = path.join(artifactsDir, "playwright-report.json"); - const env = envWithArtifacts(artifactsDir, { - JABTERM_SMOKE: "1", - SMOKE_PER_TEST_TIMEOUT_MS: String(perTestTimeoutMs), - SMOKE_TOTAL_TIMEOUT_MS: String(totalTimeoutMs), - PW_JSON_OUTPUT_FILE: jsonReport, - }); - - const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; - if (!skipBuild) { - const build = await spawnLogged(PM, ["run", "build"], { - cwd: ROOT, - env, - logStream, - passthrough: false, - }); - if (build.code !== 0) { - console.log( - `SMOKE FAIL: build (exit ${build.code ?? "?"}) | Artifacts: ${artifactsDir}`, - ); - process.exit(build.code || 1); - } - } - - const warn = setTimeout(() => { - /* marker only; we keep success output to one line */ - }, warnAfterMs); - warn.unref?.(); - - const smokeArgs = [ - "exec", - "playwright", - "test", - "--grep", - "@scenario", - "--grep-invert", - "@docs", - "--grep-invert", - "@slow", - "--project", - process.env.JABTERM_HUMAN === "1" ? "human" : "default", - "--max-failures=1", - "--quiet", - ]; - - const pw = await spawnLoggedWithTimeout(PM, smokeArgs, { - cwd: ROOT, - env, - logStream, - timeoutMs: totalTimeoutMs, - }); - clearTimeout(warn); - - const elapsedS = (Date.now() - startedAt) / 1000; - const slow = elapsedS * 1000 > warnAfterMs; - - if (pw.timedOut) { - console.log( - `SMOKE FAIL: scenario (timeout after ${Math.round(elapsedS)}s) | Artifacts: ${artifactsDir}`, - ); - process.exit(1); - } - - if (pw.code === 0) { - let passed = 0; - let total = 0; - try { - const raw = await fsp.readFile(jsonReport, "utf8"); - const report = JSON.parse(raw); - const tests = []; - collectTestsFromSuites(report?.suites, tests); - total = tests.length; - for (const t of tests) { - if (getFinalStatus(t) === "passed") passed++; - } - } catch { - // best-effort; keep smoke output stable even if JSON isn't available - } - - const ratio = total > 0 ? `${passed}/${total}` : "all"; - const warnSuffix = slow ? " (warn: exceeded SMOKE_WARN_AFTER_MS)" : ""; - console.log(`SMOKE: ${ratio} passed in ${elapsedS.toFixed(1)}s${warnSuffix}`); - process.exit(0); - } - - // Failure - let failName = "scenario"; - let reason = `exit ${pw.code ?? "?"}`; - try { - const raw = await fsp.readFile(jsonReport, "utf8"); - const report = JSON.parse(raw); - const tests = []; - collectTestsFromSuites(report?.suites, tests); - const firstFail = tests.find((t) => getFinalStatus(t) === "failed"); - if (firstFail) { - failName = firstFail.title ? String(firstFail.title) : failName; - const lastResult = (firstFail.results || []).at(-1); - const err = lastResult?.error?.message || lastResult?.error?.value; - if (err) reason = truncateOneLine(err); - } - } catch { - /* ignore */ - } - - console.log( - `SMOKE FAIL: ${truncateOneLine(failName, 80)} (${reason}) | Artifacts: ${artifactsDir}`, - ); - process.exit(pw.code || 1); -} - -async function main() { - const args = process.argv.slice(2); - const suite = args[0]; - if (!suite) { - throw new Error( - "Usage: node scripts/test-suite.mjs [--human]", - ); - } - - const human = args.includes("--human"); - - const artifactsDir = artifactsDirFor(suite); - await rmrf(artifactsDir); - await mkdirp(artifactsDir); - - const logPath = path.join(artifactsDir, "run.log"); - const logStream = fs.createWriteStream(logPath, { flags: "a" }); - - try { - if (suite === "smoke") { - if (human) process.env.JABTERM_HUMAN = "1"; - await runSmoke({ artifactsDir, logStream }); - return; - } - - if (suite === "unit") { - const res = await runUnit({ artifactsDir, logStream }); - process.exit(res.code || 0); - } - - if (suite === "scenario") { - const { grep, grepInvert } = scenarioFiltersFromEnv(); - const res = await runPlaywright({ - artifactsDir, - logStream, - grep, - grepInvert, - smoke: false, - human, - }); - process.exit(res.code || 0); - } - - if (suite === "e2e") { - const res = await runPlaywright({ - artifactsDir, - logStream, - grep: "@e2e", - smoke: false, - human, - }); - process.exit(res.code || 0); - } - - if (suite === "integration") { - logStream.write( - Buffer.from( - "This repo intentionally has no integration tests. (contract script exists)\n", - ), - ); - process.stdout.write( - "No integration tests in this repo. (intentional)\n", - ); - process.exit(0); - } - - if (suite === "all") { - // Unit first for fast feedback. - const unit = await spawnLogged(PM, ["exec", "vitest", "run"], { - cwd: ROOT, - env: envWithArtifacts(path.join(artifactsDir, "unit")), - logStream, - passthrough: true, - }); - if (unit.code !== 0) process.exit(unit.code || 1); - - const skipBuild = process.env.JABTERM_SKIP_BUILD === "1"; - if (!skipBuild) { - const build = await spawnLogged(PM, ["run", "build"], { - cwd: ROOT, - env: envWithArtifacts(artifactsDir), - logStream, - passthrough: true, - }); - if (build.code !== 0) process.exit(build.code || 1); - } - - const pwEnvBase = { - ...process.env, - JABTERM_SKIP_BUILD: "1", - }; - - const scenarioDir = path.join(artifactsDir, "scenario"); - await rmrf(scenarioDir); - await mkdirp(scenarioDir); - const scenario = await spawnLogged( - PM, - [ - "exec", - "playwright", - "test", - "--grep", - "@scenario", - "--grep-invert", - "@docs", - "--grep-invert", - "@slow", - "--project", - "default", - ], - { - cwd: ROOT, - env: envWithArtifacts(scenarioDir, pwEnvBase), - logStream, - passthrough: true, - }, - ); - if (scenario.code !== 0) process.exit(scenario.code || 1); - - const e2eDir = path.join(artifactsDir, "e2e"); - await rmrf(e2eDir); - await mkdirp(e2eDir); - const e2e = await spawnLogged(PM, ["exec", "playwright", "test", "--grep", "@e2e", "--project", "default"], { - cwd: ROOT, - env: envWithArtifacts(e2eDir, pwEnvBase), - logStream, - passthrough: true, - }); - if (e2e.code !== 0) process.exit(e2e.code || 1); - - process.exit(0); - } - - process.exit(1); - } finally { - await new Promise((resolve) => logStream.end(resolve)); - } -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/tests/echo.spec.ts b/tests/echo.e2e.ts similarity index 95% rename from tests/echo.spec.ts rename to tests/echo.e2e.ts index 2dfc8dc..fda620c 100644 --- a/tests/echo.spec.ts +++ b/tests/echo.e2e.ts @@ -18,7 +18,7 @@ const WS_URL = defaultWsUrl(); test.describe.configure({ mode: "serial" }); -test.describe("@e2e Terminal — echo round-trip (WS protocol)", () => { +test.describe("Terminal — echo round-trip (WS protocol)", () => { test("echo command returns output", async () => { let ws!: WsClient; try { @@ -59,3 +59,4 @@ test.describe("@e2e Terminal — echo round-trip (WS protocol)", () => { expect(result).toBe("open"); }); }); + diff --git a/tests/helpers/human.ts b/tests/helpers/human.ts deleted file mode 100644 index 8c35f52..0000000 --- a/tests/helpers/human.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function isHumanMode(): boolean { - return process.env.JABTERM_HUMAN === "1"; -} - -export async function breath(ms = 350): Promise { - if (!isHumanMode()) return; - await new Promise((resolve) => setTimeout(resolve, ms)); -} - diff --git a/tests/react-demo.spec.ts b/tests/react-demo.scenario.e2e.ts similarity index 96% rename from tests/react-demo.spec.ts rename to tests/react-demo.scenario.e2e.ts index 4e94cc8..1e38b24 100644 --- a/tests/react-demo.spec.ts +++ b/tests/react-demo.scenario.e2e.ts @@ -1,14 +1,14 @@ /** - * React demo smoke test. + * React demo scenario. * * Ensures the demo page exercises `jabterm/react` (mount/unmount, layout resize, * and unexpected close UI) rather than a plain xterm CDN implementation. */ import { test, expect } from "@playwright/test"; -import { breath } from "./helpers/human.js"; +import { breath } from "ai-tests/human"; -test.describe("@scenario React demo page", () => { +test.describe("React demo page", () => { test("mount/unmount and layout resize work", async ({ page }) => { const consoleErrors: string[] = []; page.on("pageerror", (err) => consoleErrors.push(err.message)); @@ -71,7 +71,6 @@ test.describe("@scenario React demo page", () => { expect(after).not.toBeNull(); expect(Math.abs(after!.width - before!.width)).toBeGreaterThan(20); - expect(consoleErrors).toEqual([]); }); diff --git a/tests/screenshot.spec.ts b/tests/screenshot.docs.e2e.ts similarity index 96% rename from tests/screenshot.spec.ts rename to tests/screenshot.docs.e2e.ts index 1951f97..45d58f0 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.docs.e2e.ts @@ -16,7 +16,7 @@ test.beforeAll(() => { fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true }); }); -test.describe("@scenario @docs @slow Terminal — README screenshots", () => { +test.describe("Terminal — README screenshots", () => { test("capture echo terminal screenshot", async ({ page }) => { await page.goto("/"); @@ -87,13 +87,17 @@ test.describe("@scenario @docs @slow Terminal — README screenshots", () => { // Type in terminal 1 await term1.click(); - await page.keyboard.type('echo "Terminal 1 — Hello from JabTerm"', { delay: 20 }); + await page.keyboard.type('echo "Terminal 1 — Hello from JabTerm"', { + delay: 20, + }); await page.keyboard.press("Enter"); await page.waitForTimeout(1000); // Type in terminal 2 await term2.click(); - await page.keyboard.type('echo "Terminal 2 — Independent session"', { delay: 20 }); + await page.keyboard.type('echo "Terminal 2 — Independent session"', { + delay: 20, + }); await page.keyboard.press("Enter"); await page.waitForTimeout(1000); @@ -103,3 +107,4 @@ test.describe("@scenario @docs @slow Terminal — README screenshots", () => { }); }); }); + diff --git a/tests/server-api.spec.ts b/tests/server-api.e2e.ts similarity index 99% rename from tests/server-api.spec.ts rename to tests/server-api.e2e.ts index 174949e..6462f2e 100644 --- a/tests/server-api.spec.ts +++ b/tests/server-api.e2e.ts @@ -57,7 +57,7 @@ async function waitForMatch( test.describe.configure({ mode: "serial" }); -test.describe("@e2e Server API — createJabtermServer", () => { +test.describe("Server API — createJabtermServer", () => { test("supports ephemeral port and deterministic shutdown", async () => { const server = createJabtermServer({ host: "127.0.0.1", diff --git a/tests/video.spec.ts b/tests/video.docs.e2e.ts similarity index 98% rename from tests/video.spec.ts rename to tests/video.docs.e2e.ts index 4a04fa7..7000ff5 100644 --- a/tests/video.spec.ts +++ b/tests/video.docs.e2e.ts @@ -80,7 +80,7 @@ test.use({ }, }); -test.describe("@scenario @docs @slow Terminal - demo video", () => { +test.describe("Terminal - demo video", () => { test("records terminal usage flow", async ({ page }) => { test.setTimeout(120_000); @@ -161,3 +161,4 @@ test.describe("@scenario @docs @slow Terminal - demo video", () => { expect(gifSize).toBeGreaterThan(0); }); }); + diff --git a/tests/zombie.spec.ts b/tests/zombie.e2e.ts similarity index 98% rename from tests/zombie.spec.ts rename to tests/zombie.e2e.ts index 53b730b..d7256c9 100644 --- a/tests/zombie.spec.ts +++ b/tests/zombie.e2e.ts @@ -14,7 +14,7 @@ const CLEANUP_DELAY_MS = 1500; test.describe.configure({ mode: "serial" }); -test.describe("@e2e Terminal — zombie process prevention", () => { +test.describe("Terminal — zombie process prevention", () => { test("pty process is killed when websocket closes", async () => { const ws = new WsClient(WS_URL); @@ -139,3 +139,4 @@ test.describe("@e2e Terminal — zombie process prevention", () => { } }); }); + From c4fba25252a7f13ea8d51cab285b5e3cc89c9b54 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 14:18:13 +0000 Subject: [PATCH 7/7] chore: rename ai-tests to test-runner Co-authored-by: Alexander Nazarov --- package.json | 16 +++++++------- packages/{ai-tests => test-runner}/README.md | 8 +++---- .../bin/test-runner.mjs} | 4 ++-- .../{ai-tests => test-runner}/package.json | 4 ++-- .../{ai-tests => test-runner}/src/human.mjs | 2 +- pnpm-lock.yaml | 22 +++++++++---------- tests/react-demo.scenario.e2e.ts | 2 +- 7 files changed, 29 insertions(+), 29 deletions(-) rename packages/{ai-tests => test-runner}/README.md (87%) rename packages/{ai-tests/bin/ai-tests.mjs => test-runner/bin/test-runner.mjs} (99%) rename packages/{ai-tests => test-runner}/package.json (82%) rename packages/{ai-tests => test-runner}/src/human.mjs (77%) diff --git a/package.json b/package.json index 12db0e7..7a46c75 100644 --- a/package.json +++ b/package.json @@ -25,15 +25,15 @@ "scripts": { "build": "tsc -p tsconfig.server.json && tsc -p tsconfig.react.json", "prepack": "pnpm build", - "test": "pnpm run build && ai-tests all", - "test:unit": "ai-tests unit", - "test:e2e": "pnpm run build && ai-tests e2e", - "test:scenario": "pnpm run build && ai-tests scenario", - "test:smoke": "pnpm run build && ai-tests scenario --smoke", + "test": "pnpm run build && test-runner all", + "test:unit": "test-runner unit", + "test:e2e": "pnpm run build && test-runner e2e", + "test:scenario": "pnpm run build && test-runner scenario", + "test:smoke": "pnpm run build && test-runner scenario --smoke", "test:integration": "node -e \"console.log('No integration tests in this repo. (intentional)')\"", - "test:docs": "pnpm run build && ai-tests e2e --project docs", + "test:docs": "pnpm run build && test-runner e2e --project docs", "test:client": "pnpm run test:unit", - "test:server:coverage": "rm -rf .cache/v8-coverage && pnpm run build && NODE_V8_COVERAGE=.cache/v8-coverage ai-tests scenario && NODE_V8_COVERAGE=.cache/v8-coverage ai-tests e2e", + "test:server:coverage": "rm -rf .cache/v8-coverage && pnpm run build && NODE_V8_COVERAGE=.cache/v8-coverage test-runner scenario && NODE_V8_COVERAGE=.cache/v8-coverage test-runner e2e", "test:client:coverage": "rm -rf .cache/coverage/client && pnpm exec vitest run --coverage", "coverage:server:report": "rm -rf .cache/coverage/server && pnpm exec c8 report --temp-directory .cache/v8-coverage --reporter=text-summary --reporter=json-summary --report-dir .cache/coverage/server --all --src src/server --exclude \"**/playwright.config.ts\" --exclude \"tests/**\" --exclude \"src/react/**\" && node scripts/normalize-coverage-summary.mjs .cache/coverage/server/coverage-summary.json", "coverage:merge": "node scripts/merge-coverage-summaries.mjs .cache/coverage/server/coverage-summary.json .cache/coverage/client/coverage-summary.json coverage/coverage-summary.json", @@ -65,7 +65,7 @@ "@testing-library/jest-dom": "^6.9.0", "@testing-library/react": "^16.3.0", "@vitest/coverage-v8": "^3.2.4", - "ai-tests": "file:packages/ai-tests", + "test-runner": "file:packages/test-runner", "c8": "^10.1.3", "jsdom": "^26.1.0", "react": "^19.2.0", diff --git a/packages/ai-tests/README.md b/packages/test-runner/README.md similarity index 87% rename from packages/ai-tests/README.md rename to packages/test-runner/README.md index 97e2e55..5fd0ced 100644 --- a/packages/ai-tests/README.md +++ b/packages/test-runner/README.md @@ -1,11 +1,11 @@ -# ai-tests +# test-runner Small, policy-light test runner intended to be copied between repos (or published). ## CLI ```bash -ai-tests [--smoke] [--human] [--pkg ...] [--project ] [--allow-missing-project] +test-runner [--smoke] [--human] [--pkg ...] [--project ] [--allow-missing-project] ``` Suites: @@ -27,7 +27,7 @@ Smoke rules (`--smoke`) apply to Playwright suites only: Human execution (`--human`) is orthogonal: - Playwright: `--headed`, `--workers=1`, `--trace=on` -- Exposes `AI_TEST_HUMAN=1` for test utilities (see `ai-tests/human`) +- Exposes `TEST_RUNNER_HUMAN=1` for test utilities (see `test-runner/human`) ## Artifacts @@ -62,7 +62,7 @@ export default defineConfig({ ## Test helper: `breath()` ```js -import { breath } from "ai-tests/human"; +import { breath } from "test-runner/human"; await breath(300); ``` diff --git a/packages/ai-tests/bin/ai-tests.mjs b/packages/test-runner/bin/test-runner.mjs similarity index 99% rename from packages/ai-tests/bin/ai-tests.mjs rename to packages/test-runner/bin/test-runner.mjs index 8ab271c..f198abe 100755 --- a/packages/ai-tests/bin/ai-tests.mjs +++ b/packages/test-runner/bin/test-runner.mjs @@ -205,7 +205,7 @@ async function runPlaywright({ const env = { ...process.env, TEST_ARTIFACTS_DIR: artifactsDir, - ...(human ? { AI_TEST_HUMAN: "1" } : {}), + ...(human ? { TEST_RUNNER_HUMAN: "1" } : {}), }; const pwOut = path.join(artifactsDir, "pw-output"); @@ -249,7 +249,7 @@ async function runPlaywright({ } const argv = await yargs(hideBin(process.argv)) - .scriptName("ai-tests") + .scriptName("test-runner") .command( "$0 ", "Run tests", diff --git a/packages/ai-tests/package.json b/packages/test-runner/package.json similarity index 82% rename from packages/ai-tests/package.json rename to packages/test-runner/package.json index 5d58109..e47b290 100644 --- a/packages/ai-tests/package.json +++ b/packages/test-runner/package.json @@ -1,11 +1,11 @@ { - "name": "ai-tests", + "name": "test-runner", "version": "0.0.0", "description": "Universal test runner (unit/e2e/scenario/smoke/human) with stable artifacts.", "type": "module", "license": "MIT", "bin": { - "ai-tests": "./bin/ai-tests.mjs" + "test-runner": "./bin/test-runner.mjs" }, "exports": { "./human": "./src/human.mjs" diff --git a/packages/ai-tests/src/human.mjs b/packages/test-runner/src/human.mjs similarity index 77% rename from packages/ai-tests/src/human.mjs rename to packages/test-runner/src/human.mjs index 2f155fc..c105c92 100644 --- a/packages/ai-tests/src/human.mjs +++ b/packages/test-runner/src/human.mjs @@ -1,5 +1,5 @@ export function isHumanMode() { - return process.env.AI_TEST_HUMAN === "1"; + return process.env.TEST_RUNNER_HUMAN === "1"; } export async function breath(ms = 250) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e7f51c4..85742d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,9 +45,6 @@ importers: '@vitest/coverage-v8': specifier: ^3.2.4 version: 3.2.4(vitest@3.2.4(@types/node@22.19.11)(jsdom@26.1.0)) - ai-tests: - specifier: file:packages/ai-tests - version: file:packages/ai-tests c8: specifier: ^10.1.3 version: 10.1.3 @@ -60,6 +57,9 @@ importers: react-dom: specifier: ^19.2.0 version: 19.2.4(react@19.2.4) + test-runner: + specifier: file:packages/test-runner + version: file:packages/test-runner typescript: specifier: ^5 version: 5.9.3 @@ -547,10 +547,6 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} - ai-tests@file:packages/ai-tests: - resolution: {directory: packages/ai-tests, type: directory} - hasBin: true - ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -1045,6 +1041,10 @@ packages: resolution: {integrity: sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==} engines: {node: '>=18'} + test-runner@file:packages/test-runner: + resolution: {directory: packages/test-runner, type: directory} + hasBin: true + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -1624,10 +1624,6 @@ snapshots: agent-base@7.1.4: {} - ai-tests@file:packages/ai-tests: - dependencies: - yargs: 18.0.0 - ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -2134,6 +2130,10 @@ snapshots: glob: 10.5.0 minimatch: 9.0.5 + test-runner@file:packages/test-runner: + dependencies: + yargs: 18.0.0 + tinybench@2.9.0: {} tinyexec@0.3.2: {} diff --git a/tests/react-demo.scenario.e2e.ts b/tests/react-demo.scenario.e2e.ts index 1e38b24..1e48258 100644 --- a/tests/react-demo.scenario.e2e.ts +++ b/tests/react-demo.scenario.e2e.ts @@ -6,7 +6,7 @@ */ import { test, expect } from "@playwright/test"; -import { breath } from "ai-tests/human"; +import { breath } from "test-runner/human"; test.describe("React demo page", () => { test("mount/unmount and layout resize work", async ({ page }) => {