From 46d5aec83d532df4094dd2ff7422e62c2838016e Mon Sep 17 00:00:00 2001 From: Delicious233 <101502465+DeliciousBuding@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:19:27 +0800 Subject: [PATCH] test(chat): add visual qa artifact reports --- .../scripts/manual-chat-flow-check.mjs | 64 ++++++++++++-- app/web/scripts/manual-chat-flow-check.mjs | 67 ++++++++++++-- docs/progress/MASTER.md | 1 + scripts/verify/chat-acceptance.mjs | 12 ++- .../scripts/verify-chat-acceptance.mjs | 4 + .../scripts/verify-chat-visual-qa.mjs | 87 +++++++++++++++++++ 6 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 tests/contract/scripts/verify-chat-visual-qa.mjs diff --git a/app/desktop/scripts/manual-chat-flow-check.mjs b/app/desktop/scripts/manual-chat-flow-check.mjs index 1d168a9f..b005fb90 100644 --- a/app/desktop/scripts/manual-chat-flow-check.mjs +++ b/app/desktop/scripts/manual-chat-flow-check.mjs @@ -8,6 +8,8 @@ const ownsServer = !process.env.AGENTHUB_MANUAL_URL; const baseURL = process.env.AGENTHUB_MANUAL_URL ?? `http://127.0.0.1:${port}`; const outputDir = path.resolve(process.cwd(), '.tmp', 'manual-chat-flow-uiux'); const screenshot = path.join(outputDir, 'desktop-1440x810-chat-flow.png'); +const metricsPath = path.join(outputDir, 'desktop-1440x810-chat-flow.metrics.json'); +const reportPath = path.join(outputDir, 'desktop-chat-flow-visual-qa.json'); fs.mkdirSync(outputDir, { recursive: true }); @@ -125,6 +127,48 @@ function assertMetrics(result) { } } +function writeVisualQaReport({ status, viewport, metrics, failure }) { + const report = { + schema: 'agenthub.chat_visual_qa.v1', + surface: 'desktop', + status, + evidence_level: 'visual-qa', + real_tested: false, + generated_at: new Date().toISOString(), + baseURL, + viewport, + screenshot, + metricsPath, + reportPath, + metrics, + failure: failure ? { message: failure } : null, + inspection: { + screenshot, + metricsPath, + checks: [ + 'Confirm the transcript stays in chronological order.', + 'Confirm user bubbles do not disappear during optimistic send.', + 'Confirm approval and preview cards render as one clean stack.', + 'Confirm no mock, debug, or data-mode text appears inside the transcript.', + ], + }, + boundaries: { + real_tokendance_id_login: false, + real_cli_or_model_api: false, + packaged_desktop: false, + signing: false, + release_upload: false, + production_deploy: false, + }, + }; + fs.writeFileSync(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`); + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(`Visual QA screenshot: ${screenshot}`); + console.log(`Visual QA metrics: ${metricsPath}`); + console.log(`Visual QA report: ${reportPath}`); + return report; +} + let server = null; let browser = null; @@ -283,14 +327,22 @@ try { }; }, { firstMessage, secondMessage }); - const report = { - baseURL, + let failure = null; + try { + assertMetrics(result); + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + } + const report = writeVisualQaReport({ + status: failure ? 'failed' : 'passed', viewport: page.viewportSize(), - screenshot, - ...result, - }; + metrics: result, + failure, + }); console.log(JSON.stringify(report, null, 2)); - assertMetrics(result); + if (failure) { + throw new Error(failure); + } } finally { if (browser) await browser.close(); stopDevServer(server); diff --git a/app/web/scripts/manual-chat-flow-check.mjs b/app/web/scripts/manual-chat-flow-check.mjs index 29331fa0..2aaa960a 100644 --- a/app/web/scripts/manual-chat-flow-check.mjs +++ b/app/web/scripts/manual-chat-flow-check.mjs @@ -9,6 +9,8 @@ const baseURL = process.env.AGENTHUB_WEB_MANUAL_URL ?? `http://127.0.0.1:${port} const hubOrigin = 'http://localhost:8080'; const outputDir = path.resolve(process.cwd(), '.tmp', 'manual-chat-flow-uiux'); const screenshot = path.join(outputDir, 'web-1440x810-chat-flow.png'); +const metricsPath = path.join(outputDir, 'web-1440x810-chat-flow.metrics.json'); +const reportPath = path.join(outputDir, 'web-chat-flow-visual-qa.json'); const sessionId = 'session-web-manual-chat-flow'; const taskId = 'task-web-manual-chat-flow'; @@ -294,6 +296,49 @@ function assertMetrics(result) { } } +function writeVisualQaReport({ status, viewport, metrics, failure }) { + const report = { + schema: 'agenthub.chat_visual_qa.v1', + surface: 'web', + status, + evidence_level: 'visual-qa', + real_tested: false, + generated_at: new Date().toISOString(), + baseURL, + dataSource: 'stubbed-hub-session', + viewport, + screenshot, + metricsPath, + reportPath, + metrics, + failure: failure ? { message: failure } : null, + inspection: { + screenshot, + metricsPath, + checks: [ + 'Confirm transcript order is user, tool result, then agent reply.', + 'Confirm Markdown tables render in the transcript.', + 'Confirm subagent report details stay in the inspector.', + 'Confirm no mock, debug, or data-mode text appears inside the transcript.', + ], + }, + boundaries: { + real_tokendance_id_login: false, + real_cli_or_model_api: false, + packaged_desktop: false, + signing: false, + release_upload: false, + production_deploy: false, + }, + }; + fs.writeFileSync(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`); + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(`Visual QA screenshot: ${screenshot}`); + console.log(`Visual QA metrics: ${metricsPath}`); + console.log(`Visual QA report: ${reportPath}`); + return report; +} + let server = null; let browser = null; @@ -410,16 +455,22 @@ try { }; }); - const report = { - baseURL, + let failure = null; + try { + assertMetrics(result); + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + } + const report = writeVisualQaReport({ + status: failure ? 'failed' : 'passed', viewport: page.viewportSize(), - screenshot, - dataSource: 'stubbed-hub-session', - real_tested: false, - ...result, - }; + metrics: result, + failure, + }); console.log(JSON.stringify(report, null, 2)); - assertMetrics(result); + if (failure) { + throw new Error(failure); + } } finally { if (browser) await browser.close(); stopDevServer(server); diff --git a/docs/progress/MASTER.md b/docs/progress/MASTER.md index cf863577..156e86a6 100644 --- a/docs/progress/MASTER.md +++ b/docs/progress/MASTER.md @@ -116,3 +116,4 @@ Per-task telemetry is stored in GitHub issue comments before task closure. Adapt | 2026-06-29 | Phase 3 complete | #388 merged via #410 and closed manually because non-default base did not auto-close it; milestone #19 is closed at 3/3 with adaptive drift_score 2, so Phase 4 requires a lightweight checkpoint before T4.1. | | 2026-06-29 | T4.1 implementation | Added a Node-based focused chat acceptance bundle for shared unit, Desktop/Web Playwright, and Desktop/Web Visual QA; package entry passed with `real_tested=false`; PR pending. | | 2026-06-29 | Phase 4 sync | #389 merged via #412 and closed manually because non-default base did not auto-close it; milestone #20 is 1/3 complete with drift_score 1, and #390 is the active next task with a drift warning. | +| 2026-06-29 | T4.2 implementation | Added Desktop/Web Visual QA metrics and report artifacts, locked their paths in the chat acceptance manifest, and verified the full chat acceptance bundle with `real_tested=false`; PR pending. | diff --git a/scripts/verify/chat-acceptance.mjs b/scripts/verify/chat-acceptance.mjs index 571ea4fc..e3fdf211 100644 --- a/scripts/verify/chat-acceptance.mjs +++ b/scripts/verify/chat-acceptance.mjs @@ -79,7 +79,11 @@ await runGate({ cwd: path.join(repoRoot, 'app', 'desktop'), command: corepack, args: ['pnpm', '--dir', path.join(repoRoot, 'app', 'desktop'), 'run', 'test:visual:chat-flow'], - artifacts: ['app/desktop/.tmp/manual-chat-flow-uiux/desktop-1440x810-chat-flow.png'], + artifacts: [ + 'app/desktop/.tmp/manual-chat-flow-uiux/desktop-1440x810-chat-flow.png', + 'app/desktop/.tmp/manual-chat-flow-uiux/desktop-1440x810-chat-flow.metrics.json', + 'app/desktop/.tmp/manual-chat-flow-uiux/desktop-chat-flow-visual-qa.json', + ], skip: args.skipDesktopVisualQa, skipReason: 'skipped by --skip-desktop-visual-qa', }); @@ -92,7 +96,11 @@ await runGate({ cwd: path.join(repoRoot, 'app', 'web'), command: corepack, args: ['pnpm', '--dir', path.join(repoRoot, 'app', 'web'), 'run', 'test:visual:chat-flow'], - artifacts: ['app/web/.tmp/manual-chat-flow-uiux/web-1440x810-chat-flow.png'], + artifacts: [ + 'app/web/.tmp/manual-chat-flow-uiux/web-1440x810-chat-flow.png', + 'app/web/.tmp/manual-chat-flow-uiux/web-1440x810-chat-flow.metrics.json', + 'app/web/.tmp/manual-chat-flow-uiux/web-chat-flow-visual-qa.json', + ], skip: args.skipWebVisualQa, skipReason: 'skipped by --skip-web-visual-qa', }); diff --git a/tests/contract/scripts/verify-chat-acceptance.mjs b/tests/contract/scripts/verify-chat-acceptance.mjs index 73ec1ad4..f0fd4048 100644 --- a/tests/contract/scripts/verify-chat-acceptance.mjs +++ b/tests/contract/scripts/verify-chat-acceptance.mjs @@ -70,6 +70,10 @@ try { assert(json.rows.filter((row) => row.name === 'web-chat-playwright' && row.evidence_level === 'playwright-ui').length === 1, 'manifest has Web Playwright row'); assert(json.rows.filter((row) => row.name === 'desktop-chat-visual-qa' && row.evidence_level === 'visual-qa').length === 1, 'manifest has Desktop Visual QA row'); assert(json.rows.filter((row) => row.name === 'web-chat-visual-qa' && row.evidence_level === 'visual-qa').length === 1, 'manifest has Web Visual QA row'); + assert(json.rows.find((row) => row.name === 'desktop-chat-visual-qa')?.artifacts?.includes('app/desktop/.tmp/manual-chat-flow-uiux/desktop-1440x810-chat-flow.metrics.json'), 'Desktop Visual QA row includes metrics artifact'); + assert(json.rows.find((row) => row.name === 'desktop-chat-visual-qa')?.artifacts?.includes('app/desktop/.tmp/manual-chat-flow-uiux/desktop-chat-flow-visual-qa.json'), 'Desktop Visual QA row includes report artifact'); + assert(json.rows.find((row) => row.name === 'web-chat-visual-qa')?.artifacts?.includes('app/web/.tmp/manual-chat-flow-uiux/web-1440x810-chat-flow.metrics.json'), 'Web Visual QA row includes metrics artifact'); + assert(json.rows.find((row) => row.name === 'web-chat-visual-qa')?.artifacts?.includes('app/web/.tmp/manual-chat-flow-uiux/web-chat-flow-visual-qa.json'), 'Web Visual QA row includes report artifact'); assert(!/sk-[A-Za-z0-9]|client_secret|Authorization:\s*Bearer/.test(jsonText), 'manifest is redacted'); } diff --git a/tests/contract/scripts/verify-chat-visual-qa.mjs b/tests/contract/scripts/verify-chat-visual-qa.mjs new file mode 100644 index 00000000..4449198e --- /dev/null +++ b/tests/contract/scripts/verify-chat-visual-qa.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const repoRoot = path.resolve(parseRepoRoot(process.argv.slice(2))); +const desktopScriptPath = path.join(repoRoot, 'app', 'desktop', 'scripts', 'manual-chat-flow-check.mjs'); +const webScriptPath = path.join(repoRoot, 'app', 'web', 'scripts', 'manual-chat-flow-check.mjs'); +const acceptancePath = path.join(repoRoot, 'scripts', 'verify', 'chat-acceptance.mjs'); +let failed = 0; + +assertVisualScriptContract('Desktop', desktopScriptPath, { + surface: 'desktop', + screenshot: 'desktop-1440x810-chat-flow.png', + metrics: 'desktop-1440x810-chat-flow.metrics.json', + report: 'desktop-chat-flow-visual-qa.json', +}); + +assertVisualScriptContract('Web', webScriptPath, { + surface: 'web', + screenshot: 'web-1440x810-chat-flow.png', + metrics: 'web-1440x810-chat-flow.metrics.json', + report: 'web-chat-flow-visual-qa.json', +}); + +assertAcceptanceContract(acceptancePath); + +process.exit(failed > 0 ? 1 : 0); + +function assertVisualScriptContract(label, scriptPath, expected) { + assert(fs.existsSync(scriptPath), `${label} Visual QA script exists`); + if (!fs.existsSync(scriptPath)) return; + + const script = fs.readFileSync(scriptPath, 'utf8'); + assert(script.includes(expected.screenshot), `${label} script keeps stable screenshot path`); + assert(script.includes(expected.metrics), `${label} script writes stable metrics path`); + assert(script.includes(expected.report), `${label} script writes stable report path`); + assert(script.includes('agenthub.chat_visual_qa.v1'), `${label} script writes stable report schema`); + assert(script.includes(`surface: '${expected.surface}'`), `${label} report records ${expected.surface} surface`); + assert(script.includes('metricsPath'), `${label} report includes metricsPath`); + assert(script.includes('reportPath'), `${label} report includes reportPath`); + assert(script.includes('inspection'), `${label} report includes concise inspection instructions`); + assert(script.includes('real_tested: false'), `${label} report records real_tested=false`); + assert(script.includes('evidence_level: \'visual-qa\''), `${label} report records visual-qa evidence level`); + assert(script.includes('fs.writeFileSync(metricsPath'), `${label} script persists metrics JSON`); + assert(script.includes('fs.writeFileSync(reportPath'), `${label} script persists report JSON`); + assert(script.includes('Visual QA screenshot:'), `${label} stdout prints screenshot path`); + assert(script.includes('Visual QA metrics:'), `${label} stdout prints metrics path`); + assert(script.includes('Visual QA report:'), `${label} stdout prints report path`); + assert(!/real_tested:\s*true|approved-real.+real_tested:\s*true|pnpm\s+tauri\s+build/i.test(script), `${label} script does not claim approved-real or packaged Desktop evidence`); +} + +function assertAcceptanceContract(scriptPath) { + assert(fs.existsSync(scriptPath), 'chat acceptance runner exists'); + if (!fs.existsSync(scriptPath)) return; + + const script = fs.readFileSync(scriptPath, 'utf8'); + for (const artifact of [ + 'desktop-1440x810-chat-flow.png', + 'desktop-1440x810-chat-flow.metrics.json', + 'desktop-chat-flow-visual-qa.json', + 'web-1440x810-chat-flow.png', + 'web-1440x810-chat-flow.metrics.json', + 'web-chat-flow-visual-qa.json', + ]) { + assert(script.includes(artifact), `chat acceptance manifest includes ${artifact}`); + } +} + +function assert(condition, message) { + if (condition) { + console.log(`PASS: ${message}`); + return; + } + failed += 1; + console.error(`FAIL: ${message}`); +} + +function parseRepoRoot(args) { + for (let i = 0; i < args.length; i += 1) { + if (args[i] === '--repo-root' || args[i] === '-RepoRoot') { + return args[i + 1]; + } + } + return '.'; +}