From 6f66c09608d29863ffb46a112b1790238105dbeb Mon Sep 17 00:00:00 2001 From: default Date: Sat, 25 Jul 2026 17:32:58 +0000 Subject: [PATCH 1/6] test(vrt): add selector contract fixtures Assisted-by: multi-agent-shogun-aki-tweak --- .../scripts/fixtures/vrt-selector/index.json | 95 +++++++++++++++++++ .../fixtures/vrt-selector/owner-paths.json | 16 ++++ .../fixtures/vrt-selector/preview-graph.json | 38 ++++++++ 3 files changed, 149 insertions(+) create mode 100644 .github/scripts/fixtures/vrt-selector/index.json create mode 100644 .github/scripts/fixtures/vrt-selector/owner-paths.json create mode 100644 .github/scripts/fixtures/vrt-selector/preview-graph.json diff --git a/.github/scripts/fixtures/vrt-selector/index.json b/.github/scripts/fixtures/vrt-selector/index.json new file mode 100644 index 000000000..d51cb7e5a --- /dev/null +++ b/.github/scripts/fixtures/vrt-selector/index.json @@ -0,0 +1,95 @@ +{ + "v": 5, + "entries": { + "auth-passwordinput--docs": { + "id": "auth-passwordinput--docs", + "type": "docs", + "importPath": "./stories/auth/PasswordInput.stories.ts" + }, + "auth-passwordinput--password-hidden": { + "id": "auth-passwordinput--password-hidden", + "type": "story", + "importPath": "./stories/auth/PasswordInput.stories.ts" + }, + "auth-passwordinput--password-visible": { + "id": "auth-passwordinput--password-visible", + "type": "story", + "importPath": "./stories/auth/PasswordInput.stories.ts" + }, + "auth-signinform--default": { + "id": "auth-signinform--default", + "type": "story", + "importPath": "./stories/auth/SignInForm.stories.ts" + }, + "auth-signupform--default": { + "id": "auth-signupform--default", + "type": "story", + "importPath": "./stories/auth/SignUpForm.stories.ts" + }, + "pages-resetpassword--request-form": { + "id": "pages-resetpassword--request-form", + "type": "story", + "importPath": "./stories/pages/ResetPassword.stories.ts" + }, + "pages-resetpassword--request-success": { + "id": "pages-resetpassword--request-success", + "type": "story", + "importPath": "./stories/pages/ResetPassword.stories.ts" + }, + "pages-resetpassword--complete-form": { + "id": "pages-resetpassword--complete-form", + "type": "story", + "importPath": "./stories/pages/ResetPassword.stories.ts" + }, + "pages-resetpassword--invalid-token": { + "id": "pages-resetpassword--invalid-token", + "type": "story", + "importPath": "./stories/pages/ResetPassword.stories.ts" + }, + "pages-signin--default": { + "id": "pages-signin--default", + "type": "story", + "importPath": "./stories/pages/SignIn.stories.ts" + }, + "pages-signin--login-error": { + "id": "pages-signin--login-error", + "type": "story", + "importPath": "./stories/pages/SignIn.stories.ts" + }, + "pages-signin--login-success": { + "id": "pages-signin--login-success", + "type": "story", + "importPath": "./stories/pages/SignIn.stories.ts" + }, + "pages-signup--default": { + "id": "pages-signup--default", + "type": "story", + "importPath": "./stories/pages/SignUp.stories.ts" + }, + "pages-signup--register-error": { + "id": "pages-signup--register-error", + "type": "story", + "importPath": "./stories/pages/SignUp.stories.ts" + }, + "pages-signup--register-submitting": { + "id": "pages-signup--register-submitting", + "type": "story", + "importPath": "./stories/pages/SignUp.stories.ts" + }, + "pages-signup--register-success": { + "id": "pages-signup--register-success", + "type": "story", + "importPath": "./stories/pages/SignUp.stories.ts" + }, + "pages-signup--register-success-201": { + "id": "pages-signup--register-success-201", + "type": "story", + "importPath": "./stories/pages/SignUp.stories.ts" + }, + "tasks-taskcard--default": { + "id": "tasks-taskcard--default", + "type": "story", + "importPath": "./stories/tasks/TaskCard.stories.ts" + } + } +} diff --git a/.github/scripts/fixtures/vrt-selector/owner-paths.json b/.github/scripts/fixtures/vrt-selector/owner-paths.json new file mode 100644 index 000000000..6d5db55d6 --- /dev/null +++ b/.github/scripts/fixtures/vrt-selector/owner-paths.json @@ -0,0 +1,16 @@ +{ + "cases": [ + { + "moduleId": "/repo/node_modules/.pnpm/vite@8.0.0/node_modules/vite/dist/client.mjs", + "owner": "vite" + }, + { + "moduleId": "/repo/node_modules/.pnpm/@storybook+core@10.5.2/node_modules/@storybook/core/dist/index.js", + "owner": "@storybook/core" + }, + { + "moduleId": "/repo/node_modules/.pnpm/@vueuse+core@14.3.0_vue@3.5.40/node_modules/@vueuse/core/index.mjs", + "owner": "@vueuse/core" + } + ] +} diff --git a/.github/scripts/fixtures/vrt-selector/preview-graph.json b/.github/scripts/fixtures/vrt-selector/preview-graph.json new file mode 100644 index 000000000..9c691888c --- /dev/null +++ b/.github/scripts/fixtures/vrt-selector/preview-graph.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "modules": { + "apps/frontend/src/components/auth/PasswordInput.vue": { + "reasons": [] + }, + "apps/frontend/src/components/auth/SignInForm.vue": { + "reasons": ["apps/frontend/src/components/auth/PasswordInput.vue"] + }, + "apps/frontend/src/components/auth/SignUpForm.vue": { + "reasons": ["apps/frontend/src/components/auth/PasswordInput.vue"] + }, + "./stories/auth/PasswordInput.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/PasswordInput.vue"] + }, + "./stories/auth/SignInForm.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/SignInForm.vue"] + }, + "./stories/auth/SignUpForm.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/SignUpForm.vue"] + }, + "./stories/pages/ResetPassword.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/PasswordInput.vue"] + }, + "./stories/pages/SignIn.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/SignInForm.vue"] + }, + "./stories/pages/SignUp.stories.ts": { + "reasons": ["apps/frontend/src/components/auth/SignUpForm.vue"] + }, + "apps/frontend/src/components/tasks/TaskCard.vue": { + "reasons": [] + }, + "./stories/tasks/TaskCard.stories.ts": { + "reasons": ["apps/frontend/src/components/tasks/TaskCard.vue"] + } + } +} From c6508e00d7e2be6391d9dc2405e80b186dda1343 Mon Sep 17 00:00:00 2001 From: default Date: Sat, 25 Jul 2026 17:33:12 +0000 Subject: [PATCH 2/6] feat(vrt): select Argos screenshots from baseline diff Resolve Argos eligible baselines before capture, classify FULL/PARTIAL/NONE through a provider-neutral graph selector, fail closed on uncertain coverage, and gate uploads on exact execution evidence. Part of #449 Assisted-by: multi-agent-shogun-aki-tweak --- .github/scripts/argos-vrt-adapter.mts | 194 +++++++++++ .github/scripts/argos-vrt-adapter.test.mts | 51 +++ .github/scripts/argos-vrt-gate.mts | 90 +++++ .github/scripts/vrt-selector.mts | 376 +++++++++++++++++++++ .github/scripts/vrt-selector.test.mts | 197 +++++++++++ .github/workflows/argos.yml | 41 ++- .gitignore | 1 + apps/frontend/.storybook/main.ts | 11 + apps/frontend/.storybook/test-runner.ts | 10 +- apps/frontend/buildSrc/vrtGraphPlugin.ts | 51 +++ 10 files changed, 1014 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/argos-vrt-adapter.mts create mode 100644 .github/scripts/argos-vrt-adapter.test.mts create mode 100644 .github/scripts/argos-vrt-gate.mts create mode 100644 .github/scripts/vrt-selector.mts create mode 100644 .github/scripts/vrt-selector.test.mts create mode 100644 apps/frontend/buildSrc/vrtGraphPlugin.ts diff --git a/.github/scripts/argos-vrt-adapter.mts b/.github/scripts/argos-vrt-adapter.mts new file mode 100644 index 000000000..38966e375 --- /dev/null +++ b/.github/scripts/argos-vrt-adapter.mts @@ -0,0 +1,194 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + parseGitNameStatus, + reachableStoryIds, + selectVrt, + stableJson, + type PreviewGraph, + type StoryIndex, +} from "./vrt-selector.mts"; + +type ArgosBuild = { + head: { sha: string; branch: string | null }; +}; + +const repositoryRoot = process.cwd(); +const evidenceDirectory = path.join(repositoryRoot, ".vrt"); +const storybookDirectory = path.join(repositoryRoot, "apps/frontend/storybook-static"); +const sourceIndexPath = path.join(storybookDirectory, "index.json"); +const graphPath = path.join(evidenceDirectory, "preview-graph.json"); +const manifestPath = path.join(evidenceDirectory, "selection-manifest.json"); +const fixtureDirectory = path.join(repositoryRoot, ".github/scripts/fixtures/vrt-selector"); + +const git = (...args: string[]) => + execFileSync("git", args, { cwd: repositoryRoot, encoding: "utf8" }).trim(); + +const changedPathsBetween = (baseline: string | null, head: string) => { + if (!baseline) { + return parseGitNameStatus( + execFileSync("git", ["show", "--format=", "--name-status", "-z", "--find-renames", head], { + cwd: repositoryRoot, + encoding: "utf8", + }), + ); + } + return parseGitNameStatus( + execFileSync( + "git", + [ + "log", + "--format=", + "--name-status", + "-z", + "--find-renames", + `${baseline}..${head}`, + ], + { + cwd: repositoryRoot, + encoding: "utf8", + }, + ), + ); +}; + +async function resolveArgosBaseline(head: string): Promise { + const token = process.env.ARGOS_TOKEN; + if (!token) return null; + const referenceBranch = process.env.ARGOS_REFERENCE_BRANCH ?? "main"; + let base = head; + try { + base = git("merge-base", `origin/${referenceBranch}`, head); + } catch { + // The API result remains authoritative; a missing local merge-base fails + // closed because no candidate can be submitted. + return null; + } + const commits = git("rev-list", base).split("\n").filter(Boolean); + if (commits.length === 0) return null; + const response = await fetch( + `${process.env.ARGOS_API_BASE_URL ?? "https://api.argos-ci.com/v2/"}baseline`, + { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + commits, + name: process.env.ARGOS_BUILD_NAME ?? "default", + mode: "ci", + }), + }, + ); + if (!response.ok) { + throw new Error(`argos_baseline_http_${response.status}`); + } + const payload = (await response.json()) as { baseline: ArgosBuild | null }; + const baseline = payload.baseline?.head.sha ?? null; + if (baseline) { + try { + git("merge-base", "--is-ancestor", baseline, head); + } catch { + throw new Error("argos_baseline_not_ancestor"); + } + } + return baseline; +} + +const headCommit = git("rev-parse", "HEAD"); +let baselineCommit: string | null = null; +const reasonCodes: string[] = []; +try { + baselineCommit = await resolveArgosBaseline(headCommit); +} catch (error) { + reasonCodes.push(error instanceof Error ? error.message : "argos_baseline_unknown_error"); +} + +const [index, graph, fixtureIndex, fixtureGraph] = await Promise.all([ + readFile(sourceIndexPath, "utf8").then((value) => JSON.parse(value) as StoryIndex), + readFile(graphPath, "utf8").then((value) => JSON.parse(value) as PreviewGraph), + readFile(path.join(fixtureDirectory, "index.json"), "utf8").then( + (value) => JSON.parse(value) as StoryIndex, + ), + readFile(path.join(fixtureDirectory, "preview-graph.json"), "utf8").then( + (value) => JSON.parse(value) as PreviewGraph, + ), +]); +const expectedFixtureStories = [ + "auth-passwordinput--password-hidden", + "auth-passwordinput--password-visible", + "auth-signinform--default", + "auth-signupform--default", + "pages-resetpassword--request-form", + "pages-resetpassword--request-success", + "pages-resetpassword--complete-form", + "pages-resetpassword--invalid-token", + "pages-signin--default", + "pages-signin--login-error", + "pages-signin--login-success", + "pages-signup--default", + "pages-signup--register-error", + "pages-signup--register-submitting", + "pages-signup--register-success", + "pages-signup--register-success-201", +].sort(); +const actualFixtureStories = reachableStoryIds(fixtureIndex, fixtureGraph, [ + "apps/frontend/src/components/auth/PasswordInput.vue", +]); +const preflightErrors = + actualFixtureStories.join("\0") === expectedFixtureStories.join("\0") + ? [] + : ["password_input_exact_set_fixture_mismatch"]; +const changedPaths = changedPathsBetween(baselineCommit, headCommit); +const isMain = + process.env.GITHUB_EVENT_NAME === "push" && process.env.GITHUB_REF === "refs/heads/main"; +const headRef = process.env.GITHUB_HEAD_REF ?? ""; +const isDependencyUpdate = + process.env.VRT_DEPENDENCY_UPDATE === "true" || headRef.startsWith("renovate/"); + +const selection = selectVrt({ + baselineCommit, + headCommit, + changedPaths, + isMain, + isDependencyUpdate, + dependencyPhase: process.env.VRT_DEPENDENCY_PHASE === "1" ? 1 : 0, + index, + graph, + preflightErrors, +}); +selection.manifest.reason_codes.push(...reasonCodes); + +await mkdir(evidenceDirectory, { recursive: true }); +await writeFile( + path.join(evidenceDirectory, "source-index.json"), + `${stableJson(index)}\n`, + "utf8", +); +await writeFile(manifestPath, `${stableJson(selection.manifest)}\n`, "utf8"); +if (selection.filteredIndex) { + await writeFile(sourceIndexPath, `${stableJson(selection.filteredIndex)}\n`, "utf8"); +} +await writeFile(path.join(evidenceDirectory, "executed-story-ids.txt"), "", "utf8"); +await writeFile(path.join(evidenceDirectory, "captured-story-ids.txt"), "", "utf8"); + +const githubOutput = process.env.GITHUB_OUTPUT; +if (githubOutput) { + await writeFile(githubOutput, `mode=${selection.manifest.mode}\nmanifest=${manifestPath}\n`, { + encoding: "utf8", + flag: "a", + }); +} + +console.log( + JSON.stringify({ + mode: selection.manifest.mode, + baselineCommit, + headCommit, + selectedCount: selection.manifest.selected_story_ids.length, + reasonCodes: selection.manifest.reason_codes, + manifestPath, + }), +); diff --git a/.github/scripts/argos-vrt-adapter.test.mts b/.github/scripts/argos-vrt-adapter.test.mts new file mode 100644 index 000000000..b68e03449 --- /dev/null +++ b/.github/scripts/argos-vrt-adapter.test.mts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { selectVrt, type PreviewGraph, type StoryIndex } from "./vrt-selector.mts"; + +const index: StoryIndex = { + entries: { + "one--default": { + id: "one--default", + type: "story", + importPath: "./stories/One.stories.ts", + }, + "two--default": { + id: "two--default", + type: "story", + importPath: "./stories/Two.stories.ts", + }, + }, +}; +const graph: PreviewGraph = { + schemaVersion: 1, + modules: { + "apps/frontend/src/One.vue": { reasons: [] }, + "apps/frontend/stories/One.stories.ts": { + reasons: ["apps/frontend/src/One.vue"], + }, + "apps/frontend/src/Two.vue": { reasons: [] }, + "apps/frontend/stories/Two.stories.ts": { + reasons: ["apps/frontend/src/Two.vue"], + }, + }, +}; + +test("provider adapter inputs remain provider-neutral at selector boundary", () => { + const result = selectVrt({ + baselineCommit: "1".repeat(40), + headCommit: "2".repeat(40), + changedPaths: ["apps/frontend/src/One.vue"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(result.manifest.mode, "PARTIAL"); + assert.deepEqual(result.manifest.selected_story_ids, ["one--default"]); + assert.equal( + Object.hasOwn(result.manifest, "argosBuild"), + false, + "provider payload must not leak into selector manifest", + ); +}); diff --git a/.github/scripts/argos-vrt-gate.mts b/.github/scripts/argos-vrt-gate.mts new file mode 100644 index 000000000..43dc4d665 --- /dev/null +++ b/.github/scripts/argos-vrt-gate.mts @@ -0,0 +1,90 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + assertCaptureGate, + sha256, + stableJson, + type SelectionManifest, + type StoryIndex, +} from "./vrt-selector.mts"; + +const repositoryRoot = process.cwd(); +const evidenceDirectory = path.join(repositoryRoot, ".vrt"); +const manifest = JSON.parse( + await readFile(path.join(evidenceDirectory, "selection-manifest.json"), "utf8"), +) as SelectionManifest; +const servedIndex = JSON.parse( + await readFile(path.join(repositoryRoot, "apps/frontend/storybook-static/index.json"), "utf8"), +) as StoryIndex; + +const readIds = async (name: string) => + (await readFile(path.join(evidenceDirectory, name), "utf8")) + .split("\n") + .map((value) => value.trim()) + .filter(Boolean); + +const [executedStoryIds, capturedStoryIds] = await Promise.all([ + readIds("executed-story-ids.txt"), + readIds("captured-story-ids.txt"), +]); + +let pending = 0; +let skipped = 0; +let failed = 0; +const testResultsPath = path.join(evidenceDirectory, "test-results.json"); +try { + const testResults = JSON.parse(await readFile(testResultsPath, "utf8")) as { + numPendingTests?: number; + numTodoTests?: number; + numFailedTests?: number; + }; + pending = testResults.numTodoTests ?? 0; + skipped = testResults.numPendingTests ?? 0; + failed = testResults.numFailedTests ?? 0; +} catch (error) { + if (manifest.mode !== "NONE") { + throw new Error(`test_results_missing:${error instanceof Error ? error.message : "unknown"}`); + } +} + +const selected = new Set(manifest.selected_story_ids); +const screenshotsOutsideManifest = capturedStoryIds.filter( + (storyId) => !selected.has(storyId), +).length; +const servedIndexSha256 = sha256(stableJson(servedIndex)); +const evidence = { + mode: manifest.mode, + selected_story_ids: manifest.selected_story_ids, + executed_story_ids: executedStoryIds, + captured_story_ids: capturedStoryIds, + served_index_sha256: servedIndexSha256, + pending, + skipped, + failed, + screenshots_outside_manifest: screenshotsOutsideManifest, +}; + +await writeFile( + path.join(evidenceDirectory, "capture-gate.json"), + `${stableJson(evidence)}\n`, + "utf8", +); +assertCaptureGate({ + manifest, + executedStoryIds, + capturedStoryIds, + servedIndexSha256, + pending, + skipped, + failed, + screenshotsOutsideManifest, +}); +console.log( + JSON.stringify({ + mode: manifest.mode, + selected: manifest.selected_story_ids.length, + executed: executedStoryIds.length, + captured: capturedStoryIds.length, + uploadGate: "passed", + }), +); diff --git a/.github/scripts/vrt-selector.mts b/.github/scripts/vrt-selector.mts new file mode 100644 index 000000000..04d17466f --- /dev/null +++ b/.github/scripts/vrt-selector.mts @@ -0,0 +1,376 @@ +import { createHash } from "node:crypto"; + +export type VrtMode = "FULL" | "PARTIAL" | "NONE"; + +export type StoryIndexEntry = { + id: string; + type: string; + importPath: string; + [key: string]: unknown; +}; + +export type StoryIndex = { + entries: Record; + [key: string]: unknown; +}; + +export type PreviewGraph = { + schemaVersion: number; + modules: Record; +}; + +export type SelectionManifest = { + mode: VrtMode; + baseline_commit: string | null; + head_commit: string; + changed_paths: string[]; + in_scope_paths: string[]; + selected_story_ids: string[]; + reason_codes: string[]; + source_index_sha256: string; + filtered_index_sha256: string | null; + graph_artifact_sha256: string; +}; + +export type SelectionInput = { + baselineCommit: string | null; + headCommit: string; + changedPaths: string[]; + isMain: boolean; + isDependencyUpdate: boolean; + dependencyPhase: 0 | 1; + index: StoryIndex; + graph: PreviewGraph; + preflightErrors?: string[]; +}; + +const normalizePath = (value: string) => value.replaceAll("\\", "/").replace(/^\.?\//, ""); + +const canonicalModuleId = (value: string) => { + const normalized = normalizePath(value).split(/[?#]/, 1)[0] ?? ""; + const frontendMarker = "/apps/frontend/"; + const markerIndex = normalized.lastIndexOf(frontendMarker); + if (markerIndex >= 0) { + return normalized.slice(markerIndex + frontendMarker.length); + } + return normalized.replace(/^apps\/frontend\//, ""); +}; + +const sortedUnique = (values: Iterable) => + [...new Set(values)].sort((left, right) => left.localeCompare(right)); + +export function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +export function stableJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((item) => stableJson(item)).join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +export function normalizeStoryIndex(index: StoryIndex): StoryIndexEntry[] { + if (!index || typeof index !== "object" || !index.entries) { + throw new Error("index_entries_missing"); + } + const storyEntries = Object.values(index.entries).filter((entry) => entry.type === "story"); + const ids = storyEntries.map((entry) => entry.id); + if ( + storyEntries.some( + (entry) => + typeof entry.id !== "string" || + entry.id.length === 0 || + typeof entry.importPath !== "string" || + entry.importPath.length === 0, + ) + ) { + throw new Error("index_story_shape_invalid"); + } + if (new Set(ids).size !== ids.length) { + throw new Error("index_story_id_duplicate"); + } + return storyEntries.sort((left, right) => left.id.localeCompare(right.id)); +} + +export function validatePreviewGraph(graph: PreviewGraph): void { + if (graph?.schemaVersion !== 1 || !graph.modules) { + throw new Error("graph_schema_invalid"); + } + for (const [moduleId, module] of Object.entries(graph.modules)) { + if ( + moduleId.length === 0 || + !module || + !Array.isArray(module.reasons) || + module.reasons.some((reason) => typeof reason !== "string") + ) { + throw new Error("graph_module_invalid"); + } + } +} + +export function reachableModules(graph: PreviewGraph, seedModules: string[]): Set { + validatePreviewGraph(graph); + const canonicalToModule = new Map( + Object.keys(graph.modules).map((moduleId) => [canonicalModuleId(moduleId), moduleId]), + ); + const queue: string[] = []; + for (const seed of seedModules) { + const moduleId = canonicalToModule.get(canonicalModuleId(seed)); + if (!moduleId) { + throw new Error(`graph_seed_unresolved:${seed}`); + } + queue.push(moduleId); + } + + const importers = new Map(); + for (const [moduleId, module] of Object.entries(graph.modules)) { + for (const reason of module.reasons) { + const canonicalReason = canonicalToModule.get(canonicalModuleId(reason)); + if (!canonicalReason) { + throw new Error(`graph_reason_unresolved:${reason}`); + } + const values = importers.get(canonicalReason) ?? []; + values.push(moduleId); + importers.set(canonicalReason, values); + } + } + + const reached = new Set(); + while (queue.length > 0) { + const moduleId = queue.shift(); + if (!moduleId || reached.has(moduleId)) continue; + reached.add(moduleId); + queue.push(...(importers.get(moduleId) ?? [])); + } + return reached; +} + +export function reachableStoryIds( + index: StoryIndex, + graph: PreviewGraph, + seedModules: string[], +): string[] { + const reached = reachableModules(graph, seedModules); + const reachedCanonical = new Set([...reached].map(canonicalModuleId)); + return normalizeStoryIndex(index) + .filter((entry) => reachedCanonical.has(canonicalModuleId(entry.importPath))) + .map((entry) => entry.id); +} + +export function filterStoryIndex(index: StoryIndex, selectedStoryIds: string[]): StoryIndex { + const selected = new Set(selectedStoryIds); + const entries = Object.fromEntries( + Object.entries(index.entries).filter( + ([, entry]) => entry.type === "story" && selected.has(entry.id), + ), + ); + return { ...index, entries }; +} + +export function packageOwnerFromModuleId(moduleId: string): string | null { + const segments = normalizePath(moduleId).split("/"); + const nodeModulesPositions = segments + .map((segment, index) => (segment === "node_modules" ? index : -1)) + .filter((index) => index >= 0); + if (nodeModulesPositions.length < 2) return null; + const ownerStart = nodeModulesPositions[1] + 1; + const first = segments[ownerStart]; + if (!first) return null; + if (first.startsWith("@")) { + const second = segments[ownerStart + 1]; + return second ? `${first}/${second}` : null; + } + return first; +} + +export function parseGitNameStatus(raw: string): string[] { + const fields = raw.split("\0").filter(Boolean); + const paths: string[] = []; + for (let index = 0; index < fields.length;) { + const status = fields[index++]; + if (!status) break; + const firstPath = fields[index++]; + if (!firstPath) { + throw new Error("git_name_status_path_missing"); + } + paths.push(firstPath); + if (status.startsWith("R") || status.startsWith("C")) { + const secondPath = fields[index++]; + if (!secondPath) { + throw new Error("git_name_status_rename_target_missing"); + } + paths.push(secondPath); + } + } + return sortedUnique(paths.map(normalizePath)); +} + +export function isInScopePath(path: string): boolean { + const normalized = normalizePath(path); + return ( + normalized.startsWith("apps/frontend/") || + normalized === "pnpm-lock.yaml" || + normalized === "pnpm-workspace.yaml" || + normalized === ".github/workflows/argos.yml" + ); +} + +export function unsafePathReason(path: string): string | null { + const normalized = normalizePath(path); + if ( + normalized === "pnpm-lock.yaml" || + normalized === "pnpm-workspace.yaml" || + normalized === "apps/frontend/pnpm-lock.yaml" || + normalized === "apps/frontend/package.json" + ) { + return "denylist_build_chain"; + } + if ( + normalized.startsWith("apps/frontend/.storybook/") || + normalized === ".github/workflows/argos.yml" + ) { + return "denylist_storybook_provider"; + } + if (/\.(css|scss|sass|less)$/.test(normalized)) { + return "denylist_global_style"; + } + if ( + normalized.includes("/icons/") || + normalized.includes("/icon/") || + normalized.includes("virtual:") || + normalized.includes("\0") + ) { + return "denylist_visual_or_virtual"; + } + return null; +} + +export function sourceSeedsFromPaths(paths: string[]): string[] { + return sortedUnique( + paths.filter( + (path) => + normalizePath(path).startsWith("apps/frontend/src/") && /\.(vue|[cm]?[jt]sx?)$/.test(path), + ), + ); +} + +export function selectVrt(input: SelectionInput): { + manifest: SelectionManifest; + filteredIndex: StoryIndex | null; +} { + const changedPaths = sortedUnique(input.changedPaths.map(normalizePath)); + const inScopePaths = changedPaths.filter(isInScopePath); + const sourceIndexSha = sha256(stableJson(input.index)); + const graphSha = sha256(stableJson(input.graph)); + const allStoryIds = normalizeStoryIndex(input.index).map((entry) => entry.id); + const unsafeReasons = sortedUnique( + inScopePaths.map(unsafePathReason).filter((reason): reason is string => !!reason), + ); + + let selectedStoryIds: string[] = []; + let mode: VrtMode = "FULL"; + let reasonCodes: string[] = []; + + if (inScopePaths.length === 0) { + mode = "NONE"; + reasonCodes = ["out_of_scope"]; + } else if (input.isMain) { + selectedStoryIds = allStoryIds; + reasonCodes = ["main_full_backstop"]; + } else if (input.preflightErrors?.length) { + selectedStoryIds = allStoryIds; + reasonCodes = ["preflight_fail_closed", ...input.preflightErrors]; + } else if (!input.baselineCommit) { + selectedStoryIds = allStoryIds; + reasonCodes = ["baseline_unconfirmed"]; + } else if (unsafeReasons.length > 0) { + selectedStoryIds = allStoryIds; + reasonCodes = unsafeReasons; + } else { + const seeds = sourceSeedsFromPaths(inScopePaths); + try { + selectedStoryIds = reachableStoryIds(input.index, input.graph, seeds); + if (input.isDependencyUpdate && selectedStoryIds.length === 0) { + mode = "NONE"; + reasonCodes = ["dependency_graph_zero_reach"]; + } else if (input.isDependencyUpdate && input.dependencyPhase === 0) { + selectedStoryIds = allStoryIds; + reasonCodes = ["dependency_pr_full_gate"]; + } else if (selectedStoryIds.length === 0) { + selectedStoryIds = allStoryIds; + reasonCodes = ["zero_reach_fail_closed"]; + } else if (selectedStoryIds.length === allStoryIds.length) { + reasonCodes = ["all_stories_reached"]; + } else { + mode = "PARTIAL"; + reasonCodes = [ + input.isDependencyUpdate ? "dependency_pr_reachable_gate" : "exact_reachability", + ]; + } + } catch (error) { + selectedStoryIds = allStoryIds; + reasonCodes = [ + "preflight_fail_closed", + error instanceof Error ? error.message : "unknown_preflight_error", + ]; + } + } + + selectedStoryIds = sortedUnique(selectedStoryIds); + const filteredIndex = mode === "PARTIAL" ? filterStoryIndex(input.index, selectedStoryIds) : null; + const filteredIndexSha = filteredIndex ? sha256(stableJson(filteredIndex)) : null; + + return { + manifest: { + mode, + baseline_commit: input.baselineCommit, + head_commit: input.headCommit, + changed_paths: changedPaths, + in_scope_paths: inScopePaths, + selected_story_ids: selectedStoryIds, + reason_codes: reasonCodes, + source_index_sha256: sourceIndexSha, + filtered_index_sha256: filteredIndexSha, + graph_artifact_sha256: graphSha, + }, + filteredIndex, + }; +} + +export function assertCaptureGate(input: { + manifest: SelectionManifest; + executedStoryIds: string[]; + capturedStoryIds: string[]; + servedIndexSha256: string; + pending: number; + skipped: number; + failed: number; + screenshotsOutsideManifest: number; +}): void { + const selected = sortedUnique(input.manifest.selected_story_ids); + const executed = sortedUnique(input.executedStoryIds); + const captured = sortedUnique(input.capturedStoryIds); + const expectedIndexSha = + input.manifest.filtered_index_sha256 ?? input.manifest.source_index_sha256; + const failures = [ + selected.join("\0") !== executed.join("\0") ? "selected_executed_mismatch" : "", + selected.join("\0") !== captured.join("\0") ? "selected_captured_mismatch" : "", + input.servedIndexSha256 !== expectedIndexSha ? "served_index_sha_mismatch" : "", + input.pending !== 0 ? "pending_nonzero" : "", + input.skipped !== 0 ? "skipped_nonzero" : "", + input.failed !== 0 ? "failed_nonzero" : "", + input.screenshotsOutsideManifest !== 0 ? "screenshots_outside_manifest" : "", + input.manifest.mode !== "NONE" && selected.length === 0 ? "capture_set_empty" : "", + ].filter(Boolean); + if (failures.length > 0) { + throw new Error(failures.join(",")); + } +} diff --git a/.github/scripts/vrt-selector.test.mts b/.github/scripts/vrt-selector.test.mts new file mode 100644 index 000000000..86cf7949c --- /dev/null +++ b/.github/scripts/vrt-selector.test.mts @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { test } from "vitest"; +import { + assertCaptureGate, + filterStoryIndex, + normalizeStoryIndex, + packageOwnerFromModuleId, + parseGitNameStatus, + reachableStoryIds, + selectVrt, + sha256, + stableJson, + type PreviewGraph, + type StoryIndex, +} from "./vrt-selector.mts"; + +const fixturePath = (name: string) => + path.resolve( + process.cwd(), + `../../.github/scripts/fixtures/vrt-selector/${name}`, + ); + +const loadJson = async (name: string): Promise => + JSON.parse(await readFile(fixturePath(name), "utf8")) as T; + +const expectedPasswordStories = [ + "auth-passwordinput--password-hidden", + "auth-passwordinput--password-visible", + "auth-signinform--default", + "auth-signupform--default", + "pages-resetpassword--request-form", + "pages-resetpassword--request-success", + "pages-resetpassword--complete-form", + "pages-resetpassword--invalid-token", + "pages-signin--default", + "pages-signin--login-error", + "pages-signin--login-success", + "pages-signup--default", + "pages-signup--register-error", + "pages-signup--register-submitting", + "pages-signup--register-success", + "pages-signup--register-success-201", +].sort(); + +test("PasswordInput fixture reaches the exact 16-story set and excludes docs", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + assert.deepEqual( + reachableStoryIds(index, graph, ["apps/frontend/src/components/auth/PasswordInput.vue"]), + expectedPasswordStories, + ); + assert.equal( + normalizeStoryIndex(index).some((entry) => entry.id.endsWith("--docs")), + false, + ); +}); + +test("unknown source fails closed to FULL", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const { manifest } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/src/components/Unknown.vue"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(manifest.mode, "FULL"); + assert.equal(manifest.reason_codes[0], "preflight_fail_closed"); +}); + +test("safe source change selects a PARTIAL exact set", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const { manifest, filteredIndex } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/src/components/auth/PasswordInput.vue"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(manifest.mode, "PARTIAL"); + assert.deepEqual(manifest.selected_story_ids, expectedPasswordStories); + assert.deepEqual( + normalizeStoryIndex(filteredIndex as StoryIndex).map((entry) => entry.id), + expectedPasswordStories, + ); +}); + +test("main remains a FULL per-merge backstop", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const { manifest } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/src/components/auth/PasswordInput.vue"], + isMain: true, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(manifest.mode, "FULL"); + assert.deepEqual(manifest.reason_codes, ["main_full_backstop"]); +}); + +test("owner normalization uses the package below the second node_modules", async () => { + const fixture = await loadJson<{ + cases: { moduleId: string; owner: string }[]; + }>("owner-paths.json"); + for (const item of fixture.cases) { + assert.equal(packageOwnerFromModuleId(item.moduleId), item.owner); + } + assert.equal(packageOwnerFromModuleId("/repo/node_modules/.pnpm/vite@8/index.js"), null); +}); + +test("B..H name-status union includes both sides of renames and copies", () => { + assert.deepEqual( + parseGitNameStatus( + [ + "M", + "apps/frontend/src/Changed.vue", + "R100", + "apps/frontend/src/Old.vue", + "apps/frontend/src/New.vue", + "C090", + "apps/frontend/src/Source.vue", + "apps/frontend/src/Copy.vue", + "D", + "apps/frontend/src/Deleted.vue", + "", + ].join("\0"), + ), + [ + "apps/frontend/src/Changed.vue", + "apps/frontend/src/Copy.vue", + "apps/frontend/src/Deleted.vue", + "apps/frontend/src/New.vue", + "apps/frontend/src/Old.vue", + "apps/frontend/src/Source.vue", + ], + ); +}); + +test("capture gate requires exact selected/executed/captured sets and SHA", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const filteredIndex = filterStoryIndex(index, expectedPasswordStories); + const filteredSha = sha256(stableJson(filteredIndex)); + const { manifest } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/src/components/auth/PasswordInput.vue"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(manifest.filtered_index_sha256, filteredSha); + assert.doesNotThrow(() => + assertCaptureGate({ + manifest, + executedStoryIds: expectedPasswordStories, + capturedStoryIds: expectedPasswordStories, + servedIndexSha256: filteredSha, + pending: 0, + skipped: 0, + failed: 0, + screenshotsOutsideManifest: 0, + }), + ); + assert.throws(() => + assertCaptureGate({ + manifest, + executedStoryIds: expectedPasswordStories.slice(1), + capturedStoryIds: expectedPasswordStories, + servedIndexSha256: filteredSha, + pending: 0, + skipped: 0, + failed: 0, + screenshotsOutsideManifest: 0, + }), + ); +}); + +test("fixture files are versioned beside the test", () => { + assert.match(fixturePath("index.json"), /vrt-selector/); +}); diff --git a/.github/workflows/argos.yml b/.github/workflows/argos.yml index b6ebe038b..448a8caa4 100644 --- a/.github/workflows/argos.yml +++ b/.github/workflows/argos.yml @@ -4,16 +4,13 @@ on: push: branches: [main] pull_request: - paths: - - apps/frontend/** - - .github/workflows/argos.yml permissions: contents: read concurrency: group: argos-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: argos: @@ -44,10 +41,17 @@ jobs: - name: Build Storybook run: pnpm build-storybook + - name: Select visual test scope + id: selector + working-directory: . + run: node .github/scripts/argos-vrt-adapter.mts + - name: Install Playwright browsers + if: steps.selector.outputs.mode != 'NONE' run: pnpm exec playwright install --with-deps chromium - name: Serve Storybook and capture every story + if: steps.selector.outputs.mode != 'NONE' run: | pnpm exec http-server storybook-static --port 6006 --silent & for attempt in {1..30}; do @@ -58,10 +62,33 @@ jobs: fi sleep 1 done - pnpm test-storybook + pnpm test-storybook --json --outputFile ../../.vrt/test-results.json env: ARGOS_ENABLED: "true" + VRT_EVIDENCE_DIR: ../../.vrt + + - name: Verify exact capture manifest + id: gate + if: always() && steps.selector.outcome == 'success' + working-directory: . + run: node .github/scripts/argos-vrt-gate.mts - name: Upload screenshots to Argos - if: env.ARGOS_TOKEN != '' - run: pnpm exec argos upload screenshots + if: >- + steps.gate.outcome == 'success' && + steps.selector.outputs.mode != 'NONE' && + env.ARGOS_TOKEN != '' + run: | + upload_args=() + if [ "${{ steps.selector.outputs.mode }}" = "PARTIAL" ]; then + upload_args+=(--subset) + fi + pnpm exec argos upload screenshots "${upload_args[@]}" + + - name: Upload VRT selection evidence + if: always() && steps.selector.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: vrt-selection-${{ github.run_id }}-${{ github.run_attempt }} + path: .vrt/ + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 2f2932521..cd1253cd0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ docs/retype.manifest .worktrees/ node_modules +.vrt/ diff --git a/apps/frontend/.storybook/main.ts b/apps/frontend/.storybook/main.ts index fbe00f265..e9354686c 100644 --- a/apps/frontend/.storybook/main.ts +++ b/apps/frontend/.storybook/main.ts @@ -1,5 +1,10 @@ import type { StorybookConfig } from '@storybook/vue3-vite'; import { mergeConfig } from 'vite'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { vrtGraphPlugin } from '../buildSrc/vrtGraphPlugin.js'; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..'); const config: StorybookConfig = { stories: ['../stories/**/*.mdx', '../stories/**/*.stories.@(js|jsx|mjs|ts|tsx)'], @@ -12,6 +17,12 @@ const config: StorybookConfig = { framework: '@storybook/vue3-vite', viteFinal: async (config) => mergeConfig(config, { + plugins: [ + vrtGraphPlugin({ + repositoryRoot, + outputFile: path.join(repositoryRoot, '.vrt/preview-graph.json'), + }), + ], resolve: { alias: { 'vike/client/router': new URL('./mocks/vike-client-router.ts', import.meta.url).pathname, diff --git a/apps/frontend/.storybook/test-runner.ts b/apps/frontend/.storybook/test-runner.ts index 37c12311e..f6349a469 100644 --- a/apps/frontend/.storybook/test-runner.ts +++ b/apps/frontend/.storybook/test-runner.ts @@ -1,12 +1,19 @@ import { argosScreenshot } from '@argos-ci/playwright'; import type { TestRunnerConfig } from '@storybook/test-runner'; +import { appendFile } from 'node:fs/promises'; +import path from 'node:path'; const VIEWPORT = { width: 1440, height: 900 }; +const evidenceDirectory = path.resolve(process.env.VRT_EVIDENCE_DIR ?? '../../.vrt'); + +const recordStory = (file: string, storyId: string) => + appendFile(path.join(evidenceDirectory, file), `${storyId}\n`, 'utf8'); const config: TestRunnerConfig = { - async preVisit(page) { + async preVisit(page, context) { if (process.env.ARGOS_ENABLED === 'true') { await page.setViewportSize(VIEWPORT); + await recordStory('executed-story-ids.txt', context.id); } }, async postVisit(page, context) { @@ -15,6 +22,7 @@ const config: TestRunnerConfig = { await argosScreenshot(page, context.id, { fullPage: true, }); + await recordStory('captured-story-ids.txt', context.id); }, }; diff --git a/apps/frontend/buildSrc/vrtGraphPlugin.ts b/apps/frontend/buildSrc/vrtGraphPlugin.ts new file mode 100644 index 000000000..7c1007141 --- /dev/null +++ b/apps/frontend/buildSrc/vrtGraphPlugin.ts @@ -0,0 +1,51 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { Plugin } from 'vite'; + +type PreviewGraph = { + schemaVersion: number; + modules: Record; +}; + +const stripQuery = (value: string) => value.split(/[?#]/, 1)[0] ?? value; + +export function vrtGraphPlugin(options: { repositoryRoot: string; outputFile: string }): Plugin { + const normalize = (moduleId: string) => { + const clean = stripQuery(moduleId); + if (path.isAbsolute(clean)) { + const relative = path.relative(options.repositoryRoot, clean); + return relative.startsWith('..') ? clean : relative.replaceAll('\\', '/'); + } + return clean.replaceAll('\\', '/'); + }; + + return { + name: 'vrt-preview-graph', + apply: 'build', + async generateBundle() { + const modules: PreviewGraph['modules'] = {}; + const moduleIds = [...this.getModuleIds()].filter((moduleId) => !moduleId.startsWith('\0')); + const included = new Set(moduleIds.map(normalize)); + for (const moduleId of moduleIds) { + const info = this.getModuleInfo(moduleId); + if (!info) continue; + const normalizedId = normalize(moduleId); + const existingReasons = modules[normalizedId]?.reasons ?? []; + modules[normalizedId] = { + reasons: [ + ...new Set([ + ...existingReasons, + ...[...info.importedIds, ...info.dynamicallyImportedIds] + .filter((importedId) => !importedId.startsWith('\0')) + .map(normalize) + .filter((importedId) => included.has(importedId)), + ]), + ].sort(), + }; + } + const graph: PreviewGraph = { schemaVersion: 1, modules }; + await mkdir(path.dirname(options.outputFile), { recursive: true }); + await writeFile(options.outputFile, `${JSON.stringify(graph)}\n`, 'utf8'); + }, + }; +} From 254a0d86750cd0884c448c9fab2e85832e47d098 Mon Sep 17 00:00:00 2001 From: default Date: Sun, 26 Jul 2026 12:24:00 +0000 Subject: [PATCH 3/6] fix(vrt): select added story files --- .../scripts/fixtures/vrt-selector/index.json | 10 ++ .../fixtures/vrt-selector/preview-graph.json | 6 + .github/scripts/vrt-selector.mts | 2 +- .github/scripts/vrt-selector.test.mts | 49 ++++++ .../visual-regression-new-stories.md | 92 +++++++++++ .../visual-regression-selection-design.md | 146 ++++++++++++++++++ 6 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 docs/development/visual-regression-new-stories.md create mode 100644 docs/development/visual-regression-selection-design.md diff --git a/.github/scripts/fixtures/vrt-selector/index.json b/.github/scripts/fixtures/vrt-selector/index.json index d51cb7e5a..ab6d353d7 100644 --- a/.github/scripts/fixtures/vrt-selector/index.json +++ b/.github/scripts/fixtures/vrt-selector/index.json @@ -90,6 +90,16 @@ "id": "tasks-taskcard--default", "type": "story", "importPath": "./stories/tasks/TaskCard.stories.ts" + }, + "newstory--default": { + "id": "newstory--default", + "type": "story", + "importPath": "./stories/NewStory.stories.ts" + }, + "newstory--compact": { + "id": "newstory--compact", + "type": "story", + "importPath": "./stories/NewStory.stories.ts" } } } diff --git a/.github/scripts/fixtures/vrt-selector/preview-graph.json b/.github/scripts/fixtures/vrt-selector/preview-graph.json index 9c691888c..48ce83b22 100644 --- a/.github/scripts/fixtures/vrt-selector/preview-graph.json +++ b/.github/scripts/fixtures/vrt-selector/preview-graph.json @@ -33,6 +33,12 @@ }, "./stories/tasks/TaskCard.stories.ts": { "reasons": ["apps/frontend/src/components/tasks/TaskCard.vue"] + }, + "apps/frontend/src/components/NewStory.vue": { + "reasons": [] + }, + "./stories/NewStory.stories.ts": { + "reasons": ["apps/frontend/src/components/NewStory.vue"] } } } diff --git a/.github/scripts/vrt-selector.mts b/.github/scripts/vrt-selector.mts index 04d17466f..52af56d5b 100644 --- a/.github/scripts/vrt-selector.mts +++ b/.github/scripts/vrt-selector.mts @@ -257,7 +257,7 @@ export function sourceSeedsFromPaths(paths: string[]): string[] { return sortedUnique( paths.filter( (path) => - normalizePath(path).startsWith("apps/frontend/src/") && /\.(vue|[cm]?[jt]sx?)$/.test(path), + normalizePath(path).startsWith("apps/frontend/") && /\.(vue|[cm]?[jt]sx?)$/.test(path), ), ); } diff --git a/.github/scripts/vrt-selector.test.mts b/.github/scripts/vrt-selector.test.mts index 86cf7949c..fed3c924f 100644 --- a/.github/scripts/vrt-selector.test.mts +++ b/.github/scripts/vrt-selector.test.mts @@ -44,6 +44,8 @@ const expectedPasswordStories = [ "pages-signup--register-success-201", ].sort(); +const expectedNewStoryEntries = ["newstory--compact", "newstory--default"]; + test("PasswordInput fixture reaches the exact 16-story set and excludes docs", async () => { const index = await loadJson("index.json"); const graph = await loadJson("preview-graph.json"); @@ -57,6 +59,53 @@ test("PasswordInput fixture reaches the exact 16-story set and excludes docs", a ); }); +test("a newly added story file selects its story entries as PARTIAL", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const { manifest } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/stories/NewStory.stories.ts"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + assert.equal(manifest.mode, "PARTIAL"); + assert.deepEqual(manifest.selected_story_ids, expectedNewStoryEntries); + assert.deepEqual(manifest.reason_codes, ["exact_reachability"]); +}); + +test("a story entry added to an existing file remains explicit in the exact gate", async () => { + const index = await loadJson("index.json"); + const graph = await loadJson("preview-graph.json"); + const { manifest, filteredIndex } = selectVrt({ + baselineCommit: "a".repeat(40), + headCommit: "b".repeat(40), + changedPaths: ["apps/frontend/stories/NewStory.stories.ts"], + isMain: false, + isDependencyUpdate: false, + dependencyPhase: 0, + index, + graph, + }); + const filteredSha = sha256(stableJson(filteredIndex)); + assert.deepEqual(manifest.selected_story_ids, expectedNewStoryEntries); + assert.doesNotThrow(() => + assertCaptureGate({ + manifest, + executedStoryIds: expectedNewStoryEntries, + capturedStoryIds: expectedNewStoryEntries, + servedIndexSha256: filteredSha, + pending: 0, + skipped: 0, + failed: 0, + screenshotsOutsideManifest: 0, + }), + ); +}); + test("unknown source fails closed to FULL", async () => { const index = await loadJson("index.json"); const graph = await loadJson("preview-graph.json"); diff --git a/docs/development/visual-regression-new-stories.md b/docs/development/visual-regression-new-stories.md new file mode 100644 index 000000000..d3e56e648 --- /dev/null +++ b/docs/development/visual-regression-new-stories.md @@ -0,0 +1,92 @@ +# 新しい Storybook story の視覚テスト運用 + +このガイドは、新しい story ファイルまたは既存ファイルの story entry を追加する開発者を対象とする。 + +選択処理の仕組みとプロバイダを載せ替えるときの境界は、[視覚テストの差分選択設計](visual-regression-selection-design.md)に記載している。 + +## Pull Request で起きること + +新しい story は、比較対象の baseline に同名の screenshot がないため、Argos では added と表示される。 + +added は撮影や比較の失敗を意味しない。 +「baseline に同名の screenshot が存在しない」という比較結果である。 + +新しい story ファイルを追加すると、selector は変更された story ファイルを preview graph の seed にする。 +そのファイルに属する `type=story` の entry を `selected_story_ids` へ入れ、PARTIAL run として撮影する。 + +既存の story ファイルに entry を追加した場合も、その entry は独立した story ID として manifest、実行記録、撮影記録に現れる。 +ただし、selector はソース差分を entry の構文単位では解析しない。 +変更されたファイルに属する全 story entry を選ぶため、「追加した entry だけを撮影する」という意味での entry 単位選択ではない。 + +docs entry は `type=docs` なので選択対象に入らない。 +同じ import path に docs entry があっても、story ID の集合には混ざらない。 + +## PARTIAL run と baseline + +PARTIAL run は Argos CLI の `--subset` として upload する。 + +Argos の baseline 候補検索は `subset=true` の build を明示的に除外する。 +したがって、PARTIAL run を承認しても、その build が将来の baseline へ昇格することはない。 + +新しい story は、次の条件をすべて満たす非 subset build で初めて baseline 候補に入る。 + +- build が完了している。 +- framework test が成功し、screenshot bucket が valid である。 +- build が rejected ではない。 +- reference build、orphan build、承認済みの check build、または merge 済み Pull Request の check build である。 + +現在の workflow では、main への push が全 story を撮る FULL backstop であり、main の reference build は自動承認される。 +そのため、Pull Request の PARTIAL run で added を確認し、変更が main へ入った後の FULL run が新しい story を含む baseline 候補になる。 + +## 開発者が確認すること + +Argos の added screenshot が意図した初期表示であることを確認する。 +意図した表示なら、通常の visual review と同じように承認する。 + +この承認は Pull Request の review 結果を確定する操作であり、PARTIAL build を baseline へ昇格させる操作ではない。 + +意図しない表示なら、story、fixture、mock、時刻固定を修正してから再実行する。 +added であることだけを理由に失敗として扱ったり、baseline 不在を隠すために既存 screenshot 名を流用したりしない。 + +## 撮影されなかったときの調査 + +GitHub Actions の `vrt-selection--` artifact を取得し、次の順序で確認する。 + +1. `.vrt/selection-manifest.json` の `changed_paths` と `in_scope_paths` に story ファイルがあるか確認する。 +2. `mode`、`reason_codes`、`baseline_commit`、`head_commit` を確認する。 +3. `selected_story_ids` に追加した story ID があるか確認する。 +4. `.vrt/executed-story-ids.txt` と `.vrt/captured-story-ids.txt` に同じ ID があるか確認する。 +5. `.vrt/capture-gate.json` の `pending`、`skipped`、`failed`、`screenshots_outside_manifest` がすべて 0 か確認する。 +6. `source_index_sha256`、`filtered_index_sha256`、`served_index_sha256` の関係を確認する。 + +PARTIAL の gate は次の不変条件を要求する。 + +```text +selected_story_ids == executed_story_ids == captured_story_ids +``` + +新しい story に baseline がなくても、この三集合が一致し、served index の SHA が manifest と一致すれば gate は通る。 +gate は visual baseline の有無を合否条件にしていないため、Argos 側では upload 後に added と判定できる。 + +`selected_story_ids` に ID がなければ、Storybook build の `index.json` で entry の `type`、`id`、`importPath` を確認する。 +`type=docs` の entry は仕様どおり除外される。 + +次に `.vrt/preview-graph.json` で `importPath` に対応する story module を確認する。 +preview graph は Storybook の Vite build が module graph を解決した後に生成されるため、新しいファイルが Storybook の `stories` glob に一致しなければ graph に載らない。 + +graph の seed が解決できない場合、selector は撮影漏れを避けるため FULL へ退避し、`preflight_fail_closed` または `zero_reach_fail_closed` を記録する。 +この場合は reason を消すために manifest を手で書き換えず、Storybook の glob、ファイル拡張子、import path、graph の正規化を修正する。 + +SHA が一致しない場合、selector が作った filtered index と、test runner が実際に配信された index が異なる。 +古い `storybook-static` の再利用、selector 実行後の index 書換え、別ディレクトリの配信を疑う。 + +## 現在の検出限界 + +selector が直接 seed にできるのは、`apps/frontend/` 以下の Vue、JavaScript、TypeScript module である。 +CSS、Storybook 設定、lockfile、workflow などの変更は影響範囲を安全に限定できないため FULL になる。 + +MDX は docs 用 glob に含まれるが、selector の story module seed には含まれない。 +MDX 変更で visual story を追加する運用を導入する場合は、index と graph の対応を fixture で先に証明してから selector の対象拡張が必要になる。 + +rename と copy は変更前後のパスを差分集合へ入れる。 +変更前の module が現行 graph に存在しない場合は解決失敗として FULL へ退避するため、撮影漏れにはならない。 diff --git a/docs/development/visual-regression-selection-design.md b/docs/development/visual-regression-selection-design.md new file mode 100644 index 000000000..7842421cd --- /dev/null +++ b/docs/development/visual-regression-selection-design.md @@ -0,0 +1,146 @@ +# 視覚テストの差分選択設計 + +この文書は、視覚テストの撮影集合を変更差分から決める contract と、visual testing provider を載せ替えるときの境界を定める。 + +新しい story を追加するときの開発者向け手順は、[新しい Storybook story の視覚テスト運用](visual-regression-new-stories.md)に記載している。 + +## 設計上の不変条件 + +視覚差分がないことを selector 自体は保証しない。 +selector が保証するのは、provider が実際に比較する baseline commit から HEAD までの変更を入力にし、選んだ story を過不足なく実行して撮影したことだけである。 + +撮影工程は次の不変条件を満たさなければ upload へ進めない。 + +```text +selected_story_ids == executed_story_ids == captured_story_ids +pending == skipped == failed == screenshots_outside_manifest == 0 +served_index_sha256 == expected_index_sha256 +``` + +不明な module、壊れた graph、未確認の baseline を「影響なし」とみなさない。 +影響範囲を限定できない入力は FULL へ退避する。 + +PARTIAL build を baseline 候補にしない。 +部分集合を完全な screenshot 集合として再利用すると、選ばれなかった story の差分が将来の比較から消えるためである。 + +## 入出力 contract + +**provider-neutral selector** は `.github/scripts/vrt-selector.mts` に置く。 +selector は provider の build ID、review payload、API response を受け取らない。 + +selector の入力は次の値である。 + +- provider が選んだ `baselineCommit`。確認できない場合は `null`。 +- `headCommit`。 +- baseline から HEAD までの rename と copy の両側を含む `changedPaths`。 +- main、dependency update、dependency phase の分類。 +- Storybook の `index.json`。 +- Storybook build が生成した preview graph。 +- adapter が検出した preflight error。 + +selector の出力は selection manifest と、PARTIAL の場合だけ filtered Storybook index である。 +manifest は mode、commit、変更パス、story ID、reason code、source index、filtered index、graph の SHA-256 を持つ。 + +SHA は安定順序の JSON から計算する。 +artifact の内容と gate が実際に配信した index を同じ値で照合するためである。 + +## preview graph + +`apps/frontend/buildSrc/vrtGraphPlugin.ts` は Storybook の Vite build 内で `getModuleIds` と `getModuleInfo` を読み、schema version 1 の graph を生成する。 + +各 module の `reasons` は、その module が import する静的 module と動的 module である。 +selector は辺を逆向きにたどり、変更 module を import する story module を完全 BFS で求める。 + +新しい story ファイルは Storybook build が index と graph を作った後に selector へ渡る。 +selector は `apps/frontend/` 以下の Vue、JavaScript、TypeScript module を seed にできるため、`apps/frontend/stories/` の追加ファイルも graph から選べる。 + +index は `type=story` だけを正規化する。 +docs entry を同じ import path の story と混同しない。 + +同じ story ファイルに複数 entry がある場合、各 entry は別の story ID として manifest に入る。 +ただし、変更差分を story export の構文単位には分解せず、変更されたファイルに属する全 story ID を選ぶ。 + +## FULL、PARTIAL、NONE + +**FULL** は全 story を撮影する。 +main の per-merge backstop、baseline 未確認、preflight error、denylist、zero reach、graph 解決失敗で選ぶ。 + +**PARTIAL** は graph から到達した story だけを撮影する。 +filtered index を Storybook の配信対象へ置き換え、manifest と filtered index の SHA を gate で照合する。 + +**NONE** は視覚テストの対象外であることを証跡化し、撮影と upload を行わない。 +変更パスがすべて対象外の場合、または dependency graph が phase 1 で到達 story 0 と証明した場合に選ぶ。 + +NONE でも manifest と gate artifact を残す。 +「workflow が動かなかった」のか「対象外と判定した」のかを区別するためである。 + +## baseline と変更パス + +Argos adapter は `/v2/baseline` へ HEAD の ancestor commit を近い順に渡し、Argos が選んだ eligible build の HEAD SHA を `baselineCommit` とする。 + +baseline 候補は complete、valid、非 subset、非 skipped、非 rejected でなければならない。 +reference または orphan は承認済みとして扱い、check は承認済み review または merge 済み Pull Request を必要とする。 + +Argos の実 build 44 では、HEAD `5313b21ea28a5cb2b73514d3d5624550a07d8d11` に対し、build 43 の `18e85b91d2916d75fa9114e8dc651c6536eddf42` が `baseBuild.head.sha` と `baseScreenshotBucket.commit` の両方に現れた。 +adapter が利用する baseline API と Argos build の公開情報が同じ commit を示すことを確認した。 + +変更パスは baseline を B、HEAD を H とする B..H の commit 履歴から集める。 +rename と copy は変更前後の両方を含める。 +Pull Request の base SHA だけを使わないのは、stacked Pull Request や main の build gap で provider の baseline が base SHA より古くなる場合があるためである。 + +## fail-closed 分類 + +次の変更は graph の到達集合だけでは安全に限定できないため FULL にする。 + +- lockfile、workspace 定義、frontend package 定義。 +- Storybook 設定、provider workflow。 +- CSS、SCSS、Sass、Less。 +- icon と virtual module。 +- graph schema 不正、未解決 seed、未解決 reason。 + +story module の追加や変更は graph で解決できるため PARTIAL にできる。 +解決できなければ `preflight_fail_closed` または `zero_reach_fail_closed` で FULL に戻る。 + +dependency update は phase 0 を FULL とする。 +phase 1 では package owner と graph 到達集合を使い、到達 story があれば PARTIAL、0 と証明できれば NONE にできる。 +pnpm module ID の package owner は、二つ目の `node_modules` より下の package 名から求める。 + +## capture と証跡 + +test runner は story の訪問前に `.vrt/executed-story-ids.txt` へ ID を記録し、screenshot 完了後に `.vrt/captured-story-ids.txt` へ同じ ID を記録する。 + +gate は test result、served index、manifest、二つの ID 集合を読み、exact-set と SHA を検証する。 +一つでも一致しなければ upload を止める。 + +workflow は PARTIAL のときだけ provider adapter へ subset を伝える。 +Argos では CLI の `--subset` がこの役割を持つ。 + +selection manifest、source index、graph、実行ID、撮影ID、test result、gate result は Actions artifact として残す。 +独立した品質確認では、被検証者が作った集計値を信用せず、同じ commit から fixture と selector を再実行して集合と SHA を再生成する。 + +## provider の載せ替え + +載せ替え時に再利用する範囲は次のとおりである。 + +- Storybook index の正規化と docs 除外。 +- preview graph の生成と完全 BFS。 +- FULL、PARTIAL、NONE の分類。 +- selection manifest と SHA。 +- filtered index。 +- executed、captured、served index の exact gate。 +- fail-closed の denylist と reason code。 + +書き換える範囲は provider adapter と upload step である。 + +- provider が実際に選ぶ baseline commit の取得。 +- eligible baseline の条件確認。 +- subset build を baseline 候補から除外する方法。 +- reference branch と review の承認規則。 +- provider 固有の認証、build name、upload option。 +- provider UI の added、changed、removed の表示と review 操作。 + +載せ替え先が「比較に使った baseline commit」を取得できなければ、PARTIAL を有効にしない。 +PR base や latest main を代用品にすると、provider の実 baseline との gap にある変更を選択集合から落とす可能性があるためである。 + +載せ替え先が subset build の baseline 除外を保証できなければ、PARTIAL upload を行わない。 +この条件を満たせない provider では FULL のみを使うか、完全集合を保持する自前の baseline store が必要になる。 From 9948a700b2a1539208cb7c107b7be43bd9ffdc75 Mon Sep 17 00:00:00 2001 From: default Date: Sun, 26 Jul 2026 13:40:08 +0000 Subject: [PATCH 4/6] fix(frontend): scope node tsconfig sources --- apps/frontend/tsconfig.node.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/tsconfig.node.json b/apps/frontend/tsconfig.node.json index 257f66612..95924edd8 100644 --- a/apps/frontend/tsconfig.node.json +++ b/apps/frontend/tsconfig.node.json @@ -9,5 +9,5 @@ "types": ["node"], "skipLibCheck": true }, - "include": ["vite.config.ts", "buildSrc/**/*"] + "include": ["vite.config.ts", "buildSrc/env.ts"] } From 1cf2e7f0390e46508b0ce087cdf908f54e73a16b Mon Sep 17 00:00:00 2001 From: default Date: Sun, 26 Jul 2026 14:47:23 +0000 Subject: [PATCH 5/6] =?UTF-8?q?docs(vrt):=20MDX=E6=8B=A1=E5=BC=B5=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=81=AE=E4=B8=BB=E8=BF=B0=E3=82=92=E6=95=B4=E3=81=88?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/development/visual-regression-new-stories.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/visual-regression-new-stories.md b/docs/development/visual-regression-new-stories.md index d3e56e648..6a235fc87 100644 --- a/docs/development/visual-regression-new-stories.md +++ b/docs/development/visual-regression-new-stories.md @@ -86,7 +86,7 @@ selector が直接 seed にできるのは、`apps/frontend/` 以下の Vue、Ja CSS、Storybook 設定、lockfile、workflow などの変更は影響範囲を安全に限定できないため FULL になる。 MDX は docs 用 glob に含まれるが、selector の story module seed には含まれない。 -MDX 変更で visual story を追加する運用を導入する場合は、index と graph の対応を fixture で先に証明してから selector の対象拡張が必要になる。 +MDX 変更で visual story を追加する運用を導入する場合は、index と graph の対応を fixture で先に証明してから selector の対象を拡張する必要がある。 rename と copy は変更前後のパスを差分集合へ入れる。 変更前の module が現行 graph に存在しない場合は解決失敗として FULL へ退避するため、撮影漏れにはならない。 From c35418d0b101fb72781dcc0dff347a7d4a8652f0 Mon Sep 17 00:00:00 2001 From: default Date: Sun, 26 Jul 2026 14:48:13 +0000 Subject: [PATCH 6/6] =?UTF-8?q?docs:=20Storybook=E8=A6=96=E8=A6=9A?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=AC=E3=82=A4=E3=83=89=E3=81=B8?= =?UTF-8?q?=E3=81=AE=E5=B0=8E=E7=B7=9A=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index 4762595d2..75a477c6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,6 +62,7 @@ cargo test --workspace --lib - バグ修正 PR には**修正前の main で fail する回帰テスト**を付ける（バグの証明として機能させる） - 統合テストは `tests/common` の `TestApp` を使う。拒否系（403/404）と対照の成功系（200/201、過剰拒否でないこと）をセットで書く +- Storybook の story を追加・変更するときは、視覚テストが変更差分から撮影対象を選ぶため、`docs/development/visual-regression-new-stories.md` を参照する（新しい story の扱い・未撮影時の調査手順） - エラーは握り潰さず `?` で伝播する（`unwrap_or(false)` / `let _ =` でのもみ消しが実バグを隠した前例あり） - コミットは Conventional Commits + 日本語（例: `fix(backend): …` / `refactor(workspace): …`）。1 Phase・1 関心 = 1 PR - PR 本文も日本語。「概要 / 変更内容 / 挙動の変化 / テスト」の構成