[AGENT] Select browser CI separately from visual evidence

.github/scripts/pr-verification-report.cjs

@@ -1,6 +1,6 @@
 "use strict";
 
-const { COMPARISON_JOBS, evaluateVerification, selectVisuals } = require("./visual-ci.cjs");
+const { BROWSER_JOBS, COMPARISON_JOBS, evaluateVerification, selectEvidence } = require("./visual-ci.cjs");
 
 const REPORT_MARKER = "<!-- vrdex-pr-verification-report -->";
 const LEGACY_MARKERS = [
@@ -14,6 +14,15 @@ const LEGACY_MARKERS = [
 ];
 
 const REPORT_ROWS = [
+  { job: "lint-web", label: "Web lint" },
+  { job: "verify-docs", label: "Docs" },
+  { job: "verify-api-mcp", label: "API and MCP" },
+  { job: "typecheck-web", label: "Web types and unit tests" },
+  { job: "typecheck-backend", label: "Backend types and unit tests" },
+  { job: "verify-backend-local", label: "Backend bootstrap" },
+  { job: "restream-local-checks", label: "Restream checks" },
+  { job: "build-web", label: "Web build" },
+  { job: "build-storybook", label: "Storybook build" },
   { artifact: "playwright-public-smoke", job: "playwright-public-smoke", label: "Public route smoke" },
   {
     artifact: "playwright-data-flow",
@@ -138,11 +147,14 @@ function buildReport({
       row.optional && job.result === "success" && job.outputs?.configured === "false";
     const result = optionalWasSkipped ? "skipped" : job.result || "unknown";
 
-    if (COMPARISON_JOBS.has(row.job) && evaluation.selected === false && job.result === "skipped") {
-      return `| ${row.label} | NOT SELECTED | Not requested |`;
+    const expected = COMPARISON_JOBS.has(row.job) ? evaluation.visuals
+      : BROWSER_JOBS.includes(row.job) || row.job === "build-storybook" ? evaluation.browsers : null;
+    if (expected === false && job.result === "skipped") {
+      return `| ${row.label.replace("screenshots", "preview")} | NOT SELECTED | Not requested |`;
     }
-    const label = evaluation.selected === false ? row.label.replace("screenshots", "assertions only") : row.label;
-    let evidence = "Missing evidence";
+    const label = evaluation.visuals === false && evaluation.browsers === true
+      ? row.label.replace("screenshots", "assertions only") : row.label;
+    let evidence = row.artifact ? "Missing evidence" : `[Open run](${runUrl})`;
     if (artifact) {
       evidence = `[Open artifact](${artifactUrl({ artifact, owner, repo, runId })})`;
     } else if (result === "skipped") {
@@ -168,14 +180,16 @@ function buildReport({
 
   return [
     REPORT_MARKER,
+    `<!-- vrdex-test-selection:${JSON.stringify({ head: headSha, browsers: evaluation.browsers, visuals: evaluation.visuals })} -->`,
     "## PR verification report",
     "",
     needsAttention
       ? "One or more reported checks need attention."
-      : "All configured preview and verification checks passed.",
+      : "All required checks and selected coverage passed.",
     "",
-    `Visuals selected for this run: ${evaluation.selected === null ? "UNKNOWN" : evaluation.selected ? "yes" : "no (assertions only)"}.`,
-    ...(selectionChanged ? ["Visual selection changed since this run. Run results below describe the original selection; request a new normal PR run for current visual evidence.", ""] : []),
+    `Browsers selected for this run: ${evaluation.browsers === null ? "UNKNOWN" : evaluation.browsers ? "yes" : "no"}.`,
+    `Visuals selected for this run: ${evaluation.visuals === null ? "UNKNOWN" : evaluation.visuals ? "yes" : "no"}.`,
+    ...(selectionChanged ? ["Test selection changed since this run. Run results below describe the original selection; request a new normal PR run for current selected coverage.", ""] : []),
     ...(needsAttention ? evaluation.errors.map(error => `- ${error}`) : []),
     "",
     "| Check | Result | Evidence |",
@@ -197,7 +211,6 @@ async function updatePrVerificationReport({ core, context, github, needs }) {
     core.info("Ignoring verification evidence for an outdated PR head.");
     return;
   }
-  const evaluationSelection = needs["visual-selection"]?.outputs?.selected;
 
   let artifacts = [];
   let files = [];
@@ -246,7 +259,9 @@ async function updatePrVerificationReport({ core, context, github, needs }) {
     core.info("Ignoring verification evidence because the PR head changed during collection.");
     return;
   }
-  const selectionChanged = String(selectVisuals("pull_request", latest.labels)) !== evaluationSelection;
+  const currentSelection = selectEvidence("pull_request", latest.labels || []);
+  const selectionChanged = evaluation.browsers !== null && evaluation.visuals !== null
+    && (currentSelection.browsers !== evaluation.browsers || currentSelection.visuals !== evaluation.visuals);
   const body = buildReport({
     artifacts,
     files,

.github/scripts/pr-verification-report.test.cjs

@@ -1,180 +1,116 @@
 "use strict";
-
 const assert = require("node:assert/strict");
 const test = require("node:test");
-
-const {
-  LEGACY_MARKERS,
-  REPORT_MARKER,
-  buildReport,
-  updatePrVerificationReport,
-} = require("./pr-verification-report.cjs");
-
-function successfulNeeds() {
-  return {
-    "visual-selection": { result: "success", outputs: { selected: "true" } },
-    "playwright-public-smoke": { result: "success" },
-    "playwright-data-flow": { outputs: {}, result: "success" },
-    "playwright-hosted-data-flow": { outputs: { configured: "false" }, result: "success" },
-    "playwright-image-diff": { outputs: {}, result: "success" },
-    "playwright-public-preview": { outputs: {}, result: "success" },
-    "storybook-component-preview": { outputs: {}, result: "success" },
-    "storybook-image-diff": { outputs: {}, result: "success" },
-  };
+const { LEGACY_MARKERS, REPORT_MARKER, buildReport, updatePrVerificationReport } = require("./pr-verification-report.cjs");
+const coreJobs = ["lint-web", "verify-docs", "verify-api-mcp", "typecheck-web", "typecheck-backend", "verify-backend-local", "restream-local-checks", "build-web"];
+const browserJobs = ["playwright-public-smoke", "playwright-data-flow", "playwright-public-preview", "storybook-component-preview"];
+function fixture(browsers = true, visuals = true) {
+  const needs = Object.fromEntries(coreJobs.map(job => [job, { result: "success" }]));
+  needs["visual-selection"] = { result: "success", outputs: { browsers: String(browsers), visuals: String(visuals) } };
+  needs["build-storybook"] = { result: browsers ? "success" : "skipped" };
+  for (const job of browserJobs) needs[job] = { result: browsers ? "success" : "skipped" };
+  needs["playwright-hosted-data-flow"] = { result: browsers ? "success" : "skipped", outputs: browsers ? { configured: "false" } : {} };
+  for (const job of ["playwright-image-diff", "storybook-image-diff"]) needs[job] = { result: visuals ? "success" : "skipped" };
+  const artifacts = [...(browsers ? browserJobs : []), ...(visuals ? ["playwright-image-diff", "storybook-image-diff"] : [])].map((name, id) => ({ name, id, expired: false }));
+  return { needs, artifacts, headSha: "0123456789abcdef", owner: "BASIC-BIT", repo: "VRDex", runId: 456 };
 }
 
-test("buildReport collates checks, artifacts, previews, and changed baselines", () => {
-  const baselineFiles = Array.from({ length: 14 }, (_, index) => ({
-    filename: `apps/web/e2e/__screenshots__/desktop-chromium/route-${index}.png`,
-    status: "modified",
-  }));
-  const report = buildReport({
-    artifacts: [
-      { expired: false, id: 101, name: "playwright-data-flow" },
-      { expired: false, id: 102, name: "playwright-image-diff" },
-    ],
-    files: baselineFiles,
-    headSha: "0123456789abcdef",
-    needs: successfulNeeds(),
-    owner: "BASIC-BIT",
-    repo: "VRDex",
-    runAttempt: 2,
-    runId: 456,
-  });
+test("default report marks every browser and Storybook build NOT SELECTED without claiming browser success", () => {
+  const report = buildReport(fixture(false, false));
+  assert.match(report, /All required checks and selected coverage passed/);
+  assert.match(report, /Browsers selected for this run: no/);
+  for (const label of ["Storybook build", "Public route smoke", "Mutation data flow", "Hosted data flow", "Public route preview", "Storybook preview", "Public route image diff", "Storybook image diff"]) {
+    assert.match(report, new RegExp(`${label} \\| NOT SELECTED \\| Not requested`));
+  }
+  assert.match(report, /Web build \| PASSED/);
+  assert.doesNotMatch(report, /Missing evidence|Not configured|assertions only/);
+});
 
-  assert.match(report, new RegExp(REPORT_MARKER));
-  assert.match(report, /Mutation data flow \| PASSED \| \[Open artifact\]/);
+test("browsers-only report keeps assertion success distinct from unselected captures and comparisons", () => {
+  const report = buildReport(fixture(true, false));
+  assert.match(report, /Browsers selected for this run: yes/);
+  assert.match(report, /Visuals selected for this run: no/);
+  assert.match(report, /Public route assertions only \| PASSED \| \[Open artifact\]/);
+  assert.match(report, /Storybook assertions only \| PASSED/);
+  assert.match(report, /Public route image diff \| NOT SELECTED/);
   assert.match(report, /Hosted data flow \| SKIPPED \| Not configured/);
-  assert.doesNotMatch(report, /Vercel preview/);
-  assert.match(report, /<details><summary>Changed visual baselines \(14\)<\/summary>/);
+  assert.doesNotMatch(report, /need attention|screenshots \| PASSED/);
+});
+
+test("visual report collates evidence and every changed baseline with immutable mode metadata", () => {
+  const input = fixture();
+  const files = Array.from({ length: 14 }, (_, index) => ({ filename: `apps/web/e2e/__screenshots__/desktop-chromium/route-${index}.png`, status: "modified" }));
+  const report = buildReport({ ...input, files, runAttempt: 2 });
+  assert.match(report, new RegExp(REPORT_MARKER));
+  assert.match(report, /Public route screenshots \| PASSED/);
+  assert.match(report, /Visuals selected for this run: yes/);
+  assert.match(report, /vrdex-test-selection:.*"head":"0123456789abcdef".*"browsers":true.*"visuals":true/);
   assert.equal(report.match(/<img /g)?.length, 14);
-  assert.match(report, /desktop-chromium \/ route-0/);
   assert.match(report, /desktop-chromium \/ route-13/);
-  assert.doesNotMatch(report, /more in the image-diff artifacts/);
   assert.match(report, /attempt 2/);
 });
 
-test("buildReport calls out a skipped required check", () => {
-  const needs = successfulNeeds();
-  needs["playwright-public-preview"].result = "skipped";
-
-  const report = buildReport({
-    artifacts: [],
-    headSha: "0123456789abcdef",
-    needs,
-    owner: "BASIC-BIT",
-    repo: "VRDex",
-    runId: 456,
-  });
+test("default core failures and selected unexpected skips remain failures", () => {
+  const input = fixture(false, false); input.needs["build-web"].result = "failure";
+  assert.match(buildReport(input), /One or more reported checks need attention/);
+  assert.match(buildReport(input), /Web build \| FAILED/);
+  const selected = fixture(); selected.needs["playwright-image-diff"].result = "skipped";
+  selected.artifacts = selected.artifacts.filter(a => a.name !== "playwright-image-diff");
+  assert.match(buildReport(selected), /Public route image diff \| SKIPPED \| Not generated \(unexpected skip\)/);
+  const failedDefault = fixture(false, false); failedDefault.needs["playwright-data-flow"].result = "failure";
+  assert.match(buildReport(failedDefault), /Mutation data flow \| FAILED/);
+});
 
-  assert.match(report, /One or more reported checks need attention/);
-  assert.match(report, /Public route screenshots \| SKIPPED/);
+test("invalid selection stays UNKNOWN and never relabels skipped jobs NOT SELECTED", () => {
+  const input = fixture(false, false); input.needs["visual-selection"].outputs = {};
+  const report = buildReport(input);
+  assert.match(report, /Browsers selected for this run: UNKNOWN/);
+  assert.match(report, /Visuals selected for this run: UNKNOWN/);
+  assert.doesNotMatch(report, /NOT SELECTED/);
 });
 
-test("updatePrVerificationReport updates one report and deletes legacy comments", async () => {
-  const updated = [];
-  const deleted = [];
-  const methods = {};
+async function publish({ input = fixture(), heads = ["0123456789abcdef"], labels = [[{ name: "ci:visuals" }]], comments = [] } = {}) {
+  let reads = 0;
+  const updates = [], creates = [], deletes = [], failures = [], methods = {};
   const github = {
-    paginate: async (method) => {
-      if (method === methods.listWorkflowRunArtifacts) return [];
-      if (method === methods.listFiles) return [];
-      return [
-        { body: `${REPORT_MARKER}\nold`, id: 1, user: { type: "Bot" } },
-        { body: `${LEGACY_MARKERS[0]}\nold`, id: 2, user: { type: "Bot" } },
-        { body: `${LEGACY_MARKERS[1]}\nhuman`, id: 3, user: { type: "User" } },
-      ];
-    },
+    paginate: async method => method === methods.artifacts ? input.artifacts : method === methods.files ? [] : comments,
     rest: {
-      actions: {
-        listWorkflowRunArtifacts: (methods.listWorkflowRunArtifacts = () => {}),
-      },
-      issues: {
-        createComment: async () =>
-          assert.fail("existing report should be updated"),
-        deleteComment: async ({ comment_id }) => deleted.push(comment_id),
-        listComments: (methods.listComments = () => {}),
-        updateComment: async (input) => updated.push(input),
-      },
-      pulls: {
-        get: async () => ({ data: { head: { sha: "0123456789abcdef" }, labels: [{ name: "ci:visuals" }] } }),
-        listFiles: (methods.listFiles = () => {}),
-      },
+      pulls: { get: async () => { const index = reads++; return { data: { head: { sha: heads[Math.min(index, heads.length - 1)] }, labels: labels[Math.min(index, labels.length - 1)] } }; }, listFiles: methods.files = () => {} },
+      actions: { listWorkflowRunArtifacts: methods.artifacts = () => {} },
+      issues: { listComments: methods.comments = () => {}, updateComment: async value => updates.push(value), createComment: async value => creates.push(value), deleteComment: async value => deletes.push(value.comment_id) },
     },
   };
-  const context = {
-    issue: { number: 168 },
-    payload: { pull_request: { head: { sha: "0123456789abcdef" } } },
-    repo: { owner: "BASIC-BIT", repo: "VRDex" },
-    runAttempt: 1,
-    runId: 456,
-  };
-
   await updatePrVerificationReport({
-    context,
-    core: { warning: assert.fail, setFailed: () => {} },
-    github,
-    needs: successfulNeeds(),
+    context: { issue: { number: 168 }, repo: { owner: "BASIC-BIT", repo: "VRDex" }, payload: { pull_request: { head: { sha: "0123456789abcdef" } } }, runId: 456, runAttempt: 1 },
+    github, core: { warning: assert.fail, info: () => {}, setFailed: message => failures.push(message) }, needs: input.needs,
   });
+  return { reads, updates, creates, deletes, failures };
+}
 
-  assert.equal(updated.length, 1);
-  assert.equal(updated[0].comment_id, 1);
-  assert.deepEqual(deleted, [2]);
-});
-
-
-test("default report distinguishes assertion success from unselected image comparisons", () => {
-  const needs = successfulNeeds();
-  needs["visual-selection"].outputs.selected = "false";
-  needs["playwright-image-diff"].result = "skipped";
-  needs["storybook-image-diff"].result = "skipped";
-  const artifacts = ["playwright-public-smoke", "playwright-data-flow", "playwright-public-preview", "storybook-component-preview"].map(name => ({ name, id: 1 }));
-  const report = buildReport({ artifacts, headSha: "0123456789abcdef", needs, owner: "BASIC-BIT", repo: "VRDex", runId: 456 });
-  assert.match(report, /assertions only/i);
-  assert.match(report, /Public route image diff \| NOT SELECTED/);
-  assert.doesNotMatch(report, /need attention/);
-  assert.doesNotMatch(report, /Public route screenshots \| PASSED/);
-});
-
-test("old-head runs never publish a report", async () => {
-  await updatePrVerificationReport({
-    context: { issue: { number: 1 }, repo: { owner: "BASIC-BIT", repo: "VRDex" }, payload: { pull_request: { head: { sha: "old-head" } } } },
-    github: { rest: { pulls: { get: async () => ({ data: { head: { sha: "new-head" }, labels: [] } }) } } },
-    core: { info: () => {}, warning: assert.fail, setFailed: assert.fail },
-    needs: successfulNeeds(),
-  });
+test("publisher updates one bot report, removes legacy comments and preserves human comments", async () => {
+  const result = await publish({ comments: [
+    { id: 1, body: REPORT_MARKER, user: { type: "Bot" } },
+    { id: 2, body: LEGACY_MARKERS[0], user: { type: "Bot" } },
+    { id: 3, body: LEGACY_MARKERS[1], user: { type: "User" } },
+  ] });
+  assert.equal(result.updates.length, 1); assert.equal(result.updates[0].comment_id, 1);
+  assert.deepEqual(result.deletes, [2]); assert.deepEqual(result.creates, []); assert.deepEqual(result.failures, []);
 });
 
-test("report preserves run selection when the label changes after it started", () => {
-  const report = buildReport({ artifacts: [], headSha: "0123456789abcdef", needs: successfulNeeds(), owner: "BASIC-BIT", repo: "VRDex", runId: 456, selectionChanged: true });
-  assert.match(report, /selection changed since this run/i);
+test("old or changed heads never publish stale evidence", async () => {
+  for (const heads of [["new-head"], ["0123456789abcdef", "new-head"]]) {
+    const result = await publish({ heads });
+    assert.deepEqual(result.updates, []); assert.deepEqual(result.creates, []);
+  }
 });
 
-
-test("a head changed during artifact collection cannot publish stale evidence", async () => {
-  let reads = 0;
-  const github = {
-    paginate: async () => [],
-    rest: {
-      pulls: { get: async () => ({ data: { head: { sha: ++reads === 1 ? "old-head" : "new-head" }, labels: [] } }), listFiles: () => {} },
-      actions: { listWorkflowRunArtifacts: () => {} },
-      issues: { listComments: () => {}, createComment: assert.fail, updateComment: assert.fail },
-    },
-  };
-  await updatePrVerificationReport({
-    context: { issue: { number: 1 }, repo: { owner: "BASIC-BIT", repo: "VRDex" }, payload: { pull_request: { head: { sha: "old-head" } } } },
-    github, core: { info: () => {}, warning: assert.fail, setFailed: () => {} }, needs: successfulNeeds(),
-  });
-  assert.equal(reads, 2);
-});
-test("missing selection is UNKNOWN, while a selected unexpected skip is not called unconfigured", () => {
-  const needs = successfulNeeds();
-  needs["visual-selection"].outputs.selected = "";
-  const input = { artifacts: [], headSha: "0123456789abcdef", needs, owner: "BASIC-BIT", repo: "VRDex", runId: 456 };
-  const unknown = buildReport(input);
-  assert.match(unknown, /Visuals selected for this run: UNKNOWN/);
-  assert.doesNotMatch(unknown, /no \(assertions only\)/);
-  needs["visual-selection"].outputs.selected = "true";
-  needs["playwright-image-diff"].result = "skipped";
-  assert.match(buildReport(input), /Public route image diff \| SKIPPED \| Not generated \(unexpected skip\)/);
+test("publisher preserves run snapshots when browser or visual selection changes during collection", async () => {
+  for (const labels of [[{ name: "ci:browsers" }], [{ name: "ci:visuals" }]]) {
+    const result = await publish({ input: fixture(false, false), labels: [[], labels] });
+    assert.equal(result.creates.length, 1);
+    assert.match(result.creates[0].body, /selection changed since this run/i);
+    assert.match(result.creates[0].body, /Browsers selected for this run: no/);
+    assert.match(result.creates[0].body, /Visuals selected for this run: no/);
+    assert.deepEqual(result.failures, []);
+  }
 });