diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 279c3d2..aac1334 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,6 +9,11 @@ "name": "vaadin-playwright-test", "source": "./skills/vaadin-playwright-test", "description": "Generate Playwright integration tests for Vaadin views using the DramaFinder library." + }, + { + "name": "visual-verification", + "source": "./skills/visual-verification", + "description": "Visually verify Vaadin UI changes with a temporary DramaFinder test that batch-captures screenshots against the running app." } ] } diff --git a/README.md b/README.md index 804cfe9..aac1c74 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,25 @@ The documentation can be found in this repository and also deployed on https://parttio-dramafinder.mintlify.app/ if you find some errors please file an issue. -## Claude Code skill +## Claude Code skills -This repository ships a Claude Code skill that generates Playwright integration -tests for Vaadin views using Drama Finder. Install it as a plugin: +This repository ships two Claude Code skills. Install them as plugins: ``` /plugin marketplace add parttio/dramafinder /plugin install vaadin-playwright-test@dramafinder +/plugin install visual-verification@dramafinder ``` +- **vaadin-playwright-test** — generates Playwright integration tests for + Vaadin views using Drama Finder. +- **visual-verification** — visually verifies UI changes by writing a temporary + Drama Finder test that batch-captures screenshots against the running app, + then reviewing them. Backed by the `org.vaadin.addons.dramafinder.agent` + helpers (`VisualVerificationTest`, `AgentReporting`, `ComponentSnapshot`), which + write a screenshot, semantic component snapshot, and stack trace to + `target/agent-report/` on failure. + To get later updates, run `/plugin marketplace update dramafinder`. ## Usage diff --git a/skills/visual-verification/.claude-plugin/plugin.json b/skills/visual-verification/.claude-plugin/plugin.json new file mode 100644 index 0000000..224e6b2 --- /dev/null +++ b/skills/visual-verification/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "visual-verification", + "description": "Visually verify Vaadin UI changes by writing a temporary DramaFinder test that batch-captures screenshots against the running app, then reviewing them — a cheap alternative to interactive Playwright MCP.", + "version": "0.1.0", + "author": { + "name": "jcgueriaud1" + }, + "homepage": "https://github.com/parttio/dramafinder", + "repository": "https://github.com/parttio/dramafinder" +} diff --git a/skills/visual-verification/SKILL.md b/skills/visual-verification/SKILL.md new file mode 100644 index 0000000..8de938d --- /dev/null +++ b/skills/visual-verification/SKILL.md @@ -0,0 +1,132 @@ +--- +name: visual-verification +description: Visually verify an implemented use case by writing a temporary DramaFinder test (Java/Playwright) against the running app that batch-captures screenshots, then reviewing the screenshots. Use after implementing UI changes. Escalate to Playwright MCP only if the report is insufficient to diagnose a problem. +--- + +# Visual Verification + +Verify what the user sees. The screenshot is the ground truth; DOM and CSS are +helpers. But **do not drive the browser interactively via MCP by default** — +write a temporary DramaFinder test that performs the whole flow and captures all +screenshots in one batch run, then review the images. One run replaces dozens of +MCP round-trips and avoids accessibility-tree dumps entirely. + +Unless the use case specifies otherwise, use a **1920x1080** viewport. + +## Prerequisite: the application must be running + +The temp test connects to an **already running** application — it never boots +the app itself. Keeping the app out of the test run is what makes iterations +cheap. Before running the verification test: + +1. Check whether the app already responds on its URL. +2. If it doesn't, start it the way this project runs its app — the project + knows how (consult its README / `CLAUDE.md` / `AGENTS.md`). Start it in the + background, wait for it to answer, and remember that you started it so you + can stop it afterwards. + +## The loop + +1. Ensure the app is running with the required state (see prerequisite above + and "Reaching the state" below). +2. Write the temp test at + `src/test/java//agent/AgentVerifyIT.java` + (fixed name, fixed `agent` sub-package — overwrite the previous one, never + commit it). Extend `org.vaadin.addons.dramafinder.agent.VisualVerificationTest`, + which already wires in the `AgentReporting` extension. +3. Run only that test with the project's build tool, with quiet output: + - Maven: `mvn -q surefire:test -Dtest=AgentVerifyIT` + - Gradle: `./gradlew test --tests '*.AgentVerifyIT' --console=plain -q` + If a `scripts/agent-verify.*` wrapper exists in the project, prefer it — it + prints only pass/fail, assertion messages, and the report path. + Point the test at a non-default host/port with + `-Ddramafinder.agent.baseUrl=http://localhost:9000` (or the + `DRAMAFINDER_BASE_URL` env var). Run with `-Dheadless=false` to watch it. +4. Read the report directory — `target/agent-report/` (Maven) or + `build/agent-report/` (Gradle): view each numbered screenshot + (`01-…png`, `02-…png`, …) and apply the visual validation rules below. On + failure the `AgentReporting` extension also writes `failure.txt` (assertion + message + trimmed stack trace + URL), `failure.png` (full-page screenshot), + and `component-snapshot.txt` (semantic component snapshot). +5. Record results in the per-use-case checklist. Delete or overwrite the temp + test when done. + +## Writing the temp test + +Write the test as a normal DramaFinder test — for how to locate Vaadin +components and assert on them, use the **vaadin-playwright-test** skill. This +skill only adds the visual-capture concerns on top of it: + +- Extend `org.vaadin.addons.dramafinder.agent.VisualVerificationTest`, which + connects to the already-running app and wires in the `AgentReporting` + extension. It never boots the app or a Spring context. +- Navigate with `open("route")` (relative to the base URL) or plain Playwright + `page.navigate(...)`. The base URL defaults to `http://localhost:8080` and is + overridable via `-Ddramafinder.agent.baseUrl=...` or `DRAMAFINDER_BASE_URL`. +- Use **DramaFinder locators** for all Vaadin components — never hand-rolled + shadow-DOM selectors. +- The base class sets a 1920x1080 viewport for you; override per test, and add + 375x812 and 768x1024 passes when the use case has responsive requirements. +- `shot("name")` at every **key interaction point** and each **unique visual + state** — named descriptively (`01-login`, `02-order-list`, + `03-submit-dialog`). Screenshots are auto-numbered into the report directory. +- Assert behaviour only lightly (enough to know the flow progressed). Behaviour + is covered by the browserless tests (pyramid layer 3) — don't duplicate those + assertions here. Screenshots of unique visual states are the deliverable. + +## Reaching the state + +- **Deep-link with stable selectors** where possible; drive elements by + DramaFinder locators, button text, `aria-label`, or stable `name` attributes. +- If the screen under test needs data or a logged-in user, script the minimal + setup steps inside the same temp test — still one batch run, never set up + state interactively via MCP. Screenshot only the states under test, not the + setup steps. + +## Validating visual appearance + +Review every screenshot in the report directory against: + +1. Layout matches expectations (spacing, alignment, sizing) +2. Spacing & padding are consistent — appropriate breathing room, no cramped or + excessively spaced areas. Nested layouts (AppLayout > VerticalLayout > card) + don't double-up or collapse padding. Similar views (e.g., all admin views) + share the same content padding. +3. Typography is readable and consistent +4. Interactive elements are clearly identifiable +5. Responsive behaviour works at common breakpoints when required by the use + case (mobile, tablet, desktop) +6. Text contrast and readability + - All text clearly readable against its background (titles, labels, values, + badges) + - Colored text (warning/error values, status badges) has sufficient contrast + - Elements inheriting a different color scheme (dark sidebar vs light + content) render correctly — CSS custom properties like + `var(--vaadin-background-color)` may resolve differently per inherited + scheme + - No backgrounds swallow their content text + +## Escalation — Playwright MCP as last resort + +Use the MCP **only** when the batch run cannot answer the question: + +- a failure isn't diagnosable from `failure.txt` + screenshots + the semantic + snapshot, or +- the state is genuinely exploratory (unknown UI, need to poke around + interactively). + +When escalating, start from the evidence the report already produced (open the +failing route directly, don't replay the whole flow), scope any +`browser_snapshot` you take (`depth`, `filename`), and return to the batch loop +as soon as the cause is understood. + +## Steps (per use case) + +All steps must be done; thoroughness over speed. + +1. App running (started by you if needed) with the required state +2. Temp test navigates every route in the use case's UI/Routes section +3. Temp test performs each step of the main flow +4. Screenshots captured at key interaction points and unique visual states +5. Screenshots validated against the rules above +6. Results recorded — note any visual issues in the per-use-case checklist diff --git a/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReport.java b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReport.java new file mode 100644 index 0000000..f3a4abf --- /dev/null +++ b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReport.java @@ -0,0 +1,205 @@ +package org.vaadin.addons.dramafinder.agent; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; + +import com.microsoft.playwright.Page; + +/** + * Writes agent-oriented verification artifacts for a single test into a report + * directory ({@code target/agent-report/} for Maven, {@code build/agent-report/} + * for Gradle). + *

+ * It produces two kinds of output: + *

+ * A far cheaper agent loop than driving Playwright MCP interactively: one run + * batches every action and screenshot, then the agent reads a terse report. + */ +public class AgentReport { + + private static final int MAX_STACK_FRAMES = 15; + + private final Page page; + private final Path directory; + private int shotCounter = 0; + + /** + * Create a report writing into the build-tool default directory. + * + * @param page the Playwright page to screenshot and inspect + */ + public AgentReport(Page page) { + this(page, defaultReportDirectory()); + } + + /** + * Create a report writing into the given directory. + * + * @param page the Playwright page to screenshot and inspect + * @param directory the report output directory + */ + public AgentReport(Page page, Path directory) { + this.page = page; + this.directory = directory; + } + + /** + * Resolve the default report directory for the current build tool: + * {@code build/agent-report} when a Gradle build file is present, otherwise + * {@code target/agent-report} (Maven). + * + * @return the resolved report directory (not created) + */ + public static Path defaultReportDirectory() { + boolean gradle = Files.exists(Path.of("build.gradle")) + || Files.exists(Path.of("build.gradle.kts")) + || Files.exists(Path.of("settings.gradle")) + || Files.exists(Path.of("settings.gradle.kts")); + Path base = gradle ? Path.of("build") : Path.of("target"); + return base.resolve("agent-report"); + } + + /** + * The directory this report writes to. + * + * @return the report directory + */ + public Path directory() { + return directory; + } + + /** + * Capture a numbered screenshot of the current page state. + *

+ * The file name is {@code NN-.png}, where {@code NN} is a + * zero-padded, incrementing counter, giving deterministic ordering that + * matches the sequence of interaction points in the test. + * + * @param name a short descriptive name for the interaction point + * @return the written screenshot path + */ + public synchronized Path shot(String name) { + ensureDirectory(); + String fileName = String.format(Locale.ROOT, "%02d-%s.png", + ++shotCounter, sanitize(name)); + Path file = directory.resolve(fileName); + page.screenshot(new Page.ScreenshotOptions().setPath(file)); + return file; + } + + /** + * Write the failure bundle for a failed test: {@code failure.txt} (message + * + trimmed stack trace + page URL), {@code failure.png} (full-page + * screenshot), and {@code component-snapshot.txt} (semantic snapshot). + *

+ * Best-effort: a failure to capture the screenshot or snapshot (e.g. the + * page is already closed) never masks the original test failure. + * + * @param error the failure cause (may be {@code null}) + */ + public void captureFailure(Throwable error) { + ensureDirectory(); + + StringBuilder txt = new StringBuilder(); + txt.append("URL: ").append(safeUrl()).append(System.lineSeparator()); + txt.append(System.lineSeparator()); + txt.append(error == null ? "Test failed (no throwable)" : error.toString()); + txt.append(System.lineSeparator()).append(System.lineSeparator()); + txt.append(trimmedStackTrace(error)); + write(directory.resolve("failure.txt"), txt.toString()); + + try { + page.screenshot(new Page.ScreenshotOptions() + .setPath(directory.resolve("failure.png")) + .setFullPage(true)); + } catch (RuntimeException e) { + write(directory.resolve("failure.png.txt"), + "Screenshot capture failed: " + e.getMessage()); + } + + try { + String snapshot = ComponentSnapshot.capture(page); + write(directory.resolve("component-snapshot.txt"), + snapshot.isEmpty() ? "(no Vaadin components found)" : snapshot); + } catch (RuntimeException e) { + write(directory.resolve("component-snapshot.txt"), + "Component snapshot failed: " + e.getMessage()); + } + } + + private void ensureDirectory() { + try { + Files.createDirectories(directory); + } catch (IOException e) { + throw new UncheckedIOException( + "Could not create report directory " + directory, e); + } + } + + private void write(Path file, String content) { + try { + Files.writeString(file, content, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("Could not write " + file, e); + } + } + + private String safeUrl() { + try { + return page.url(); + } catch (RuntimeException e) { + return "(unknown)"; + } + } + + private static String trimmedStackTrace(Throwable error) { + if (error == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + StackTraceElement[] frames = error.getStackTrace(); + int shown = 0; + for (StackTraceElement frame : frames) { + if (shown >= MAX_STACK_FRAMES) { + sb.append("\t... ").append(frames.length - shown) + .append(" more").append(System.lineSeparator()); + break; + } + String cls = frame.getClassName(); + // Drop reflective/JUnit framework noise; keep test + library frames. + if (cls.startsWith("java.") + || cls.startsWith("jdk.") + || cls.startsWith("sun.") + || cls.startsWith("org.junit.") + || cls.startsWith("org.springframework.")) { + continue; + } + sb.append("\tat ").append(frame).append(System.lineSeparator()); + shown++; + } + Throwable cause = error.getCause(); + if (cause != null && cause != error) { + sb.append("Caused by: ").append(cause).append(System.lineSeparator()); + } + return sb.toString(); + } + + private static String sanitize(String name) { + if (name == null || name.isBlank()) { + return "shot"; + } + return name.trim().replaceAll("[^A-Za-z0-9._-]+", "-"); + } +} diff --git a/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReportProvider.java b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReportProvider.java new file mode 100644 index 0000000..f56a624 --- /dev/null +++ b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReportProvider.java @@ -0,0 +1,32 @@ +package org.vaadin.addons.dramafinder.agent; + +import com.microsoft.playwright.Page; + +/** + * Implemented by tests that want the {@link AgentReporting} extension to write a + * failure bundle using a specific page and report instance. + *

+ * {@link VisualVerificationTest} implements this for you. Tests that use + * {@code @ExtendWith(AgentReporting.class)} directly may implement it to control + * exactly which page is captured; otherwise the extension falls back to + * reflecting a {@link Page} or {@link AgentReport} field off the test instance. + */ +public interface AgentReportProvider { + + /** + * The page the extension should screenshot and inspect on failure. + * + * @return the active Playwright page, or {@code null} if none is available + */ + Page agentPage(); + + /** + * The report the extension should write the failure bundle into. + * + * @return the report instance, or {@code null} to let the extension create + * one from {@link #agentPage()} + */ + default AgentReport agentReport() { + return null; + } +} diff --git a/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReporting.java b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReporting.java new file mode 100644 index 0000000..2705e35 --- /dev/null +++ b/src/main/java/org/vaadin/addons/dramafinder/agent/AgentReporting.java @@ -0,0 +1,91 @@ +package org.vaadin.addons.dramafinder.agent; + +import java.lang.reflect.Field; +import java.util.Optional; + +import com.microsoft.playwright.Page; +import org.junit.jupiter.api.extension.AfterTestExecutionCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * JUnit 5 extension that, on test failure, writes an agent-oriented report to + * {@code target/agent-report/} (Maven) or {@code build/agent-report/} (Gradle): + * a {@code failure.txt} with the assertion message and a trimmed stack trace, a + * screenshot of the page at failure time, and a semantic + * {@link ComponentSnapshot component snapshot}. + *

+ * The capture runs in {@link AfterTestExecutionCallback}, i.e. before + * {@code @AfterEach} closes the page, so the screenshot and snapshot reflect the + * exact failing state. + *

+ * Register it either by extending {@link VisualVerificationTest} (which already carries + * {@code @ExtendWith(AgentReporting.class)} and implements + * {@link AgentReportProvider}) or by annotating any Playwright test with + * {@code @ExtendWith(AgentReporting.class)}. In the latter case the extension + * locates the page/report by implementing {@link AgentReportProvider}, or, as a + * fallback, by reflecting an {@link AgentReport} or {@link Page} field off the + * test instance. + */ +public class AgentReporting implements AfterTestExecutionCallback { + + @Override + public void afterTestExecution(ExtensionContext context) { + Optional failure = context.getExecutionException(); + if (failure.isEmpty()) { + return; + } + Object test = context.getTestInstance().orElse(null); + if (test == null) { + return; + } + locateReport(test).ifPresent(report -> report.captureFailure(failure.get())); + } + + private Optional locateReport(Object test) { + if (test instanceof AgentReportProvider provider) { + AgentReport report = provider.agentReport(); + if (report != null) { + return Optional.of(report); + } + Page page = provider.agentPage(); + if (page != null) { + return Optional.of(new AgentReport(page)); + } + } + return reflectReport(test); + } + + // Fallback for @ExtendWith users who don't implement AgentReportProvider: + // find an AgentReport field, or build one from a Page field. + private Optional reflectReport(Object test) { + AgentReport report = firstFieldValue(test, AgentReport.class); + if (report != null) { + return Optional.of(report); + } + Page page = firstFieldValue(test, Page.class); + if (page != null) { + return Optional.of(new AgentReport(page)); + } + return Optional.empty(); + } + + @SuppressWarnings("unchecked") + private T firstFieldValue(Object test, Class type) { + for (Class c = test.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + for (Field field : c.getDeclaredFields()) { + if (type.isAssignableFrom(field.getType())) { + try { + field.setAccessible(true); + Object value = field.get(test); + if (value != null) { + return (T) value; + } + } catch (ReflectiveOperationException | RuntimeException ignore) { + // Skip inaccessible fields. + } + } + } + } + return null; + } +} diff --git a/src/main/java/org/vaadin/addons/dramafinder/agent/ComponentSnapshot.java b/src/main/java/org/vaadin/addons/dramafinder/agent/ComponentSnapshot.java new file mode 100644 index 0000000..ccccb18 --- /dev/null +++ b/src/main/java/org/vaadin/addons/dramafinder/agent/ComponentSnapshot.java @@ -0,0 +1,165 @@ +package org.vaadin.addons.dramafinder.agent; + +import java.util.List; + +import com.microsoft.playwright.Page; + +/** + * Produces a compact, human- and agent-readable summary of the Vaadin + * components currently present on a page. + *

+ * Instead of dumping a full accessibility tree (which is large and noisy for + * Vaadin's shadow DOM and teleported overlays), this walks the light DOM for + * {@code vaadin-*} elements and emits one terse line per meaningful component, + * e.g.: + * 

+ * vaadin-grid — 12 rows, columns [Date, Amount, Status], 1 selected
+ * vaadin-combo-box "Category" = "Travel"
+ * notification: "Saved"
+ * vaadin-dialog-overlay open — "Confirm"
+ *
+ * The result is orders of magnitude smaller than an accessibility-tree dump and + * captures exactly the state an agent needs to reason about the UI. It is also + * embedded in failure reports written by {@link AgentReporting}. + */ +public final class ComponentSnapshot { + + private ComponentSnapshot() { + } + + // Walks the DOM in the browser and returns one string per component. + // @formatter:off + private static final String SNAPSHOT_SCRIPT = """ + () => { + const out = []; + // Structural / internal tags we never report on their own. + const SKIP = new Set([ + 'vaadin-grid-cell-content','vaadin-grid-column','vaadin-grid-column-group', + 'vaadin-grid-sorter','vaadin-grid-tree-toggle','vaadin-grid-flow-selection-column', + 'vaadin-grid-selection-column','vaadin-combo-box-item','vaadin-item', + 'vaadin-select-item','vaadin-list-box','vaadin-menu-bar-item','vaadin-menu-bar-button', + 'vaadin-context-menu-item','vaadin-tab','vaadin-notification-container', + 'vaadin-dev-tools','vaadin-connection-indicator','vaadin-overlay','vaadin-scroller', + 'vaadin-horizontal-layout','vaadin-vertical-layout','vaadin-form-layout', + 'vaadin-form-item','vaadin-app-layout','vaadin-split-layout','vaadin-icon', + 'vaadin-avatar-group' + ]); + // Overlays whose descendants are just list items, not real state. + const ITEM_OVERLAYS = 'vaadin-combo-box-overlay,vaadin-multi-select-combo-box-overlay,' + + 'vaadin-select-overlay,vaadin-menu-bar-overlay,vaadin-context-menu-overlay'; + + const quote = (v) => (v === undefined || v === null || v === '') ? '' : ' = "' + v + '"'; + const labelOf = (el) => (el.label || el.getAttribute('aria-label') || '').trim(); + + const els = Array.from(document.querySelectorAll('*')) + .filter(e => e.tagName.toLowerCase().startsWith('vaadin-')); + + for (const el of els) { + const tag = el.tagName.toLowerCase(); + if (SKIP.has(tag)) continue; + // Skip items nested inside selection overlays. + if (el.closest(ITEM_OVERLAYS) && !el.matches(ITEM_OVERLAYS)) continue; + // Skip anything living inside a grid (cells, editors) except the grid itself. + const grid = el.closest('vaadin-grid,vaadin-tree-grid'); + if (grid && grid !== el) continue; + + const label = labelOf(el); + const named = label ? ' "' + label + '"' : ''; + + if (tag === 'vaadin-grid' || tag === 'vaadin-tree-grid') { + const rows = el._flatSize ?? el._effectiveSize ?? el.size ?? 0; + const headers = []; + const sr = el.shadowRoot; + if (sr) { + sr.querySelectorAll('thead th').forEach(th => { + const slot = th.querySelector('slot'); + if (!slot || !slot.assignedNodes) return; + const txt = slot.assignedNodes().map(n => n.textContent || '').join('').trim(); + if (txt) headers.push(txt); + }); + } + const selected = (el.selectedItems || []).length; + out.push(tag + named + ' — ' + rows + ' rows, columns [' + + headers.join(', ') + ']' + (selected ? ', ' + selected + ' selected' : '')); + continue; + } + + if (tag === 'vaadin-virtual-list') { + const items = (el.items || []).length; + out.push(tag + named + ' — ' + items + ' items'); + continue; + } + + if (tag === 'vaadin-notification-card') { + const txt = (el.textContent || '').replace(/\\s+/g, ' ').trim(); + if (txt) out.push('notification: "' + txt + '"'); + continue; + } + + if (tag.endsWith('-overlay')) { + const opened = el.opened || el.hasAttribute('opened'); + if (!opened) continue; + const header = (el.headerTitle || '').trim() + || (el.querySelector('[slot="title"]')?.textContent || '').trim(); + out.push(tag + ' open' + (header ? ' — "' + header + '"' : '')); + continue; + } + + const flags = []; + if (el.disabled) flags.push('disabled'); + if (el.readonly) flags.push('read-only'); + if (el.required) flags.push('required'); + if (el.invalid) flags.push('invalid'); + const flagStr = flags.length ? ' (' + flags.join(', ') + ')' : ''; + + if (tag === 'vaadin-checkbox') { + out.push(tag + named + ' = ' + (el.checked ? 'checked' : 'unchecked') + flagStr); + continue; + } + if (tag === 'vaadin-checkbox-group' || tag === 'vaadin-radio-group') { + const val = Array.isArray(el.value) ? el.value.join(', ') : (el.value || ''); + out.push(tag + named + quote(val) + flagStr); + continue; + } + + if (tag === 'vaadin-button') { + const txt = (el.textContent || '').replace(/\\s+/g, ' ').trim(); + if (txt) out.push('vaadin-button "' + txt + '"' + flagStr); + continue; + } + + // Generic field-like components expose a value / label. + if ('value' in el || label) { + let val = el.value; + if (Array.isArray(val)) val = val.join(', '); + out.push(tag + named + quote(val) + flagStr); + } + } + return out; + } + """; + // @formatter:on + + /** + * Capture a compact snapshot of all meaningful Vaadin components on the + * page. + * + * @param page the Playwright page to inspect + * @return a multi-line summary, one component per line (empty string when + * no components are found) + */ + @SuppressWarnings("unchecked") + public static String capture(Page page) { + Object result = page.evaluate(SNAPSHOT_SCRIPT); + if (!(result instanceof List list) || list.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (Object line : list) { + if (line != null) { + sb.append(line).append(System.lineSeparator()); + } + } + return sb.toString().stripTrailing(); + } +} diff --git a/src/main/java/org/vaadin/addons/dramafinder/agent/VisualVerificationTest.java b/src/main/java/org/vaadin/addons/dramafinder/agent/VisualVerificationTest.java new file mode 100644 index 0000000..5c52bb1 --- /dev/null +++ b/src/main/java/org/vaadin/addons/dramafinder/agent/VisualVerificationTest.java @@ -0,0 +1,152 @@ +package org.vaadin.addons.dramafinder.agent; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserType.LaunchOptions; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.vaadin.addons.dramafinder.AbstractBasePlaywrightIT; + +/** + * Base class for temporary, agent-driven visual verification tests that run + * against an already running application. + *

+ * Unlike the Spring-based integration test base, this class does not boot the + * application — keeping the app out of the test run is what makes iterations + * cheap. It launches a Playwright browser, opens a fresh {@link Page} per test + * at a 1920×1080 viewport, and wires in {@link AgentReporting} so a failure + * automatically produces a screenshot, semantic snapshot, and stack trace. + *

+ * Typical use (a throwaway {@code AgentVerifyIT}): + * 

{@code
+ * class AgentVerifyIT extends VisualVerificationTest {
+ *     @Test
+ *     void verifyOrders() {
+ *         open("orders");
+ *         shot("01-order-list");
+ *         GridElement.get(page).assertRowCount(12);
+ *     }
+ * }
+ * }
+ * The base URL defaults to {@code http://localhost:8080} and can be overridden + * with the {@code dramafinder.agent.baseUrl} system property or the + * {@code DRAMAFINDER_BASE_URL} environment variable. Headless mode follows the + * same {@code headless} / {@code HEADLESS} convention as the rest of the + * library. + */ +@ExtendWith(AgentReporting.class) +public abstract class VisualVerificationTest implements AgentReportProvider { + + private static final String DEFAULT_BASE_URL = "http://localhost:8080"; + + private static Playwright playwright; + private static Browser browser; + + /** The active page for the current test. */ + protected Page page; + + /** The report writer for the current test. */ + protected AgentReport report; + + @BeforeAll + static void startBrowser() { + playwright = Playwright.create(); + browser = playwright.chromium() + .launch(new LaunchOptions().setHeadless(isHeadless())); + } + + @AfterAll + static void stopBrowser() { + if (browser != null) { + browser.close(); + browser = null; + } + if (playwright != null) { + playwright.close(); + playwright = null; + } + } + + @BeforeEach + void openPage() { + page = browser.newPage(); + page.setViewportSize(1920, 1080); + page.setDefaultTimeout(15000); + report = new AgentReport(page); + } + + @AfterEach + void closePage() { + if (page != null) { + page.close(); + page = null; + } + } + + /** + * Navigate to a path relative to the configured base URL and wait for + * Vaadin to finish loading. + * + * @param path the route path (with or without a leading slash) + */ + protected void open(String path) { + String normalized = path.startsWith("/") ? path.substring(1) : path; + page.navigate(baseUrl() + "/" + normalized); + page.waitForFunction(AbstractBasePlaywrightIT.WAIT_FOR_VAADIN_SCRIPT); + } + + /** + * Capture a numbered screenshot into the report directory. + * + * @param name a short descriptive name for the interaction point + */ + protected void shot(String name) { + report.shot(name); + } + + /** + * The base URL the test connects to. + * + * @return the configured base URL (default {@code http://localhost:8080}) + */ + protected String baseUrl() { + String property = System.getProperty("dramafinder.agent.baseUrl"); + if (property != null && !property.isBlank()) { + return trimTrailingSlash(property); + } + String env = System.getenv("DRAMAFINDER_BASE_URL"); + if (env != null && !env.isBlank()) { + return trimTrailingSlash(env); + } + return DEFAULT_BASE_URL; + } + + @Override + public Page agentPage() { + return page; + } + + @Override + public AgentReport agentReport() { + return report; + } + + private static String trimTrailingSlash(String url) { + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + private static boolean isHeadless() { + String value = System.getProperty("headless"); + if (value == null || value.isBlank()) { + value = System.getenv("HEADLESS"); + } + if (value == null || value.isBlank()) { + return true; + } + return Boolean.parseBoolean(value); + } +} diff --git a/src/test/java/org/vaadin/addons/dramafinder/tests/it/AgentReportingIT.java b/src/test/java/org/vaadin/addons/dramafinder/tests/it/AgentReportingIT.java new file mode 100644 index 0000000..a504842 --- /dev/null +++ b/src/test/java/org/vaadin/addons/dramafinder/tests/it/AgentReportingIT.java @@ -0,0 +1,145 @@ +package org.vaadin.addons.dramafinder.tests.it; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Proxy; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import com.microsoft.playwright.Page; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.vaadin.addons.dramafinder.agent.AgentReport; +import org.vaadin.addons.dramafinder.agent.AgentReportProvider; +import org.vaadin.addons.dramafinder.agent.AgentReporting; +import org.vaadin.addons.dramafinder.agent.ComponentSnapshot; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the agent reporting utilities ({@link ComponentSnapshot} and + * {@link AgentReport}) against a real rendered page. + */ +@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) +public class AgentReportingIT extends SpringPlaywrightIT { + + @Override + public String getView() { + return "grid-basic"; + } + + @Test + public void snapshotSummarisesGrids() { + String snapshot = ComponentSnapshot.capture(page); + + assertFalse(snapshot.isBlank(), "Snapshot should not be empty"); + assertTrue(snapshot.contains("vaadin-grid"), + "Snapshot should mention grids: " + snapshot); + assertTrue(snapshot.contains("columns [First Name, Last Name, Email]"), + "Snapshot should list the grid columns: " + snapshot); + assertTrue(snapshot.contains("100 rows"), + "Snapshot should report the basic grid row count: " + snapshot); + } + + @Test + public void snapshotOmitsInternalCellElements() { + String snapshot = ComponentSnapshot.capture(page); + assertFalse(snapshot.contains("vaadin-grid-cell-content"), + "Snapshot should not include internal cell elements: " + snapshot); + } + + @Test + public void shotWritesNumberedScreenshot(@org.junit.jupiter.api.io.TempDir Path tempDir) + throws Exception { + Path dir = tempDir.resolve("agent-report"); + AgentReport report = new AgentReport(page, dir); + + Path first = report.shot("login"); + Path second = report.shot("order list"); + + assertTrue(Files.exists(first), "First screenshot should exist"); + assertTrue(Files.exists(second), "Second screenshot should exist"); + assertEquals("01-login.png", first.getFileName().toString()); + assertEquals("02-order-list.png", second.getFileName().toString()); + assertTrue(Files.size(first) > 0, "Screenshot should not be empty"); + } + + @Test + public void captureFailureWritesBundle(@org.junit.jupiter.api.io.TempDir Path tempDir) + throws Exception { + Path dir = tempDir.resolve("agent-report"); + AgentReport report = new AgentReport(page, dir); + + report.captureFailure(new AssertionError("expected 12 rows but was 100")); + + Path failureTxt = dir.resolve("failure.txt"); + assertTrue(Files.exists(failureTxt), "failure.txt should exist"); + assertTrue(Files.readString(failureTxt).contains("expected 12 rows but was 100"), + "failure.txt should contain the assertion message"); + assertTrue(Files.exists(dir.resolve("failure.png")), + "failure.png should exist"); + + Path snapshot = dir.resolve("component-snapshot.txt"); + assertTrue(Files.exists(snapshot), "component-snapshot.txt should exist"); + assertTrue(Files.readString(snapshot).contains("vaadin-grid"), + "component snapshot should describe the page"); + } + + @Test + public void extensionCapturesFailureFromProvider( + @org.junit.jupiter.api.io.TempDir Path tempDir) throws Exception { + Path dir = tempDir.resolve("agent-report"); + AgentReport report = new AgentReport(page, dir); + + // A test instance the extension can resolve the page/report from. + AgentReportProvider testInstance = new AgentReportProvider() { + @Override + public Page agentPage() { + return page; + } + + @Override + public AgentReport agentReport() { + return report; + } + }; + + AssertionError failure = new AssertionError("row count mismatch"); + ExtensionContext context = failingContext(testInstance, failure); + + new AgentReporting().afterTestExecution(context); + + assertTrue(Files.exists(dir.resolve("failure.txt")), + "extension should have written the failure bundle"); + assertTrue(Files.readString(dir.resolve("failure.txt")).contains("row count mismatch")); + } + + @Test + public void extensionSkipsWhenTestPassed( + @org.junit.jupiter.api.io.TempDir Path tempDir) { + Path dir = tempDir.resolve("agent-report"); + AgentReportProvider testInstance = () -> page; + ExtensionContext context = failingContext(testInstance, null); + + new AgentReporting().afterTestExecution(context); + + assertFalse(Files.exists(dir), "no report should be written on success"); + } + + // Minimal ExtensionContext exposing only the execution exception and the + // test instance, which is all AgentReporting reads. + private static ExtensionContext failingContext(Object testInstance, Throwable error) { + InvocationHandler handler = (proxy, method, args) -> switch (method.getName()) { + case "getExecutionException" -> Optional.ofNullable(error); + case "getTestInstance" -> Optional.ofNullable(testInstance); + default -> method.getReturnType() == Optional.class ? Optional.empty() : null; + }; + return (ExtensionContext) Proxy.newProxyInstance( + AgentReportingIT.class.getClassLoader(), + new Class[] { ExtensionContext.class }, handler); + } +}