From 04c47e24340269daca69cd2ed46e8a487bb4ab30 Mon Sep 17 00:00:00 2001 From: Alexander Nazarov Date: Wed, 25 Feb 2026 00:04:08 +0300 Subject: [PATCH 1/6] feat: rename player to studio-player; add CLI scenario autoplay - Move Electron app from apps/player to apps/studio-player and update package/scripts - Add nested two-cursor cursor-proof scenario + E2E proof artifact - Support passing *.scenario.ts via CLI to auto-load and auto-play Co-authored-by: Cursor --- .../{player => studio-player}/assets/icon.png | Bin .../electron/main.ts | 79 +++++- .../electron/preload.d.ts | 0 .../electron/preload.js | 0 apps/{player => studio-player}/index.html | 2 +- apps/{player => studio-player}/package.json | 2 +- .../playwright.config.ts | 0 .../public/favicon.png | Bin .../{player => studio-player}/server/cache.ts | 0 .../server/executor.ts | 44 +++- .../{player => studio-player}/server/index.ts | 15 +- apps/{player => studio-player}/src/App.tsx | 27 +- .../src/components/controls.tsx | 0 .../src/components/cursor-overlay.tsx | 0 .../src/components/preview.tsx | 0 .../src/components/scenario-grid.tsx | 0 .../src/components/scenario-picker.tsx | 0 .../src/components/step-graph.tsx | 0 .../src/components/studio-grid.tsx | 0 .../src/hooks/use-player.ts | 0 apps/{player => studio-player}/src/index.css | 0 apps/{player => studio-player}/src/main.tsx | 0 .../src/vite-env.d.ts | 0 .../tests/cli-autoplay.e2e.test.ts | 97 +++++++ .../tests/cursor-proof.e2e.test.ts | 126 +++++++++ .../tests/electron.e2e.test.ts | 0 .../tests/player-self-test.e2e.test.ts | 0 .../tests/player.scenario.e2e.test.ts | 0 .../tests/smoke.e2e.test.ts | 2 +- .../tsconfig.app.json | 0 apps/{player => studio-player}/tsconfig.json | 0 .../tsconfig.node.json | 0 apps/{player => studio-player}/vite.config.ts | 0 package.json | 9 +- packages/browser2video/session.ts | 43 +++- pnpm-lock.yaml | 2 +- tests/fixtures/simple-page.html | 68 +++++ tests/scenarios/collab.test.ts | 2 +- tests/scenarios/cursor-proof.scenario.ts | 240 ++++++++++++++++++ .../mcp-generated/all-in-one.test.ts | 2 +- tests/scenarios/player-self-test.scenario.ts | 2 +- tests/scenarios/simple-click.scenario.ts | 43 ++++ 42 files changed, 769 insertions(+), 36 deletions(-) rename apps/{player => studio-player}/assets/icon.png (100%) rename apps/{player => studio-player}/electron/main.ts (79%) rename apps/{player => studio-player}/electron/preload.d.ts (100%) rename apps/{player => studio-player}/electron/preload.js (100%) rename apps/{player => studio-player}/index.html (90%) rename apps/{player => studio-player}/package.json (95%) rename apps/{player => studio-player}/playwright.config.ts (100%) rename apps/{player => studio-player}/public/favicon.png (100%) rename apps/{player => studio-player}/server/cache.ts (100%) rename apps/{player => studio-player}/server/executor.ts (86%) rename apps/{player => studio-player}/server/index.ts (98%) rename apps/{player => studio-player}/src/App.tsx (81%) rename apps/{player => studio-player}/src/components/controls.tsx (100%) rename apps/{player => studio-player}/src/components/cursor-overlay.tsx (100%) rename apps/{player => studio-player}/src/components/preview.tsx (100%) rename apps/{player => studio-player}/src/components/scenario-grid.tsx (100%) rename apps/{player => studio-player}/src/components/scenario-picker.tsx (100%) rename apps/{player => studio-player}/src/components/step-graph.tsx (100%) rename apps/{player => studio-player}/src/components/studio-grid.tsx (100%) rename apps/{player => studio-player}/src/hooks/use-player.ts (100%) rename apps/{player => studio-player}/src/index.css (100%) rename apps/{player => studio-player}/src/main.tsx (100%) rename apps/{player => studio-player}/src/vite-env.d.ts (100%) create mode 100644 apps/studio-player/tests/cli-autoplay.e2e.test.ts create mode 100644 apps/studio-player/tests/cursor-proof.e2e.test.ts rename apps/{player => studio-player}/tests/electron.e2e.test.ts (100%) rename apps/{player => studio-player}/tests/player-self-test.e2e.test.ts (100%) rename apps/{player => studio-player}/tests/player.scenario.e2e.test.ts (100%) rename apps/{player => studio-player}/tests/smoke.e2e.test.ts (97%) rename apps/{player => studio-player}/tsconfig.app.json (100%) rename apps/{player => studio-player}/tsconfig.json (100%) rename apps/{player => studio-player}/tsconfig.node.json (100%) rename apps/{player => studio-player}/vite.config.ts (100%) create mode 100644 tests/fixtures/simple-page.html create mode 100644 tests/scenarios/cursor-proof.scenario.ts create mode 100644 tests/scenarios/simple-click.scenario.ts diff --git a/apps/player/assets/icon.png b/apps/studio-player/assets/icon.png similarity index 100% rename from apps/player/assets/icon.png rename to apps/studio-player/assets/icon.png diff --git a/apps/player/electron/main.ts b/apps/studio-player/electron/main.ts similarity index 79% rename from apps/player/electron/main.ts rename to apps/studio-player/electron/main.ts index 92f54ec..4fb7161 100644 --- a/apps/player/electron/main.ts +++ b/apps/studio-player/electron/main.ts @@ -1,7 +1,7 @@ /** - * Electron main process for the b2v Player. + * Electron main process for Studio Player. * - * - Creates the main BrowserWindow with the player React UI + * - Creates the main BrowserWindow with the Studio Player React UI * - Manages a WebContentsView for embedding scenario pages directly * - Runs the HTTP+WS server in-process (for direct onRequestPage callbacks) * - Exposes CDP port so Playwright (in the session) can connect to @@ -82,36 +82,74 @@ const SERVER_PORT = parseInt(process.env.PORT ?? "9521", 10); const isEmbedded = process.env.B2V_EMBEDDED === "1"; +function parseAutoScenarioFromCli(argv: string[]): { file: string | null; autoplay: boolean } { + // Electron argv usually looks like: + // [electronExe, appPath, ...userArgs] + // We support both explicit `--scenario` and positional `*.scenario.ts`. + let file: string | null = null; + let autoplay = false; + + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--no-play" || a === "--no-autoplay") autoplay = false; + if (a === "--play" || a === "--autoplay") autoplay = true; + + if (a === "--scenario" && argv[i + 1]) { + file = argv[i + 1]; + i++; + continue; + } + if (a.startsWith("--scenario=")) { + file = a.slice("--scenario=".length); + continue; + } + + if (a.endsWith(".scenario.ts") || a.endsWith(".scenario.js") || a.endsWith(".scenario.mjs")) { + file = a; + // If a scenario is provided positionally, default to autoplay unless explicitly disabled. + if (!argv.includes("--no-play") && !argv.includes("--no-autoplay")) autoplay = true; + } + } + + return { file, autoplay }; +} + function createMainWindow() { mainWindow = new BrowserWindow({ // When running embedded inside another player (self-test), hide the window // completely. The UI is served via HTTP and rendered in the parent player's // scenario WebContentsView. On macOS, show:false alone can still flash; // we also use off-screen position and minimal size. - width: isEmbedded ? 1 : 1440, - height: isEmbedded ? 1 : 900, + // Embedded instances need a real surface for CDP capture/screencast. + // Keep the window off-screen and transparent instead of tiny/minimized. + width: isEmbedded ? 1280 : 1440, + height: isEmbedded ? 720 : 900, x: isEmbedded ? -10000 : undefined, y: isEmbedded ? -10000 : undefined, show: !isEmbedded, skipTaskbar: isEmbedded, // Prevent embedded window from appearing in Mission Control / Expose ...(isEmbedded ? { type: "toolbar" as any, focusable: false, hasShadow: false } : {}), - title: "b2v Player", + title: "Studio Player", icon: path.join(__dirname, "..", "assets", "icon.png"), webPreferences: { preload: path.join(__dirname, "preload.js"), contextIsolation: true, nodeIntegration: false, + // Nested StudioPlayer runs hidden/offscreen; keep timers/rendering alive + // so CDP screencasts still produce frames. + backgroundThrottling: !isEmbedded, }, }); // For embedded instances: aggressively hide the window. // macOS can show windows during loadURL or other async operations. if (isEmbedded) { - mainWindow.hide(); + // Keep it effectively invisible, but still "shown" so Chromium paints frames. + try { mainWindow.setOpacity(0); } catch { } + try { mainWindow.setIgnoreMouseEvents(true); } catch { } mainWindow.setVisibleOnAllWorkspaces(false); - // Minimize to ensure it never appears in front of the parent - mainWindow.minimize(); + mainWindow.showInactive(); } mainWindow.webContents.setWindowOpenHandler(() => ({ action: "deny" as const })); @@ -174,8 +212,12 @@ export async function createScenarioView( contextIsolation: false, nodeIntegration: false, sandbox: false, + backgroundThrottling: false, }, }); + if (isEmbedded) { + try { scenarioView.webContents.setBackgroundThrottling(false); } catch { } + } scenarioView.webContents.setWindowOpenHandler(() => ({ action: "deny" as const })); @@ -193,9 +235,16 @@ export async function createScenarioView( callback({ responseHeaders: headers }); }); - // Start hidden (zero-size). The React ElectronScenarioView component - // will send the correct bounds via IPC once it mounts. - scenarioView.setBounds({ x: 0, y: 0, width: 0, height: 0 }); + // Default bounds: + // - Normal mode: start hidden (0×0). The React ElectronScenarioView component + // will send the correct bounds via IPC once it mounts. + // - Embedded mode (nested StudioPlayer): ElectronScenarioView is NOT mounted, + // so the view would stay 0×0 forever and CDP screencasts would produce + // no frames. In that case, size it immediately to the requested viewport. + const initialBounds = isEmbedded + ? { x: 0, y: 0, width: Math.max(1, viewport.width), height: Math.max(1, viewport.height) } + : { x: 0, y: 0, width: 0, height: 0 }; + scenarioView.setBounds(initialBounds); mainWindow.contentView.addChildView(scenarioView); await scenarioView.webContents.loadURL(url); @@ -258,7 +307,13 @@ app.whenReady().then(async () => { } // Now navigate to the real player URL - const playerUrl = `http://localhost:${SERVER_PORT}`; + const { file: autoScenarioFile, autoplay } = parseAutoScenarioFromCli(process.argv); + const params = new URLSearchParams(); + if (autoScenarioFile) { + params.set("scenario", autoScenarioFile); + if (autoplay) params.set("autoplay", "1"); + } + const playerUrl = `http://localhost:${SERVER_PORT}${params.size ? `/?${params.toString()}` : ""}`; console.error(`[electron ${elt()}] Loading player UI: ${playerUrl}`); mainWindow!.loadURL(playerUrl); // Re-hide after loadURL for embedded instances diff --git a/apps/player/electron/preload.d.ts b/apps/studio-player/electron/preload.d.ts similarity index 100% rename from apps/player/electron/preload.d.ts rename to apps/studio-player/electron/preload.d.ts diff --git a/apps/player/electron/preload.js b/apps/studio-player/electron/preload.js similarity index 100% rename from apps/player/electron/preload.js rename to apps/studio-player/electron/preload.js diff --git a/apps/player/index.html b/apps/studio-player/index.html similarity index 90% rename from apps/player/index.html rename to apps/studio-player/index.html index b9bbc9d..3f81c48 100644 --- a/apps/player/index.html +++ b/apps/studio-player/index.html @@ -4,7 +4,7 @@ - b2v Player + Studio Player
diff --git a/apps/player/package.json b/apps/studio-player/package.json similarity index 95% rename from apps/player/package.json rename to apps/studio-player/package.json index 357db48..4cb7e9c 100644 --- a/apps/player/package.json +++ b/apps/studio-player/package.json @@ -1,5 +1,5 @@ { - "name": "@browser2video/player", + "name": "@browser2video/studio-player", "version": "0.1.0", "private": true, "type": "module", diff --git a/apps/player/playwright.config.ts b/apps/studio-player/playwright.config.ts similarity index 100% rename from apps/player/playwright.config.ts rename to apps/studio-player/playwright.config.ts diff --git a/apps/player/public/favicon.png b/apps/studio-player/public/favicon.png similarity index 100% rename from apps/player/public/favicon.png rename to apps/studio-player/public/favicon.png diff --git a/apps/player/server/cache.ts b/apps/studio-player/server/cache.ts similarity index 100% rename from apps/player/server/cache.ts rename to apps/studio-player/server/cache.ts diff --git a/apps/player/server/executor.ts b/apps/studio-player/server/executor.ts similarity index 86% rename from apps/player/server/executor.ts rename to apps/studio-player/server/executor.ts index e6c4b82..f785981 100644 --- a/apps/player/server/executor.ts +++ b/apps/studio-player/server/executor.ts @@ -95,13 +95,20 @@ export class Executor { private async ensureSession(mode: "human" | "fast"): Promise { if (!this.session) { + // Starting fresh — clear any stale abort flag from a previous session. + // This prevents late-arriving cancel messages from blocking new executions. + this._aborted = false; + const isEmbedded = process.env.B2V_EMBEDDED === "1"; const prevCwd = process.cwd(); if (this.projectRoot) process.chdir(this.projectRoot); let newSession: Session | null = null; try { newSession = await createSession({ mode, - record: mode === "human", + // Embedded (nested StudioPlayer) needs live frames via CDP screencast. + // Session recording in Electron mode also uses CDP screencast internally, + // so enabling both would conflict and produce 0 live frames. + record: mode === "human" && !isEmbedded, narration: { enabled: true, realtime: true }, ...this.descriptor.sessionOpts, ...this.sessionOpts, @@ -147,7 +154,6 @@ export class Executor { } // Start screencasting for video mode, or when embedded (no ElectronView overlay) - const isEmbedded = process.env.B2V_EMBEDDED === "1"; if (this.onLiveFrame && (this.viewMode === "video" || isEmbedded)) { await this.startScreencast(); } @@ -263,10 +269,38 @@ export class Executor { const panes: Map = (this.session as any).panes; const firstPane = panes?.values().next().value; if (firstPane?.page) { - const buf = await firstPane.page.screenshot({ type: "png" }); - screenshot = buf.toString("base64"); + const isEmbedded = process.env.B2V_EMBEDDED === "1"; + + // Embedded Electron pages can hang on Playwright's screenshot pipeline + // ("waiting for fonts to load..."). In that case, use a raw CDP capture + // which is fast and doesn't depend on window visibility. + if (isEmbedded && this.cdpEndpoint) { + try { + const cdp = await firstPane.page.context().newCDPSession(firstPane.page); + const timeoutMs = 5000; + const res = await Promise.race([ + cdp.send("Page.captureScreenshot", { format: "png", fromSurface: true }), + new Promise((_, reject) => setTimeout(() => reject(new Error(`timeout ${timeoutMs}ms`)), timeoutMs)), + ]); + await cdp.detach().catch(() => { }); + if (res?.data) { + screenshot = String(res.data); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[executor] CDP screenshot failed (falling back): ${message}`); + } + } + + if (!screenshot) { + const buf = await firstPane.page.screenshot({ type: "png", timeout: 10_000 }); + screenshot = buf.toString("base64"); + } } - } catch { /* page may be closed */ } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[executor] Screenshot capture failed: ${message}`); + } return { screenshot, durationMs }; } diff --git a/apps/player/server/index.ts b/apps/studio-player/server/index.ts similarity index 98% rename from apps/player/server/index.ts rename to apps/studio-player/server/index.ts index bfe6691..d657a55 100644 --- a/apps/player/server/index.ts +++ b/apps/studio-player/server/index.ts @@ -155,6 +155,13 @@ function findScenarioFiles(dir: string, base: string): string[] { return results.sort(); } +function listPlayerScenarioFiles(): string[] { + // Only include player-compatible scenarios. + // We intentionally do NOT scan the entire repo because some `*.scenario.ts` + // files are standalone scripts (top-level await) rather than defineScenario(). + return findScenarioFiles(path.join(PROJECT_ROOT, "tests", "scenarios"), PROJECT_ROOT); +} + // --------------------------------------------------------------------------- // Vite dev proxy // --------------------------------------------------------------------------- @@ -430,7 +437,7 @@ httpServer.on("upgrade", (req, socket, head) => { wss.on("connection", (ws) => { console.error("[player] Client connected"); - const files = findScenarioFiles(PROJECT_ROOT, PROJECT_ROOT); + const files = listPlayerScenarioFiles(); send(ws, { type: "scenarioFiles", files }); send(ws, { type: "viewMode", mode: currentViewMode }); if (terminalServer) { @@ -590,7 +597,7 @@ wss.on("connection", (ws) => { } case "listScenarios": { - send(ws, { type: "scenarioFiles", files: findScenarioFiles(PROJECT_ROOT, PROJECT_ROOT) }); + send(ws, { type: "scenarioFiles", files: listPlayerScenarioFiles() }); break; } @@ -632,7 +639,7 @@ wss.on("connection", (ws) => { case "importArtifacts": { const artifactsDir = path.isAbsolute(msg.dir) ? msg.dir : path.resolve(PROJECT_ROOT, msg.dir); - const scenarioFiles = findScenarioFiles(PROJECT_ROOT, PROJECT_ROOT); + const scenarioFiles = listPlayerScenarioFiles(); const imported = cache.importAllFromDir(artifactsDir, scenarioFiles); const scenarios = [...imported.keys()]; console.error(`[player] Imported artifacts for ${imported.size} scenario(s): ${scenarios.join(", ")}`); @@ -655,7 +662,7 @@ wss.on("connection", (ws) => { } case "downloadArtifacts": { - const scenarioFiles = findScenarioFiles(PROJECT_ROOT, PROJECT_ROOT); + const scenarioFiles = listPlayerScenarioFiles(); console.error(`[player] Downloading CI artifacts from GitHub...`); send(ws, { type: "status", loaded: !!executor, executedUpTo: executor?.lastExecutedIndex ?? -1 }); try { diff --git a/apps/player/src/App.tsx b/apps/studio-player/src/App.tsx similarity index 81% rename from apps/player/src/App.tsx rename to apps/studio-player/src/App.tsx index 7f4601c..9689fad 100644 --- a/apps/player/src/App.tsx +++ b/apps/studio-player/src/App.tsx @@ -1,3 +1,4 @@ +import { useEffect, useRef } from "react"; import { usePlayer } from "./hooks/use-player"; import { StepGraph } from "./components/step-graph"; import { Preview } from "./components/preview"; @@ -33,6 +34,30 @@ export default function App() { const activeScreenshot = activeStep >= 0 ? screenshots[activeStep] : null; const activeCaption = activeStep >= 0 && scenario ? scenario.steps[activeStep]?.caption : undefined; + const autoRunInitRef = useRef(false); + const autoRunRef = useRef<{ file: string; autoplay: boolean } | null>(null); + if (!autoRunInitRef.current) { + autoRunInitRef.current = true; + const params = new URLSearchParams(window.location.search); + const file = params.get("scenario") ?? params.get("file"); + if (file) { + const autoplay = params.get("autoplay") !== "0" && params.get("play") !== "0"; + autoRunRef.current = { file, autoplay }; + } + } + + useEffect(() => { + const auto = autoRunRef.current; + if (!auto) return; + if (!connected) return; + + loadScenario(auto.file); + if (auto.autoplay) runAll(); + + // Only do this once per app launch. + autoRunRef.current = null; + }, [connected, loadScenario, runAll]); + return (
-

b2v Player Studio

+

Studio Player

Select a scenario to record and replay.
diff --git a/apps/player/src/components/controls.tsx b/apps/studio-player/src/components/controls.tsx similarity index 100% rename from apps/player/src/components/controls.tsx rename to apps/studio-player/src/components/controls.tsx diff --git a/apps/player/src/components/cursor-overlay.tsx b/apps/studio-player/src/components/cursor-overlay.tsx similarity index 100% rename from apps/player/src/components/cursor-overlay.tsx rename to apps/studio-player/src/components/cursor-overlay.tsx diff --git a/apps/player/src/components/preview.tsx b/apps/studio-player/src/components/preview.tsx similarity index 100% rename from apps/player/src/components/preview.tsx rename to apps/studio-player/src/components/preview.tsx diff --git a/apps/player/src/components/scenario-grid.tsx b/apps/studio-player/src/components/scenario-grid.tsx similarity index 100% rename from apps/player/src/components/scenario-grid.tsx rename to apps/studio-player/src/components/scenario-grid.tsx diff --git a/apps/player/src/components/scenario-picker.tsx b/apps/studio-player/src/components/scenario-picker.tsx similarity index 100% rename from apps/player/src/components/scenario-picker.tsx rename to apps/studio-player/src/components/scenario-picker.tsx diff --git a/apps/player/src/components/step-graph.tsx b/apps/studio-player/src/components/step-graph.tsx similarity index 100% rename from apps/player/src/components/step-graph.tsx rename to apps/studio-player/src/components/step-graph.tsx diff --git a/apps/player/src/components/studio-grid.tsx b/apps/studio-player/src/components/studio-grid.tsx similarity index 100% rename from apps/player/src/components/studio-grid.tsx rename to apps/studio-player/src/components/studio-grid.tsx diff --git a/apps/player/src/hooks/use-player.ts b/apps/studio-player/src/hooks/use-player.ts similarity index 100% rename from apps/player/src/hooks/use-player.ts rename to apps/studio-player/src/hooks/use-player.ts diff --git a/apps/player/src/index.css b/apps/studio-player/src/index.css similarity index 100% rename from apps/player/src/index.css rename to apps/studio-player/src/index.css diff --git a/apps/player/src/main.tsx b/apps/studio-player/src/main.tsx similarity index 100% rename from apps/player/src/main.tsx rename to apps/studio-player/src/main.tsx diff --git a/apps/player/src/vite-env.d.ts b/apps/studio-player/src/vite-env.d.ts similarity index 100% rename from apps/player/src/vite-env.d.ts rename to apps/studio-player/src/vite-env.d.ts diff --git a/apps/studio-player/tests/cli-autoplay.e2e.test.ts b/apps/studio-player/tests/cli-autoplay.e2e.test.ts new file mode 100644 index 0000000..abac39e --- /dev/null +++ b/apps/studio-player/tests/cli-autoplay.e2e.test.ts @@ -0,0 +1,97 @@ +import { test, expect, _electron, type ElectronApplication, type Page } from "@playwright/test"; +import { execSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +const PROJECT_ROOT = path.resolve(import.meta.dirname, "../../.."); +const PLAYER_DIR = path.resolve(import.meta.dirname, ".."); + +function findScenarioFiles(dir: string, base: string): string[] { + const results: string[] = []; + if (!fs.existsSync(dir)) return results; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory() && entry.name !== "node_modules" && entry.name !== ".git") { + results.push(...findScenarioFiles(full, base)); + } else if (entry.isFile() && entry.name.endsWith(".scenario.ts")) { + results.push(path.relative(base, full)); + } + } + return results.sort(); +} + +function killPort(port: number) { + try { + const pids = execSync(`lsof -ti :${port} 2>/dev/null`, { encoding: "utf8" }).trim(); + for (const pid of pids.split("\n").filter(Boolean)) { + try { execSync(`kill -9 ${pid} 2>/dev/null`); } catch { } + } + } catch { } +} + +async function closeElectron(electronApp: ElectronApplication) { + try { + const proc = electronApp.process(); + const pid = proc.pid; + if (pid && proc.exitCode === null && proc.signalCode === null) { + process.kill(pid, "SIGTERM"); + await new Promise((resolve) => { + proc.on("exit", () => resolve()); + setTimeout(() => { + try { process.kill(pid, "SIGKILL"); } catch { } + resolve(); + }, 5_000); + }); + } + } catch { /* already exited */ } +} + +const scenarioFiles = findScenarioFiles(path.join(PROJECT_ROOT, "tests", "scenarios"), PROJECT_ROOT); + +test.describe.configure({ mode: "serial" }); + +for (const [i, scenarioFile] of scenarioFiles.entries()) { + test(`CLI scenario autoplay: ${scenarioFile}`, async () => { + test.setTimeout(180_000); + + // Use per-test ports to avoid cross-test interference. + const TEST_PORT = 9661 + i * 4; + const TEST_CDP_PORT = 9461 + i * 4; + killPort(TEST_PORT); + killPort(TEST_CDP_PORT); + + let electronApp: ElectronApplication | null = null; + try { + electronApp = await _electron.launch({ + // Provide scenario file as CLI arg (positional) + args: [PLAYER_DIR, scenarioFile], + cwd: PROJECT_ROOT, + env: { + ...process.env, + NODE_OPTIONS: "--experimental-strip-types --no-warnings", + PORT: String(TEST_PORT), + B2V_CDP_PORT: String(TEST_CDP_PORT), + // Keep it fast; we only validate the CLI auto-load + auto-start wiring. + B2V_MODE: "fast", + }, + timeout: 60_000, + }); + + const page: Page = await electronApp.firstWindow(); + await page.waitForLoadState("domcontentloaded"); + + // Scenario should auto-load (step cards appear). + await page.waitForSelector("[data-testid='step-card-0']", { timeout: 90_000 }); + + // Autoplay should either start (Play -> Stop) or fail with an error banner. + // Some scenarios have heavy setup and may error quickly in CI environments. + await Promise.race([ + page.waitForSelector("[data-testid='ctrl-stop']", { timeout: 120_000 }), + page.waitForSelector(".bg-red-950", { timeout: 120_000 }), + ]); + } finally { + if (electronApp) await closeElectron(electronApp); + } + }); +} + diff --git a/apps/studio-player/tests/cursor-proof.e2e.test.ts b/apps/studio-player/tests/cursor-proof.e2e.test.ts new file mode 100644 index 0000000..62a5a01 --- /dev/null +++ b/apps/studio-player/tests/cursor-proof.e2e.test.ts @@ -0,0 +1,126 @@ +/** + * Cursor Proof E2E — Launches the outer player, loads cursor-proof scenario + * (which spawns the inner player), and verifies both cursors are visible. + * + * Uses the SAME ports as the self-test (9561 outer, 9591 inner) since the + * self-test's nested architecture is proven to work. + */ + +import { test, expect, _electron, type ElectronApplication, type Page } from "@playwright/test"; +import { execSync } from "node:child_process"; +import { existsSync, mkdirSync, statSync } from "node:fs"; +import path from "node:path"; + +const PROJECT_ROOT = path.resolve(import.meta.dirname, "../../.."); +const PLAYER_DIR = path.resolve(import.meta.dirname, ".."); +const TEST_PORT = 9561; +const TEST_CDP_PORT = 9365; +const ARTIFACTS_DIR = path.resolve(PROJECT_ROOT, ".cache/tests/test-e2e__electron/cursor-proof"); +const PROOF_PATH = path.join(ARTIFACTS_DIR, "b2v-cursor-proof.png"); + +let electronApp: ElectronApplication; +let page: Page; + +test.describe.configure({ mode: "serial" }); + +test.beforeAll(async () => { + const t0 = performance.now(); + const ms = () => `${((performance.now() - t0) / 1000).toFixed(1)}s`; + + mkdirSync(ARTIFACTS_DIR, { recursive: true }); + + // Kill stale processes on both outer and inner ports + for (const port of [TEST_PORT, TEST_PORT + 1, TEST_CDP_PORT, 9581, 9582, 9385]) { + try { + const pids = execSync(`lsof -ti :${port} 2>/dev/null`, { encoding: "utf8" }).trim(); + for (const pid of pids.split("\n").filter(Boolean)) { + try { execSync(`kill -9 ${pid} 2>/dev/null`); } catch { } + } + } catch { } + } + + console.log(`[cursor-proof ${ms()}] Launching Electron player...`); + electronApp = await _electron.launch({ + args: [PLAYER_DIR], + cwd: PROJECT_ROOT, + env: { + ...process.env, + NODE_OPTIONS: "--experimental-strip-types --no-warnings", + PORT: String(TEST_PORT), + B2V_CDP_PORT: String(TEST_CDP_PORT), + B2V_TEST_ARTIFACTS_DIR: ARTIFACTS_DIR, + }, + timeout: 60_000, + }); + console.log(`[cursor-proof ${ms()}] Electron launched`); + + // Pipe process output so we can see scenario + inner player logs + const proc = electronApp.process(); + proc.stdout?.on("data", (d) => console.log(`[electron-out] ${d.toString().trimEnd()}`)); + proc.stderr?.on("data", (d) => console.error(`[electron-err] ${d.toString().trimEnd()}`)); + + page = await electronApp.firstWindow(); + await page.waitForLoadState("domcontentloaded"); + console.log(`[cursor-proof ${ms()}] domcontentloaded`); +}); + +test.afterAll(async () => { + if (!electronApp) return; + try { + const proc = electronApp.process(); + const pid = proc.pid; + if (pid && proc.exitCode === null && proc.signalCode === null) { + process.kill(pid, "SIGTERM"); + await new Promise((resolve) => { + proc.on("exit", () => resolve()); + setTimeout(() => { + try { process.kill(pid, "SIGKILL"); } catch { } + resolve(); + }, 5_000); + }); + } + } catch { /* already exited */ } +}); + +test("cursor proof — both cursors visible in nested player", async () => { + test.setTimeout(600_000); // 10 min — inner player + Vite takes time + + // Wait for player's picker to appear + await page.waitForSelector("[data-testid='picker-select']", { timeout: 90_000 }); + console.log("[cursor-proof] Player UI ready"); + + // Load cursor-proof scenario + await page.selectOption("[data-testid='picker-select']", { + label: "tests/scenarios/cursor-proof.scenario.ts", + }); + await page.waitForTimeout(2000); + await page.waitForSelector("[data-testid='step-card-0']", { timeout: 30_000 }); + const stepCards = page.locator("[data-testid^='step-card-']"); + const finalCount = await stepCards.count(); + console.log(`[cursor-proof] Loaded scenario with ${finalCount} steps`); + + // Click Play All + await page.click("[data-testid='ctrl-play-all']"); + console.log("[cursor-proof] Play All clicked — waiting for all steps"); + + // Poll for step completion + const pollIntervalMs = 3000; + for (let tick = 0; tick < 120; tick++) { + await page.waitForTimeout(pollIntervalMs); + const count = await stepCards.count(); + let doneCount = 0; + for (let i = 0; i < count; i++) { + const cls = await stepCards.nth(i).getAttribute("class") ?? ""; + if (cls.includes("emerald")) doneCount++; + } + const log = `${doneCount}/${count} done`; + console.log(`[cursor-proof] ${log}`); + if (doneCount === count) break; + } + + // Verify proof files exist + expect(existsSync(PROOF_PATH)).toBe(true); + const size = statSync(PROOF_PATH).size; + console.log(`[cursor-proof] Proof screenshot: ${size} bytes`); + expect(size).toBeGreaterThan(1000); +}); diff --git a/apps/player/tests/electron.e2e.test.ts b/apps/studio-player/tests/electron.e2e.test.ts similarity index 100% rename from apps/player/tests/electron.e2e.test.ts rename to apps/studio-player/tests/electron.e2e.test.ts diff --git a/apps/player/tests/player-self-test.e2e.test.ts b/apps/studio-player/tests/player-self-test.e2e.test.ts similarity index 100% rename from apps/player/tests/player-self-test.e2e.test.ts rename to apps/studio-player/tests/player-self-test.e2e.test.ts diff --git a/apps/player/tests/player.scenario.e2e.test.ts b/apps/studio-player/tests/player.scenario.e2e.test.ts similarity index 100% rename from apps/player/tests/player.scenario.e2e.test.ts rename to apps/studio-player/tests/player.scenario.e2e.test.ts diff --git a/apps/player/tests/smoke.e2e.test.ts b/apps/studio-player/tests/smoke.e2e.test.ts similarity index 97% rename from apps/player/tests/smoke.e2e.test.ts rename to apps/studio-player/tests/smoke.e2e.test.ts index 3e83c9d..acfb0a0 100644 --- a/apps/player/tests/smoke.e2e.test.ts +++ b/apps/studio-player/tests/smoke.e2e.test.ts @@ -42,7 +42,7 @@ test("player opens and closes without zombie processes", async () => { // Wait for the real page to load (splash page has no title) await page.waitForFunction(() => document.title.length > 0, { timeout: 30_000 }); const title = await page.title(); - expect(title.toLowerCase()).toContain("b2v"); + expect(title.toLowerCase()).toContain("studio"); // Close via SIGTERM (exercises the graceful shutdown path) const pid = electronApp.process().pid; diff --git a/apps/player/tsconfig.app.json b/apps/studio-player/tsconfig.app.json similarity index 100% rename from apps/player/tsconfig.app.json rename to apps/studio-player/tsconfig.app.json diff --git a/apps/player/tsconfig.json b/apps/studio-player/tsconfig.json similarity index 100% rename from apps/player/tsconfig.json rename to apps/studio-player/tsconfig.json diff --git a/apps/player/tsconfig.node.json b/apps/studio-player/tsconfig.node.json similarity index 100% rename from apps/player/tsconfig.node.json rename to apps/studio-player/tsconfig.node.json diff --git a/apps/player/vite.config.ts b/apps/studio-player/vite.config.ts similarity index 100% rename from apps/player/vite.config.ts rename to apps/studio-player/vite.config.ts diff --git a/package.json b/package.json index 193f5bb..6be22f8 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,12 @@ "run:console-logs": "node tests/scenarios/console-logs.test.ts", "gen:docs": "node scripts/gen-docs.ts", "gen:video-page": "node scripts/gen-video-page.ts", - "player": "pnpm -C apps/player dev", - "self-test": "pnpm -C apps/player exec playwright test player-self-test", - "self-test:human": "B2V_HUMAN=1 pnpm -C apps/player exec playwright test player-self-test --headed" + "studio-player": "pnpm -C apps/studio-player dev", + "player": "pnpm -C apps/studio-player dev", + "self-test": "pnpm -C apps/studio-player exec playwright test player-self-test", + "self-test:human": "B2V_HUMAN=1 pnpm -C apps/studio-player exec playwright test player-self-test --headed", + "cursor-proof": "pnpm -C apps/studio-player exec playwright test cursor-proof", + "cursor-proof:human": "B2V_HUMAN=1 pnpm -C apps/studio-player exec playwright test cursor-proof --headed" }, "devDependencies": { "@playwright/test": "^1.50.0", diff --git a/packages/browser2video/session.ts b/packages/browser2video/session.ts index e0d5789..2373cc2 100644 --- a/packages/browser2video/session.ts +++ b/packages/browser2video/session.ts @@ -538,11 +538,13 @@ export class Session { // a WebContentsView and then locate it via CDP. const targetUrl = opts.url ?? "about:blank"; - // Snapshot existing page URLs before creating the new view - const existingUrls = new Set(); + // Snapshot existing page REFERENCES before creating the new view. + // Using URLs doesn't work when the same URL is reused across sessions + // (e.g., the same demo app URL for a new WebContentsView). + const existingPages = new Set(); for (const ctx of this.browser!.contexts()) { for (const p of ctx.pages()) { - existingUrls.add(p.url()); + existingPages.add(p); } } @@ -553,7 +555,7 @@ export class Session { for (let attempt = 0; attempt < 60; attempt++) { for (const ctx of this.browser!.contexts()) { for (const p of ctx.pages()) { - if (!existingUrls.has(p.url())) { + if (!existingPages.has(p) && !p.isClosed()) { found = p; break; } @@ -569,6 +571,34 @@ export class Session { context = page.context(); console.error(`[session] Found Electron-managed page via CDP: ${page.url()}`); + // Sync Playwright's viewport tracking with the requested viewport. + // Electron WebContentsViews may start at 0×0 and Playwright can keep + // stale metrics, causing locator actions to fail as "outside viewport". + try { + await page.setViewportSize({ width: vpW, height: vpH }); + } catch { /* best-effort */ } + + // Init scripts for Electron-managed pages (page is already navigated by Electron, + // so we must use evaluate() for the current page AND addInitScript for future navs) + if (this.mode === "human") { + await page.evaluate(HIDE_CURSOR_INIT_SCRIPT).catch((e: any) => console.error("[session] HIDE_CURSOR eval failed:", e.message)); + await page.addInitScript(HIDE_CURSOR_INIT_SCRIPT); + await page.evaluate(CURSOR_OVERLAY_SCRIPT).catch((e: any) => console.error("[session] CURSOR_OVERLAY eval failed:", e.message)); + await page.addInitScript(CURSOR_OVERLAY_SCRIPT); + if (this.cursorColor) { + const { fill, stroke } = this.cursorColor; + const colorScript = `window.__b2v_setCursorColor?.('default', '${fill}', '${stroke}')`; + await page.evaluate(colorScript).catch((e: any) => console.error("[session] cursor color eval failed:", e.message)); + await page.addInitScript(colorScript); + console.error(`[session] Cursor color registered: fill=${fill} stroke=${stroke}`); + } + console.error("[session] Electron cursor overlay injected successfully"); + } + if (this.mode === "fast") { + await page.evaluate(FAST_MODE_INIT_SCRIPT).catch((e: any) => console.error("[session] FAST_MODE eval failed:", e.message)); + await page.addInitScript(FAST_MODE_INIT_SCRIPT); + } + // Start CDP screencast recording for Electron pages if (this.record) { rawVideoPath = path.join(this.artifactDir, `${id}.raw.webm`); @@ -692,6 +722,11 @@ export class Session { if (!found) throw new Error("Could not find terminal page via CDP"); page = found; context = page.context(); + + // Best-effort: keep viewport metrics consistent for locator operations. + try { + await page.setViewportSize({ width: vpW, height: vpH }); + } catch { /* best-effort */ } } else { const ctxOpts: { viewport: { width: number; height: number }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6383a4..09f4957 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,7 +103,7 @@ importers: specifier: ^3.5.0 version: 3.5.0(vite@6.4.1(@types/node@24.10.13)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.46.0)(tsx@4.21.0)) - apps/player: + apps/studio-player: dependencies: '@xterm/xterm': specifier: ^6.0.0 diff --git a/tests/fixtures/simple-page.html b/tests/fixtures/simple-page.html new file mode 100644 index 0000000..69b6851 --- /dev/null +++ b/tests/fixtures/simple-page.html @@ -0,0 +1,68 @@ + + + + + + Simple Page + + + +

+

Simple Form

+ +
+ +
✓ Done!
+
+ + diff --git a/tests/scenarios/collab.test.ts b/tests/scenarios/collab.test.ts index f824256..fc92367 100644 --- a/tests/scenarios/collab.test.ts +++ b/tests/scenarios/collab.test.ts @@ -7,5 +7,5 @@ if (isDirectRun) { runScenario(descriptor).then(() => process.exit(0)).catch((e) => { console.error(e); process.exit(1); }); } else { const { test } = await import("@playwright/test"); - test.skip("collab (requires Electron — run via apps/player E2E)", async () => { test.setTimeout(180_000); await runScenario(descriptor); }); + test.skip("collab (requires Electron — run via apps/studio-player E2E)", async () => { test.setTimeout(180_000); await runScenario(descriptor); }); } diff --git a/tests/scenarios/cursor-proof.scenario.ts b/tests/scenarios/cursor-proof.scenario.ts new file mode 100644 index 0000000..c10c94e --- /dev/null +++ b/tests/scenarios/cursor-proof.scenario.ts @@ -0,0 +1,240 @@ +/** + * Cursor Proof scenario — spawns an inner player, loads simple-click, + * plays the scenario, then captures a proof screenshot while BOTH cursors + * are visible at once: + * + * - Outer cursor: InjectedActor (pink) rendered in the inner player's UI DOM + * - Inner cursor: scenario Actor cursor (from B2V_CURSOR_COLOR) rendered inside + * the scenario preview image (live screencast frame) + * + * Modeled EXACTLY after player-self-test.scenario.ts setup. + */ +import { defineScenario, InjectedActor, type Session } from "browser2video"; +import path from "node:path"; +import { spawn, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; + +const PLAYER_DIR = path.resolve(import.meta.dirname, "../../apps/studio-player"); +const INNER_PORT = 9581; +const INNER_CDP_PORT = 9385; + +interface Ctx { + page: Awaited>["page"]; + injected: InjectedActor; + innerProcess: ChildProcess; +} + +async function waitForPort(port: number, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const ok = await new Promise((resolve) => { + const req = http.get(`http://localhost:${port}`, (res) => { + res.resume(); + resolve(res.statusCode !== undefined); + }); + req.on("error", () => resolve(false)); + req.setTimeout(1000, () => { req.destroy(); resolve(false); }); + }); + if (ok) return; + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error(`Port ${port} did not become available within ${timeoutMs}ms`); +} + +export default defineScenario("Cursor Proof", (s) => { + s.setup(async (session: Session) => { + const t0 = Date.now(); + const elapsed = () => `${((Date.now() - t0) / 1000).toFixed(1)}s`; + + const { execSync } = await import("node:child_process"); + + // Kill stale processes on inner ports (same as self-test) + for (const port of [INNER_PORT, INNER_CDP_PORT]) { + try { + const pids = execSync(`lsof -ti :${port} 2>/dev/null`, { encoding: "utf8" }).trim(); + if (pids) { + for (const pid of pids.split("\n").filter(Boolean)) { + if (pid === String(process.pid)) continue; + try { execSync(`kill -9 ${pid} 2>/dev/null`); } catch { } + } + await new Promise((r) => setTimeout(r, 300)); + } + } catch { } + } + console.error(`[cursor-proof ${elapsed()}] Port cleanup done`); + + const electronPath = path.resolve(PLAYER_DIR, "node_modules/.bin/electron"); + console.error(`[cursor-proof ${elapsed()}] Spawning inner player on port ${INNER_PORT}...`); + const innerProcess = spawn( + electronPath, + [PLAYER_DIR], + { + cwd: path.resolve(PLAYER_DIR, "../.."), + env: { + ...process.env, + NODE_OPTIONS: "--experimental-strip-types --no-warnings", + PORT: String(INNER_PORT), + B2V_CDP_PORT: String(INNER_CDP_PORT), + B2V_EMBEDDED: "1", + B2V_CURSOR_COLOR: "#fb923c,#9a3412", // orange scenario cursor + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + innerProcess.stdout?.on("data", (d: Buffer) => process.stderr.write(`[inner] ${d}`)); + innerProcess.stderr?.on("data", (d: Buffer) => process.stderr.write(`[inner] ${d}`)); + + session.addCleanup(async () => { + if (!innerProcess.killed && innerProcess.exitCode === null) { + try { innerProcess.kill("SIGTERM"); } catch { } + await new Promise((r) => setTimeout(r, 1000)); + try { innerProcess.kill("SIGKILL"); } catch { } + } + }); + + console.error(`[cursor-proof ${elapsed()}] Waiting for inner player HTTP...`); + await waitForPort(INNER_PORT, 60_000); + console.error(`[cursor-proof ${elapsed()}] Inner player HTTP is up, opening page...`); + + const { page } = await session.openPage({ + url: `http://localhost:${INNER_PORT}`, + viewport: { width: 1280, height: 720 }, + }); + console.error(`[cursor-proof ${elapsed()}] Page created`); + await page.waitForLoadState("domcontentloaded"); + console.error(`[cursor-proof ${elapsed()}] domcontentloaded`); + + // Wait for studio-react mode (same 3-retry pattern as self-test) + for (let attempt = 0; attempt < 3; attempt++) { + try { + await page.waitForSelector("[data-preview-mode='studio-react']", { timeout: 15_000 }); + break; + } catch { + console.error(`[cursor-proof ${elapsed()}] Attempt ${attempt + 1}: studio not ready, reloading...`); + await page.reload(); + await page.waitForLoadState("domcontentloaded"); + } + } + console.error(`[cursor-proof ${elapsed()}] Inner player UI ready!`); + + // Pink InjectedActor cursor (tester cursor) over inner player page + const injected = new InjectedActor(page, "tester", { + mode: session.modeRef, + cursorColor: { fill: "#ff69b4", stroke: "#c2185b" }, + }); + await injected.init(); + await page.setViewportSize({ width: 1280, height: 720 }); + + return { page, injected, innerProcess }; + }); + + // Step 1: Load simple-click scenario into the inner player + s.step("Load simple-click scenario", async ({ injected, page }) => { + await page.selectOption("[data-testid='picker-select']", { + label: "tests/scenarios/simple-click.scenario.ts", + }); + await page.waitForTimeout(2000); + await page.waitForSelector("[data-testid='step-card-0']", { timeout: 30_000 }); + console.error("[cursor-proof] simple-click loaded"); + await injected.breathe(); + }); + + // Step 2: Run the inner scenario (step 2) and capture proof screenshot + s.step("Play and capture cursor proof", async ({ injected, page }) => { + // Play all steps so step screenshots are generated inside the inner UI. + // We'll use the step-2 thumbnail (hover confirm) as the "inner cursor" proof surface. + await injected.click("[data-testid='ctrl-play-all']"); + console.error("[cursor-proof] Play All clicked in inner player"); + + // Wait for step-card-1 to receive a screenshot thumbnail (means stepComplete arrived). + const stepCard = page.locator("[data-testid='step-card-1']"); + await stepCard.locator("img").first().waitFor({ state: "visible", timeout: 120_000 }); + + const artifactsDir = process.env.B2V_TEST_ARTIFACTS_DIR || "/tmp"; + try { fs.mkdirSync(artifactsDir, { recursive: true }); } catch { /* ignore */ } + const proofPath = path.join(artifactsDir, "b2v-cursor-proof.png"); + + // Poll until BOTH cursors are visible: + // - Outer (InjectedActor) cursor: DOM element `#__b2v_cursor_tester` + // - Inner (scenario Actor) cursor: orange pixels inside the step-card-1 screenshot thumbnail + const t0 = Date.now(); + const deadline = t0 + 20_000; + const target = { r: 0xfb, g: 0x92, b: 0x3c }; // #fb923c (inner cursor fill) + const tol = 28; + const minMatches = 140; + + let lastDebug = ""; + let outerVisible = false; + let innerVisible = false; + + while (Date.now() < deadline) { + const res = await page.evaluate(({ target, tol, minMatches }) => { + const outerEl = document.getElementById("__b2v_cursor_tester") as HTMLElement | null; + const outerVisible = + !!outerEl && + getComputedStyle(outerEl).display !== "none" && + outerEl.getBoundingClientRect().width > 0; + + const stepImg = document.querySelector("[data-testid='step-card-1'] img") as HTMLImageElement | null; + const img = stepImg ?? null; + if (!img || !img.complete || img.naturalWidth < 10 || img.naturalHeight < 10) { + return { outerVisible, innerMatches: 0, previewMode: "step-card-1", hasImg: !!img }; + } + + const canvas = document.createElement("canvas"); + canvas.width = img.naturalWidth; + canvas.height = img.naturalHeight; + const ctx = canvas.getContext("2d", { willReadFrequently: true } as any); + if (!ctx) return { outerVisible, innerMatches: 0, previewMode: "step-card-1", hasImg: true }; + + ctx.drawImage(img, 0, 0); + const { data } = ctx.getImageData(0, 0, canvas.width, canvas.height); + + let matches = 0; + const stride = 2; // sample every 2 pixels for speed + const w = canvas.width; + const h = canvas.height; + for (let y = 0; y < h; y += stride) { + for (let x = 0; x < w; x += stride) { + const i = (y * w + x) * 4; + const a = data[i + 3]; + if (a < 200) continue; + const r = data[i], g = data[i + 1], b = data[i + 2]; + if (Math.abs(r - target.r) <= tol && Math.abs(g - target.g) <= tol && Math.abs(b - target.b) <= tol) { + matches++; + if (matches >= minMatches) { + return { outerVisible, innerMatches: matches, previewMode: "step-card-1", hasImg: true }; + } + } + } + } + return { outerVisible, innerMatches: matches, previewMode: "step-card-1", hasImg: true }; + }, { target, tol, minMatches }); + + outerVisible = res.outerVisible; + innerVisible = res.innerMatches >= minMatches; + const dbg = `mode=${res.previewMode} img=${res.hasImg} outer=${outerVisible} innerMatches=${res.innerMatches}`; + if (dbg !== lastDebug) { + console.error(`[cursor-proof] ${dbg}`); + lastDebug = dbg; + } + + if (outerVisible && innerVisible) break; + await page.waitForTimeout(250); + } + + if (!outerVisible) { + throw new Error("Outer InjectedActor cursor did not become visible (expected #__b2v_cursor_tester)."); + } + if (!innerVisible) { + throw new Error(`Inner scenario cursor was not detected in preview image within ${(Date.now() - t0) / 1000}s.`); + } + + await page.screenshot({ path: proofPath, type: "png" }); + console.error(`[cursor-proof] ✅ Proof screenshot saved: ${proofPath}`); + + await page.waitForTimeout(500); + await injected.breathe(); + }); +}); diff --git a/tests/scenarios/mcp-generated/all-in-one.test.ts b/tests/scenarios/mcp-generated/all-in-one.test.ts index 8cf9ae5..7afe9f7 100644 --- a/tests/scenarios/mcp-generated/all-in-one.test.ts +++ b/tests/scenarios/mcp-generated/all-in-one.test.ts @@ -7,5 +7,5 @@ if (isDirectRun) { runScenario(descriptor).then(() => process.exit(0)).catch((e) => { console.error(e); process.exit(1); }); } else { const { test } = await import("@playwright/test"); - test.skip("all-in-one (requires Electron — run via apps/player E2E)", async () => { test.setTimeout(300_000); await runScenario(descriptor); }); + test.skip("all-in-one (requires Electron — run via apps/studio-player E2E)", async () => { test.setTimeout(300_000); await runScenario(descriptor); }); } diff --git a/tests/scenarios/player-self-test.scenario.ts b/tests/scenarios/player-self-test.scenario.ts index 264db52..a5b13a3 100644 --- a/tests/scenarios/player-self-test.scenario.ts +++ b/tests/scenarios/player-self-test.scenario.ts @@ -21,7 +21,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { defineScenario, type Actor, type Page } from "browser2video"; import { InjectedActor } from "browser2video/injected-actor"; -const PLAYER_DIR = path.resolve(import.meta.dirname, "../../apps/player"); +const PLAYER_DIR = path.resolve(import.meta.dirname, "../../apps/studio-player"); const INNER_PORT = 9591; const INNER_CDP_PORT = 9395; const DEMO_VITE_PORT = 5199; diff --git a/tests/scenarios/simple-click.scenario.ts b/tests/scenarios/simple-click.scenario.ts new file mode 100644 index 0000000..80c06f0 --- /dev/null +++ b/tests/scenarios/simple-click.scenario.ts @@ -0,0 +1,43 @@ +/** + * Simple Click scenario — opens a static HTML page and clicks the Confirm button. + * Used by cursor-proof test to verify cursor overlay visibility. + */ +import { defineScenario, startServer, type Actor } from "browser2video"; + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +interface Ctx { + actor: Actor; +} + +export default defineScenario("Simple Click", (s) => { + s.setup(async (session) => { + const server = await startServer({ type: "static", root: "tests/fixtures" }); + if (!server) throw new Error("Failed to start static server"); + session.addCleanup(() => server.stop()); + const { actor } = await session.openPage({ + url: `${server.baseURL}/simple-page.html`, + viewport: { width: 650 }, + }); + return { actor }; + }); + + s.step("Wait for page", async ({ actor }) => { + await actor.waitFor('[data-testid="btn-confirm"]'); + }); + + s.step("Hover confirm (cursor proof)", async ({ actor }) => { + await actor.hover('[data-testid="btn-confirm"]'); + // Keep the cursor visible long enough for the player preview screencast + // to capture a frame where the cursor is clearly present. + await sleep(1500); + }); + + s.step("Click confirm button", async ({ actor }) => { + const btn = actor.page.locator('[data-testid="btn-confirm"]'); + await actor.clickLocator(btn); + await actor.waitFor('[data-testid="done-msg"].show', 10_000); + }); +}); From 475ea4fb1b8c5bf40b82cd671b2ab0bde404a058 Mon Sep 17 00:00:00 2001 From: Alexander Nazarov Date: Wed, 25 Feb 2026 05:42:07 +0300 Subject: [PATCH 2/6] feat: fix stop button audio, add --headless mode, concurrent chat typing - AudioDirector now tracks spawned audio processes and exposes stop() to kill them immediately on abort. Sleep timers are also cancelled so the scenario unblocks without waiting for narration to finish. - Session.abort() calls audioDirector.stop() before closing pages. - New --headless CLI flag for Studio Player: hides the Electron window, disables realtime audio, auto-exits after scenario completes. - Chat scenario: Veronica's message typing now runs concurrently with Bob's terminal typing in both Scene 1 and Scene 3 (via typeInTerminalViaInject). Bob stays engaged after checking calendar. - E2E test for stop button verifying no lingering audio processes. Co-authored-by: Cursor --- AGENTS.md | 17 + apps/demo/src/App.tsx | 56 +- apps/demo/src/components/iphone-chrome.tsx | 122 +++ apps/demo/src/components/macos-chrome.tsx | 117 +++ apps/demo/src/components/pixel-chrome.tsx | 77 ++ apps/demo/src/pages/chat.tsx | 198 ++++- apps/demo/src/pages/movie.tsx | 127 +++ apps/demo/src/pages/wiki.tsx | 141 ++++ apps/studio-player/electron/main.ts | 39 +- apps/studio-player/server/executor.ts | 9 +- apps/studio-player/server/index.ts | 87 +- apps/studio-player/src/App.tsx | 7 +- .../src/components/audio-settings.tsx | 155 ++++ .../studio-player/src/components/controls.tsx | 15 +- apps/studio-player/src/hooks/use-player.ts | 25 +- .../tests/all-scenarios.e2e.test.ts | 139 ++++ .../tests/chat-language.e2e.test.ts | 162 ++++ apps/studio-player/tests/electron.e2e.test.ts | 39 + .../tests/tui-terminals.e2e.test.ts | 88 ++ packages/browser2video/actor.ts | 2 +- packages/browser2video/index.ts | 3 +- packages/browser2video/narrator.ts | 569 ++++++++++++- packages/browser2video/schemas/narration.ts | 7 +- packages/browser2video/session.ts | 22 +- packages/browser2video/terminal-actor.ts | 40 +- .../browser2video/tts-language-presets.ts | 170 ++++ tests/scenarios/chat.scenario.ts | 757 ++++++++++++++---- tests/scenarios/chat.test.ts | 11 + .../mcp-generated/all-in-one.scenario.ts | 51 +- tests/scenarios/tui-terminals.scenario.ts | 24 +- 30 files changed, 2993 insertions(+), 283 deletions(-) create mode 100644 apps/demo/src/components/iphone-chrome.tsx create mode 100644 apps/demo/src/components/macos-chrome.tsx create mode 100644 apps/demo/src/components/pixel-chrome.tsx create mode 100644 apps/demo/src/pages/movie.tsx create mode 100644 apps/demo/src/pages/wiki.tsx create mode 100644 apps/studio-player/src/components/audio-settings.tsx create mode 100644 apps/studio-player/tests/all-scenarios.e2e.test.ts create mode 100644 apps/studio-player/tests/chat-language.e2e.test.ts create mode 100644 apps/studio-player/tests/tui-terminals.e2e.test.ts create mode 100644 packages/browser2video/tts-language-presets.ts create mode 100644 tests/scenarios/chat.test.ts diff --git a/AGENTS.md b/AGENTS.md index c647502..fb4b767 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,3 +2,20 @@ The default agent for this repo is agents/DEFAULT.AGENT.md ^ Read this file on session start Testing strategy: agents/testing-strategy.md + +## Player scenario concurrency rule + +When driving **Studio Player** (Electron app) via UI automation, WebSocket messages, or CLI: + +- Never start multiple scenarios concurrently in a single player instance. +- Always serialize scenario commands (`load`, `runAll`, `runStep`, `reset`, `cancel`). +- Do not issue a second `runAll`/`runStep` while a run is in progress; wait for completion or send `cancel` and wait for cancellation acknowledgement before starting another run. +- If you implement new automation, ensure it cannot trigger overlapping executions (race conditions between `load` and `runAll`, double-clicks, reconnect retries, etc.). + +## Scenario debugging workflow rule + +When validating “all scenarios run without errors” and you find a failure: + +- First reproduce by playing **only the failing scenario** (do not rerun the full suite yet). +- Fix the scenario (or the underlying library/app bug), then rerun **only that scenario** until it passes reliably. +- Only after the single-scenario run is stable, rerun the **all-scenarios** suite again. diff --git a/apps/demo/src/App.tsx b/apps/demo/src/App.tsx index 3f26bbe..e70e5c6 100644 --- a/apps/demo/src/App.tsx +++ b/apps/demo/src/App.tsx @@ -1,5 +1,5 @@ import { Suspense, useMemo } from "react"; -import { Routes, Route, Navigate, useLocation, useNavigate } from "react-router-dom"; +import { Routes, Route, Navigate, useLocation, useNavigate, useSearchParams } from "react-router-dom"; import { AnimatePresence, motion } from "framer-motion"; import { LayoutDashboard, ListTodo, TerminalSquare, Columns3, MessageCircle, CalendarDays } from "lucide-react"; import { Button } from "@/components/ui/button"; @@ -10,6 +10,10 @@ import TerminalsPage from "@/pages/terminals"; import KanbanPage from "@/pages/kanban"; import ChatPage from "@/pages/chat"; import CalendarPage from "@/pages/calendar"; +import MoviePage from "@/pages/movie"; +import WikiPage from "@/pages/wiki"; +import IPhoneChrome from "@/components/iphone-chrome"; +import PixelChrome from "@/components/pixel-chrome"; import { RepoContext } from "@/lib/use-automerge"; import { createRepo } from "@/lib/use-automerge"; @@ -51,23 +55,43 @@ function NavMenu({ onNavigate }: { onNavigate: (path: string) => void }) { ); } -function AppLayout({ children }: { children: React.ReactNode }) { +function PlainLayout({ children }: { children: React.ReactNode }) { return (
-
- {children} -
+
{children}
); } +function DeviceLayout({ role, children }: { role: string | null; children: React.ReactNode }) { + if (role === "veronica") return {children}; + if (role === "bob") return {children}; + return {children}; +} + +function PageRoutes() { + const location = useLocation(); + return ( + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + ); +} + export default function App() { const location = useLocation(); - const wsUrl = useMemo(() => { - const params = new URLSearchParams(location.search); - return params.get("ws") ?? undefined; - }, [location.search]); + const [params] = useSearchParams(); + const role = params.get("role"); + const wsUrl = useMemo(() => params.get("ws") ?? undefined, [params]); const repo = useMemo(() => createRepo({ wsUrl }), [wsUrl]); return ( @@ -81,17 +105,9 @@ export default function App() { exit={{ opacity: 0 }} transition={{ duration: 0.2 }} > - - - } /> - } /> - } /> - } /> - } /> - } /> - } /> - - + + + diff --git a/apps/demo/src/components/iphone-chrome.tsx b/apps/demo/src/components/iphone-chrome.tsx new file mode 100644 index 0000000..02b0e21 --- /dev/null +++ b/apps/demo/src/components/iphone-chrome.tsx @@ -0,0 +1,122 @@ +/** + * @description iPhone / iOS-style device chrome. + * Wraps page content with a status bar (Dynamic Island), dock, and home indicator. + */ +import { useCallback, type ReactNode } from "react"; +import { useNavigate, useLocation, useSearchParams } from "react-router-dom"; +import { + Phone, + Globe, + MessageCircle, + Camera, + Music, + Map, + Settings, + Wifi, + BatteryMedium, + Signal, +} from "lucide-react"; + +const DOCK_APPS = [ + { id: "phone", icon: Phone, label: "Phone", color: "from-green-400 to-green-600" }, + { id: "safari", icon: Globe, label: "Safari", color: "from-sky-400 to-blue-600" }, + { id: "messages", icon: MessageCircle, label: "Messages", color: "from-green-400 to-emerald-600", testId: "dock-messages" }, + { id: "camera", icon: Camera, label: "Camera", color: "from-zinc-500 to-zinc-700" }, +] as const; + +interface Props { + children: ReactNode; + onMessengerClick?: () => void; +} + +function StatusBar() { + const now = new Date(); + const time = now.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false }); + + return ( +
+ {time} + + {/* Dynamic Island */} +
+ +
+ + + +
+
+ ); +} + +export default function IPhoneChrome({ children, onMessengerClick }: Props) { + const navigate = useNavigate(); + const location = useLocation(); + const [params] = useSearchParams(); + const role = params.get("role") ?? "veronica"; + const ws = params.get("ws") ?? ""; + + const handleDockClick = useCallback( + (appId: string) => { + if (appId === "messages") { + if (onMessengerClick) { + onMessengerClick(); + } else { + navigate(`/chat?role=${encodeURIComponent(role)}&ws=${encodeURIComponent(ws)}`); + } + } + }, + [navigate, role, ws, onMessengerClick], + ); + + const isMessengerActive = location.pathname === "/chat"; + + return ( +
+
+ +
+ + {/* Content */} +
{children}
+ + {/* Dock */} +
+
+ {DOCK_APPS.map((app) => { + const Icon = app.icon; + const isActive = app.id === "messages" && isMessengerActive; + return ( + + ); + })} +
+
+ + {/* Home indicator */} +
+
+
+
+ ); +} diff --git a/apps/demo/src/components/macos-chrome.tsx b/apps/demo/src/components/macos-chrome.tsx new file mode 100644 index 0000000..ea3b79e --- /dev/null +++ b/apps/demo/src/components/macos-chrome.tsx @@ -0,0 +1,117 @@ +/** + * @description macOS-style device chrome with title bar and dock. + * Wraps page content to look like a native macOS window. + */ +import { useCallback, type ReactNode } from "react"; +import { useNavigate, useLocation, useSearchParams } from "react-router-dom"; +import { + FolderOpen, + Globe, + MessageCircle, + Image, + Music, + FileText, + Settings, + Compass, +} from "lucide-react"; + +const DOCK_APPS = [ + { id: "finder", icon: FolderOpen, label: "Finder", color: "from-sky-400 to-sky-600" }, + { id: "safari", icon: Compass, label: "Safari", color: "from-blue-400 to-indigo-600" }, + { id: "messages", icon: MessageCircle, label: "Messages", color: "from-green-400 to-emerald-600", testId: "dock-messages" }, + { id: "photos", icon: Image, label: "Photos", color: "from-orange-300 via-pink-400 to-violet-500" }, + { id: "music", icon: Music, label: "Music", color: "from-pink-500 to-red-500" }, + { id: "notes", icon: FileText, label: "Notes", color: "from-amber-300 to-yellow-500" }, + { id: "settings", icon: Settings, label: "Settings", color: "from-zinc-400 to-zinc-600" }, +] as const; + +interface Props { + children: ReactNode; + title?: string; + onMessengerClick?: () => void; +} + +export default function MacOSChrome({ children, title, onMessengerClick }: Props) { + const navigate = useNavigate(); + const location = useLocation(); + const [params] = useSearchParams(); + const role = params.get("role") ?? "veronica"; + const ws = params.get("ws") ?? ""; + + const windowTitle = title ?? deriveTitle(location.pathname); + + const handleDockClick = useCallback( + (appId: string) => { + if (appId === "messages") { + if (onMessengerClick) { + onMessengerClick(); + } else { + navigate(`/chat?role=${encodeURIComponent(role)}&ws=${encodeURIComponent(ws)}`); + } + } + }, + [navigate, role, ws, onMessengerClick], + ); + + const isMessengerActive = location.pathname === "/chat"; + + return ( +
+ {/* Title bar */} +
+
+
+
+
+
+ {windowTitle} +
+ + {/* Content */} +
{children}
+ + {/* Dock */} +
+
+ {DOCK_APPS.map((app) => { + const Icon = app.icon; + const isActive = app.id === "messages" && isMessengerActive; + return ( + + ); + })} +
+
+
+ ); +} + +function deriveTitle(pathname: string): string { + if (pathname === "/movie") return "Movies & TV"; + if (pathname === "/chat") return "Messages"; + if (pathname === "/calendar") return "Calendar"; + return "Finder"; +} diff --git a/apps/demo/src/components/pixel-chrome.tsx b/apps/demo/src/components/pixel-chrome.tsx new file mode 100644 index 0000000..cbbeaaf --- /dev/null +++ b/apps/demo/src/components/pixel-chrome.tsx @@ -0,0 +1,77 @@ +/** + * @description Google Pixel / Android-style device chrome. + * Wraps page content with a status bar and 3-button navigation bar. + */ +import { type ReactNode } from "react"; +import { useLocation } from "react-router-dom"; +import { Wifi, BatteryMedium, Signal } from "lucide-react"; + +interface Props { + children: ReactNode; +} + +function StatusBar() { + const now = new Date(); + const time = now.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false }); + + return ( +
+ {time} +
+ + + + 82% +
+
+ ); +} + +function NavBar() { + return ( +
+ {/* Back */} +
+ {/* Home */} +
+ {/* Recents */} +
+
+ ); +} + +export default function PixelChrome({ children }: Props) { + const location = useLocation(); + const pageTitle = deriveTitle(location.pathname); + + return ( +
+ + + {/* App bar */} +
+ {pageTitle} +
+ + {/* Content */} +
{children}
+ + {/* Gesture pill + nav */} + +
+ ); +} + +function deriveTitle(pathname: string): string { + if (pathname === "/chat") return "Messages"; + if (pathname === "/calendar") return "Calendar"; + if (pathname === "/wiki") return "Chrome"; + if (pathname === "/") return "Home"; + return "App"; +} diff --git a/apps/demo/src/pages/chat.tsx b/apps/demo/src/pages/chat.tsx index cd3f293..1b9f5fe 100644 --- a/apps/demo/src/pages/chat.tsx +++ b/apps/demo/src/pages/chat.tsx @@ -1,10 +1,10 @@ /** * @description Chat page synced between two users via Automerge. - * URL params: ?role=alice|bob&ws=... + #automerge:docHash + * URL params: ?role=veronica|bob&ws=... + #automerge:docHash */ import { useState, useCallback, useRef, useEffect, type KeyboardEvent } from "react"; import { motion, AnimatePresence } from "framer-motion"; -import { Send, Wifi, MessageCircle } from "lucide-react"; +import { Send, Wifi, MessageCircle, Pencil } from "lucide-react"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; import { @@ -20,8 +20,9 @@ import { interface ChatMessage { id: string; - sender: "alice" | "bob"; + sender: "veronica" | "bob"; text: string; + image?: string; ts: number; } @@ -33,11 +34,11 @@ interface ChatDoc { /* Helpers */ /* ------------------------------------------------------------------ */ -function getRoleFromURL(): "alice" | "bob" { +function getRoleFromURL(): "veronica" | "bob" { const params = new URLSearchParams(window.location.search); const role = params.get("role"); - if (role === "alice" || role === "bob") return role; - return "alice"; + if (role === "veronica" || role === "bob") return role; + return "veronica"; } let msgCounter = 0; @@ -47,7 +48,7 @@ function nextMsgId(): string { } const ROLE_COLORS = { - alice: { + veronica: { bg: "bg-violet-500/20", border: "border-violet-500/30", name: "text-violet-400", @@ -63,7 +64,7 @@ const ROLE_COLORS = { }, } as const; -const ROLE_LABELS = { alice: "Alice 👩", bob: "Bob 👨‍💻" } as const; +const ROLE_LABELS = { veronica: "Veronica 👩", bob: "Bob 👨‍💻" } as const; /* ------------------------------------------------------------------ */ /* Hook: find or create the shared Automerge document via URL hash */ @@ -96,10 +97,14 @@ function MessageBubble({ msg, isMine, index, + liked, + onLike, }: { msg: ChatMessage; isMine: boolean; index: number; + liked: boolean; + onLike: (id: string) => void; }) { const colors = ROLE_COLORS[msg.sender]; const time = new Date(msg.ts).toLocaleTimeString([], { @@ -120,13 +125,46 @@ function MessageBubble({
- {msg.sender === "alice" ? "A" : "B"} + {msg.sender === "veronica" ? "V" : "B"}
- {/* Bubble */} -
-

{msg.text}

-

{time}

+ {/* Bubble + reaction */} +
+
+ {msg.image && ( + sketch + )} +

{msg.text}

+
+

{time}

+ {!isMine && ( + + )} +
+
+ + {liked && ( + + ❤️ + + )} +
@@ -153,6 +191,89 @@ function NotificationBadge({ count }: { count: number }) { ); } +/* ------------------------------------------------------------------ */ +/* Sketchpad */ +/* ------------------------------------------------------------------ */ + +function Sketchpad({ onSend }: { onSend?: (dataUrl: string) => void }) { + const canvasRef = useRef(null); + const drawing = useRef(false); + const lastPt = useRef<{ x: number; y: number } | null>(null); + + function getPos(e: React.PointerEvent) { + const c = canvasRef.current!; + const r = c.getBoundingClientRect(); + return { + x: (e.clientX - r.left) * (c.width / r.width), + y: (e.clientY - r.top) * (c.height / r.height), + }; + } + + function onDown(e: React.PointerEvent) { + drawing.current = true; + lastPt.current = getPos(e); + (e.target as HTMLCanvasElement).setPointerCapture(e.pointerId); + } + + function onMove(e: React.PointerEvent) { + if (!drawing.current) return; + const ctx = canvasRef.current?.getContext("2d"); + if (!ctx || !lastPt.current) return; + const pt = getPos(e); + ctx.strokeStyle = "#c084fc"; + ctx.lineWidth = 3; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + ctx.beginPath(); + ctx.moveTo(lastPt.current.x, lastPt.current.y); + ctx.lineTo(pt.x, pt.y); + ctx.stroke(); + lastPt.current = pt; + } + + function onUp() { + drawing.current = false; + lastPt.current = null; + } + + function handleSend() { + const c = canvasRef.current; + if (!c || !onSend) return; + onSend(c.toDataURL("image/png")); + } + + return ( +
+ + {onSend && ( +
+ +
+ )} +
+ ); +} + /* ------------------------------------------------------------------ */ /* Chat view (needs docUrl) */ /* ------------------------------------------------------------------ */ @@ -162,6 +283,8 @@ function ChatView({ docUrl }: { docUrl: AutomergeUrl }) { const [doc, changeDoc] = useDocument(docUrl, { suspense: true }); const [inputValue, setInputValue] = useState(""); const [lastSeenCount, setLastSeenCount] = useState(0); + const [sketchOpen, setSketchOpen] = useState(false); + const [likedMessages, setLikedMessages] = useState>(new Set()); const scrollRef = useRef(null); const inputRef = useRef(null); @@ -199,6 +322,19 @@ function ChatView({ docUrl }: { docUrl: AutomergeUrl }) { inputRef.current?.focus(); }, [inputValue, changeDoc, role]); + const sendSketch = useCallback((dataUrl: string) => { + changeDoc((d) => { + d.messages.push({ + id: nextMsgId(), + sender: role, + text: "", + image: dataUrl, + ts: Date.now(), + }); + }); + setSketchOpen(false); + }, [changeDoc, role]); + const handleKeyDown = useCallback( (e: KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { @@ -209,10 +345,19 @@ function ChatView({ docUrl }: { docUrl: AutomergeUrl }) { [sendMessage], ); + const toggleLike = useCallback((msgId: string) => { + setLikedMessages((prev) => { + const next = new Set(prev); + if (next.has(msgId)) next.delete(msgId); + else next.add(msgId); + return next; + }); + }, []); + const colors = ROLE_COLORS[role]; return ( -
+
{/* Header */}
@@ -249,15 +394,40 @@ function ChatView({ docUrl }: { docUrl: AutomergeUrl }) { msg={msg} isMine={msg.sender === role} index={idx} + liked={likedMessages.has(msg.id)} + onLike={toggleLike} /> ))} )}
+ {/* Sketchpad */} + + {sketchOpen && ( + + + + )} + + {/* Input bar */}
+ +
+
+
+
+
+

+ 3 Body Problem +

+
+ 2024 + · + + TV-MA + + · + 1 Season +
+
+ {[1, 2, 3, 4].map((i) => ( + + ))} + + 8.0 / 10 +
+
+
+
+ + {/* ── Actions ──────────────────────────────────────────── */} +
+ + +
+ + {/* ── Genres ───────────────────────────────────────────── */} +
+ {GENRES.map((g) => ( + + {g} + + ))} +
+ + {/* ── Synopsis ─────────────────────────────────────────── */} +
+

Synopsis

+

+ A young woman’s fateful decision in 1960s China reverberates across space and time + to a group of brilliant scientists in the present day. As the laws of nature unravel + before their eyes, five former classmates reunite to confront the greatest threat in + humanity’s history. Based on the acclaimed novel by Liu Cixin. +

+
+ + {/* ── Creators ─────────────────────────────────────────── */} +
+

Creators

+

+ David Benioff · D.B. Weiss · Alexander Woo +

+
+ + {/* ── Episode info ─────────────────────────────────────── */} +
+
+ + 52–65 min / episode + + 8 episodes +
+
+ + {/* ── Cast ─────────────────────────────────────────────── */} +
+

Cast

+
+ {CAST.map((c) => ( +
+
+ {c.initials} +
+ + {c.name} + +
+ ))} +
+
+
+
+ ); +} diff --git a/apps/demo/src/pages/wiki.tsx b/apps/demo/src/pages/wiki.tsx new file mode 100644 index 0000000..3dfe4b0 --- /dev/null +++ b/apps/demo/src/pages/wiki.tsx @@ -0,0 +1,141 @@ +/** + * @description Simplified Wikipedia-style article about Armillaria ostoyae. + * Used by the chat scenario — Bob reads this on his Pixel phone. + */ + +const TAXONOMY = [ + { label: "Kingdom", value: "Fungi", link: true }, + { label: "Division", value: "Basidiomycota", link: true }, + { label: "Order", value: "Agaricales", link: true }, + { label: "Family", value: "Physalacriaceae", link: true }, + { label: "Genus", value: "Armillaria", link: true, italic: true }, + { label: "Species", value: "A.\u00a0ostoyae", bold: true, italic: true }, +] as const; + +export default function WikiPage() { + return ( +
+
+ {/* Wikipedia header bar */} +
+ + + + Wikipedia +
+ +
+ {/* Article title */} +

+ Armillaria ostoyae +

+

+ From Wikipedia, the free encyclopedia +

+ + {/* Infobox */} +
+
+ Armillaria ostoyae +
+
+ 🍄 +
+
+ Scientific classification +
+ + + {TAXONOMY.map((row) => ( + + + + + ))} + +
+ {row.label}: + + + {row.value} + +
+
+ + {/* Lead paragraph */} +

+ Armillaria ostoyae (synonym A. solidipes) is a + species of pathogenic fungus in the family Physalacriaceae. The mycelium invades the + sapwood of trees and is able to disseminate over great distances under the bark or + between trees in the form of black rhizomorphs (“shoestrings”). +

+ +

+ A specimen in northeastern Oregon’s{" "} + Malheur National Forest is possibly the{" "} + largest living organism on Earth by mass, area, and volume; it covers{" "} + 3.5 square miles (9.1 km²) and weighs as much as{" "} + 35,000 tons. It is estimated to be some 8,000 years old. +

+ + {/* Description section */} +

+ Description +

+

+ The species grows and spreads primarily underground, such that the bulk of the organism + is not visible from the surface. In the autumn, the subterranean parts of the organism + bloom “honey mushrooms” as surface fruits. Low competition for land and + nutrients often allow this fungus to grow to huge proportions. +

+ + {/* Pathogenicity section */} +

+ Pathogenicity +

+

+ This species is of particular interest to forest managers, as it is highly pathogenic + to a number of commercial softwoods, notably Douglas-fir, true firs, pine trees, and + Western Hemlock. The fungus is able to remain viable in stumps for 50 years. +

+

+ Pathogenicity of the fungus is seen to differ among trees of varying age and location. + Younger conifer trees at age 10 and below are more susceptible to infection, while more + mature trees have an increased chance of survival. +

+ + {/* Distribution section */} +

+ Distribution and habitat +

+

+ Armillaria ostoyae is mostly common in the cooler regions of the northern + hemisphere. In North America, this fungus is found on host coniferous trees in the + forests of British Columbia and the Pacific Northwest. +

+

+ A mushroom colony in the Malheur National Forest in the Strawberry Mountains of eastern + Oregon was found to be the largest fungal colony in the world, spanning an area of + 3.5 square miles (2,200 acres; 9.1 km²). If considered a single + organism, it is one of the largest known organisms in the world by area. +

+
+
+
+ ); +} diff --git a/apps/studio-player/electron/main.ts b/apps/studio-player/electron/main.ts index 4fb7161..6d7092d 100644 --- a/apps/studio-player/electron/main.ts +++ b/apps/studio-player/electron/main.ts @@ -81,18 +81,25 @@ let scenarioView: WebContentsView | null = null; const SERVER_PORT = parseInt(process.env.PORT ?? "9521", 10); const isEmbedded = process.env.B2V_EMBEDDED === "1"; +const cliArgs = (() => { // parse early so createMainWindow can use headless flag + return parseAutoScenarioFromCli(process.argv); +})(); +const isHeadless = cliArgs.headless; +const isHidden = isEmbedded || isHeadless; -function parseAutoScenarioFromCli(argv: string[]): { file: string | null; autoplay: boolean } { +function parseAutoScenarioFromCli(argv: string[]): { file: string | null; autoplay: boolean; headless: boolean } { // Electron argv usually looks like: // [electronExe, appPath, ...userArgs] // We support both explicit `--scenario` and positional `*.scenario.ts`. let file: string | null = null; let autoplay = false; + let headless = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--no-play" || a === "--no-autoplay") autoplay = false; if (a === "--play" || a === "--autoplay") autoplay = true; + if (a === "--headless") headless = true; if (a === "--scenario" && argv[i + 1]) { file = argv[i + 1]; @@ -111,7 +118,7 @@ function parseAutoScenarioFromCli(argv: string[]): { file: string | null; autopl } } - return { file, autoplay }; + return { file, autoplay, headless }; } function createMainWindow() { @@ -122,12 +129,13 @@ function createMainWindow() { // we also use off-screen position and minimal size. // Embedded instances need a real surface for CDP capture/screencast. // Keep the window off-screen and transparent instead of tiny/minimized. + // --headless also hides the window but keeps normal sizing for video output. width: isEmbedded ? 1280 : 1440, height: isEmbedded ? 720 : 900, x: isEmbedded ? -10000 : undefined, y: isEmbedded ? -10000 : undefined, - show: !isEmbedded, - skipTaskbar: isEmbedded, + show: !isHidden, + skipTaskbar: isHidden, // Prevent embedded window from appearing in Mission Control / Expose ...(isEmbedded ? { type: "toolbar" as any, focusable: false, hasShadow: false } : {}), title: "Studio Player", @@ -138,13 +146,13 @@ function createMainWindow() { nodeIntegration: false, // Nested StudioPlayer runs hidden/offscreen; keep timers/rendering alive // so CDP screencasts still produce frames. - backgroundThrottling: !isEmbedded, + backgroundThrottling: !isHidden, }, }); - // For embedded instances: aggressively hide the window. + // For headless / embedded instances: aggressively hide the window. // macOS can show windows during loadURL or other async operations. - if (isEmbedded) { + if (isHidden) { // Keep it effectively invisible, but still "shown" so Chromium paints frames. try { mainWindow.setOpacity(0); } catch { } try { mainWindow.setIgnoreMouseEvents(true); } catch { } @@ -215,7 +223,7 @@ export async function createScenarioView( backgroundThrottling: false, }, }); - if (isEmbedded) { + if (isHidden) { try { scenarioView.webContents.setBackgroundThrottling(false); } catch { } } @@ -284,18 +292,19 @@ app.whenReady().then(async () => { app.dock.setIcon(iconPath); } + if (isHeadless) console.error(`[electron ${elt()}] Running in headless mode`); console.error(`[electron ${elt()}] Creating main window...`); createMainWindow(); console.error(`[electron ${elt()}] Main window created`); process.env.PORT = String(SERVER_PORT); process.env.B2V_CDP_PORT = String(CDP_PORT); + if (isHeadless) process.env.B2V_HEADLESS = "1"; // Load a minimal splash page immediately. This unblocks Playwright's // firstWindow() which otherwise waits ~15s for the first navigation. mainWindow!.loadURL("data:text/html,
Starting…
"); - // Re-hide after loadURL for embedded instances (macOS can show the window) - if (isEmbedded) mainWindow!.hide(); + if (isHidden) mainWindow!.hide(); // Import and start the server in-process (~0.5s) console.error(`[electron ${elt()}] Importing server module...`); @@ -307,17 +316,15 @@ app.whenReady().then(async () => { } // Now navigate to the real player URL - const { file: autoScenarioFile, autoplay } = parseAutoScenarioFromCli(process.argv); const params = new URLSearchParams(); - if (autoScenarioFile) { - params.set("scenario", autoScenarioFile); - if (autoplay) params.set("autoplay", "1"); + if (cliArgs.file) { + params.set("scenario", cliArgs.file); + if (cliArgs.autoplay) params.set("autoplay", "1"); } const playerUrl = `http://localhost:${SERVER_PORT}${params.size ? `/?${params.toString()}` : ""}`; console.error(`[electron ${elt()}] Loading player UI: ${playerUrl}`); mainWindow!.loadURL(playerUrl); - // Re-hide after loadURL for embedded instances - if (isEmbedded) mainWindow!.hide(); + if (isHidden) mainWindow!.hide(); // Verify CDP port is actually listening const http = await import("node:http"); diff --git a/apps/studio-player/server/executor.ts b/apps/studio-player/server/executor.ts index f785981..3d02d6a 100644 --- a/apps/studio-player/server/executor.ts +++ b/apps/studio-player/server/executor.ts @@ -68,9 +68,10 @@ export class Executor { this.onRequestPage = opts?.onRequestPage ?? null; const hasNarration = descriptor.steps.some((s) => !!s.narration || !!s.narrationFn); - if (hasNarration && !process.env.OPENAI_API_KEY) { - console.warn("\n WARNING: Scenario has narrated steps but OPENAI_API_KEY is not set."); - console.warn(" Set the env var to enable text-to-speech narration.\n"); + if (hasNarration && !process.env.OPENAI_API_KEY && !process.env.GOOGLE_TTS_API_KEY) { + console.warn("\n WARNING: Scenario has narrated steps but no cloud TTS key is set."); + console.warn(" Will try system TTS (macOS say / Windows SAPI) or Piper as fallback."); + console.warn(" For best quality, set OPENAI_API_KEY or GOOGLE_TTS_API_KEY.\n"); } } @@ -109,7 +110,7 @@ export class Executor { // Session recording in Electron mode also uses CDP screencast internally, // so enabling both would conflict and produce 0 live frames. record: mode === "human" && !isEmbedded, - narration: { enabled: true, realtime: true }, + narration: { enabled: true, realtime: process.env.B2V_HEADLESS !== "1" }, ...this.descriptor.sessionOpts, ...this.sessionOpts, headed: false, diff --git a/apps/studio-player/server/index.ts b/apps/studio-player/server/index.ts index d657a55..8b7f97e 100644 --- a/apps/studio-player/server/index.ts +++ b/apps/studio-player/server/index.ts @@ -67,6 +67,15 @@ const PROJECT_ROOT = findProjectRoot(); // Message types // --------------------------------------------------------------------------- +interface AudioSettings { + provider?: string; + voice?: string; + speed?: number; + model?: string; + language?: string; + realtime?: boolean; +} + type ClientMsg = | { type: "load"; file: string } | { type: "runStep"; index: number } @@ -76,6 +85,8 @@ type ClientMsg = | { type: "listScenarios" } | { type: "clearCache" } | { type: "setViewMode"; mode: ViewMode } + | { type: "setAudioSettings"; settings: AudioSettings } + | { type: "getAudioSettings" } | { type: "importArtifacts"; dir: string } | { type: "downloadArtifacts"; runId?: string; artifactName?: string }; @@ -95,7 +106,17 @@ type ServerMsg = | { type: "cancelled" } | { type: "viewMode"; mode: ViewMode } | { type: "replayEvent"; event: ReplayEvent } - | { type: "artifactsImported"; count: number; scenarios: string[] }; + | { type: "artifactsImported"; count: number; scenarios: string[] } + | { type: "audioSettings"; settings: AudioSettings; detected: string }; + +function detectTtsProvider(): string { + if (process.env.B2V_TTS_PROVIDER && process.env.B2V_TTS_PROVIDER !== "auto") return process.env.B2V_TTS_PROVIDER; + if (process.env.GOOGLE_TTS_API_KEY) return "google"; + if (process.env.OPENAI_API_KEY) return "openai"; + if (process.platform === "darwin") return "system"; + if (process.platform === "win32") return "system"; + return "none"; +} function send(ws: WebSocket, msg: ServerMsg) { if (ws.readyState === WebSocket.OPEN) { @@ -202,6 +223,12 @@ let viteProcess: ChildProcess | null = null; let terminalServer: TerminalServer | null = null; let currentViewMode: ViewMode = "live"; +function getRunMode(): "human" | "fast" { + const raw = (process.env.B2V_MODE ?? "").toLowerCase(); + if (raw === "fast") return "fast"; + return "human"; +} + // Electron mode: when B2V_CDP_PORT is set, Playwright connects via CDP const electronCdpPort = process.env.B2V_CDP_PORT ? parseInt(process.env.B2V_CDP_PORT, 10) : 0; const electronCdpEndpoint = electronCdpPort > 0 ? `http://localhost:${electronCdpPort}` : null; @@ -440,6 +467,18 @@ wss.on("connection", (ws) => { const files = listPlayerScenarioFiles(); send(ws, { type: "scenarioFiles", files }); send(ws, { type: "viewMode", mode: currentViewMode }); + send(ws, { + type: "audioSettings", + settings: { + provider: process.env.B2V_TTS_PROVIDER, + voice: process.env.B2V_NARRATION_VOICE ?? process.env.B2V_VOICE, + speed: process.env.B2V_NARRATION_SPEED ? parseFloat(process.env.B2V_NARRATION_SPEED) : undefined, + model: process.env.B2V_NARRATION_MODEL, + language: process.env.B2V_NARRATION_LANGUAGE, + realtime: process.env.B2V_REALTIME_AUDIO === "true" ? true : undefined, + }, + detected: detectTtsProvider(), + }); if (terminalServer) { // Send the player's own /terminal URL as the base for terminal iframes const terminalPageUrl = `http://localhost:${PORT}`; @@ -536,7 +575,7 @@ wss.on("connection", (ws) => { } await executor.runTo( msg.index, - "human", + getRunMode(), (index, fastForward) => send(ws, { type: "stepStart", index, fastForward }), (result) => { send(ws, { type: "stepComplete", ...result }); @@ -555,7 +594,7 @@ wss.on("connection", (ws) => { for (let i = 0; i < executor.stepCount; i++) { await executor.runTo( i, - "human", + getRunMode(), (index, fastForward) => send(ws, { type: "stepStart", index, fastForward }), (result) => { send(ws, { type: "stepComplete", ...result }); @@ -578,6 +617,10 @@ wss.on("connection", (ws) => { } } send(ws, { type: "finished", videoPath: videoPath ?? (currentCacheDir ? cache.getVideoPath(currentCacheDir) : null) ?? undefined }); + if (process.env.B2V_HEADLESS === "1") { + console.error("[player] Headless run complete, exiting."); + setTimeout(() => process.exit(0), 500); + } } catch (err) { if ((err as Error).message?.includes("aborted")) { console.error("[player] Execution aborted by user"); @@ -613,6 +656,44 @@ wss.on("connection", (ws) => { break; } + case "setAudioSettings": { + const s = msg.settings; + if (s.provider) process.env.B2V_TTS_PROVIDER = s.provider; + else delete process.env.B2V_TTS_PROVIDER; + + if (s.voice) process.env.B2V_NARRATION_VOICE = s.voice; + else delete process.env.B2V_NARRATION_VOICE; + + if (s.speed != null) process.env.B2V_NARRATION_SPEED = String(s.speed); + else delete process.env.B2V_NARRATION_SPEED; + + if (s.model) process.env.B2V_NARRATION_MODEL = s.model; + else delete process.env.B2V_NARRATION_MODEL; + + if (s.language) process.env.B2V_NARRATION_LANGUAGE = s.language; + else delete process.env.B2V_NARRATION_LANGUAGE; + + if (s.realtime != null) process.env.B2V_REALTIME_AUDIO = s.realtime ? "true" : "false"; + else delete process.env.B2V_REALTIME_AUDIO; + + console.error(`[player] Audio settings updated: provider=${s.provider ?? "auto"}`); + send(ws, { type: "audioSettings", settings: s, detected: detectTtsProvider() }); + break; + } + + case "getAudioSettings": { + const settings: AudioSettings = { + provider: process.env.B2V_TTS_PROVIDER, + voice: process.env.B2V_NARRATION_VOICE ?? process.env.B2V_VOICE, + speed: process.env.B2V_NARRATION_SPEED ? parseFloat(process.env.B2V_NARRATION_SPEED) : undefined, + model: process.env.B2V_NARRATION_MODEL, + language: process.env.B2V_NARRATION_LANGUAGE, + realtime: process.env.B2V_REALTIME_AUDIO === "true" ? true : undefined, + }; + send(ws, { type: "audioSettings", settings, detected: detectTtsProvider() }); + break; + } + case "cancel": { if (executor) { console.error("[player] Cancelling current execution..."); diff --git a/apps/studio-player/src/App.tsx b/apps/studio-player/src/App.tsx index 9689fad..c018025 100644 --- a/apps/studio-player/src/App.tsx +++ b/apps/studio-player/src/App.tsx @@ -8,7 +8,7 @@ import { ScenarioPicker } from "./components/scenario-picker"; const WS_URL = `ws://${window.location.host}/ws`; export default function App() { - const { state, cursor, loadScenario, runStep, runAll, reset, cancel, clearCache, setViewMode, importArtifacts, downloadArtifacts, sendStudioEvent } = usePlayer(WS_URL); + const { state, cursor, loadScenario, runStep, runAll, reset, cancel, clearCache, setViewMode, importArtifacts, downloadArtifacts, sendStudioEvent, setAudioSettings } = usePlayer(WS_URL); const { scenario, scenarioFiles, @@ -29,6 +29,8 @@ export default function App() { importing, importResult, cacheSize, + audioSettings, + detectedProvider, } = state; const activeScreenshot = activeStep >= 0 ? screenshots[activeStep] : null; @@ -130,6 +132,8 @@ export default function App() { connected={connected} importing={importing} importResult={importResult} + audioSettings={audioSettings} + detectedProvider={detectedProvider} onRunStep={runStep} onRunAll={runAll} onReset={reset} @@ -137,6 +141,7 @@ export default function App() { onClearCache={clearCache} onImportArtifacts={importArtifacts} onDownloadArtifacts={downloadArtifacts} + onAudioSettingsChange={setAudioSettings} /> )}
diff --git a/apps/studio-player/src/components/audio-settings.tsx b/apps/studio-player/src/components/audio-settings.tsx new file mode 100644 index 0000000..9ad5561 --- /dev/null +++ b/apps/studio-player/src/components/audio-settings.tsx @@ -0,0 +1,155 @@ +/** + * @description Audio / narration settings panel for Studio Player. + * Allows choosing TTS provider, voice, speed, language, and realtime playback. + */ +import { useState, useEffect } from "react"; +import { Volume2, X } from "lucide-react"; +import type { AudioSettings } from "../hooks/use-player"; + +const PROVIDERS = [ + { value: "", label: "Auto (best available)" }, + { value: "google", label: "Google Cloud TTS" }, + { value: "openai", label: "OpenAI" }, + { value: "system", label: "System (macOS / Windows)" }, + { value: "piper", label: "Piper (free, offline)" }, +] as const; + +interface Props { + settings: AudioSettings; + detectedProvider: string; + onUpdate: (settings: AudioSettings) => void; +} + +export function AudioSettingsPanel({ settings, detectedProvider, onUpdate }: Props) { + const [open, setOpen] = useState(false); + const [local, setLocal] = useState(settings); + + useEffect(() => { setLocal(settings); }, [settings]); + + const apply = (patch: Partial) => { + const next = { ...local, ...patch }; + setLocal(next); + onUpdate(next); + }; + + const providerLabel = PROVIDERS.find((p) => p.value === (detectedProvider || ""))?.label ?? detectedProvider; + + return ( +
+ + + {open && ( +
+
+ Audio Settings + +
+ +
+ {/* Provider */} + + + + Active: {providerLabel} + + + + {/* Voice */} + + apply({ voice: e.target.value || undefined })} + placeholder="auto (provider default)" + className="w-full bg-zinc-900 border border-zinc-700 rounded-md px-2 py-1 text-xs text-zinc-300 placeholder:text-zinc-600" + data-testid="audio-voice" + /> + + + {/* Speed */} + + apply({ speed: parseFloat(e.target.value) })} + className="w-full accent-blue-500" + data-testid="audio-speed" + /> + + + {/* Language */} + + apply({ language: e.target.value || undefined })} + placeholder="none (original text)" + className="w-full bg-zinc-900 border border-zinc-700 rounded-md px-2 py-1 text-xs text-zinc-300 placeholder:text-zinc-600" + data-testid="audio-language" + /> + + + {/* Model (OpenAI only) */} + + + + + {/* Realtime */} + + + +
+
+ )} +
+ ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+ + {children} +
+ ); +} diff --git a/apps/studio-player/src/components/controls.tsx b/apps/studio-player/src/components/controls.tsx index 5067b9e..932846f 100644 --- a/apps/studio-player/src/components/controls.tsx +++ b/apps/studio-player/src/components/controls.tsx @@ -1,6 +1,7 @@ import { useState } from "react"; import { Play, Square, SkipForward, SkipBack, RotateCcw, Trash2, Download, FolderInput } from "lucide-react"; -import type { StepState } from "../hooks/use-player"; +import type { StepState, AudioSettings } from "../hooks/use-player"; +import { AudioSettingsPanel } from "./audio-settings"; interface ControlsProps { stepCount: number; @@ -9,6 +10,8 @@ interface ControlsProps { connected: boolean; importing: boolean; importResult: { count: number; scenarios: string[] } | null; + audioSettings: AudioSettings; + detectedProvider: string; onRunStep: (index: number) => void; onRunAll: () => void; onReset: () => void; @@ -16,6 +19,7 @@ interface ControlsProps { onClearCache: () => void; onImportArtifacts: (dir: string) => void; onDownloadArtifacts: (runId?: string) => void; + onAudioSettingsChange: (settings: AudioSettings) => void; } export function Controls({ @@ -25,6 +29,8 @@ export function Controls({ connected, importing, importResult, + audioSettings, + detectedProvider, onRunStep, onRunAll, onReset, @@ -32,6 +38,7 @@ export function Controls({ onClearCache, onImportArtifacts, onDownloadArtifacts, + onAudioSettingsChange, }: ControlsProps) { const isRunning = stepStates.some((s) => s === "running" || s === "fast-forwarding"); const allDone = stepStates.every((s) => s === "done"); @@ -141,6 +148,12 @@ export function Controls({ )}
+ + + ) +} + +function CarouselNext({ + className, + variant = "outline", + size = "icon", + ...props +}: React.ComponentProps) { + const { orientation, scrollNext, canScrollNext } = useCarousel() + + return ( + + ) +} + +export { + type CarouselApi, + Carousel, + CarouselContent, + CarouselItem, + CarouselPrevious, + CarouselNext, +} diff --git a/apps/demo/src/pages/slides.tsx b/apps/demo/src/pages/slides.tsx new file mode 100644 index 0000000..82847a3 --- /dev/null +++ b/apps/demo/src/pages/slides.tsx @@ -0,0 +1,124 @@ +import * as React from "react"; +import { + Carousel, + CarouselContent, + CarouselItem, + CarouselNext, + CarouselPrevious, + type CarouselApi, +} from "@/components/ui/carousel"; +import { Card, CardContent } from "@/components/ui/card"; +import { StarsBackground } from "@/components/animate-ui/stars-background"; + +const SLIDES = [ + { + title: "Welcome", + description: "An introduction to the slide carousel experience.", + bg: "from-indigo-600 to-violet-700", + }, + { + title: "Design", + description: "Beautiful interfaces built with care and attention.", + bg: "from-rose-500 to-pink-700", + }, + { + title: "Develop", + description: "Clean code, tested and ready for production.", + bg: "from-emerald-500 to-teal-700", + }, + { + title: "Deploy", + description: "Ship fast with confidence and reliability.", + bg: "from-amber-500 to-orange-700", + }, + { + title: "Iterate", + description: "Measure, learn, and continuously improve.", + bg: "from-cyan-500 to-blue-700", + }, +]; + +export default function SlidesPage() { + const [api, setApi] = React.useState(); + const [current, setCurrent] = React.useState(0); + + React.useEffect(() => { + if (!api) return; + + const onSelect = () => setCurrent(api.selectedScrollSnap()); + + setCurrent(api.selectedScrollSnap()); + api.on("select", onSelect); + return () => { api.off("select", onSelect); }; + }, [api]); + + return ( + +
+

+ Slide {current + 1} of {SLIDES.length} +

+ + + + {SLIDES.map((s, i) => ( + + + +
+

+ {s.title} +

+

+ {s.description} +

+
+ + + + ))} + + + + + + + {/* Dot indicators */} +
+ {SLIDES.map((_, i) => ( +
+
+ + ); +} diff --git a/packages/browser2video/actor.ts b/packages/browser2video/actor.ts index e7bf886..6d9720a 100644 --- a/packages/browser2video/actor.ts +++ b/packages/browser2video/actor.ts @@ -268,20 +268,23 @@ export const CURSOR_OVERLAY_SCRIPT = ` document.documentElement.style.scrollBehavior = 'smooth'; } + window.__b2v_laserTrails = {}; + window.__b2v_moveCursor = function(x, y, actorId) { - var el = getCursorEl(actorId || 'default'); + var id = actorId || 'default'; + var el = getCursorEl(id); if (!el) return; // body not ready var wasHidden = el.style.display === 'none'; if (wasHidden) { - // First appearance: teleport without transition to avoid sliding from corner el.style.transition = 'none'; el.style.transform = 'translate(' + (x - 2) + 'px,' + (y - 2) + 'px)'; el.style.display = ''; - // Re-enable transition after a frame requestAnimationFrame(function() { el.style.transition = 'transform 40ms ease-in-out'; }); } else { el.style.transform = 'translate(' + (x - 2) + 'px,' + (y - 2) + 'px)'; } + var trail = window.__b2v_laserTrails[id]; + if (trail) trail.points.push({ x: x, y: y, t: performance.now() }); }; // Pre-register a custom color for an actor ID (call before first moveCursor) @@ -315,6 +318,96 @@ export const CURSOR_OVERLAY_SCRIPT = ` setTimeout(() => ring.remove(), 700); }; + window.__b2v_cursorDown = function(actorId) { + var id = actorId || 'default'; + var el = getCursorEl(id); + if (!el) return; + el.style.filter = 'drop-shadow(0 1px 2px rgba(0,0,0,0.5)) drop-shadow(0 0 6px rgba(96,165,250,0.7))'; + var svg = el.querySelector('svg'); + if (svg) svg.style.transform = 'scale(0.78)'; + var dot = document.createElement('div'); + dot.className = '__b2v_hold_dot'; + dot.id = '__b2v_hold_dot_' + id; + dot.style.cssText = 'position:absolute;left:3px;top:3px;width:10px;height:10px;border-radius:50%;background:rgba(96,165,250,0.7);animation:__b2v_hold_pulse 0.8s ease-in-out infinite;pointer-events:none;'; + el.appendChild(dot); + }; + + window.__b2v_cursorUp = function(actorId) { + var id = actorId || 'default'; + var el = getCursorEl(id); + if (!el) return; + el.style.filter = 'drop-shadow(0 1px 2px rgba(0,0,0,0.5))'; + var svg = el.querySelector('svg'); + if (svg) svg.style.transform = ''; + var dot = document.getElementById('__b2v_hold_dot_' + id); + if (dot) dot.remove(); + }; + + window.__b2v_laserOn = function(actorId) { + var id = actorId || 'default'; + if (window.__b2v_laserTrails[id]) return; + var canvas = document.createElement('canvas'); + canvas.id = '__b2v_laser_' + id; + canvas.width = window.innerWidth; + canvas.height = window.innerHeight; + canvas.style.cssText = 'position:fixed;top:0;left:0;width:100%;height:100%;z-index:999996;pointer-events:none;'; + document.body.appendChild(canvas); + var ctx = canvas.getContext('2d'); + var trail = { points: [], canvas: canvas, ctx: ctx, raf: 0 }; + window.__b2v_laserTrails[id] = trail; + var TRAIL_MS = 400; + function draw() { + var now = performance.now(); + var w = canvas.width; var h = canvas.height; + if (w !== window.innerWidth || h !== window.innerHeight) { + canvas.width = window.innerWidth; + canvas.height = window.innerHeight; + w = canvas.width; h = canvas.height; + } + ctx.clearRect(0, 0, w, h); + while (trail.points.length > 0 && now - trail.points[0].t > TRAIL_MS) trail.points.shift(); + var pts = trail.points; + if (pts.length >= 2) { + for (var i = 1; i < pts.length; i++) { + var prev = pts[i - 1]; + var cur = pts[i]; + var ageStart = (now - prev.t) / TRAIL_MS; + var ageEnd = (now - cur.t) / TRAIL_MS; + var alpha = 0.85 * (1 - (ageStart + ageEnd) / 2); + var lw = 6 * (1 - (ageStart + ageEnd) / 2 * 0.5); + if (alpha <= 0 || lw <= 0) continue; + ctx.beginPath(); + ctx.moveTo(prev.x, prev.y); + ctx.lineTo(cur.x, cur.y); + ctx.strokeStyle = 'rgba(239, 68, 68, ' + alpha + ')'; + ctx.lineWidth = lw; + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + ctx.stroke(); + ctx.beginPath(); + ctx.moveTo(prev.x, prev.y); + ctx.lineTo(cur.x, cur.y); + ctx.strokeStyle = 'rgba(239, 68, 68, ' + (alpha * 0.25) + ')'; + ctx.lineWidth = lw + 6; + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + ctx.stroke(); + } + } + trail.raf = requestAnimationFrame(draw); + } + trail.raf = requestAnimationFrame(draw); + }; + + window.__b2v_laserOff = function(actorId) { + var id = actorId || 'default'; + var trail = window.__b2v_laserTrails[id]; + if (!trail) return; + cancelAnimationFrame(trail.raf); + if (trail.canvas.parentNode) trail.canvas.parentNode.removeChild(trail.canvas); + delete window.__b2v_laserTrails[id]; + }; + if (!document.getElementById('__b2v_style')) { var ensureStyle = function() { if (!document.head) return; @@ -325,6 +418,10 @@ export const CURSOR_OVERLAY_SCRIPT = ` 0% { width: 0; height: 0; opacity: 1; } 100% { width: 80px; height: 80px; opacity: 0; } } + @keyframes __b2v_hold_pulse { + 0%, 100% { transform: scale(1); opacity: 0.7; } + 50% { transform: scale(1.4); opacity: 0.4; } + } \`; document.head.appendChild(style); }; @@ -639,7 +736,9 @@ export class Actor { if (this.mode === "human") { await this.page.mouse.down(); + await this.page.evaluate(`window.__b2v_cursorDown?.('${this.cursorId}')`); await sleep(pickMs(this.delays.clickHoldMs)); + await this.page.evaluate(`window.__b2v_cursorUp?.('${this.cursorId}')`); await this.page.mouse.up(); } else { await this.page.mouse.click(x, y); @@ -834,6 +933,7 @@ export class Actor { await this.page.mouse.move(from.x, from.y); await this.page.mouse.down(); + await this.page.evaluate(`window.__b2v_cursorDown?.('${this.cursorId}')`); this._emitClick(from.x, from.y); await sleep(pickMs(this.delays.clickHoldMs)); @@ -850,6 +950,7 @@ export class Actor { } await sleep(pickMs(this.delays.afterClickMs)); + await this.page.evaluate(`window.__b2v_cursorUp?.('${this.cursorId}')`); await this.page.mouse.up(); this.cursorX = to.x; @@ -955,6 +1056,7 @@ export class Actor { await this.page.mouse.move(absPoints[0].x, absPoints[0].y); await this.page.mouse.down(); + await this.page.evaluate(`window.__b2v_cursorDown?.('${this.cursorId}')`); for (let i = 1; i < absPoints.length; i++) { const segSteps = this.mode === "human" ? 12 : 1; @@ -972,6 +1074,7 @@ export class Actor { } } + await this.page.evaluate(`window.__b2v_cursorUp?.('${this.cursorId}')`); await this.page.mouse.up(); this.cursorX = absPoints[absPoints.length - 1].x; this.cursorY = absPoints[absPoints.length - 1].y; @@ -1053,6 +1156,113 @@ export class Actor { this.cursorY = Math.round(prevY); } + /** + * Highlight an element with a laser-pointer trail spiraling around it. + * Enables the laser trail, performs circleAround, then disables the trail. + * Fast mode: no-op (same as circleAround). + */ + async highlight(selector: string, opts?: { durationMs?: number }) { + if (this.mode !== "human") return; + await this.page.evaluate(`window.__b2v_laserOn?.('${this.cursorId}')`); + await this.circleAround(selector, opts); + await this.page.evaluate(`window.__b2v_laserOff?.('${this.cursorId}')`); + } + + /** + * Draw on a transparent full-page overlay without dispatching real pointer + * events (so underlying page elements are not affected). Injects a canvas, + * draws strokes via JS evaluate, and animates the cursor visually. + * Points use 0-1 normalized coordinates relative to the viewport. + */ + async drawOnPage( + points: Array<{ x: number; y: number }>, + opts?: { color?: string; lineWidth?: number; clear?: boolean }, + ) { + if (points.length < 2) return; + const color = opts?.color ?? "rgba(239, 68, 68, 0.85)"; + const lineWidth = opts?.lineWidth ?? 3; + + const vp = this.page.viewportSize()!; + const absPoints = points.map((p) => ({ + x: Math.round(p.x * vp.width), + y: Math.round(p.y * vp.height), + })); + + await this.page.evaluate( + ({ color, lineWidth }) => { + let c = document.getElementById("__b2v_draw_overlay") as HTMLCanvasElement | null; + if (!c) { + c = document.createElement("canvas"); + c.id = "__b2v_draw_overlay"; + c.width = window.innerWidth; + c.height = window.innerHeight; + c.style.cssText = + "position:fixed;top:0;left:0;width:100%;height:100%;z-index:999997;pointer-events:none;"; + document.body.appendChild(c); + } + const ctx = c.getContext("2d")!; + ctx.strokeStyle = color; + ctx.lineWidth = lineWidth; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + }, + { color, lineWidth }, + ); + + if (this.mode === "human") { + const movePoints = windMouse({ x: this.cursorX, y: this.cursorY }, absPoints[0]); + for (let i = 0; i < movePoints.length; i++) { + const p = movePoints[i]!; + await this.page.evaluate(`window.__b2v_moveCursor?.(${p.x}, ${p.y}, '${this.cursorId}')`); + this._emitCursorMove(p.x, p.y); + await sleep(easedStepMs(pickMs(this.delays.mouseMoveStepMs), i, movePoints.length)); + } + } + + await this.page.evaluate(`window.__b2v_cursorDown?.('${this.cursorId}')`); + + for (let i = 1; i < absPoints.length; i++) { + const prev = absPoints[i - 1]; + const cur = absPoints[i]; + const segSteps = this.mode === "human" ? 12 : 1; + const segPoints = linearPath(prev, cur, segSteps); + + for (let j = 0; j < segPoints.length; j++) { + const p = segPoints[j]!; + await this.page.evaluate( + ({ x, y }) => { + const c = document.getElementById("__b2v_draw_overlay") as HTMLCanvasElement | null; + if (!c) return; + const ctx = c.getContext("2d")!; + ctx.lineTo(x, y); + ctx.stroke(); + ctx.beginPath(); + ctx.moveTo(x, y); + }, + { x: p.x, y: p.y }, + ); + if (this.mode === "human") { + await this.page.evaluate(`window.__b2v_moveCursor?.(${p.x}, ${p.y}, '${this.cursorId}')`); + this._emitCursorMove(p.x, p.y); + await sleep(easedStepMs(pickMs(this.delays.mouseMoveStepMs), j, segPoints.length, 2)); + } + } + } + + await this.page.evaluate(`window.__b2v_cursorUp?.('${this.cursorId}')`); + this.cursorX = absPoints[absPoints.length - 1].x; + this.cursorY = absPoints[absPoints.length - 1].y; + await sleep(pickMs(this.delays.afterDragMs)); + + if (opts?.clear) { + await sleep(1500); + await this.page.evaluate(() => { + const c = document.getElementById("__b2v_draw_overlay"); + if (c) c.remove(); + }); + } + } + /** * Press a keyboard key with a human-like pause afterwards. * Useful for TUI / terminal interactions where raw key presses are needed. @@ -1078,7 +1288,9 @@ export class Actor { this._emitClick(x, y); await sleep(pickMs(this.delays.clickEffectMs)); await this.page.mouse.down(); + await this.page.evaluate(`window.__b2v_cursorDown?.('${this.cursorId}')`); await sleep(pickMs(this.delays.clickHoldMs)); + await this.page.evaluate(`window.__b2v_cursorUp?.('${this.cursorId}')`); await this.page.mouse.up(); await sleep(pickMs(this.delays.afterClickMs)); } else { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09f4957..f843285 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,12 +50,18 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 + embla-carousel-react: + specifier: ^8.6.0 + version: 8.6.0(react@19.2.4) framer-motion: specifier: ^11.18.0 version: 11.18.2(react-dom@19.2.4(react@19.2.4))(react@19.2.4) lucide-react: specifier: ^0.469.0 version: 0.469.0(react@19.2.4) + motion: + specifier: ^12.34.3 + version: 12.34.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) radix-ui: specifier: ^1.4.3 version: 1.4.3(@types/react-dom@19.2.3(@types/react@19.2.13))(@types/react@19.2.13)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -4448,6 +4454,19 @@ packages: engines: {node: '>= 12.20.55'} hasBin: true + embla-carousel-react@8.6.0: + resolution: {integrity: sha512-0/PjqU7geVmo6F734pmPqpyHqiM99olvyecY7zdweCw+6tKEXnrE90pBiBbMMU8s5tICemzpQ3hi5EpxzGW+JA==} + peerDependencies: + react: ^16.8.0 || ^17.0.1 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + + embla-carousel-reactive-utils@8.6.0: + resolution: {integrity: sha512-fMVUDUEx0/uIEDM0Mz3dHznDhfX+znCCDCeIophYb1QGVM7YThSWX+wz11zlYwWFOr74b4QLGg0hrGPJeG2s4A==} + peerDependencies: + embla-carousel: 8.6.0 + + embla-carousel@8.6.0: + resolution: {integrity: sha512-SjWyZBHJPbqxHOzckOfo8lHisEaJWmwd23XppYFYVh10bU66/Pn5tkVkbkCMZVdbUE5eTCI2nD8OyIP4Z+uwkA==} + emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -4780,6 +4799,20 @@ packages: react-dom: optional: true + framer-motion@12.34.3: + resolution: {integrity: sha512-v81ecyZKYO/DfpTwHivqkxSUBzvceOpoI+wLfgCgoUIKxlFKEXdg0oR9imxwXumT4SFy8vRk9xzJ5l3/Du/55Q==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + fresh@0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} @@ -5843,9 +5876,29 @@ packages: motion-dom@11.18.1: resolution: {integrity: sha512-g76KvA001z+atjfxczdRtw/RXOM3OMSdd1f4DL77qCTF/+avrRJiawSG4yDibEQ215sr9kpinSlX2pCTJ9zbhw==} + motion-dom@12.34.3: + resolution: {integrity: sha512-sYgFe+pR9aIM7o4fhs2aXtOI+oqlUd33N9Yoxcgo1Fv7M20sRkHtCmzE/VRNIcq7uNJ+qio+Xubt1FXH3pQ+eQ==} + motion-utils@11.18.1: resolution: {integrity: sha512-49Kt+HKjtbJKLtgO/LKj9Ld+6vw9BjH5d9sc40R/kVyH8GLAXgT42M2NnuPcJNuA3s9ZfZBUcwIgpmZWGEE+hA==} + motion-utils@12.29.2: + resolution: {integrity: sha512-G3kc34H2cX2gI63RqU+cZq+zWRRPSsNIOjpdl9TN4AQwC4sgwYPl/Q/Obf/d53nOm569T0fYK+tcoSV50BWx8A==} + + motion@12.34.3: + resolution: {integrity: sha512-xZIkBGO7v/Uvm+EyaqYd+9IpXu0sZqLywVlGdCFrrMiaO9JI4Kx51mO9KlHSWwll+gZUVY5OJsWgYI5FywJ/tw==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + mrmime@2.0.1: resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} engines: {node: '>=10'} @@ -12889,6 +12942,18 @@ snapshots: transitivePeerDependencies: - supports-color + embla-carousel-react@8.6.0(react@19.2.4): + dependencies: + embla-carousel: 8.6.0 + embla-carousel-reactive-utils: 8.6.0(embla-carousel@8.6.0) + react: 19.2.4 + + embla-carousel-reactive-utils@8.6.0(embla-carousel@8.6.0): + dependencies: + embla-carousel: 8.6.0 + + embla-carousel@8.6.0: {} + emoji-regex@10.6.0: {} emoji-regex@8.0.0: {} @@ -13312,6 +13377,15 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) + framer-motion@12.34.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + motion-dom: 12.34.3 + motion-utils: 12.29.2 + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + fresh@0.5.2: {} fresh@2.0.0: {} @@ -14682,8 +14756,22 @@ snapshots: dependencies: motion-utils: 11.18.1 + motion-dom@12.34.3: + dependencies: + motion-utils: 12.29.2 + motion-utils@11.18.1: {} + motion-utils@12.29.2: {} + + motion@12.34.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + framer-motion: 12.34.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + mrmime@2.0.1: {} ms@2.0.0: {} diff --git a/tests/scenarios/drawing.scenario.ts b/tests/scenarios/drawing.scenario.ts new file mode 100644 index 0000000..982bb54 --- /dev/null +++ b/tests/scenarios/drawing.scenario.ts @@ -0,0 +1,95 @@ +/** + * Drawing scenario — tests laser-pointer highlight and freehand drawing + * annotations on a slide carousel with an animated starfield background. + */ +import { defineScenario, startServer } from "browser2video"; +import type { Page } from "playwright-core"; + +interface Ctx { + actor: import("browser2video").Actor; +} + +const narrations = { + intro: "This scenario demonstrates the laser pointer and drawing overlay features.", + highlight: "First, let's highlight the slide title using the laser pointer.", + drawing: "Now let's draw some annotations right on the page.", + outro: "And that's it!", +}; + +const CHECKMARK_POINTS = [ + { x: 0.42, y: 0.52 }, + { x: 0.46, y: 0.58 }, + { x: 0.48, y: 0.60 }, + { x: 0.54, y: 0.48 }, + { x: 0.60, y: 0.40 }, +]; + +const STAR_POINTS = [ + { x: 0.50, y: 0.30 }, + { x: 0.53, y: 0.42 }, + { x: 0.62, y: 0.42 }, + { x: 0.55, y: 0.50 }, + { x: 0.58, y: 0.62 }, + { x: 0.50, y: 0.54 }, + { x: 0.42, y: 0.62 }, + { x: 0.45, y: 0.50 }, + { x: 0.38, y: 0.42 }, + { x: 0.47, y: 0.42 }, + { x: 0.50, y: 0.30 }, +]; + +async function assertSlide(page: Page, expected: number) { + const expectedText = `Slide ${expected} of 5`; + await page.waitForFunction( + (text: string) => + document.querySelector('[data-testid="slides-current"]')?.textContent?.trim() === text, + expectedText, + { timeout: 5000 }, + ); +} + +export default defineScenario("Drawing", (s) => { + s.setup(async (session) => { + const server = await startServer({ type: "vite", root: "apps/demo" }); + if (!server) throw new Error("Failed to start Vite server"); + session.addCleanup(() => server.stop()); + + const { actor } = await session.openPage({ + url: `${server.baseURL}/slides`, + viewport: { width: 650 }, + }); + + for (const text of Object.values(narrations)) { + await session.audio.warmup(text); + } + + return { actor }; + }); + + s.step("Introduction", narrations.intro, async ({ actor }) => { + await actor.waitFor('[data-testid="slides-page"]'); + await assertSlide(actor.page, 1); + }); + + s.step("Highlight slide title", narrations.highlight, async ({ actor }) => { + await actor.highlight('[data-testid="slides-title-0"]'); + }); + + s.step("Draw annotation", narrations.drawing, async ({ actor }) => { + await actor.drawOnPage(STAR_POINTS, { + color: "rgba(250, 204, 21, 0.9)", + lineWidth: 3, + }); + await actor.drawOnPage(CHECKMARK_POINTS, { + color: "rgba(74, 222, 128, 0.9)", + lineWidth: 4, + }); + }); + + s.step("Outro", narrations.outro, async ({ actor }) => { + await actor.page.evaluate(() => { + const c = document.getElementById("__b2v_draw_overlay"); + if (c) c.remove(); + }); + }); +}); diff --git a/tests/scenarios/slides-and-narration.scenario.ts b/tests/scenarios/slides-and-narration.scenario.ts new file mode 100644 index 0000000..b136835 --- /dev/null +++ b/tests/scenarios/slides-and-narration.scenario.ts @@ -0,0 +1,88 @@ +/** + * Slides and Narration scenario — tests narrator speech and mouse interactions + * (button clicks + swipe via drag) on a slide carousel with an animated + * starfield background. + */ +import { defineScenario, startServer } from "browser2video"; +import type { Page } from "playwright-core"; + +interface Ctx { + actor: import("browser2video").Actor; +} + +const narrations = { + intro: "This scenario tests narration and simple mouse interactions with a slide carousel.", + buttons: "First, let's navigate through the slides using the forward and back buttons.", + swipe: "Now let's try swiping left and right to change slides, just like on a touchscreen.", + outro: "And that's it!", +}; + +const DRAG_SELECTOR = '[data-slot="carousel-content"]'; + +async function assertSlide(page: Page, expected: number) { + const expectedText = `Slide ${expected} of 5`; + await page.waitForFunction( + (text: string) => + document.querySelector('[data-testid="slides-current"]')?.textContent?.trim() === text, + expectedText, + { timeout: 5000 }, + ); +} + +export default defineScenario("Slides and Narration", (s) => { + s.setup(async (session) => { + const server = await startServer({ type: "vite", root: "apps/demo" }); + if (!server) throw new Error("Failed to start Vite server"); + session.addCleanup(() => server.stop()); + + const { actor } = await session.openPage({ + url: `${server.baseURL}/slides`, + viewport: { width: 650 }, + }); + + for (const text of Object.values(narrations)) { + await session.audio.warmup(text); + } + + return { actor }; + }); + + s.step("Introduction", narrations.intro, async ({ actor }) => { + await actor.waitFor('[data-testid="slides-page"]'); + await assertSlide(actor.page, 1); + }); + + s.step("Navigate forward with buttons", narrations.buttons, async ({ actor }) => { + await actor.click('[data-testid="slides-next"]'); + await assertSlide(actor.page, 2); + + await actor.click('[data-testid="slides-next"]'); + await assertSlide(actor.page, 3); + + await actor.click('[data-testid="slides-next"]'); + await assertSlide(actor.page, 4); + }); + + s.step("Navigate backward with buttons", async ({ actor }) => { + await actor.click('[data-testid="slides-prev"]'); + await assertSlide(actor.page, 3); + + await actor.click('[data-testid="slides-prev"]'); + await assertSlide(actor.page, 2); + }); + + s.step("Swipe forward", narrations.swipe, async ({ actor }) => { + await actor.dragByOffset(DRAG_SELECTOR, -300, 0); + await assertSlide(actor.page, 3); + + await actor.dragByOffset(DRAG_SELECTOR, -300, 0); + await assertSlide(actor.page, 4); + }); + + s.step("Swipe backward", async ({ actor }) => { + await actor.dragByOffset(DRAG_SELECTOR, 300, 0); + await assertSlide(actor.page, 3); + }); + + s.step("Outro", narrations.outro, async () => {}); +}); From d2075b4f7fef9498871d7b004864e42100a69ea6 Mon Sep 17 00:00:00 2001 From: Alexander Nazarov Date: Wed, 25 Feb 2026 16:55:47 +0300 Subject: [PATCH 4/6] refactor: move laser pointer highlight to slides scenario, keep drawing separate Co-authored-by: Cursor --- tests/scenarios/drawing.scenario.ts | 7 +------ tests/scenarios/slides-and-narration.scenario.ts | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/scenarios/drawing.scenario.ts b/tests/scenarios/drawing.scenario.ts index 982bb54..993ed13 100644 --- a/tests/scenarios/drawing.scenario.ts +++ b/tests/scenarios/drawing.scenario.ts @@ -10,8 +10,7 @@ interface Ctx { } const narrations = { - intro: "This scenario demonstrates the laser pointer and drawing overlay features.", - highlight: "First, let's highlight the slide title using the laser pointer.", + intro: "This scenario demonstrates the drawing overlay feature.", drawing: "Now let's draw some annotations right on the page.", outro: "And that's it!", }; @@ -71,10 +70,6 @@ export default defineScenario("Drawing", (s) => { await assertSlide(actor.page, 1); }); - s.step("Highlight slide title", narrations.highlight, async ({ actor }) => { - await actor.highlight('[data-testid="slides-title-0"]'); - }); - s.step("Draw annotation", narrations.drawing, async ({ actor }) => { await actor.drawOnPage(STAR_POINTS, { color: "rgba(250, 204, 21, 0.9)", diff --git a/tests/scenarios/slides-and-narration.scenario.ts b/tests/scenarios/slides-and-narration.scenario.ts index b136835..96e7d16 100644 --- a/tests/scenarios/slides-and-narration.scenario.ts +++ b/tests/scenarios/slides-and-narration.scenario.ts @@ -14,6 +14,7 @@ const narrations = { intro: "This scenario tests narration and simple mouse interactions with a slide carousel.", buttons: "First, let's navigate through the slides using the forward and back buttons.", swipe: "Now let's try swiping left and right to change slides, just like on a touchscreen.", + highlight: "Let me highlight the title of this slide using the laser pointer.", outro: "And that's it!", }; @@ -84,5 +85,9 @@ export default defineScenario("Slides and Narration", (s) => { await assertSlide(actor.page, 3); }); + s.step("Highlight slide title", narrations.highlight, async ({ actor }) => { + await actor.highlight('[data-testid="slides-title-2"]'); + }); + s.step("Outro", narrations.outro, async () => {}); }); From 7b3b36be18dda31e49314b902c3359cd40b745db Mon Sep 17 00:00:00 2001 From: Alexander Nazarov Date: Wed, 25 Feb 2026 17:12:10 +0300 Subject: [PATCH 5/6] feat: show cache size on Clear Cache button, add self-test assertion - Server broadcasts cache size on connect and after cache writes - Controls button displays formatted size (e.g. "Clear Cache 185.0 MB") - Player hook handles new cacheSize message type - Self-test scenario verifies cache button shows non-zero size after steps Co-authored-by: Cursor --- apps/studio-player/server/index.ts | 11 +++++++++++ apps/studio-player/src/App.tsx | 1 + .../studio-player/src/components/controls.tsx | 13 ++++++++++++- apps/studio-player/src/hooks/use-player.ts | 6 ++++++ tests/scenarios/player-self-test.scenario.ts | 19 +++++++++++++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/apps/studio-player/server/index.ts b/apps/studio-player/server/index.ts index 8b7f97e..57785b4 100644 --- a/apps/studio-player/server/index.ts +++ b/apps/studio-player/server/index.ts @@ -103,6 +103,7 @@ type ServerMsg = | { type: "paneLayout"; layout: PaneLayoutInfo } | { type: "cachedData"; screenshots: (string | null)[]; stepDurations: (number | null)[]; stepHasAudio: boolean[]; videoPath?: string | null } | { type: "cacheCleared"; cacheSize?: number } + | { type: "cacheSize"; size: number } | { type: "cancelled" } | { type: "viewMode"; mode: ViewMode } | { type: "replayEvent"; event: ReplayEvent } @@ -136,11 +137,19 @@ function persistStepCache(index: number, screenshot: string, durationMs: number, contentHash: currentContentHash, steps: currentStepMetas, }); + broadcastCacheSize(); } catch (err) { console.error("[player] Cache write error:", err); } } +function broadcastCacheSize() { + const size = cache.getCacheSize(); + for (const client of wss.clients) { + send(client as WebSocket, { type: "cacheSize", size }); + } +} + // --------------------------------------------------------------------------- // Dynamic scenario import // --------------------------------------------------------------------------- @@ -466,6 +475,7 @@ wss.on("connection", (ws) => { const files = listPlayerScenarioFiles(); send(ws, { type: "scenarioFiles", files }); + send(ws, { type: "cacheSize", size: cache.getCacheSize() }); send(ws, { type: "viewMode", mode: currentViewMode }); send(ws, { type: "audioSettings", @@ -616,6 +626,7 @@ wss.on("connection", (ws) => { console.error("[player] Failed to save video to cache:", err); } } + broadcastCacheSize(); send(ws, { type: "finished", videoPath: videoPath ?? (currentCacheDir ? cache.getVideoPath(currentCacheDir) : null) ?? undefined }); if (process.env.B2V_HEADLESS === "1") { console.error("[player] Headless run complete, exiting."); diff --git a/apps/studio-player/src/App.tsx b/apps/studio-player/src/App.tsx index c018025..52f2985 100644 --- a/apps/studio-player/src/App.tsx +++ b/apps/studio-player/src/App.tsx @@ -138,6 +138,7 @@ export default function App() { onRunAll={runAll} onReset={reset} onCancel={cancel} + cacheSize={cacheSize} onClearCache={clearCache} onImportArtifacts={importArtifacts} onDownloadArtifacts={downloadArtifacts} diff --git a/apps/studio-player/src/components/controls.tsx b/apps/studio-player/src/components/controls.tsx index 932846f..988c5ca 100644 --- a/apps/studio-player/src/components/controls.tsx +++ b/apps/studio-player/src/components/controls.tsx @@ -3,6 +3,13 @@ import { Play, Square, SkipForward, SkipBack, RotateCcw, Trash2, Download, Folde import type { StepState, AudioSettings } from "../hooks/use-player"; import { AudioSettingsPanel } from "./audio-settings"; +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} + interface ControlsProps { stepCount: number; activeStep: number; @@ -12,6 +19,7 @@ interface ControlsProps { importResult: { count: number; scenarios: string[] } | null; audioSettings: AudioSettings; detectedProvider: string; + cacheSize: number; onRunStep: (index: number) => void; onRunAll: () => void; onReset: () => void; @@ -31,6 +39,7 @@ export function Controls({ importResult, audioSettings, detectedProvider, + cacheSize, onRunStep, onRunAll, onReset, @@ -172,7 +181,9 @@ export function Controls({ data-testid="ctrl-clear-cache" > - Clear cache + + {cacheSize > 0 ? `Clear Cache ${formatBytes(cacheSize)}` : "Clear Cache"} +
diff --git a/apps/studio-player/src/hooks/use-player.ts b/apps/studio-player/src/hooks/use-player.ts index cedd316..4542b5f 100644 --- a/apps/studio-player/src/hooks/use-player.ts +++ b/apps/studio-player/src/hooks/use-player.ts @@ -88,6 +88,7 @@ type Action = | { type: "importStart" } | { type: "artifactsImported"; count: number; scenarios: string[] } | { type: "audioSettings"; settings: AudioSettings; detected: string } + | { type: "cacheSize"; size: number } ; const initial: PlayerState = { @@ -205,6 +206,8 @@ function reducer(state: PlayerState, action: Action): PlayerState { return { ...state, importing: false, importResult: { count: action.count, scenarios: action.scenarios } }; case "audioSettings": return { ...state, audioSettings: action.settings, detectedProvider: action.detected }; + case "cacheSize": + return { ...state, cacheSize: action.size }; case "reset": return { ...state, @@ -345,6 +348,9 @@ export function usePlayer(wsUrl: string) { case "audioSettings": dispatch({ type: "audioSettings", settings: msg.settings, detected: msg.detected }); break; + case "cacheSize": + dispatch({ type: "cacheSize", size: msg.size }); + break; case "error": dispatch({ type: "error", message: msg.message }); break; diff --git a/tests/scenarios/player-self-test.scenario.ts b/tests/scenarios/player-self-test.scenario.ts index a5b13a3..4af061a 100644 --- a/tests/scenarios/player-self-test.scenario.ts +++ b/tests/scenarios/player-self-test.scenario.ts @@ -538,6 +538,25 @@ export default defineScenario("Player Self-Test", (s) => { } }); + s.step("Cache button shows size", async ({ page }) => { + const btn = page.locator("[data-testid='ctrl-clear-cache']"); + await btn.waitFor({ timeout: 5_000 }); + + await page.waitForFunction( + () => /Clear Cache \d/.test( + document.querySelector('[data-testid="ctrl-clear-cache"]')?.textContent ?? "", + ), + undefined, + { timeout: 5_000 }, + ); + + const text = await btn.textContent(); + if (!text || !(/Clear Cache \d/.test(text))) { + throw new Error(`Cache button should show size after steps ran, got: "${text}"`); + } + console.error(`[self-test] Cache button text: "${text}"`); + }); + // ═══════════════════════════════════════════════════════════════════ // Phase 6 — Cleanup & verification // ═══════════════════════════════════════════════════════════════════ From 4f30bd7c0ed3fe9bc0c12b06fce50ca059d44b2d Mon Sep 17 00:00:00 2001 From: Alexander Nazarov Date: Wed, 25 Feb 2026 22:04:25 +0300 Subject: [PATCH 6/6] feat: per-mode step durations, loading overlay, and timing display Store fast and human mode durations separately in cache so both are preserved across runs. Display as "humanDur / fastDur" with N/A fallback. Progress bar falls back to the other mode when the active mode has no data. Add semi-transparent overlay for scenario loading and step fast-forwarding states. Track loading boolean in player state. Co-authored-by: Cursor --- apps/studio-player/server/cache.ts | 26 +++++-- apps/studio-player/server/index.ts | 62 ++++++++++++----- apps/studio-player/src/App.tsx | 23 ++++++- .../src/components/step-graph.tsx | 69 ++++++++++++++----- apps/studio-player/src/hooks/use-player.ts | 57 ++++++++++++--- .../slides-and-narration.scenario.ts | 2 +- 6 files changed, 185 insertions(+), 54 deletions(-) diff --git a/apps/studio-player/server/cache.ts b/apps/studio-player/server/cache.ts index 6f07ae1..8dd6a4c 100644 --- a/apps/studio-player/server/cache.ts +++ b/apps/studio-player/server/cache.ts @@ -6,6 +6,8 @@ import { execFileSync } from "node:child_process"; export interface StepMeta { index: number; durationMs: number; + durationMsFast?: number; + durationMsHuman?: number; hasAudio: boolean; } @@ -110,6 +112,8 @@ export class PlayerCache { loadCachedData(scenarioAbsPath: string, scenarioRelPath: string, stepCount: number): { screenshots: (string | null)[]; stepDurations: (number | null)[]; + stepDurationsFast: (number | null)[]; + stepDurationsHuman: (number | null)[]; stepHasAudio: boolean[]; cacheDir: string; contentHash: string; @@ -121,18 +125,22 @@ export class PlayerCache { const screenshots: (string | null)[] = []; const stepDurations: (number | null)[] = []; + const stepDurationsFast: (number | null)[] = []; + const stepDurationsHuman: (number | null)[] = []; const stepHasAudio: boolean[] = []; for (let i = 0; i < stepCount; i++) { screenshots.push(this.loadScreenshot(dir, i)); const stepMeta = meta.steps.find((s) => s.index === i); stepDurations.push(stepMeta?.durationMs ?? null); + stepDurationsFast.push(stepMeta?.durationMsFast ?? null); + stepDurationsHuman.push(stepMeta?.durationMsHuman ?? null); stepHasAudio.push(stepMeta?.hasAudio ?? false); } const videoPath = this.getVideoPath(dir); - return { screenshots, stepDurations, stepHasAudio, cacheDir: dir, contentHash: hash, videoPath }; + return { screenshots, stepDurations, stepDurationsFast, stepDurationsHuman, stepHasAudio, cacheDir: dir, contentHash: hash, videoPath }; } /** @@ -153,11 +161,17 @@ export class PlayerCache { const { dir, hash } = this.getDir(scenarioAbsPath, scenarioRelPath); fs.mkdirSync(dir, { recursive: true }); - const steps: StepMeta[] = runJson.steps.map((s) => ({ - index: s.index - 1, - durationMs: s.endMs - s.startMs, - hasAudio: !!(runJson.audioEvents && runJson.audioEvents.length > 0), - })); + const isFast = runJson.mode === "fast"; + const steps: StepMeta[] = runJson.steps.map((s) => { + const dur = s.endMs - s.startMs; + return { + index: s.index - 1, + durationMs: dur, + durationMsFast: isFast ? dur : undefined, + durationMsHuman: isFast ? undefined : dur, + hasAudio: !!(runJson.audioEvents && runJson.audioEvents.length > 0), + }; + }); const videoSrc = path.join(artifactDir, "run.mp4"); if (fs.existsSync(videoSrc)) { diff --git a/apps/studio-player/server/index.ts b/apps/studio-player/server/index.ts index 57785b4..a91fc93 100644 --- a/apps/studio-player/server/index.ts +++ b/apps/studio-player/server/index.ts @@ -97,11 +97,11 @@ type ServerMsg = | { type: "stepComplete"; index: number; screenshot: string; mode: "human" | "fast"; durationMs: number } | { type: "finished"; videoPath?: string } | { type: "error"; message: string } - | { type: "status"; loaded: boolean; executedUpTo: number } + | { type: "status"; loaded: boolean; executedUpTo: number; runMode?: "human" | "fast" } | { type: "scenarioFiles"; files: string[] } | { type: "liveFrame"; data: string; paneId?: string } | { type: "paneLayout"; layout: PaneLayoutInfo } - | { type: "cachedData"; screenshots: (string | null)[]; stepDurations: (number | null)[]; stepHasAudio: boolean[]; videoPath?: string | null } + | { type: "cachedData"; screenshots: (string | null)[]; stepDurations: (number | null)[]; stepDurationsFast: (number | null)[]; stepDurationsHuman: (number | null)[]; stepHasAudio: boolean[]; videoPath?: string | null } | { type: "cacheCleared"; cacheSize?: number } | { type: "cacheSize"; size: number } | { type: "cancelled" } @@ -125,13 +125,21 @@ function send(ws: WebSocket, msg: ServerMsg) { } } -function persistStepCache(index: number, screenshot: string, durationMs: number, exec: Executor) { +function persistStepCache(index: number, screenshot: string, durationMs: number, mode: "human" | "fast", exec: Executor) { if (!currentCacheDir || !currentContentHash || !currentScenarioFile) return; try { if (screenshot) cache.saveScreenshot(currentCacheDir, index, screenshot); const hasAudio = !!exec.steps[index]?.narration; + const existing = currentStepMetas.find((m) => m.index === index); currentStepMetas = currentStepMetas.filter((m) => m.index !== index); - currentStepMetas.push({ index, durationMs, hasAudio }); + const meta: StepMeta = { + index, + durationMs, + durationMsFast: mode === "fast" ? durationMs : (existing?.durationMsFast), + durationMsHuman: mode === "human" ? durationMs : (existing?.durationMsHuman), + hasAudio, + }; + currentStepMetas.push(meta); cache.saveMeta(currentCacheDir, { scenarioFile: currentScenarioFile, contentHash: currentContentHash, @@ -496,7 +504,7 @@ wss.on("connection", (ws) => { } if (executor) { - send(ws, { type: "status", loaded: true, executedUpTo: -1 }); + send(ws, { type: "status", loaded: true, executedUpTo: -1, runMode: getRunMode() }); } // Serialize message processing: async handlers fired by WebSocket @@ -524,7 +532,6 @@ wss.on("connection", (ws) => { if (electronMain) electronMain.destroyScenarioView(); currentScenarioFile = msg.file; - currentStepMetas = []; const descriptor = await loadScenarioDescriptor(msg.file); executor = new Executor(descriptor, { projectRoot: PROJECT_ROOT, @@ -540,6 +547,8 @@ wss.on("connection", (ws) => { const absPath = path.isAbsolute(msg.file) ? msg.file : path.resolve(PROJECT_ROOT, msg.file); const { dir, hash } = cache.getDir(absPath, msg.file); + const existingMeta = cache.loadMeta(dir); + currentStepMetas = (existingMeta && existingMeta.contentHash === hash) ? existingMeta.steps : []; currentCacheDir = dir; currentContentHash = hash; @@ -555,15 +564,20 @@ wss.on("connection", (ws) => { type: "cachedData", screenshots: cached.screenshots, stepDurations: cached.stepDurations, + stepDurationsFast: cached.stepDurationsFast, + stepDurationsHuman: cached.stepDurationsHuman, stepHasAudio: cached.stepHasAudio, videoPath: cached.videoPath, }); } else { const hasAudio = executor.steps.map((s) => !!s.narration); + const empty = executor.steps.map(() => null); send(ws, { type: "cachedData", screenshots: executor.steps.map(() => null), - stepDurations: executor.steps.map(() => null), + stepDurations: empty, + stepDurationsFast: empty, + stepDurationsHuman: empty, stepHasAudio: hasAudio, }); } @@ -581,15 +595,21 @@ wss.on("connection", (ws) => { executor.onLiveFrame = (data, paneId) => send(ws, { type: "liveFrame", data, paneId }); executor.onPaneLayout = (layout) => send(ws, { type: "paneLayout", layout }); executor.onReplayEvent = (event) => send(ws, { type: "replayEvent", event }); - currentStepMetas = []; + if (currentCacheDir && currentContentHash) { + const meta = cache.loadMeta(currentCacheDir); + currentStepMetas = (meta && meta.contentHash === currentContentHash) ? meta.steps : []; + } else { + currentStepMetas = []; + } } + const runMode = getRunMode(); await executor.runTo( msg.index, - getRunMode(), + runMode, (index, fastForward) => send(ws, { type: "stepStart", index, fastForward }), (result) => { send(ws, { type: "stepComplete", ...result }); - persistStepCache(result.index, result.screenshot, result.durationMs, executor!); + persistStepCache(result.index, result.screenshot, result.durationMs, runMode, executor!); }, ); break; @@ -601,14 +621,15 @@ wss.on("connection", (ws) => { break; } try { + const runAllMode = getRunMode(); for (let i = 0; i < executor.stepCount; i++) { await executor.runTo( i, - getRunMode(), + runAllMode, (index, fastForward) => send(ws, { type: "stepStart", index, fastForward }), (result) => { send(ws, { type: "stepComplete", ...result }); - persistStepCache(result.index, result.screenshot, result.durationMs, executor!); + persistStepCache(result.index, result.screenshot, result.durationMs, runAllMode, executor!); }, ); } @@ -646,7 +667,7 @@ wss.on("connection", (ws) => { case "reset": { if (executor) await executor.reset(); if (electronMain) electronMain.destroyScenarioView(); - send(ws, { type: "status", loaded: !!executor, executedUpTo: -1 }); + send(ws, { type: "status", loaded: !!executor, executedUpTo: -1, runMode: getRunMode() }); break; } @@ -722,10 +743,15 @@ wss.on("connection", (ws) => { executor.onLiveFrame = (data, paneId) => send(ws, { type: "liveFrame", data, paneId }); executor.onPaneLayout = (layout) => send(ws, { type: "paneLayout", layout }); executor.onReplayEvent = (event) => send(ws, { type: "replayEvent", event }); - currentStepMetas = []; + if (currentCacheDir && currentContentHash) { + const meta = cache.loadMeta(currentCacheDir); + currentStepMetas = (meta && meta.contentHash === currentContentHash) ? meta.steps : []; + } else { + currentStepMetas = []; + } } send(ws, { type: "viewMode", mode: currentViewMode }); - send(ws, { type: "status", loaded: !!executor, executedUpTo: -1 }); + send(ws, { type: "status", loaded: !!executor, executedUpTo: -1, runMode: getRunMode() }); break; } @@ -745,6 +771,8 @@ wss.on("connection", (ws) => { type: "cachedData", screenshots: cached.screenshots, stepDurations: cached.stepDurations, + stepDurationsFast: cached.stepDurationsFast, + stepDurationsHuman: cached.stepDurationsHuman, stepHasAudio: cached.stepHasAudio, videoPath: cached.videoPath, }); @@ -756,7 +784,7 @@ wss.on("connection", (ws) => { case "downloadArtifacts": { const scenarioFiles = listPlayerScenarioFiles(); console.error(`[player] Downloading CI artifacts from GitHub...`); - send(ws, { type: "status", loaded: !!executor, executedUpTo: executor?.lastExecutedIndex ?? -1 }); + send(ws, { type: "status", loaded: !!executor, executedUpTo: executor?.lastExecutedIndex ?? -1, runMode: getRunMode() }); try { const { imported } = await cache.downloadFromGitHub(scenarioFiles, { runId: msg.runId, @@ -774,6 +802,8 @@ wss.on("connection", (ws) => { type: "cachedData", screenshots: cached.screenshots, stepDurations: cached.stepDurations, + stepDurationsFast: cached.stepDurationsFast, + stepDurationsHuman: cached.stepDurationsHuman, stepHasAudio: cached.stepHasAudio, videoPath: cached.videoPath, }); diff --git a/apps/studio-player/src/App.tsx b/apps/studio-player/src/App.tsx index 52f2985..0d5666d 100644 --- a/apps/studio-player/src/App.tsx +++ b/apps/studio-player/src/App.tsx @@ -20,8 +20,11 @@ export default function App() { studioFrames, connected, error, - stepDurations, + loading, + stepDurationsFast, + stepDurationsHuman, stepHasAudio, + runMode, viewMode, paneLayout, terminalServerUrl, @@ -33,6 +36,10 @@ export default function App() { detectedProvider, } = state; + const isFastForwarding = stepStates.some((s) => s === "fast-forwarding"); + const showOverlay = loading || isFastForwarding; + const overlayLabel = loading ? "Loading..." : "Replaying slides..."; + const activeScreenshot = activeStep >= 0 ? screenshots[activeStep] : null; const activeCaption = activeStep >= 0 && scenario ? scenario.steps[activeStep]?.caption : undefined; @@ -87,8 +94,10 @@ export default function App() { stepStates={stepStates} screenshots={screenshots} activeStep={activeStep} - stepDurations={stepDurations} + stepDurationsFast={stepDurationsFast} + stepDurationsHuman={stepDurationsHuman} stepHasAudio={stepHasAudio} + runMode={runMode} onStepClick={runStep} /> ) : ( @@ -104,7 +113,7 @@ export default function App() {
)}
-
+
+ {showOverlay && ( +
+
+
+ {overlayLabel} +
+
+ )}
diff --git a/apps/studio-player/src/components/step-graph.tsx b/apps/studio-player/src/components/step-graph.tsx index aec466a..e2f58f3 100644 --- a/apps/studio-player/src/components/step-graph.tsx +++ b/apps/studio-player/src/components/step-graph.tsx @@ -2,6 +2,11 @@ import { useEffect, useRef } from "react"; import { Volume2 } from "lucide-react"; import type { StepState, StepInfo } from "../hooks/use-player"; +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + return `${(ms / 1000).toFixed(1)}s`; +} + const stateStyles: Record = { pending: "border-zinc-700/60 bg-zinc-900/40", "fast-forwarding": "border-yellow-600/60 bg-yellow-950/30", @@ -9,13 +14,17 @@ const stateStyles: Record = { done: "border-emerald-600/60 bg-emerald-950/20", }; +const TEXT_SHADOW = "-1px -1px 0 #000, 1px -1px 0 #000, -1px 1px 0 #000, 1px 1px 0 #000"; + interface StepGraphProps { steps: StepInfo[]; stepStates: StepState[]; screenshots: (string | null)[]; activeStep: number; - stepDurations: (number | null)[]; + stepDurationsFast: (number | null)[]; + stepDurationsHuman: (number | null)[]; stepHasAudio: boolean[]; + runMode: "human" | "fast"; onStepClick: (index: number) => void; } @@ -24,8 +33,10 @@ export function StepGraph({ stepStates, screenshots, activeStep, - stepDurations, + stepDurationsFast, + stepDurationsHuman, stepHasAudio, + runMode, onStepClick, }: StepGraphProps) { const containerRef = useRef(null); @@ -35,7 +46,11 @@ export function StepGraph({ activeRef.current?.scrollIntoView({ behavior: "smooth", block: "nearest" }); }, [activeStep]); - const maxDuration = Math.max(1, ...stepDurations.filter((d): d is number => d !== null)); + const activeDurations = runMode === "fast" ? stepDurationsFast : stepDurationsHuman; + const effectiveDurations = activeDurations.map((d, i) => + (d !== null && d > 0) ? d : ((runMode === "fast" ? stepDurationsHuman[i] : stepDurationsFast[i]) ?? null), + ); + const maxDuration = Math.max(1, ...effectiveDurations.filter((d): d is number => d !== null && d > 0)); return (
@@ -43,8 +58,15 @@ export function StepGraph({ const state = stepStates[i] ?? "pending"; const isActive = i === activeStep; const screenshot = screenshots[i]; - const duration = stepDurations[i]; + const fastDur = stepDurationsFast[i]; + const humanDur = stepDurationsHuman[i]; + const activeDur = activeDurations[i]; + const barDur = effectiveDurations[i]; const hasAudio = stepHasAudio[i] ?? false; + const hasAnyDuration = (humanDur !== null && humanDur > 0) || (fastDur !== null && fastDur > 0); + + const humanLabel = humanDur !== null && humanDur > 0 ? formatDuration(humanDur) : "N/A"; + const fastLabel = fastDur !== null && fastDur > 0 ? formatDuration(fastDur) : "N/A"; return (
- {/* Widescreen 16:9 thumbnail with overlaid text */}
{screenshot ? ( )} - {/* Overlaid caption with black outline for visibility */}
- + {i + 1} - + {step.caption}
-
+
{hasAudio && } + + {hasAnyDuration && ( + + 0 ? (runMode === "human" ? "text-blue-300" : "text-zinc-300") : "text-zinc-500"}> + {humanLabel} + + / + 0 ? (runMode === "fast" ? "text-yellow-300" : "text-zinc-300") : "text-zinc-500"}> + {fastLabel} + + + )} + {state === "running" && (
)} @@ -94,11 +125,11 @@ export function StepGraph({
- {duration !== null && ( -
+ {barDur !== null && barDur > 0 && ( +
)} diff --git a/apps/studio-player/src/hooks/use-player.ts b/apps/studio-player/src/hooks/use-player.ts index 4542b5f..2123575 100644 --- a/apps/studio-player/src/hooks/use-player.ts +++ b/apps/studio-player/src/hooks/use-player.ts @@ -52,8 +52,11 @@ export interface PlayerState { stepStates: StepState[]; screenshots: (string | null)[]; stepDurations: (number | null)[]; + stepDurationsFast: (number | null)[]; + stepDurationsHuman: (number | null)[]; stepHasAudio: boolean[]; activeStep: number; + runMode: "human" | "fast"; liveFrame: string | null; liveFrames: Record; studioFrames: Record; @@ -63,6 +66,7 @@ export interface PlayerState { error: string | null; importing: boolean; importResult: { count: number; scenarios: string[] } | null; + loading: boolean; cacheSize: number; audioSettings: AudioSettings; detectedProvider: string; @@ -71,17 +75,18 @@ export interface PlayerState { type Action = | { type: "connected" } | { type: "disconnected" } + | { type: "loading" } | { type: "studioReady"; terminalServerUrl: string } | { type: "scenarioFiles"; files: string[] } | { type: "scenario"; name: string; steps: StepInfo[] } | { type: "stepStart"; index: number; fastForward: boolean } - | { type: "stepComplete"; index: number; screenshot: string; mode: string; durationMs: number } + | { type: "stepComplete"; index: number; screenshot: string; mode: "human" | "fast"; durationMs: number } | { type: "liveFrame"; data: string; paneId?: string } | { type: "paneLayout"; layout: PaneLayoutInfo } | { type: "finished"; videoPath?: string } | { type: "error"; message: string } | { type: "reset" } - | { type: "cachedData"; screenshots: (string | null)[]; stepDurations: (number | null)[]; stepHasAudio: boolean[]; videoPath?: string | null } + | { type: "cachedData"; screenshots: (string | null)[]; stepDurations: (number | null)[]; stepDurationsFast: (number | null)[]; stepDurationsHuman: (number | null)[]; stepHasAudio: boolean[]; videoPath?: string | null } | { type: "cacheCleared"; cacheSize?: number } | { type: "cancelled" } | { type: "viewMode"; mode: ViewMode } @@ -89,6 +94,7 @@ type Action = | { type: "artifactsImported"; count: number; scenarios: string[] } | { type: "audioSettings"; settings: AudioSettings; detected: string } | { type: "cacheSize"; size: number } + | { type: "status"; runMode?: "human" | "fast" } ; const initial: PlayerState = { @@ -99,8 +105,11 @@ const initial: PlayerState = { stepStates: [], screenshots: [], stepDurations: [], + stepDurationsFast: [], + stepDurationsHuman: [], stepHasAudio: [], activeStep: -1, + runMode: "human", liveFrame: null, liveFrames: {}, studioFrames: {}, @@ -110,6 +119,7 @@ const initial: PlayerState = { error: null, importing: false, importResult: null, + loading: false, cacheSize: 0, audioSettings: {}, detectedProvider: "none", @@ -121,6 +131,8 @@ function reducer(state: PlayerState, action: Action): PlayerState { return { ...state, connected: true, error: null }; case "disconnected": return { ...state, connected: false, terminalServerUrl: null }; + case "loading": + return { ...state, loading: true }; case "studioReady": return { ...state, terminalServerUrl: action.terminalServerUrl }; case "scenarioFiles": @@ -128,10 +140,13 @@ function reducer(state: PlayerState, action: Action): PlayerState { case "scenario": return { ...state, + loading: false, scenario: { name: action.name, steps: action.steps }, stepStates: action.steps.map(() => "pending" as StepState), screenshots: action.steps.map(() => null), stepDurations: action.steps.map(() => null), + stepDurationsFast: action.steps.map(() => null), + stepDurationsHuman: action.steps.map(() => null), stepHasAudio: action.steps.map(() => false), activeStep: -1, liveFrame: null, @@ -145,16 +160,22 @@ function reducer(state: PlayerState, action: Action): PlayerState { ...state, screenshots: action.screenshots.map((s, i) => s ?? state.screenshots[i] ?? null), stepDurations: action.stepDurations, + stepDurationsFast: action.stepDurationsFast, + stepDurationsHuman: action.stepDurationsHuman, stepHasAudio: action.stepHasAudio, videoPath: action.videoPath ?? state.videoPath, }; - case "cacheCleared": + case "cacheCleared": { + const empty = state.scenario?.steps.map(() => null) ?? []; return { ...state, - screenshots: state.scenario?.steps.map(() => null) ?? [], - stepDurations: state.scenario?.steps.map(() => null) ?? [], + screenshots: empty, + stepDurations: [...empty], + stepDurationsFast: [...empty], + stepDurationsHuman: [...empty], cacheSize: action.cacheSize ?? 0, }; + } case "cancelled": return { ...state, @@ -182,7 +203,13 @@ function reducer(state: PlayerState, action: Action): PlayerState { if (action.screenshot) screenshots[action.index] = action.screenshot; const stepDurations = [...state.stepDurations]; if (action.durationMs) stepDurations[action.index] = action.durationMs; - return { ...state, stepStates, screenshots, stepDurations, activeStep: action.index, liveFrame: null, liveFrames: {} }; + const stepDurationsFast = [...state.stepDurationsFast]; + const stepDurationsHuman = [...state.stepDurationsHuman]; + if (action.durationMs) { + if (action.mode === "fast") stepDurationsFast[action.index] = action.durationMs; + else stepDurationsHuman[action.index] = action.durationMs; + } + return { ...state, stepStates, screenshots, stepDurations, stepDurationsFast, stepDurationsHuman, runMode: action.mode, activeStep: action.index, liveFrame: null, liveFrames: {} }; } case "liveFrame": { const paneId = action.paneId ?? "pane-0"; @@ -197,7 +224,7 @@ function reducer(state: PlayerState, action: Action): PlayerState { case "finished": return { ...state, liveFrame: null, liveFrames: {}, videoPath: action.videoPath ?? null, error: null }; case "error": - return { ...state, error: action.message }; + return { ...state, loading: false, error: action.message }; case "viewMode": return { ...state, viewMode: action.mode }; case "importStart": @@ -208,6 +235,8 @@ function reducer(state: PlayerState, action: Action): PlayerState { return { ...state, audioSettings: action.settings, detectedProvider: action.detected }; case "cacheSize": return { ...state, cacheSize: action.size }; + case "status": + return { ...state, runMode: action.runMode ?? state.runMode }; case "reset": return { ...state, @@ -294,7 +323,15 @@ export function usePlayer(wsUrl: string) { setCursor({ x: 0, y: 0, clickEffect: false, visible: false }); break; case "cachedData": - dispatch({ type: "cachedData", screenshots: msg.screenshots, stepDurations: msg.stepDurations, stepHasAudio: msg.stepHasAudio, videoPath: msg.videoPath }); + dispatch({ + type: "cachedData", + screenshots: msg.screenshots, + stepDurations: msg.stepDurations, + stepDurationsFast: msg.stepDurationsFast ?? msg.stepDurations.map(() => null), + stepDurationsHuman: msg.stepDurationsHuman ?? msg.stepDurations.map(() => null), + stepHasAudio: msg.stepHasAudio, + videoPath: msg.videoPath, + }); break; case "cacheCleared": dispatch({ type: "cacheCleared", cacheSize: msg.cacheSize }); @@ -306,7 +343,7 @@ export function usePlayer(wsUrl: string) { dispatch({ type: "stepStart", index: msg.index, fastForward: msg.fastForward }); break; case "stepComplete": - dispatch({ type: "stepComplete", index: msg.index, screenshot: msg.screenshot, mode: msg.mode, durationMs: msg.durationMs ?? 0 }); + dispatch({ type: "stepComplete", index: msg.index, screenshot: msg.screenshot, mode: msg.mode === "fast" ? "fast" : "human", durationMs: msg.durationMs ?? 0 }); setCursor((c) => ({ ...c, clickEffect: false })); break; case "liveFrame": @@ -355,6 +392,7 @@ export function usePlayer(wsUrl: string) { dispatch({ type: "error", message: msg.message }); break; case "status": + dispatch({ type: "status", runMode: msg.runMode }); break; } } catch { /* ignore parse errors */ } @@ -379,6 +417,7 @@ export function usePlayer(wsUrl: string) { }, []); const loadScenario = useCallback((file: string) => { + dispatch({ type: "loading" }); sendMsg({ type: "load", file }); }, [sendMsg]); diff --git a/tests/scenarios/slides-and-narration.scenario.ts b/tests/scenarios/slides-and-narration.scenario.ts index 96e7d16..4ac85d1 100644 --- a/tests/scenarios/slides-and-narration.scenario.ts +++ b/tests/scenarios/slides-and-narration.scenario.ts @@ -26,7 +26,7 @@ async function assertSlide(page: Page, expected: number) { (text: string) => document.querySelector('[data-testid="slides-current"]')?.textContent?.trim() === text, expectedText, - { timeout: 5000 }, + { timeout: 10000 }, ); }