diff --git a/.gitignore b/.gitignore index 771c4bd5d..84a5a1cea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,45 +1,52 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-electron -dist-ssr -*.local -.env - -# Editor directories and files -.vscode/* -.zed/ -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? -release/** -*.kiro/ -.claude/ -# npx electron-builder --mac --win - -# Playwright -test-results -playwright-report/ - -# Vitest browser mode screenshots -__screenshots__/ - -# shell files -/shell.sh -# Nix -result -result-* -.direnv/ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-electron +dist-ssr +*.local +.env + +# Native helper build outputs +/electron/native/wgc-capture/build/ +/electron/native/bin/ + +# Editor directories and files +.vscode/* +.zed/ +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +release/** +*.kiro/ +.claude/ +# npx electron-builder --mac --win + +# Playwright +test-results +playwright-report/ + +# Vitest browser mode screenshots +__screenshots__/ + +# shell files +/shell.sh +# Nix +result +result-* +.direnv/ + +#kilocode +.kilo/ \ No newline at end of file diff --git a/docs/architecture/native-bridge.md b/docs/architecture/native-bridge.md new file mode 100644 index 000000000..ef320f777 --- /dev/null +++ b/docs/architecture/native-bridge.md @@ -0,0 +1,39 @@ +# Native Bridge Architecture + +## Goal + +Provide a single, resilient source of truth for platform-native capabilities while keeping Electron transport thin and renderer APIs unified. + +## Layers + +1. Native adapters +Platform-specific providers implement stable domain interfaces such as cursor telemetry or system asset discovery. + +2. Main-process services +Services orchestrate adapters, own runtime state, and expose domain-level operations. + +3. Unified IPC transport +Renderer code talks to a single `native-bridge:invoke` channel using versioned contracts. + +4. Renderer client +React code should consume `src/native/client.ts` rather than binding directly to ad hoc Electron APIs. + +## Principles + +- Single source of truth: runtime-native state lives in the Electron main process. +- Capability-first: renderer can query support before attempting native behavior. +- Versioned contracts: requests and responses are explicit and evolve predictably. +- Resilience: every response uses a consistent result envelope with stable error codes. + +## Current rollout + +This repository now contains the initial scaffold: + +- shared contracts in `src/native/contracts.ts` +- renderer SDK in `src/native/client.ts` +- main-process state store in `electron/native-bridge/store.ts` +- cursor telemetry adapter in `electron/native-bridge/cursor/telemetryCursorAdapter.ts` +- domain services in `electron/native-bridge/services/*` +- unified handler registration in `electron/ipc/nativeBridge.ts` + +The legacy `window.electronAPI` surface still exists for backward compatibility. New native-facing features should prefer the unified bridge client. \ No newline at end of file diff --git a/docs/engineering/windows-native-recorder-roadmap.md b/docs/engineering/windows-native-recorder-roadmap.md new file mode 100644 index 000000000..146582a5c --- /dev/null +++ b/docs/engineering/windows-native-recorder-roadmap.md @@ -0,0 +1,248 @@ +# Windows Native Recorder Roadmap + +OpenScreen's Windows recorder should be owned by one native backend. Electron capture can remain available for non-Windows platforms and temporary developer diagnostics, but Windows production recording should not silently fall back to `getDisplayMedia` / `MediaRecorder`. + +## Goals + +- Capture displays and windows through Windows Graphics Capture (WGC). +- Render the native Windows cursor as OpenScreen's high-quality scalable cursor overlay. +- Capture system audio through WASAPI loopback. +- Capture microphone audio through WASAPI. +- Mix system audio and microphone audio into the primary screen recording. +- Capture webcam video natively and compose it into the Windows helper MP4 during the native-recording migration. +- Keep preview/export aligned because screen video, audio, webcam, and cursor share one native timing origin. +- Keep exported MP4s Windows-friendly: H.264 video plus AAC audio. Opus-in-MP4 is not an acceptable Windows export target. +- Package the native helper with the Windows app. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Replacing the editor/export pipeline. A later pass can reintroduce a separate editable native `webcamVideoPath`; the current Windows-native milestone prioritizes a helper-owned multi-flux MP4 with deterministic screen/audio/mic/webcam sync. +- Adding a native fallback for macOS or Linux in this branch. + +## Target Architecture + +The renderer keeps the existing recording controls. On Windows, `useScreenRecorder` sends a complete recording request to Electron and does not assemble Windows `MediaStream` tracks with `MediaRecorder`. + +Electron owns the native recording session: + +- resolves the selected source; +- resolves output paths; +- starts cursor sampling; +- starts the helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a Windows-native capability is unavailable. + +The helper owns Windows media capture: + +- WGC screen/window frames; +- WASAPI system loopback; +- WASAPI microphone input; +- Media Foundation webcam capture; +- DirectShow webcam fallback for virtual cameras not visible to Media Foundation; +- Media Foundation encoding/muxing; +- stream timestamp normalization. + +## Helper Contract V2 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 2, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 123, + "windowHandle": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000 + }, + "audio": { + "system": { "enabled": true }, + "microphone": { "enabled": true, "deviceId": "default", "gain": 1.4 } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "Camera (NVIDIA Broadcast)", + "width": 1280, + "height": 720, + "fps": 30, + "bitrate": 18000000 + }, + "outputs": { + "screenPath": "C:\\Users\\me\\recording-123.mp4", + "manifestPath": "C:\\Users\\me\\recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 2 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "audio-device-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "unsupported-window-source", "message": "..." } +``` + +During migration, Electron also accepts the current textual helper messages so existing display-only smoke tests keep working. + +## Implementation Phases + +### 1. Native Session Boundary + +- Add a structured Windows native recording request type. +- Pass source kind, audio flags, microphone device, webcam flags, and output paths into the helper. +- On Windows, do not silently fall back to Electron capture. If the helper is unavailable or a native feature is missing, show a clear error. +- Keep Electron fallback only for non-Windows and optional developer diagnostics. + +Acceptance: + +- Display-only recording still works. +- Enabling an unsupported native feature returns an explicit native error instead of recording through Electron. + +### 2. WASAPI System Audio + +Status: initial implementation landed. The helper captures the default render endpoint with WASAPI loopback, passes the runtime mix format into `MFEncoder`, and muxes AAC audio into the primary MP4. Long-run drift correction and explicit silence insertion remain follow-up hardening work. + +- Add `WasapiLoopbackCapture`. +- Capture the default render endpoint in shared loopback mode. +- Keep `WasapiLoopbackCapture` responsible only for device activation, packet capture, and packet timestamps. +- Keep `MFEncoder` responsible for all Media Foundation stream definitions and muxing. +- Feed the endpoint mix format into `MFEncoder` as the single source of truth for audio stream shape: sample rate, channel count, bits per sample, block alignment, average bytes/sec, and subtype (`PCM` or `Float`). +- Encode the primary screen MP4 with H.264 video and AAC audio through one `IMFSinkWriter`. +- Timestamp audio from the captured frame count in 100ns units. The first implementation uses the WASAPI packet timeline; later drift correction will add explicit silence or resampling if long recordings show measurable clock skew. +- Treat microphone mixing as a later phase. System loopback must land first without introducing renderer-side audio code. + +Acceptance: + +- Screen MP4 has an AAC audio track when system audio is enabled. +- A 5-minute recording has audio/video duration drift below one frame. + +SSOT rules for this phase: + +- `src/lib/nativeWindowsRecording.ts` is the renderer/main TypeScript request contract. +- `docs/engineering/windows-native-recorder-roadmap.md` is the feature-level contract and phase checklist. +- `WgcSession::captureWidth()/captureHeight()` is the encoded screen frame size until a dedicated native scaling stage exists. +- `WasapiLoopbackCapture::inputFormat()` is the runtime audio format source used by `MFEncoder`. +- The renderer passes both the browser webcam `deviceId` and selected display label as `deviceName`; `electron/native/wgc-capture/src/webcam_capture.*` is the only place that maps those values to Media Foundation devices. +- Electron resolves the selected label to a DirectShow filter CLSID once and passes it as `webcamDirectShowClsid`; the helper must not independently guess among DirectShow filters. +- No duplicated hard-coded audio format assumptions in `main.cpp`. + +### 3. WASAPI Microphone + +Status: initial implementation in progress. The helper can open the default WASAPI capture endpoint, apply the OpenScreen microphone gain, encode mic-only audio, and mix system loopback plus microphone through a single queued `AudioMixer` timeline when both endpoints expose the same runtime format. Audio endpoints are warmed before WGC starts, the mixer drops pre-roll and begins its paced timeline on the first encoded video frame, then cuts queued tail audio on stop so the MP4 does not drift past the video. Browser `deviceId` to MMDevice id mapping, resampling between mismatched endpoint formats, and drift correction remain follow-up hardening work. + +- Add microphone device enumeration and stable device-id mapping. +- Capture selected/default microphone through WASAPI. +- Apply OpenScreen's current mic gain policy. +- Mix microphone and system audio before AAC encoding. + +Acceptance: + +- Mic-only, system-only, and mixed audio recordings produce a valid AAC track. +- Device unplug/permission failure produces an explicit error or warning. + +### 4. Webcam Capture + +- Add Media Foundation webcam source reader. +- Select requested dimensions/fps or the nearest format accepted by Media Foundation. +- Convert webcam samples to BGRA and compose them into the primary helper MP4 as an initial bottom-right picture-in-picture overlay. +- Ignore black webcam warmup frames and keep the overlay hidden until the first visible frame is available, so virtual cameras do not flash a black picture-in-picture rectangle at recording start. +- Keep the helper process as the SSOT for screen/window, WASAPI system audio, microphone, webcam, and mux timing. +- Match the requested webcam through Media Foundation friendly names first, then browser device ids/symbolic links, so UI selection remains stable across Chromium and Windows native device namespaces. +- Use the Electron-resolved DirectShow CLSID when the selected virtual camera, for example NVIDIA Broadcast, is registered for DirectShow but absent from Media Foundation enumeration. +- Later: promote the same webcam capture source to a separate editable native `webcamVideoPath` if product requirements need post-recording layout edits. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- `npm run test:wgc-webcam:win` validates the helper path when a webcam is available and skips explicitly when no webcam device exists. +- Combined webcam + system audio + microphone produces one MP4 with H.264 video and AAC audio. + +### 5. Native Window Capture + +Status: initial implementation in progress. Electron parses the `window::...` desktop source id through the shared native Windows recording contract and passes `windowHandle` to the helper. The helper resolves the `HWND`, validates it with `IsWindow`, and creates the WGC item with `CreateForWindow(HWND)`. Resize/minimize/move hardening and protected-window diagnostics remain follow-up work. + +- Resolve Electron `window:*` selections to an `HWND`. +- Use WGC `CreateForWindow(HWND)`. +- Handle window close, minimize, resize, DPI scaling, and monitor moves. +- Return clear errors for unsupported protected windows. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/mic/webcam. +- Window resize and movement do not corrupt the recording. + +### 6. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial screen/webcam outputs. +- Keep restart as stop-discard-start from Electron until the helper supports a native restart event. + +Acceptance: + +- Pause/resume keeps preview duration coherent. +- Cancel leaves no stale media/session/cursor files. + +### 7. Test Pipeline + +- `npm run test:wgc-helper:win`: display-only helper smoke test. +- `npm run test:wgc-audio:win`: validates AAC track presence and duration. +- `npm run test:wgc-window:win`: captures a fixture window by HWND. +- `npm run test:wgc-webcam:win`: validates webcam output when a webcam is available, otherwise skips explicitly. +- Packaging check: confirms the helper is in `app.asar.unpacked`. +- Export check: exported MP4s generated from native recordings keep an AAC audio track when the source has audio. +- `npm run test:wgc-mic:win`: validates default-microphone capture writes an AAC track when an input endpoint is available. +- `npm run test:wgc-mixed-audio:win`: validates system loopback plus microphone writes one mixed AAC track when endpoint formats are compatible. + +## Backlog + +### Native Cursor Click Bounce Is Not Visibly Applied + +Status: open. Do not treat Windows native cursor `Click Bounce` as shipped. + +Problem: + +- The cursor settings UI exposes `Size`, `Smoothing`, `Motion Blur`, and `Click Bounce`. +- On Windows native cursor recordings, `Size`, `Smoothing`, and `Motion Blur` are visibly applied in preview/export. +- `Click Bounce` still has no visible effect in manual packaged-app testing, even after adding click-related sample metadata. + +What has already been tried: + +- Added `interactionType: "click" | "mouseup" | "move"` to native cursor samples. +- Added polling-based left-button state through `GetAsyncKeyState`. +- Added the `GetAsyncKeyState` low-bit path to catch quick clicks between samples. +- Added a PowerShell/C# `WH_MOUSE_LL` mouse hook experiment and launched the sampler through a temporary `.ps1` file to avoid Windows command-line length limits. +- Updated `npm run test:cursor-native:win` so the diagnostic can observe a synthetic short click and emit `clickSampleCount`. + +Current diagnosis: + +- The diagnostic can observe synthetic click events, but this has not translated into a visible `Click Bounce` effect in the real packaged app. +- The test currently proves that some click metadata can be recorded, not that the full OpenScreen record -> preview -> export path displays a bounce at the expected time. +- The current native implementation may be animating from metadata that is not present in the real recording session, may be using the wrong timestamp origin, or may be applying a scale change too subtle to notice on the DOM/native cursor path. + +Next investigation when resumed: + +- Inspect the actual `.cursor.json`/session sidecar generated by a packaged-app manual recording and confirm whether real clicks produce `interactionType: "click"` at the right `timeMs`. +- Add a targeted end-to-end fixture that records a known click, loads the generated project, and asserts the preview/export cursor scale changes across adjacent frames. +- Compare the native DOM cursor path against the older `PixiCursorOverlay` click visual state and decide whether native cursor bounce should be a scale-only animation, an additional click ring, or a short explicit keyframe animation independent of sample cadence. +- If event capture remains unreliable in the PowerShell sampler, move click events into a small native cursor helper instead of PowerShell/C# script injection. + +## Ship Criteria + +- Windows display capture works with cursor, system audio, microphone, and webcam. +- Windows window capture works with cursor, system audio, microphone, and webcam. +- Preview and export show no cursor position drift. +- Preview and export show no measurable audio/video/webcam drift. +- Windows production builds do not depend on Electron capture fallback. diff --git a/docs/testing/windows-native-cursor.md b/docs/testing/windows-native-cursor.md new file mode 100644 index 000000000..f6d214b59 --- /dev/null +++ b/docs/testing/windows-native-cursor.md @@ -0,0 +1,129 @@ +# Windows native cursor test pipeline + +This branch includes two Windows-focused diagnostics for fast iteration on native cursor capture and rendering. They are intentionally local developer tools: they create short videos and JSON reports so cursor changes can be inspected without doing a full manual record/edit/export cycle. + +## Native sampler diagnostic + +```powershell +npm run test:cursor-native:win +``` + +This script does not launch OpenScreen. It: + +- starts a Windows `GetCursorInfo` sampler +- moves the real OS pointer with `SetCursorPos` +- captures native cursor handles, hotspots, assets, and standard `IDC_*` cursor types +- writes normalized `CursorRecordingData` +- generates an abstract preview video +- generates a real-screen preview video using screenshots of the current desktop + +The output directory is printed in the command result, for example: + +```text +C:\Users\\AppData\Local\Temp\openscreen-cursor-native-... +``` + +Useful files: + +- `report.json`: sample counts, asset counts, cursor handles, and generated artifact paths +- `cursor-recording-data.json`: sidecar-compatible cursor data +- `preview.webm`: abstract path/asset/hotspot preview +- `real-capture-preview.webm`: real desktop screenshot background with reconstructed cursor overlay +- `assets/*.png`: raw cursor bitmaps captured from Windows + +Environment overrides: + +```powershell +$env:CURSOR_TEST_DURATION_MS = "3000" +$env:CURSOR_TEST_SAMPLE_INTERVAL_MS = "16" +$env:CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS = "80" +$env:CURSOR_TEST_OUTPUT_DIR = "C:\temp\openscreen-cursor-test" +npm run test:cursor-native:win +``` + +## OpenScreen preview capture + +```powershell +npm run capture:openscreen-preview +``` + +This script launches the real Electron app, injects a fixture video plus cursor sidecar data, opens the editor, captures frames from the actual OpenScreen preview UI, and encodes them into a WebM. + +By default it uses the latest `cursor-recording-data.json` generated by `npm run test:cursor-native:win`. To force a specific sidecar: + +```powershell +$env:CURSOR_RECORDING_DATA_PATH = "C:\path\to\cursor-recording-data.json" +npm run capture:openscreen-preview +``` + +Useful environment overrides: + +```powershell +$env:OPENSCREEN_PREVIEW_SKIP_BUILD = "true" +$env:OPENSCREEN_PREVIEW_FRAME_COUNT = "120" +$env:OPENSCREEN_PREVIEW_FPS = "30" +$env:OPENSCREEN_PREVIEW_OUTPUT_DIR = "C:\temp\openscreen-preview" +npm run capture:openscreen-preview +``` + +Useful files: + +- `openscreen-preview.webm`: video of the real OpenScreen editor preview +- `frames/*.png`: captured preview frames +- `report.json`: fixture paths, source sidecar, frame count, and output path + +## What these tests validate + +Together, the scripts make it quick to inspect: + +- whether Windows cursor samples are visible and continuous +- whether native hotspots stay anchored when scaling to `3x` +- whether standard Windows cursors are recognized via `IDC_*` +- whether high-quality SVG cursor replacements follow the native hotspot +- whether the real OpenScreen preview renders the same cursor behavior as the diagnostic pipeline + +They are not a full substitute for an end-to-end manual recording pass. Before shipping cursor changes, also test a real capture session and export from the packaged app. + +## Known Gap + +Windows native cursor `Click Bounce` is currently backlogged. `Size`, `Smoothing`, and `Motion Blur` can be validated through preview/export, but `Click Bounce` has not shown a visible effect in packaged-app manual testing. The current diagnostic can observe synthetic click metadata, but that is not enough to validate the real OpenScreen record -> preview -> export path. + +Track the open item in `docs/engineering/windows-native-recorder-roadmap.md` under `Native Cursor Click Bounce Is Not Visibly Applied`. + +## Native Windows capture backend + +The app now routes Windows recordings through an external WGC helper instead of Electron `getDisplayMedia`. This is meant to remove the coordinate and clock split that made the reconstructed cursor drift in the preview/export path. + +Current native availability rules: + +- Windows 10 build 19041 or newer +- a helper executable is available + +The helper currently implements display/window video capture, system audio loopback, default microphone capture, Media Foundation webcam capture, and DirectShow fallback for selected virtual cameras such as NVIDIA Broadcast. Webcam frames are composed into the primary MP4 as a bottom-right picture-in-picture overlay, and black webcam warmup frames are ignored until the first visible frame is available. + +Build OpenScreen's helper locally: + +```powershell +npm run build:native:win +``` + +Smoke-test the helper directly: + +```powershell +npm run test:wgc-helper:win +npm run test:wgc-window:win +npm run test:wgc-audio:win +npm run test:wgc-mic:win +npm run test:wgc-mixed-audio:win +npm run test:wgc-webcam:win +``` + +For local diagnostics with another compatible helper, point OpenScreen at that executable: + +```powershell +$env:OPENSCREEN_WGC_CAPTURE_EXE = "C:\path\to\wgc-capture.exe" +npm run build-vite +npm run dev +``` + +The helper receives one JSON config argument, emits JSON lifecycle events, prints the legacy `Recording started` marker, accepts `stop` on stdin, and prints `Recording stopped. Output path: `. See `electron/native/README.md` for the exact contract and build output paths. diff --git a/electron-builder.json5 b/electron-builder.json5 index ad6cd1853..9299fc761 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -4,10 +4,10 @@ "appId": "com.siddharthvaddem.openscreen", "asar": true, // .node binaries can't be dlopen'd from inside an asar — must live unpacked. - "asarUnpack": [ - "node_modules/uiohook-napi/**/*", - "**/*.node" - ], + "asarUnpack": [ + "node_modules/uiohook-napi/**/*", + "**/*.node" + ], "productName": "Openscreen", "npmRebuild": true, "buildDependenciesFromSource": true, @@ -15,12 +15,12 @@ "directories": { "output": "release/${version}" }, - "files": [ - "dist", - "dist-electron", - "!*.png", - "!preview*.png", - "!*.md", + "files": [ + "dist", + "dist-electron", + "!*.png", + "!preview*.png", + "!*.md", "!README.md", "!CONTRIBUTING.md", "!LICENSE" @@ -65,12 +65,19 @@ "artifactName": "${productName}-Linux-${version}.${ext}", "category": "AudioVideo" }, - "win": { - "target": [ - "nsis" - ], - "icon": "icons/icons/win/icon.ico" - }, + "win": { + "target": [ + "nsis" + ], + "icon": "icons/icons/win/icon.ico", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["**/*"] + } + ] + }, "nsis": { "oneClick": false, "allowToChangeInstallationDirectory": true diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index d9ebab272..8810fc67d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -24,6 +24,9 @@ declare namespace NodeJS { // Used in Renderer process, expose in `preload.ts` interface Window { electronAPI: { + invokeNativeBridge: ( + request: import("../src/native/contracts").NativeBridgeRequest, + ) => Promise>; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; @@ -68,7 +71,29 @@ interface Window { message?: string; error?: string; }>; - setRecordingState: (recording: boolean, recordingId?: number) => Promise; + setRecordingState: ( + recording: boolean, + recordingId?: number, + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode, + ) => Promise; + isNativeWindowsCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: string; + error?: string; + }>; + startNativeWindowsRecording: ( + request: import("../src/lib/nativeWindowsRecording").NativeWindowsRecordingRequest, + ) => Promise; + stopNativeWindowsRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; discardCursorTelemetry: (recordingId: number) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 7361b26fc..835d3b2fe 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,10 +1,9 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; -import { createRequire } from "node:module"; import path from "node:path"; -import { fileURLToPath } from "node:url"; - -const nodeRequire = createRequire(import.meta.url); - +import { fileURLToPath, pathToFileURL } from "node:url"; +import type { DesktopCapturerSource } from "electron"; import { app, BrowserWindow, @@ -15,22 +14,32 @@ import { shell, systemPreferences, } from "electron"; +import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { - type CursorTelemetryPoint, - createCursorTelemetryBuffer, -} from "../../src/lib/cursorTelemetryBuffer"; -import { + type CursorCaptureMode, + normalizeCursorCaptureMode, normalizeProjectMedia, normalizeRecordingSession, type ProjectMedia, type RecordingSession, type StoreRecordedSessionInput, } from "../../src/lib/recordingSession"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + ProjectFileResult, + ProjectPathResult, +} from "../../src/native/contracts"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; +import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { registerNativeBridgeHandlers } from "./nativeBridge"; const PROJECT_FILE_EXTENSION = "openscreen"; const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); +const RECORDING_FILE_PREFIX = "recording-"; const RECORDING_SESSION_SUFFIX = ".session.json"; const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]); @@ -60,6 +69,19 @@ function isPathAllowed(filePath: string): boolean { return getAllowedReadDirs().some((dir) => isPathWithinDir(resolved, dir)); } +function resolveApprovedVideoPath(videoPath?: string | null): string | null { + const normalizedPath = normalizeVideoSourcePath(videoPath); + if (!normalizedPath) { + return null; + } + + if (!hasAllowedImportVideoExtension(normalizedPath) || !isPathAllowed(normalizedPath)) { + return null; + } + + return normalizedPath; +} + /** * Helper function to build dialog options with a parent window only when it's valid. * This prevents passing stale or destroyed BrowserWindow references to dialog calls. @@ -187,13 +209,26 @@ async function getApprovedProjectSession( type SelectedSource = { name: string; + id?: string; + display_id?: string; [key: string]: unknown; }; let selectedSource: SelectedSource | null = null; +let selectedDesktopSource: DesktopCapturerSource | null = null; +let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; let currentRecordingSession: RecordingSession | null = null; +/** + * Returns the cached DesktopCapturerSource set when the user picked a source. + * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. + */ +export function getSelectedDesktopSource(): DesktopCapturerSource | null { + return selectedDesktopSource; +} +let currentVideoPath: string | null = null; + function normalizePath(filePath: string) { return path.resolve(filePath); } @@ -226,528 +261,1015 @@ function isTrustedProjectPath(filePath?: string | null) { return normalizePath(filePath) === normalizePath(currentProjectPath); } -function setCurrentRecordingSessionState(session: RecordingSession | null) { - currentRecordingSession = session; -} +const CURSOR_TELEMETRY_VERSION = 2; +const CURSOR_SAMPLE_INTERVAL_MS = 33; +const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz + +let cursorRecordingSession: CursorRecordingSession | null = null; +let pendingCursorRecordingData: CursorRecordingData | null = null; +let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; +let nativeWindowsCaptureOutput = ""; +let nativeWindowsCaptureTargetPath: string | null = null; +let nativeWindowsCaptureWebcamTargetPath: string | null = null; +let nativeWindowsCaptureRecordingId: number | null = null; +let nativeWindowsCursorOffsetMs = 0; +let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; +const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000; + +function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { + if (!sample || typeof sample !== "object") { + return null; + } -function getSessionManifestPathForVideo(videoPath: string) { - const parsed = path.parse(videoPath); - const baseName = parsed.name.endsWith("-webcam") - ? parsed.name.slice(0, -"-webcam".length) - : parsed.name; - return path.join(parsed.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); + const point = sample as Partial; + const interactionType = + point.interactionType === "click" || + point.interactionType === "mouseup" || + point.interactionType === "move" + ? point.interactionType + : "move"; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5, + assetId: typeof point.assetId === "string" ? point.assetId : null, + visible: typeof point.visible === "boolean" ? point.visible : true, + cursorType: typeof point.cursorType === "string" ? point.cursorType : null, + interactionType, + }; } -async function loadRecordedSessionForVideoPath( - videoPath: string, -): Promise { - const normalizedVideoPath = normalizeVideoSourcePath(videoPath); - if (!normalizedVideoPath) { +function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { + if (!asset || typeof asset !== "object") { + return null; + } + + const candidate = asset as Partial; + if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") { return null; } + return { + id: candidate.id, + platform: + candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + imageDataUrl: candidate.imageDataUrl, + width: + typeof candidate.width === "number" && Number.isFinite(candidate.width) + ? Math.max(1, Math.round(candidate.width)) + : 1, + height: + typeof candidate.height === "number" && Number.isFinite(candidate.height) + ? Math.max(1, Math.round(candidate.height)) + : 1, + hotspotX: + typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX) + ? Math.max(0, Math.round(candidate.hotspotX)) + : 0, + hotspotY: + typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY) + ? Math.max(0, Math.round(candidate.hotspotY)) + : 0, + scaleFactor: + typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) + ? Math.max(0.1, candidate.scaleFactor) + : undefined, + cursorType: typeof candidate.cursorType === "string" ? candidate.cursorType : null, + }; +} + +async function readCursorRecordingFile(targetVideoPath: string): Promise { + const telemetryPath = `${targetVideoPath}.cursor.json`; try { - const manifestPath = getSessionManifestPathForVideo(normalizedVideoPath); - const content = await fs.readFile(manifestPath, "utf-8"); - const session = normalizeRecordingSession(JSON.parse(content)); - if (!session) { - return null; + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : []; + + const samples = rawSamples + .map((sample: unknown) => normalizeCursorSample(sample)) + .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample => + Boolean(sample), + ) + .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs); + + const assets = rawAssets + .map((asset: unknown) => normalizeCursorAsset(asset)) + .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset)); + + return { + version: + typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1, + provider: parsed?.provider === "native" ? "native" : "none", + samples, + assets, + }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { + version: CURSOR_TELEMETRY_VERSION, + provider: "none", + samples: [], + assets: [], + }; } - const normalizedSession: RecordingSession = { - ...session, - screenVideoPath: normalizeVideoSourcePath(session.screenVideoPath) ?? session.screenVideoPath, - ...(session.webcamVideoPath - ? { - webcamVideoPath: - normalizeVideoSourcePath(session.webcamVideoPath) ?? session.webcamVideoPath, - } - : {}), + console.error("Failed to load cursor telemetry:", error); + throw error; + } +} + +async function readCursorTelemetryFile(targetVideoPath: string) { + try { + const recordingData = await readCursorRecordingFile(targetVideoPath); + return { + success: true, + samples: recordingData.samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })), + }; + } catch (error) { + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], }; + } +} - const targetPath = normalizePath(normalizedVideoPath); - const screenMatches = normalizePath(normalizedSession.screenVideoPath) === targetPath; - const webcamMatches = normalizedSession.webcamVideoPath - ? normalizePath(normalizedSession.webcamVideoPath) === targetPath - : false; +function resolveAssetBasePath() { + try { + if (app.isPackaged) { + const assetPath = path.join(process.resourcesPath, "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } + const assetPath = path.join(app.getAppPath(), "public", "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } catch (err) { + console.error("Failed to resolve asset base path:", err); + return null; + } +} - return screenMatches || webcamMatches ? normalizedSession : null; - } catch { +function getSelectedSourceBounds() { + const cursor = screen.getCursorScreenPoint(); + const sourceDisplayId = Number(selectedSource?.display_id); + const sourceDisplay = Number.isFinite(sourceDisplayId) + ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) + : null; + return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; +} + +function getSelectedSourceId() { + return typeof selectedSource?.id === "string" ? selectedSource.id : null; +} + +function getSelectedDisplay() { + const sourceDisplayId = Number(selectedSource?.display_id); + if (!Number.isFinite(sourceDisplayId)) { return null; } + + return screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null; +} + +function resolveUnpackedAppPath(...segments: string[]) { + const resolved = path.join(app.getAppPath(), ...segments); + if (app.isPackaged) { + return resolved.replace(/\.asar([/\\])/, ".asar.unpacked$1"); + } + + return resolved; } -async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { - const createdAt = - typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) - ? payload.createdAt - : Date.now(); - const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); - await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); - - let webcamVideoPath: string | undefined; - if (payload.webcam) { - webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); - await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); +function resolvePackagedResourcePath(...segments: string[]) { + if (!app.isPackaged) { + return null; } - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt } - : { screenVideoPath, createdAt }; - setCurrentRecordingSessionState(session); - currentProjectPath = null; + return path.join(process.resourcesPath, ...segments); +} + +function getNativeWindowsCaptureHelperCandidates() { + const envPath = process.env.OPENSCREEN_WGC_CAPTURE_EXE?.trim(); + const archTag = process.arch === "arm64" ? "win32-arm64" : "win32-x64"; + return [ + envPath, + resolveUnpackedAppPath( + "electron", + "native", + "wgc-capture", + "build", + "Release", + "wgc-capture.exe", + ), + resolveUnpackedAppPath("electron", "native", "wgc-capture", "build", "wgc-capture.exe"), + resolveUnpackedAppPath("electron", "native", "bin", archTag, "wgc-capture.exe"), + resolvePackagedResourcePath("electron", "native", "bin", archTag, "wgc-capture.exe"), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +async function findNativeWindowsCaptureHelperPath() { + if (process.platform !== "win32") { + return null; + } - const telemetryPath = `${screenVideoPath}.cursor.json`; - const pendingBatch = cursorTelemetryBuffer.takeNextBatch(); - const pendingClicks = takeCursorClickTimestamps(); - if ((pendingBatch && pendingBatch.samples.length > 0) || pendingClicks.length > 0) { + for (const candidate of getNativeWindowsCaptureHelperCandidates()) { try { - await fs.writeFile( - telemetryPath, - JSON.stringify( - { - version: CURSOR_TELEMETRY_VERSION, - samples: pendingBatch?.samples ?? [], - clicks: pendingClicks, - }, - null, - 2, - ), - "utf-8", - ); - } catch (err) { - if (pendingBatch) cursorTelemetryBuffer.prependBatch(pendingBatch); - throw err; + await fs.access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next configured helper location. } } - const sessionManifestPath = path.join( - RECORDINGS_DIR, - `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, - ); - await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + return null; +} - return { - success: true, - path: screenVideoPath, - session, - message: "Recording session stored successfully", - }; +function isWindowsGraphicsCaptureOsSupported() { + if (process.platform !== "win32") { + return false; + } + + const [, , build] = process.getSystemVersion().split(".").map(Number); + return Number.isFinite(build) && build >= 19041; } -const CURSOR_TELEMETRY_VERSION = 1; -const CURSOR_SAMPLE_INTERVAL_MS = 100; -const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz - -let cursorCaptureInterval: NodeJS.Timeout | null = null; -let cursorCaptureStartTimeMs = 0; -const cursorTelemetryBuffer = createCursorTelemetryBuffer({ - maxActiveSamples: MAX_CURSOR_SAMPLES, -}); - -// Mouse click timestamps (macOS only — uiohook-napi behind Accessibility). -const MAX_CURSOR_CLICKS = 60 * 60 * 60; // ~1 click/sec for an hour -let cursorClickTimestampsMs: number[] = []; -let uioHookInstance: { - start: () => void; - stop: () => void; - on: (...a: unknown[]) => void; - off?: (...a: unknown[]) => void; - removeListener?: (...a: unknown[]) => void; -} | null = null; -let uioHookMouseDownHandler: ((event: { time?: number }) => void) | null = null; -let uioHookFailureLogged = false; - -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); +function normalizeNativeDeviceName(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); } -function loadUioHookForClicks(): typeof uioHookInstance { - try { - // Dynamic require + try/catch so a broken native binary doesn't crash startup. - const mod = nodeRequire("uiohook-napi"); - const candidate = mod.uIOhook ?? mod.default?.uIOhook ?? mod.uiohook ?? mod.default; - if (candidate && typeof candidate.start === "function" && typeof candidate.on === "function") { - return candidate; - } - return null; - } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] uiohook-napi unavailable:", error); - } - return null; +function scoreNativeDeviceName(candidateName: string, candidateId: string, requestedName?: string) { + const candidate = normalizeNativeDeviceName(candidateName); + const id = normalizeNativeDeviceName(candidateId); + const requested = normalizeNativeDeviceName(requestedName ?? ""); + if (!requested) { + return 0; + } + if (candidate === requested) { + return 1000; + } + if (candidate.includes(requested) || requested.includes(candidate)) { + return 900; } + if (id.includes(requested) || requested.includes(id)) { + return 800; + } + + return requested + .split(/\s+/) + .filter((word) => word.length > 1 && !["camera", "webcam", "video", "input"].includes(word)) + .reduce((score, word) => { + if (candidate.includes(word)) return score + 100; + if (id.includes(word)) return score + 50; + return score; + }, 0); } -function startClickCapture() { - if (process.platform !== "darwin") return; - if (uioHookInstance) return; +function queryDirectShowVideoInputRegistry() { + return new Promise((resolve) => { + const proc = spawn( + "reg.exe", + ["query", "HKCR\\CLSID\\{860BB310-5D01-11D0-BD3B-00A0C911CE86}\\Instance", "/s"], + { windowsHide: true }, + ); + let stdout = ""; + proc.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf16le").includes("\u0000") + ? chunk.toString("utf16le") + : chunk.toString(); + }); + proc.on("close", () => resolve(stdout)); + proc.on("error", () => resolve("")); + }); +} - // Passive check — the prompt fires from the renderer when the user toggles - // "Only on clicks" so it doesn't stack with the screen-recording prompt. - try { - if (!systemPreferences.isTrustedAccessibilityClient(false)) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn( - "[clickCapture] Accessibility permission not granted — click capture disabled.", - ); - } - return; +async function resolveDirectShowWebcamClsid(deviceName?: string) { + if (process.platform !== "win32" || !deviceName?.trim()) { + return null; + } + + const output = await queryDirectShowVideoInputRegistry(); + let current: { friendlyName?: string; clsid?: string } = {}; + const entries: Array<{ friendlyName?: string; clsid?: string }> = []; + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + if (/^HKEY_/i.test(line)) { + if (current.friendlyName || current.clsid) entries.push(current); + current = {}; + continue; } - } catch { - // fall through; uiohook will fail defensively below + const match = line.match(/^(\S+)\s+REG_SZ\s+(.+)$/); + if (!match) continue; + if (match[1] === "FriendlyName") current.friendlyName = match[2].trim(); + if (match[1] === "CLSID") current.clsid = match[2].trim(); + } + if (current.friendlyName || current.clsid) entries.push(current); + + let best: { clsid: string; friendlyName?: string; score: number } | null = null; + for (const entry of entries) { + if (!entry.clsid) continue; + const score = scoreNativeDeviceName(entry.friendlyName ?? "", entry.clsid, deviceName); + if (!best || score > best.score) { + best = { clsid: entry.clsid, friendlyName: entry.friendlyName, score }; + } + } + + if (!best || best.score <= 0) { + return null; } - const hook = loadUioHookForClicks(); - if (!hook) return; + console.info("[native-wgc] resolved DirectShow webcam filter", { + requestedName: deviceName, + filterName: best.friendlyName, + clsid: best.clsid, + score: best.score, + }); + return best.clsid; +} - uioHookMouseDownHandler = (event) => { - const elapsed = Math.max(0, Date.now() - cursorCaptureStartTimeMs); - void event; - if (cursorClickTimestampsMs.length >= MAX_CURSOR_CLICKS) return; - cursorClickTimestampsMs.push(elapsed); - }; +async function startCursorRecording(recordingId?: number) { + if (cursorRecordingSession) { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + cursorRecordingSession = null; + } + + pendingCursorRecordingData = null; + cursorRecordingSession = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: process.platform, + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), + startTimeMs: + typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, + }); try { - hook.on("mousedown", uioHookMouseDownHandler); - hook.start(); - uioHookInstance = hook; + await cursorRecordingSession.start(); } catch (error) { - if (!uioHookFailureLogged) { - uioHookFailureLogged = true; - console.warn("[clickCapture] failed to start uiohook:", error); - } - uioHookMouseDownHandler = null; + console.error("Failed to start cursor recording session:", error); + cursorRecordingSession = null; } } -function stopClickCapture() { - if (!uioHookInstance) return; +async function stopCursorRecording() { + if (!cursorRecordingSession) { + return; + } + try { - if (uioHookMouseDownHandler) { - if (typeof uioHookInstance.off === "function") { - uioHookInstance.off("mousedown", uioHookMouseDownHandler); - } else if (typeof uioHookInstance.removeListener === "function") { - uioHookInstance.removeListener("mousedown", uioHookMouseDownHandler); - } - } - uioHookInstance.stop(); + pendingCursorRecordingData = await cursorRecordingSession.stop(); } catch (error) { - console.warn("[clickCapture] failed to stop uiohook:", error); + console.error("Failed to stop cursor recording session:", error); + pendingCursorRecordingData = null; + } finally { + cursorRecordingSession = null; } - uioHookInstance = null; - uioHookMouseDownHandler = null; } -function takeCursorClickTimestamps(): number[] { - const out = cursorClickTimestampsMs; - cursorClickTimestampsMs = []; - return out; +async function writePendingCursorTelemetry(videoPath: string) { + const telemetryPath = `${videoPath}.cursor.json`; + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { + await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8"); + } + pendingCursorRecordingData = null; } -function stopCursorCapture() { - if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); - cursorCaptureInterval = null; +function shiftPendingCursorTelemetry(offsetMs: number) { + if (!pendingCursorRecordingData || !Number.isFinite(offsetMs) || offsetMs <= 0) { + return; } - stopClickCapture(); + + pendingCursorRecordingData = { + ...pendingCursorRecordingData, + samples: pendingCursorRecordingData.samples + .map((sample) => ({ + ...sample, + timeMs: Math.max(0, sample.timeMs - offsetMs), + })) + .sort((a, b) => a.timeMs - b.timeMs), + }; } -function sampleCursorPoint() { - const cursor = screen.getCursorScreenPoint(); - const sourceDisplayId = Number(selectedSource?.display_id); - const sourceDisplay = Number.isFinite(sourceDisplayId) - ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) - : null; - const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor); - const bounds = display.bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - - const cx = clamp((cursor.x - bounds.x) / width, 0, 1); - const cy = clamp((cursor.y - bounds.y) / height, 0, 1); - - cursorTelemetryBuffer.push({ - timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), - cx, - cy, +function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("Timed out waiting for native Windows capture to start")); + }, 12000); + + const onOutput = (chunk: Buffer) => { + nativeWindowsCaptureOutput += chunk.toString(); + if (nativeWindowsCaptureOutput.includes("Recording started")) { + cleanup(); + resolve(); + } + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onExit = (code: number | null) => { + cleanup(); + reject( + new Error( + nativeWindowsCaptureOutput.trim() || + `Native Windows capture exited before recording started (code=${code ?? "unknown"})`, + ), + ); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("error", onError); + proc.off("exit", onExit); + }; + + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("error", onError); + proc.once("exit", onExit); }); } -export function registerIpcHandlers( - createEditorWindow: () => void, - createSourceSelectorWindow: () => BrowserWindow, - createCountdownOverlayWindow: () => BrowserWindow, - getMainWindow: () => BrowserWindow | null, - getSourceSelectorWindow: () => BrowserWindow | null, - getCountdownOverlayWindow: () => BrowserWindow | null, - onRecordingStateChange?: (recording: boolean, sourceName: string) => void, - switchToHud?: () => void, -) { - const supportsWindowOpacity = process.platform !== "linux"; - const countdownOverlayState = { - visible: false, - value: null as number | null, - activeRunId: null as number | null, - hideCommitId: 0, - hideCommitTimer: null as ReturnType | null, - }; - const COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS = 1200; +function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + if (!proc.killed) { + proc.kill(); + } + reject( + new Error( + `Timed out waiting for native Windows capture to stop. Output path: ${ + nativeWindowsCaptureTargetPath ?? "unknown" + }. Output: ${nativeWindowsCaptureOutput.trim()}`, + ), + ); + }, NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS); + const onOutput = (chunk: Buffer) => { + nativeWindowsCaptureOutput += chunk.toString(); + }; + const onClose = (code: number | null) => { + cleanup(); + const match = nativeWindowsCaptureOutput.match(/Recording stopped\. Output path: (.+)/); + if (match?.[1]) { + resolve(match[1].trim()); + return; + } + if (code === 0 && nativeWindowsCaptureTargetPath) { + resolve(nativeWindowsCaptureTargetPath); + return; + } + reject( + new Error( + nativeWindowsCaptureOutput.trim() || + `Native Windows capture exited with code=${code ?? "unknown"}`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; - const clearCountdownOverlayHideCommit = () => { - if (countdownOverlayState.hideCommitTimer) { - clearTimeout(countdownOverlayState.hideCommitTimer); - countdownOverlayState.hideCommitTimer = null; - } - }; + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + }); +} - const commitCountdownOverlayHide = (win: BrowserWindow, hideCommitId: number) => { - if (win.isDestroyed()) { - return; - } +function readNativeWindowsWebcamFormat(output: string) { + const lines = output.split(/\r?\n/).filter((line) => line.includes('"event":"webcam-format"')); + const lastLine = lines.at(-1); + if (!lastLine) { + return null; + } - if (countdownOverlayState.visible || countdownOverlayState.hideCommitId !== hideCommitId) { - return; - } + try { + return JSON.parse(lastLine) as { + width?: number; + height?: number; + fps?: number; + deviceName?: string; + }; + } catch { + return null; + } +} - win.hide(); - if (supportsWindowOpacity) { - // Reset baseline opacity for the next show cycle. - win.setOpacity(1); +function setCurrentRecordingSessionState(session: RecordingSession | null) { + currentRecordingSession = session; + currentVideoPath = session?.screenVideoPath ?? null; +} + +function getSessionManifestPathForVideo(videoPath: string) { + const parsedPath = path.parse(videoPath); + const baseName = parsedPath.name.endsWith("-webcam") + ? parsedPath.name.slice(0, -"-webcam".length) + : parsedPath.name; + return path.join(parsedPath.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); +} + +async function loadRecordedSessionForVideoPath( + videoPath: string, +): Promise { + try { + const manifestPath = getSessionManifestPathForVideo(videoPath); + if (!isPathAllowed(manifestPath)) { + const parsedVideoPath = path.parse(videoPath); + if (!isPathWithinDir(path.resolve(manifestPath), parsedVideoPath.dir)) { + return null; + } } - }; - const flushCountdownOverlayState = (win: BrowserWindow) => { - if (win.isDestroyed()) { - return; + const content = await fs.readFile(manifestPath, "utf-8"); + const session = normalizeRecordingSession(JSON.parse(content)); + if (!session) { + return null; } - clearCountdownOverlayHideCommit(); - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); - if (!countdownOverlayState.visible) { - return; + const normalizedVideoPath = normalizePath(videoPath); + const matchesScreen = normalizePath(session.screenVideoPath) === normalizedVideoPath; + const matchesWebcam = + typeof session.webcamVideoPath === "string" && + normalizePath(session.webcamVideoPath) === normalizedVideoPath; + if (!matchesScreen && !matchesWebcam) { + return null; } - if (win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(1); + if (!isPathAllowed(session.screenVideoPath)) { + const approvedScreen = await approveReadableVideoPath(session.screenVideoPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + if (!approvedScreen) { + return null; } - return; + session.screenVideoPath = approvedScreen; } - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && !win.isVisible()) { - if (supportsWindowOpacity) { - win.setOpacity(0); - } - win.showInactive(); - - if (supportsWindowOpacity) { - setTimeout(() => { - if (!win.isDestroyed() && countdownOverlayState.visible && win.isVisible()) { - win.setOpacity(1); - } - }, 0); - } + if (session.webcamVideoPath && !isPathAllowed(session.webcamVideoPath)) { + const approvedWebcam = await approveReadableVideoPath(session.webcamVideoPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + if (!approvedWebcam) { + session.webcamVideoPath = undefined; + } else { + session.webcamVideoPath = approvedWebcam; } - }, 16); - }; - - ipcMain.handle("countdown-overlay-show", (_, value: number, runId: number) => { - countdownOverlayState.activeRunId = runId; - countdownOverlayState.visible = true; - countdownOverlayState.value = value; - - const win = getCountdownOverlayWindow() ?? createCountdownOverlayWindow(); - if (win.isDestroyed()) { - return; } - if (win.webContents.isLoading()) { - win.webContents.once("did-finish-load", () => { - if (!win.isDestroyed()) { - flushCountdownOverlayState(win); - } - }); - } else { - flushCountdownOverlayState(win); + approveFilePath(session.screenVideoPath); + if (session.webcamVideoPath) { + approveFilePath(session.webcamVideoPath); } + return session; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== "ENOENT") { + console.error("Failed to restore recording session manifest:", error); + } + return null; + } +} + +export function registerIpcHandlers( + createEditorWindow: () => void, + createSourceSelectorWindow: () => BrowserWindow, + createCountdownOverlayWindow: () => BrowserWindow, + getMainWindow: () => BrowserWindow | null, + getSourceSelectorWindow: () => BrowserWindow | null, + getCountdownOverlayWindow?: () => BrowserWindow | null, + onRecordingStateChange?: (recording: boolean, sourceName: string) => void, + _switchToHud?: () => void, +) { + ipcMain.handle("get-sources", async (_, opts) => { + const sources = await desktopCapturer.getSources(opts); + lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); + return sources.map((source) => ({ + id: source.id, + name: source.name, + display_id: source.display_id, + thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, + appIcon: source.appIcon ? source.appIcon.toDataURL() : null, + })); }); - ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => { - if (countdownOverlayState.activeRunId !== runId || !countdownOverlayState.visible) { - return; + ipcMain.handle("select-source", async (_, source: SelectedSource) => { + selectedSource = source; + // Reuse the exact source object returned during enumeration to avoid + // Windows window-source id mismatches across separate getSources() calls. + selectedDesktopSource = + typeof source.id === "string" ? (lastEnumeratedSources.get(source.id) ?? null) : null; + + if (!selectedDesktopSource && typeof source.id === "string") { + try { + const sources = await desktopCapturer.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 0, height: 0 }, + fetchWindowIcons: true, + }); + lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate])); + selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null; + } catch { + selectedDesktopSource = null; + } } + const sourceSelectorWin = getSourceSelectorWindow(); + if (sourceSelectorWin) { + sourceSelectorWin.close(); + } + return selectedSource; + }); - countdownOverlayState.value = value; + ipcMain.handle("get-selected-source", () => { + return selectedSource; + }); - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - return; + ipcMain.handle("request-camera-access", async () => { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; } - if (win.webContents.isLoading()) { - return; - } + try { + const status = systemPreferences.getMediaAccessStatus("camera"); + if (status === "granted") { + return { success: true, granted: true, status }; + } - win.webContents.send("countdown-overlay-value", value); + if (status === "not-determined") { + const granted = await systemPreferences.askForMediaAccess("camera"); + return { + success: true, + granted, + status: granted ? "granted" : systemPreferences.getMediaAccessStatus("camera"), + }; + } + + return { success: true, granted: false, status }; + } catch (error) { + console.error("Failed to request camera access:", error); + return { + success: false, + granted: false, + status: "unknown", + error: String(error), + }; + } }); - ipcMain.handle("countdown-overlay-hide", (_, runId: number) => { - if (countdownOverlayState.activeRunId !== runId) { + ipcMain.handle("open-source-selector", () => { + const sourceSelectorWin = getSourceSelectorWindow(); + if (sourceSelectorWin) { + sourceSelectorWin.focus(); return; } + createSourceSelectorWindow(); + }); - countdownOverlayState.visible = false; - countdownOverlayState.hideCommitId += 1; - const hideCommitId = countdownOverlayState.hideCommitId; - clearCountdownOverlayHideCommit(); + ipcMain.handle("switch-to-editor", () => { + const mainWin = getMainWindow(); + if (mainWin) { + mainWin.close(); + } + createEditorWindow(); + }); - const win = getCountdownOverlayWindow(); - if (!win || win.isDestroyed()) { - countdownOverlayState.value = null; + ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow(); + if (overlayWindow.isDestroyed()) { return; } - if (supportsWindowOpacity) { - // Hide visually immediately to avoid hide/show compositor flashes on rapid restart. - win.setOpacity(0); + if (!overlayWindow.isVisible()) { + overlayWindow.showInactive(); } - countdownOverlayState.value = null; - if (!win.webContents.isLoading()) { - win.webContents.send("countdown-overlay-value", countdownOverlayState.value); + if (overlayWindow.webContents.isLoading()) { + await new Promise((resolve) => { + overlayWindow.webContents.once("did-finish-load", () => resolve()); + }); } - if (!supportsWindowOpacity) { - win.hide(); + overlayWindow.webContents.send("countdown-overlay-value", value, runId); + }); + + ipcMain.handle("countdown-overlay-set-value", (_, value: number, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { return; } - countdownOverlayState.hideCommitTimer = setTimeout(() => { - countdownOverlayState.hideCommitTimer = null; - commitCountdownOverlayHide(win, hideCommitId); - }, COUNTDOWN_OVERLAY_HIDE_DEBOUNCE_MS); + overlayWindow.webContents.send("countdown-overlay-value", value, runId); }); - ipcMain.handle("switch-to-hud", () => { - if (switchToHud) switchToHud(); - }); - ipcMain.handle("start-new-recording", () => { - try { - setCurrentRecordingSessionState(null); - if (switchToHud) { - switchToHud(); - } - return { success: true }; - } catch (error) { - console.error("Failed to start new recording:", error); - return { success: false, error: String(error) }; + ipcMain.handle("countdown-overlay-hide", (_, runId: number) => { + const overlayWindow = getCountdownOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return; } - }); - ipcMain.handle("get-sources", async (_, opts) => { - const ownWindowSourceIds = new Set( - BrowserWindow.getAllWindows() - .map((win) => { - try { - return win.getMediaSourceId(); - } catch { - return null; - } - }) - .filter((id): id is string => Boolean(id)), - ); - const sources = await desktopCapturer.getSources(opts); - return sources - .filter((source) => !ownWindowSourceIds.has(source.id)) - .map((source) => ({ - id: source.id, - name: source.name, - display_id: source.display_id, - thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null, - appIcon: source.appIcon ? source.appIcon.toDataURL() : null, - })); + overlayWindow.webContents.send("countdown-overlay-value", null, runId); + overlayWindow.hide(); }); - ipcMain.handle("select-source", (_, source: SelectedSource) => { - selectedSource = source; - const sourceSelectorWin = getSourceSelectorWindow(); - if (sourceSelectorWin) { - sourceSelectorWin.close(); + ipcMain.handle("is-native-windows-capture-available", async () => { + if (!isWindowsGraphicsCaptureOsSupported()) { + return { success: true, available: false, reason: "unsupported-os" }; } - return selectedSource; - }); - ipcMain.handle("get-selected-source", () => { - return selectedSource; + const helperPath = await findNativeWindowsCaptureHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; }); - ipcMain.handle("request-camera-access", async () => { - if (process.platform !== "darwin") { - return { success: true, granted: true, status: "granted" }; - } + ipcMain.handle( + "start-native-windows-recording", + async (_, request: NativeWindowsRecordingRequest) => { + try { + if (!isWindowsGraphicsCaptureOsSupported()) { + return { + success: false, + error: "Windows Graphics Capture requires Windows 10 build 19041 or newer.", + }; + } + if (nativeWindowsCaptureProcess) { + return { success: false, error: "Native Windows capture is already running." }; + } + + const helperPath = await findNativeWindowsCaptureHelperPath(); + if (!helperPath) { + return { success: false, error: "Native Windows capture helper is not available." }; + } + + if (!request?.source?.sourceId) { + return { + success: false, + error: "Native Windows capture request is missing a source.", + }; + } + + const recordingId = + typeof request.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const webcamOutputPath = path.join( + RECORDINGS_DIR, + `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`, + ); + const sourceDisplay = + request.source.type === "display" && typeof request.source.displayId === "number" + ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? + null) + : getSelectedDisplay(); + const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const displayId = + typeof request.source.displayId === "number" && Number.isFinite(request.source.displayId) + ? request.source.displayId + : Number(selectedSource?.display_id); + const webcamDirectShowClsid = request.webcam.enabled + ? await resolveDirectShowWebcamClsid(request.webcam.deviceName) + : null; + const cursorCaptureMode = + normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; + const config = { + schemaVersion: 2, + recordingId, + outputPath, + sourceType: request.source.type, + sourceId: request.source.sourceId, + displayId: Number.isFinite(displayId) ? displayId : 0, + windowHandle: request.source.windowHandle ?? null, + fps: request.video.fps, + videoWidth: request.video.width, + videoHeight: request.video.height, + displayX: bounds.x, + displayY: bounds.y, + displayW: bounds.width, + displayH: bounds.height, + hasDisplayBounds: true, + captureSystemAudio: request.audio.system.enabled, + captureMic: request.audio.microphone.enabled, + microphoneDeviceId: request.audio.microphone.deviceId ?? null, + microphoneDeviceName: request.audio.microphone.deviceName ?? null, + microphoneGain: request.audio.microphone.gain, + webcamEnabled: request.webcam.enabled, + webcamDeviceId: request.webcam.deviceId ?? null, + webcamDeviceName: request.webcam.deviceName ?? null, + webcamDirectShowClsid, + webcamWidth: request.webcam.width, + webcamHeight: request.webcam.height, + webcamFps: request.webcam.fps, + captureCursor: cursorCaptureMode === "system", + cursorCaptureMode, + outputs: { + screenPath: outputPath, + webcamPath: webcamOutputPath, + }, + source: { + type: request.source.type, + sourceId: request.source.sourceId, + displayId: Number.isFinite(displayId) ? displayId : null, + windowHandle: request.source.windowHandle ?? null, + bounds, + }, + video: request.video, + audio: request.audio, + webcam: request.webcam, + cursor: { + mode: cursorCaptureMode, + }, + }; - try { - const status = systemPreferences.getMediaAccessStatus("camera"); - if (status === "granted") { - return { success: true, granted: true, status }; - } + console.info("[native-wgc] starting Windows capture", { + helperPath, + source: request.source, + audio: request.audio, + webcam: request.webcam, + cursor: { mode: cursorCaptureMode }, + bounds, + sourceId: selectedSource?.id ?? null, + usedDisplayMatch: Boolean(sourceDisplay), + outputPath, + }); + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + nativeWindowsCaptureOutput = ""; + nativeWindowsCaptureTargetPath = outputPath; + nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null; + nativeWindowsCaptureRecordingId = recordingId; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = cursorCaptureMode; + + const cursorStartTimeMs = Date.now(); + if (cursorCaptureMode === "editable-overlay") { + await startCursorRecording(cursorStartTimeMs); + console.info("[native-wgc] cursor sampler ready", { + cursorStartTimeMs, + warmupMs: Date.now() - cursorStartTimeMs, + }); + } else { + pendingCursorRecordingData = null; + } + + const proc = spawn(helperPath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + nativeWindowsCaptureProcess = proc; + + await waitForNativeWindowsCaptureStart(proc); + const captureStartedAtMs = Date.now(); + nativeWindowsCursorOffsetMs = + cursorCaptureMode === "editable-overlay" + ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) + : 0; + const webcamFormat = readNativeWindowsWebcamFormat(nativeWindowsCaptureOutput); + console.info("[native-wgc] capture started", { + captureStartedAtMs, + cursorOffsetMs: nativeWindowsCursorOffsetMs, + webcamFormat, + }); + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } - if (status === "not-determined") { - const granted = await systemPreferences.askForMediaAccess("camera"); return { success: true, - granted, - status: granted ? "granted" : systemPreferences.getMediaAccessStatus("camera"), + recordingId, + path: outputPath, + helperPath, }; + } catch (error) { + console.error("Failed to start native Windows recording:", error); + nativeWindowsCaptureProcess?.kill(); + nativeWindowsCaptureProcess = null; + nativeWindowsCaptureTargetPath = null; + nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureRecordingId = null; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = "editable-overlay"; + await stopCursorRecording(); + return { success: false, error: String(error) }; } + }, + ); - return { success: true, granted: false, status }; - } catch (error) { - console.error("Failed to request camera access:", error); - return { - success: false, - granted: false, - status: "unknown", - error: String(error), - }; - } - }); + ipcMain.handle("stop-native-windows-recording", async (_, discard?: boolean) => { + const proc = nativeWindowsCaptureProcess; + const preferredPath = nativeWindowsCaptureTargetPath; + const preferredWebcamPath = nativeWindowsCaptureWebcamTargetPath; + const recordingId = nativeWindowsCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = nativeWindowsCursorCaptureMode; - // macOS Accessibility prompt for global click capture. First call shows the - // system dialog; the user has to toggle the app in System Settings (no - // programmatic grant exists for Accessibility). - ipcMain.handle("request-accessibility-access", () => { - if (process.platform !== "darwin") { - return { success: true, granted: true }; + if (!proc) { + return { success: false, error: "Native Windows capture is not running." }; } + try { - const granted = systemPreferences.isTrustedAccessibilityClient(true); - return { success: true, granted }; - } catch (error) { - console.error("Failed to request accessibility access:", error); - return { success: false, granted: false, error: String(error) }; - } - }); + const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc); + proc.stdin.write("stop\n"); + const stoppedPath = await stoppedPathPromise; + const screenVideoPath = stoppedPath || preferredPath; + if (!screenVideoPath) { + throw new Error("Native Windows capture did not return an output path."); + } - ipcMain.handle("open-source-selector", () => { - const sourceSelectorWin = getSourceSelectorWindow(); - if (sourceSelectorWin) { - sourceSelectorWin.focus(); - return; - } - createSourceSelectorWindow(); - }); + if (cursorCaptureMode === "editable-overlay") { + await stopCursorRecording(); + } else { + pendingCursorRecordingData = null; + } + if (discard) { + pendingCursorRecordingData = null; + await Promise.all([ + fs.rm(screenVideoPath, { force: true }), + preferredWebcamPath ? fs.rm(preferredWebcamPath, { force: true }) : Promise.resolve(), + fs.rm(`${screenVideoPath}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } - ipcMain.handle("switch-to-editor", () => { - const mainWin = getMainWindow(); - if (mainWin) { - mainWin.close(); + if (cursorCaptureMode === "editable-overlay") { + shiftPendingCursorTelemetry(nativeWindowsCursorOffsetMs); + await writePendingCursorTelemetry(screenVideoPath); + } + let webcamVideoPath: string | undefined; + if (preferredWebcamPath) { + try { + await fs.access(preferredWebcamPath, fsConstants.R_OK); + webcamVideoPath = preferredWebcamPath; + } catch { + webcamVideoPath = undefined; + } + } + const session: RecordingSession = webcamVideoPath + ? { screenVideoPath, webcamVideoPath, createdAt: recordingId, cursorCaptureMode } + : { screenVideoPath, createdAt: recordingId, cursorCaptureMode }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native Windows recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native Windows recording:", error); + await stopCursorRecording(); + return { success: false, error: String(error) }; + } finally { + nativeWindowsCaptureProcess = null; + nativeWindowsCaptureTargetPath = null; + nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureRecordingId = null; + nativeWindowsCursorOffsetMs = 0; + nativeWindowsCursorCaptureMode = "editable-overlay"; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } } - createEditorWindow(); }); ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { @@ -763,6 +1285,48 @@ export function registerIpcHandlers( } }); + async function storeRecordedSessionFiles(payload: StoreRecordedSessionInput) { + const createdAt = + typeof payload.createdAt === "number" && Number.isFinite(payload.createdAt) + ? payload.createdAt + : Date.now(); + const cursorCaptureMode = normalizeCursorCaptureMode(payload.cursorCaptureMode); + const screenVideoPath = resolveRecordingOutputPath(payload.screen.fileName); + await fs.writeFile(screenVideoPath, Buffer.from(payload.screen.videoData)); + + let webcamVideoPath: string | undefined; + if (payload.webcam) { + webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); + await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); + } + + const session: RecordingSession = webcamVideoPath + ? { + screenVideoPath, + webcamVideoPath, + createdAt, + ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + } + : { screenVideoPath, createdAt, ...(cursorCaptureMode ? { cursorCaptureMode } : {}) }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + await writePendingCursorTelemetry(screenVideoPath); + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(payload.screen.fileName).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Recording session stored successfully", + }; + } + ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => { try { return await storeRecordedSessionFiles({ @@ -794,24 +1358,7 @@ export function registerIpcHandlers( return { success: false, message: "No recorded video found" }; } - // Sort by most recently modified to reliably get the latest recording. - // Lexicographic sort is unreliable (e.g. recording-9.webm > recording-10.webm). - let latestVideo: string | null = null; - let latestMtimeMs = 0; - for (const file of videoFiles) { - try { - const stat = await fs.stat(path.join(RECORDINGS_DIR, file)); - if (stat.mtimeMs > latestMtimeMs) { - latestMtimeMs = stat.mtimeMs; - latestVideo = file; - } - } catch { - // Skip inaccessible files. - } - } - if (!latestVideo) { - return { success: false, message: "No recorded video found" }; - } + const latestVideo = videoFiles.sort().reverse()[0]; const videoPath = path.join(RECORDINGS_DIR, latestVideo); return { success: true, path: videoPath }; @@ -821,153 +1368,38 @@ export function registerIpcHandlers( } }); - ipcMain.handle("read-binary-file", async (_, inputPath: string) => { - try { - const normalizedPath = normalizeVideoSourcePath(inputPath); - if (!normalizedPath) { - return { success: false, message: "Invalid file path" }; + ipcMain.handle( + "set-recording-state", + async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => { + const normalizedCursorCaptureMode = + normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay"; + if (recording && normalizedCursorCaptureMode === "editable-overlay") { + await startCursorRecording(recordingId); + } else { + await stopCursorRecording(); } - if (!isPathAllowed(normalizedPath)) { - console.warn( - "[read-binary-file] Rejected path outside allowed directories:", - normalizedPath, - ); - return { success: false, message: "Access denied: path outside allowed directories" }; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(recording, source.name); } - - const data = await fs.readFile(normalizedPath); - return { - success: true, - data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), - path: normalizedPath, - }; - } catch (error) { - console.error("Failed to read binary file:", error); - return { - success: false, - message: "Failed to read binary file", - error: String(error), - }; - } - }); - - ipcMain.handle("set-recording-state", (_, recording: boolean, recordingId?: number) => { - if (recording) { - stopCursorCapture(); - // The renderer is the source of truth for the recording id (it - // uses the same id as the saved fileName). Fall back to a - // timestamp only if the renderer didn't supply one, so the - // buffer always has a stable key per session. - const id = typeof recordingId === "number" ? recordingId : Date.now(); - cursorTelemetryBuffer.startSession(id); - cursorCaptureStartTimeMs = Date.now(); - cursorClickTimestampsMs = []; - startClickCapture(); - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); - } else { - stopCursorCapture(); - cursorTelemetryBuffer.endSession(); - } - - const source = selectedSource || { name: "Screen" }; - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name); - } - }); - - ipcMain.handle("discard-cursor-telemetry", (_, recordingId: number) => { - cursorTelemetryBuffer.discardBatch(recordingId); - }); + }, + ); ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { - const targetVideoPath = normalizeVideoSourcePath( + const targetVideoPath = resolveApprovedVideoPath( videoPath ?? currentRecordingSession?.screenVideoPath, ); if (!targetVideoPath) { return { success: true, samples: [] }; } - if (!isPathAllowed(targetVideoPath)) { - console.warn( - "[get-cursor-telemetry] Rejected path outside allowed directories:", - targetVideoPath, - ); - return { success: true, samples: [] }; - } - - const telemetryPath = `${targetVideoPath}.cursor.json`; - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - const rawClicks = Array.isArray(parsed?.clicks) ? parsed.clicks : []; - const clicks: number[] = rawClicks - .map((value: unknown) => - typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null, - ) - .filter((v: number | null): v is number => v !== null) - .sort((a: number, b: number) => a - b); - - return { success: true, samples, clicks }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [], clicks: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - clicks: [], - }; - } + return readCursorTelemetryFile(targetVideoPath); }); ipcMain.handle("open-external-url", async (_, url: string) => { try { - const ALLOWED_SCHEMES = ["http:", "https:", "mailto:"]; - let parsed: URL; - try { - parsed = new URL(url); - } catch { - return { success: false, error: "Invalid URL" }; - } - - if (!ALLOWED_SCHEMES.includes(parsed.protocol)) { - return { success: false, error: `Unsupported URL scheme: ${parsed.protocol}` }; - } - - await shell.openExternal(parsed.toString()); + await shell.openExternal(url); return { success: true }; } catch (error) { console.error("Failed to open URL:", error); @@ -975,15 +1407,10 @@ export function registerIpcHandlers( } }); - /** - * Handles saving an exported video file. - * Shows a save dialog, normalizes the file path for the current OS, - * ensures the directory exists, and writes the video data. - * @param _ - Unused event parameter. - * @param videoData - The exported video as an ArrayBuffer. - * @param fileName - Suggested filename for the save dialog. - * @returns Object with success status, optional file path, and error details. - */ + // Return base path for assets so renderer can resolve file:// paths in production + ipcMain.handle("get-asset-base-path", () => { + return resolveAssetBasePath(); + }); ipcMain.handle("save-exported-video", async (_, videoData: ArrayBuffer, fileName: string) => { try { @@ -1014,18 +1441,11 @@ export function registerIpcHandlers( }; } - // --- FIX: Normalize the path for Windows compatibility --- - const normalizedPath = path.normalize(result.filePath); - - // Ensure the parent directory exists (Windows may fail if the folder is missing) - await fs.mkdir(path.dirname(normalizedPath), { recursive: true }); - // --- END FIX --- - - await fs.writeFile(normalizedPath, Buffer.from(videoData)); + await fs.writeFile(result.filePath, Buffer.from(videoData)); return { success: true, - path: normalizedPath, + path: result.filePath, message: "Video exported successfully", }; } catch (error) { @@ -1037,6 +1457,7 @@ export function registerIpcHandlers( }; } }); + ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -1060,17 +1481,18 @@ export function registerIpcHandlers( return { success: false, canceled: true }; } - const approvedPath = await approveReadableVideoPath(result.filePaths[0]); - if (!approvedPath) { + const normalizedPath = await approveReadableVideoPath(result.filePaths[0]); + if (!normalizedPath) { return { success: false, - message: "Selected file is not a supported video", + message: "Selected file is not a supported readable video file", }; } + currentProjectPath = null; return { success: true, - path: approvedPath, + path: normalizedPath, }; } catch (error) { console.error("Failed to open file picker:", error); @@ -1106,78 +1528,116 @@ export function registerIpcHandlers( } }); + ipcMain.handle("read-binary-file", async (_, filePath: string) => { + try { + const normalizedPath = await approveReadableVideoPath(filePath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + const data = await fs.readFile(normalizedPath); + return { + success: true, + data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength), + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to read binary file:", error); + return { + success: false, + message: "Failed to read binary file", + error: String(error), + }; + } + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { - try { - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } + return saveProjectFile(projectData, suggestedName, existingProjectPath); + }, + ); - const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const dialogOptions = buildDialogOptions( - { - title: mainT("dialogs", "fileDialogs.saveProject"), - defaultPath: path.join(RECORDINGS_DIR, defaultName), - filters: [ - { - name: mainT("dialogs", "fileDialogs.openscreenProject"), - extensions: [PROJECT_FILE_EXTENSION], - }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }, - getMainWindow(), + async function saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ): Promise { + try { + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null; + + if (trustedExistingProjectPath) { + await fs.writeFile( + trustedExistingProjectPath, + JSON.stringify(projectData, null, 2), + "utf-8", ); - const result = await dialog.showSaveDialog(dialogOptions); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } - - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; - + currentProjectPath = trustedExistingProjectPath; return { success: true, - path: result.filePath, + path: trustedExistingProjectPath, message: "Project saved successfully", }; - } catch (error) { - console.error("Failed to save project file:", error); + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}`; + + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.saveProject"), + defaultPath: path.join(RECORDINGS_DIR, defaultName), + filters: [ + { + name: mainT("dialogs", "fileDialogs.openscreenProject"), + extensions: [PROJECT_FILE_EXTENSION], + }, + { name: "JSON", extensions: ["json"] }, + ], + properties: ["createDirectory", "showOverwriteConfirmation"], + }, + getMainWindow(), + ); + const result = await dialog.showSaveDialog(dialogOptions); + + if (result.canceled || !result.filePath) { return { success: false, - message: "Failed to save project file", - error: String(error), + canceled: true, + message: "Save project canceled", }; } - }, - ); + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); + currentProjectPath = result.filePath; + + return { + success: true, + path: result.filePath, + message: "Project saved successfully", + }; + } catch (error) { + console.error("Failed to save project file:", error); + return { + success: false, + message: "Failed to save project file", + error: String(error), + }; + } + } ipcMain.handle("load-project-file", async () => { + return loadProjectFile(); + }); + + async function loadProjectFile(): Promise { try { const dialogOptions = buildDialogOptions( { @@ -1204,9 +1664,8 @@ export function registerIpcHandlers( const filePath = result.filePaths[0]; const content = await fs.readFile(filePath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, filePath); currentProjectPath = filePath; - setCurrentRecordingSessionState(session); + setCurrentRecordingSessionState(await getApprovedProjectSession(project, filePath)); return { success: true, @@ -1221,9 +1680,13 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } ipcMain.handle("load-current-project-file", async () => { + return loadCurrentProjectFile(); + }); + + async function loadCurrentProjectFile(): Promise { try { if (!currentProjectPath) { return { success: false, message: "No active project" }; @@ -1231,8 +1694,7 @@ export function registerIpcHandlers( const content = await fs.readFile(currentProjectPath, "utf-8"); const project = JSON.parse(content); - const session = await getApprovedProjectSession(project, currentProjectPath); - setCurrentRecordingSessionState(session); + setCurrentRecordingSessionState(await getApprovedProjectSession(project, currentProjectPath)); return { success: true, path: currentProjectPath, @@ -1246,12 +1708,18 @@ export function registerIpcHandlers( error: String(error), }; } + } + + ipcMain.handle("set-current-video-path", async (_, path: string) => { + return setCurrentVideoPath(path); }); + ipcMain.handle("set-current-recording-session", (_, session: RecordingSession | null) => { - const normalized = normalizeRecordingSession(session); - setCurrentRecordingSessionState(normalized); + const normalizedSession = normalizeRecordingSession(session); + setCurrentRecordingSessionState(normalizedSession); + currentVideoPath = normalizedSession?.screenVideoPath ?? null; currentProjectPath = null; - return { success: true, session: normalized ?? undefined }; + return { success: true, session: currentRecordingSession }; }); ipcMain.handle("get-current-recording-session", () => { @@ -1260,19 +1728,17 @@ export function registerIpcHandlers( : { success: false }; }); - ipcMain.handle("set-current-video-path", async (_, path: string) => { + async function setCurrentVideoPath(path: string): Promise { const normalizedPath = normalizeVideoSourcePath(path); if (!normalizedPath || !isPathAllowed(normalizedPath)) { - return { success: false, message: "Video path has not been approved" }; + return { + success: false, + message: "Video path has not been approved", + }; } const restoredSession = await loadRecordedSessionForVideoPath(normalizedPath); if (restoredSession) { - // Approve all media paths from the restored session so they can be read later - approveFilePath(restoredSession.screenVideoPath); - if (restoredSession.webcamVideoPath) { - approveFilePath(restoredSession.webcamVideoPath); - } setCurrentRecordingSessionState(restoredSession); } else { setCurrentRecordingSessionState({ @@ -1281,20 +1747,26 @@ export function registerIpcHandlers( }); } currentProjectPath = null; - return { success: true }; - }); + return { success: true, path: currentVideoPath ?? normalizedPath }; + } ipcMain.handle("get-current-video-path", () => { - return currentRecordingSession?.screenVideoPath - ? { success: true, path: currentRecordingSession.screenVideoPath } - : { success: false }; + return getCurrentVideoPathResult(); }); + function getCurrentVideoPathResult(): ProjectPathResult { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + } + ipcMain.handle("clear-current-video-path", () => { - setCurrentRecordingSessionState(null); - return { success: true }; + return clearCurrentVideoPath(); }); + function clearCurrentVideoPath(): ProjectPathResult { + currentVideoPath = null; + return { success: true }; + } + ipcMain.handle("get-platform", () => { return process.platform; }); @@ -1317,4 +1789,21 @@ export function registerIpcHandlers( return { success: false, error: String(error) }; } }); + + registerNativeBridgeHandlers({ + getPlatform: () => process.platform, + getCurrentProjectPath: () => currentProjectPath, + getCurrentVideoPath: () => currentVideoPath, + saveProjectFile, + loadProjectFile, + loadCurrentProjectFile, + setCurrentVideoPath, + getCurrentVideoPathResult, + clearCurrentVideoPath, + resolveAssetBasePath, + resolveVideoPath: (videoPath?: string | null) => + resolveApprovedVideoPath(videoPath ?? currentVideoPath), + loadCursorRecordingData: readCursorRecordingFile, + loadCursorTelemetry: readCursorTelemetryFile, + }); } diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts new file mode 100644 index 000000000..7f7b24b51 --- /dev/null +++ b/electron/ipc/nativeBridge.ts @@ -0,0 +1,229 @@ +import { ipcMain } from "electron"; +import { + NATIVE_BRIDGE_CHANNEL, + NATIVE_BRIDGE_VERSION, + type NativeBridgeErrorCode, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectFileResult, + type ProjectPathResult, +} from "../../src/native/contracts"; +import type { CursorTelemetryLoadResult } from "../native-bridge/cursor/adapter"; +import { TelemetryCursorAdapter } from "../native-bridge/cursor/telemetryCursorAdapter"; +import { CursorService } from "../native-bridge/services/cursorService"; +import { ProjectService } from "../native-bridge/services/projectService"; +import { SystemService } from "../native-bridge/services/systemService"; +import { NativeBridgeStateStore } from "../native-bridge/store"; + +export interface NativeBridgeContext { + getPlatform: () => NodeJS.Platform; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; + resolveAssetBasePath: () => string | null; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadCursorRecordingData: ( + videoPath: string, + ) => Promise; + loadCursorTelemetry: (videoPath: string) => Promise; +} + +function normalizePlatform(platform: NodeJS.Platform): NativePlatform { + if (platform === "darwin" || platform === "win32") { + return platform; + } + + return "linux"; +} + +function createMeta(requestId?: string) { + return { + version: NATIVE_BRIDGE_VERSION, + requestId: requestId || `native-${Date.now()}`, + timestampMs: Date.now(), + } as const; +} + +function createSuccessResponse(requestId: string | undefined, data: TData) { + return { + ok: true, + data, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function createErrorResponse( + requestId: string | undefined, + code: NativeBridgeErrorCode, + message: string, + retryable = false, +) { + return { + ok: false, + error: { + code, + message, + retryable, + }, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function isBridgeRequest(value: unknown): value is NativeBridgeRequest { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return typeof candidate.domain === "string" && typeof candidate.action === "string"; +} + +export function registerNativeBridgeHandlers(context: NativeBridgeContext) { + ipcMain.removeHandler(NATIVE_BRIDGE_CHANNEL); + + const platform = normalizePlatform(context.getPlatform()); + const store = new NativeBridgeStateStore(platform); + const projectService = new ProjectService({ + store, + getCurrentProjectPath: context.getCurrentProjectPath, + getCurrentVideoPath: context.getCurrentVideoPath, + saveProjectFile: context.saveProjectFile, + loadProjectFile: context.loadProjectFile, + loadCurrentProjectFile: context.loadCurrentProjectFile, + setCurrentVideoPath: context.setCurrentVideoPath, + getCurrentVideoPathResult: context.getCurrentVideoPathResult, + clearCurrentVideoPath: context.clearCurrentVideoPath, + }); + const cursorService = new CursorService({ + store, + adapter: new TelemetryCursorAdapter({ + loadRecordingData: context.loadCursorRecordingData, + resolveVideoPath: context.resolveVideoPath, + loadTelemetry: context.loadCursorTelemetry, + }), + }); + const systemService = new SystemService({ + store, + getPlatform: () => platform, + getAssetBasePath: context.resolveAssetBasePath, + getCursorCapabilities: () => cursorService.getCapabilities(), + }); + + ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (_, request: unknown) => { + if (!isBridgeRequest(request)) { + return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request."); + } + + const requestId = request.requestId; + const domain = request.domain as string; + + try { + switch (request.domain) { + case "system": { + const action = request.action as string; + switch (request.action) { + case "getPlatform": + return createSuccessResponse(requestId, systemService.getPlatform()); + case "getAssetBasePath": + return createSuccessResponse(requestId, systemService.getAssetBasePath()); + case "getCapabilities": + return createSuccessResponse(requestId, await systemService.getCapabilities()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported system action: ${action}`, + ); + } + } + + case "project": { + const action = request.action as string; + switch (request.action) { + case "getCurrentContext": + return createSuccessResponse(requestId, projectService.getCurrentContext()); + case "saveProjectFile": + return createSuccessResponse( + requestId, + await projectService.saveProjectFile( + request.payload.projectData, + request.payload.suggestedName, + request.payload.existingProjectPath, + ), + ); + case "loadProjectFile": + return createSuccessResponse(requestId, await projectService.loadProjectFile()); + case "loadCurrentProjectFile": + return createSuccessResponse( + requestId, + await projectService.loadCurrentProjectFile(), + ); + case "setCurrentVideoPath": + return createSuccessResponse( + requestId, + await projectService.setCurrentVideoPath(request.payload.path), + ); + case "getCurrentVideoPath": + return createSuccessResponse(requestId, projectService.getCurrentVideoPath()); + case "clearCurrentVideoPath": + return createSuccessResponse(requestId, projectService.clearCurrentVideoPath()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported project action: ${action}`, + ); + } + } + + case "cursor": { + const action = request.action as string; + switch (request.action) { + case "getCapabilities": + return createSuccessResponse(requestId, await cursorService.getCapabilities()); + case "getTelemetry": + return createSuccessResponse( + requestId, + await cursorService.getTelemetry(request.payload?.videoPath), + ); + case "getRecordingData": + return createSuccessResponse( + requestId, + await cursorService.getRecordingData(request.payload?.videoPath), + ); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported cursor action: ${action}`, + ); + } + } + + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported bridge domain: ${domain}`, + ); + } + } catch (error) { + return createErrorResponse( + requestId, + "INTERNAL_ERROR", + error instanceof Error ? error.message : "Unknown native bridge error.", + true, + ); + } + }); +} diff --git a/electron/main.ts b/electron/main.ts index 0b90b8921..059db0700 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -13,7 +13,7 @@ import { Tray, } from "electron"; import { mainT, setMainLocale } from "./i18n"; -import { registerIpcHandlers } from "./ipc/handlers"; +import { getSelectedDesktopSource, registerIpcHandlers } from "./ipc/handlers"; import { createCountdownOverlayWindow, createEditorWindow, @@ -385,6 +385,23 @@ app.whenReady().then(async () => { app.dock?.show(); } + // Intercept getDisplayMedia to return the pre-selected source without the cursor. + // The source is cached synchronously at select-source time to avoid async delays here. + if (process.platform === "win32") { + session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => { + const source = getSelectedDesktopSource(); + if (!source) { + callback({}); + return; + } + callback({ + video: source, + // WASAPI loopback provides system audio capture on Windows. + audio: "loopback", + }); + }); + } + // Allow microphone/media permission checks session.defaultSession.setPermissionCheckHandler((_webContents, permission) => { const allowed = ["media", "audioCapture", "microphone", "videoCapture", "camera"]; diff --git a/electron/native-bridge/cursor/adapter.ts b/electron/native-bridge/cursor/adapter.ts new file mode 100644 index 000000000..cdb88e24a --- /dev/null +++ b/electron/native-bridge/cursor/adapter.ts @@ -0,0 +1,20 @@ +import type { + CursorCapabilities, + CursorProviderKind, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; + +export interface CursorTelemetryLoadResult { + success: boolean; + samples: CursorTelemetryPoint[]; + message?: string; + error?: string; +} + +export interface CursorNativeAdapter { + readonly kind: CursorProviderKind; + getCapabilities(): Promise; + getRecordingData(videoPath?: string | null): Promise; + getTelemetry(videoPath?: string | null): Promise; +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts new file mode 100644 index 000000000..783584117 --- /dev/null +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -0,0 +1,44 @@ +import type { Rectangle } from "electron"; +import type { CursorRecordingData } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; +import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; + +interface CreateCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + platform: NodeJS.Platform; + sampleIntervalMs: number; + sourceId?: string | null; + startTimeMs?: number; +} + +class NoopCursorRecordingSession implements CursorRecordingSession { + async start(): Promise { + // Native cursor capture is currently Windows-only. + } + + async stop(): Promise { + return { + version: 2, + provider: "none", + assets: [], + samples: [], + }; + } +} + +export function createCursorRecordingSession( + options: CreateCursorRecordingSessionOptions, +): CursorRecordingSession { + if (options.platform === "win32") { + return new WindowsNativeRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, + startTimeMs: options.startTimeMs, + }); + } + + return new NoopCursorRecordingSession(); +} diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts new file mode 100644 index 000000000..9cebe9f4c --- /dev/null +++ b/electron/native-bridge/cursor/recording/session.ts @@ -0,0 +1,6 @@ +import type { CursorRecordingData } from "../../../../src/native/contracts"; + +export interface CursorRecordingSession { + start(): Promise; + stop(): Promise; +} diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts new file mode 100644 index 000000000..e719d8ee3 --- /dev/null +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -0,0 +1,63 @@ +import { type Rectangle, screen } from "electron"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface TelemetryRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export class TelemetryRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private interval: NodeJS.Timeout | null = null; + private startTimeMs = 0; + + constructor(private readonly options: TelemetryRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.captureSample(); + this.interval = setInterval(() => { + this.captureSample(); + }, this.options.sampleIntervalMs); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private captureSample() { + const cursor = screen.getCursorScreenPoint(); + const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, display.width); + const height = Math.max(1, display.height); + + this.samples.push({ + timeMs: Math.max(0, Date.now() - this.startTimeMs), + cx: clamp((cursor.x - display.x) / width, 0, 1), + cy: clamp((cursor.y - display.y) / height, 0, 1), + visible: true, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts new file mode 100644 index 000000000..bed037a1e --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.script.ts @@ -0,0 +1,391 @@ +export function buildPowerShellScript(sampleIntervalMs: number, windowHandle?: string | null) { + const targetWindowHandle = + typeof windowHandle === "string" && /^(?:0x[0-9a-fA-F]+|\d+)$/.test(windowHandle) + ? `'${windowHandle}'` + : "$null"; + const script = String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$targetWindowHandle = ${targetWindowHandle} + +$source = @" +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorInterop { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_LBUTTONUP = 0x0202; + private static readonly object MouseSync = new object(); + private static int LeftDownCount = 0; + private static int LeftUpCount = 0; + private static IntPtr MouseHook = IntPtr.Zero; + private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback; + + public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + public struct MouseButtonEvents { + public int LeftDownCount; + public int LeftUpCount; + } + + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [StructLayout(LayoutKind.Sequential)] + public struct RECT { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + public static bool InstallMouseHook() { + if (MouseHook != IntPtr.Zero) { + return true; + } + + using (Process process = Process.GetCurrentProcess()) + using (ProcessModule module = process.MainModule) { + MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0); + } + + return MouseHook != IntPtr.Zero; + } + + public static MouseButtonEvents ConsumeMouseButtonEvents() { + lock (MouseSync) { + MouseButtonEvents events = new MouseButtonEvents { + LeftDownCount = LeftDownCount, + LeftUpCount = LeftUpCount + }; + LeftDownCount = 0; + LeftUpCount = 0; + return events; + } + } + + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) { + lock (MouseSync) { + if (message == WM_LBUTTONDOWN) { + LeftDownCount += 1; + } else { + LeftUpCount += 1; + } + } + } + } + + return CallNextHookEx(MouseHook, nCode, wParam, lParam); + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll")] + public static extern short GetAsyncKeyState(int vKey); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool IsWindow(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); +} +"@ + +Add-Type -TypeDefinition $source + +$standardCursors = @{ + arrow = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512)) + text = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513)) + wait = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514)) + crosshair = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515)) + 'up-arrow' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516)) + 'resize-nwse' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642)) + 'resize-nesw' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643)) + 'resize-ew' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644)) + 'resize-ns' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645)) + move = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646)) + 'not-allowed' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648)) + pointer = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649)) + 'app-starting' = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650)) + help = [OpenScreenCursorInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651)) +} + +function Get-StandardCursorType($cursorHandle) { + if ($cursorHandle -eq [IntPtr]::Zero) { + return $null + } + + foreach ($entry in $standardCursors.GetEnumerator()) { + if ($entry.Value -eq $cursorHandle) { + return $entry.Key + } + } + + return $null +} + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CustomCursorType($bitmap, $hotspotX, $hotspotY) { + if ($bitmap.Width -lt 24 -or $bitmap.Height -lt 24 -or $bitmap.Width -gt 64 -or $bitmap.Height -gt 64) { + return $null + } + + if ($hotspotX -lt ($bitmap.Width * 0.25) -or $hotspotX -gt ($bitmap.Width * 0.75) -or + $hotspotY -lt ($bitmap.Height * 0.15) -or $hotspotY -gt ($bitmap.Height * 0.55)) { + return $null + } + + $opaquePixels = 0 + $topHalfOpaquePixels = 0 + $left = $bitmap.Width + $top = $bitmap.Height + $right = -1 + $bottom = -1 + + for ($y = 0; $y -lt $bitmap.Height; $y++) { + for ($x = 0; $x -lt $bitmap.Width; $x++) { + if ($bitmap.GetPixel($x, $y).A -le 32) { + continue + } + + $opaquePixels += 1 + if ($y -lt ($bitmap.Height / 2)) { + $topHalfOpaquePixels += 1 + } + if ($x -lt $left) { $left = $x } + if ($x -gt $right) { $right = $x } + if ($y -lt $top) { $top = $y } + if ($y -gt $bottom) { $bottom = $y } + } + } + + if ($opaquePixels -lt 90 -or $right -lt $left -or $bottom -lt $top) { + return $null + } + + $opaqueWidth = $right - $left + 1 + $opaqueHeight = $bottom - $top + 1 + if ($opaqueWidth -lt ($bitmap.Width * 0.35) -or $opaqueWidth -gt ($bitmap.Width * 0.9) -or + $opaqueHeight -lt ($bitmap.Height * 0.45) -or $opaqueHeight -gt $bitmap.Height) { + return $null + } + + if ($top -gt ($bitmap.Height * 0.45) -or $bottom -lt ($bitmap.Height * 0.65)) { + return $null + } + + if ($topHalfOpaquePixels -gt ($opaquePixels * 0.55)) { + return 'closed-hand' + } + + return 'open-hand' +} + +function Get-TargetBounds() { + if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) { + return $null + } + + try { + $handleValue = [int64]::Parse($targetWindowHandle) + $windowHandle = [IntPtr]::new($handleValue) + if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) { + return $null + } + + $rect = New-Object OpenScreenCursorInterop+RECT + if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) { + return $null + } + + $width = $rect.Right - $rect.Left + $height = $rect.Bottom - $rect.Top + if ($width -le 0 -or $height -le 0) { + return $null + } + + return @{ + x = $rect.Left + y = $rect.Top + width = $width + height = $height + } + } + catch { + return $null + } +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + $hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + $customCursorType = Get-CustomCursorType -bitmap $bitmap -hotspotX $hotspotX -hotspotY $hotspotY + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = $hotspotX + hotspotY = $hotspotY + cursorType = $customCursorType + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +[OpenScreenCursorInterop]::InstallMouseHook() | Out-Null +[OpenScreenCursorInterop]::GetAsyncKeyState(0x01) | Out-Null +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +while ($true) { + [System.Windows.Forms.Application]::DoEvents() + $mouseEvents = [OpenScreenCursorInterop]::ConsumeMouseButtonEvents() + $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) + + if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${sampleIntervalMs} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $cursorType = Get-StandardCursorType $cursorInfo.hCursor + $leftButtonState = [OpenScreenCursorInterop]::GetAsyncKeyState(0x01) + $leftButtonDown = ($leftButtonState -band 0x8000) -ne 0 + $leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0) + $leftButtonReleased = $mouseEvents.LeftUpCount -gt 0 + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + if ($asset -and $cursorType) { + $asset.cursorType = $cursorType + } elseif ($asset -and $asset.cursorType) { + $cursorType = $asset.cursorType + } + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + cursorType = $cursorType + leftButtonDown = $leftButtonDown + leftButtonPressed = $leftButtonPressed + leftButtonReleased = $leftButtonReleased + bounds = Get-TargetBounds + asset = $asset + } + + Start-Sleep -Milliseconds ${sampleIntervalMs} +} +`; + + return script; +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts new file mode 100644 index 000000000..dd4aab070 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -0,0 +1,346 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; +import { screen } from "electron"; +import { parseWindowHandleFromSourceId } from "../../../../src/lib/nativeWindowsRecording"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; +import { buildPowerShellScript } from "./windowsNativeRecordingSession.script"; +import type { + WindowsCursorEvent, + WindowsNativeRecordingSessionOptions, +} from "./windowsNativeRecordingSession.types"; + +const READY_TIMEOUT_MS = 5_000; + +interface NormalizedSample { + sample: CursorRecordingSample; + withinBounds: boolean; +} + +export class WindowsNativeRecordingSession implements CursorRecordingSession { + private assets = new Map(); + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private helperScriptPath: string | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private sampleCount = 0; + private outOfBoundsSampleCount = 0; + private previousLeftButtonDown = false; + + constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} + + async start(): Promise { + this.assets.clear(); + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.sampleCount = 0; + this.outOfBoundsSampleCount = 0; + this.previousLeftButtonDown = false; + + const script = buildPowerShellScript( + this.options.sampleIntervalMs, + parseWindowHandleFromSourceId(this.options.sourceId), + ); + const helperScriptDir = join(tmpdir(), "openscreen-cursor-native"); + mkdirSync(helperScriptDir, { recursive: true }); + const helperScriptPath = join( + helperScriptDir, + `cursor-sampler-${process.pid}-${Date.now()}-${randomUUID()}.ps1`, + ); + writeFileSync(helperScriptPath, script, "utf8"); + this.helperScriptPath = helperScriptPath; + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + helperScriptPath, + ], + { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ); + + this.process = child; + this.logDiagnostic("spawn", { + pid: child.pid ?? null, + sampleIntervalMs: this.options.sampleIntervalMs, + sourceId: this.options.sourceId ?? null, + windowHandle: parseWindowHandleFromSourceId(this.options.sourceId), + }); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + this.handleStdoutChunk(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + this.logDiagnostic("stderr", { message }); + } + console.error("[cursor-native]", message); + }); + child.once("exit", (code, signal) => { + this.cleanupHelperScript(helperScriptPath); + this.logDiagnostic("exit", { + code, + signal, + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + this.rejectReady( + new Error(`Windows cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + }); + child.once("error", (error) => { + this.cleanupHelperScript(helperScriptPath); + this.logDiagnostic("process-error", { message: error.message }); + this.rejectReady(error); + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.terminateHelperProcess(); + this.cleanupHelperScript(helperScriptPath); + throw error; + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + this.killHelperProcess(child); + + this.logDiagnostic("stop", { + sampleCount: this.sampleCount, + assetCount: this.assets.size, + outOfBoundsSampleCount: this.outOfBoundsSampleCount, + }); + + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const payload = JSON.parse(trimmedLine) as WindowsCursorEvent; + this.handleEvent(payload); + } catch (error) { + console.error("Failed to parse Windows cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: WindowsCursorEvent) { + if (payload.type === "error") { + this.logDiagnostic("helper-error", { message: payload.message }); + console.error("Windows cursor helper error:", payload.message); + this.failHelper(new Error(payload.message)); + return; + } + + if (payload.type === "ready") { + this.logDiagnostic("ready", { timestampMs: payload.timestampMs }); + this.resolveReady(); + return; + } + + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { + const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y }); + this.assets.set(payload.asset.id, { + id: payload.asset.id, + platform: "win32", + imageDataUrl: payload.asset.imageDataUrl, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + cursorType: payload.asset.cursorType ?? payload.cursorType ?? null, + }); + this.logDiagnostic("asset", { + id: payload.asset.id, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + } + + const normalized = this.normalizeSample(payload); + this.sampleCount += 1; + if (!normalized.withinBounds) { + this.outOfBoundsSampleCount += 1; + } + + this.samples.push(normalized.sample); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } + + private normalizeSample( + payload: Extract, + ): NormalizedSample { + const bounds = + payload.bounds ?? this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (payload.x - bounds.x) / width; + const normalizedY = (payload.y - bounds.y) / height; + const withinBounds = + normalizedX >= 0 && normalizedX <= 1 && normalizedY >= 0 && normalizedY <= 1; + const leftButtonDown = payload.leftButtonDown === true; + const leftButtonPressed = payload.leftButtonPressed === true; + const leftButtonReleased = payload.leftButtonReleased === true; + const interactionType = + leftButtonPressed || (leftButtonDown && !this.previousLeftButtonDown) + ? "click" + : leftButtonReleased || (!leftButtonDown && this.previousLeftButtonDown) + ? "mouseup" + : "move"; + this.previousLeftButtonDown = leftButtonDown; + + if (this.sampleCount === 0 || (!withinBounds && this.outOfBoundsSampleCount === 0)) { + this.logDiagnostic("sample", { + rawX: payload.x, + rawY: payload.y, + normalizedX, + normalizedY, + visible: payload.visible, + withinBounds, + bounds, + handle: payload.handle, + }); + } + + return { + withinBounds, + sample: { + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: normalizedX, + cy: normalizedY, + assetId: payload.handle, + visible: payload.visible && withinBounds, + cursorType: payload.cursorType ?? payload.asset?.cursorType ?? null, + interactionType, + }, + }; + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for Windows cursor helper readiness")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private failHelper(error: Error) { + this.rejectReady(error); + this.terminateHelperProcess(); + } + + private terminateHelperProcess() { + const child = this.process; + this.process = null; + this.killHelperProcess(child); + } + + private killHelperProcess(child: ChildProcessByStdio | null) { + if (child && !child.killed) { + child.kill(); + } + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private cleanupHelperScript(scriptPath = this.helperScriptPath) { + if (!scriptPath) { + return; + } + + try { + rmSync(scriptPath, { force: true }); + } catch (error) { + this.logDiagnostic("script-cleanup-error", { + path: scriptPath, + message: error instanceof Error ? error.message : String(error), + }); + } finally { + if (this.helperScriptPath === scriptPath) { + this.helperScriptPath = null; + } + } + } + + private logDiagnostic(event: string, data: Record) { + console.info( + "[cursor-native][win32]", + JSON.stringify({ + event, + ...data, + }), + ); + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts new file mode 100644 index 000000000..f3b69da0f --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -0,0 +1,56 @@ +import type { Rectangle } from "electron"; +import type { NativeCursorType } from "../../../../src/native/contracts"; + +export interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + cursorType?: NativeCursorType | null; + leftButtonDown?: boolean; + leftButtonPressed?: boolean; + leftButtonReleased?: boolean; + bounds?: { + x: number; + y: number; + width: number; + height: number; + } | null; + asset: WindowsCursorAssetPayload | null; +} + +export interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +export interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +export interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; + cursorType?: NativeCursorType | null; +} + +export type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +export interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + sourceId?: string | null; + startTimeMs?: number; +} diff --git a/electron/native-bridge/cursor/telemetryCursorAdapter.ts b/electron/native-bridge/cursor/telemetryCursorAdapter.ts new file mode 100644 index 000000000..073b18316 --- /dev/null +++ b/electron/native-bridge/cursor/telemetryCursorAdapter.ts @@ -0,0 +1,49 @@ +import type { CursorCapabilities, CursorRecordingData } from "../../../src/native/contracts"; +import type { CursorNativeAdapter, CursorTelemetryLoadResult } from "./adapter"; + +interface TelemetryCursorAdapterOptions { + loadRecordingData: (videoPath: string) => Promise; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadTelemetry: (videoPath: string) => Promise; +} + +export class TelemetryCursorAdapter implements CursorNativeAdapter { + readonly kind = "none" as const; + + constructor(private readonly options: TelemetryCursorAdapterOptions) {} + + async getCapabilities(): Promise { + return { + telemetry: true, + systemAssets: false, + provider: this.kind, + }; + } + + async getRecordingData(videoPath?: string | null): Promise { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + version: 2, + provider: this.kind, + samples: [], + assets: [], + }; + } + + return this.options.loadRecordingData(resolvedVideoPath); + } + + async getTelemetry(videoPath?: string | null) { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + success: false, + message: "No video path is available for cursor telemetry", + samples: [], + } satisfies CursorTelemetryLoadResult; + } + + return this.options.loadTelemetry(resolvedVideoPath); + } +} diff --git a/electron/native-bridge/services/cursorService.ts b/electron/native-bridge/services/cursorService.ts new file mode 100644 index 000000000..e3e9a2552 --- /dev/null +++ b/electron/native-bridge/services/cursorService.ts @@ -0,0 +1,46 @@ +import type { + CursorCapabilities, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; +import type { CursorNativeAdapter } from "../cursor/adapter"; +import type { NativeBridgeStateStore } from "../store"; + +interface CursorServiceOptions { + store: NativeBridgeStateStore; + adapter: CursorNativeAdapter; +} + +export class CursorService { + constructor(private readonly options: CursorServiceOptions) {} + + async getCapabilities(): Promise { + const capabilities = await this.options.adapter.getCapabilities(); + this.options.store.setCursorCapabilities(capabilities); + return capabilities; + } + + async getTelemetry(videoPath?: string | null): Promise { + const result = await this.options.adapter.getTelemetry(videoPath); + if (!result.success) { + throw new Error(result.message || result.error || "Failed to load cursor telemetry"); + } + + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, result.samples.length); + } + + return result.samples; + } + + async getRecordingData(videoPath?: string | null): Promise { + const data = await this.options.adapter.getRecordingData(videoPath); + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, data.samples.length); + } + + return data; + } +} diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts new file mode 100644 index 000000000..965b4fb70 --- /dev/null +++ b/electron/native-bridge/services/projectService.ts @@ -0,0 +1,80 @@ +import type { + ProjectContext, + ProjectFileResult, + ProjectPathResult, +} from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface ProjectServiceOptions { + store: NativeBridgeStateStore; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; +} + +export class ProjectService { + constructor(private readonly options: ProjectServiceOptions) {} + + getCurrentContext(): ProjectContext { + const context = { + currentProjectPath: this.options.getCurrentProjectPath(), + currentVideoPath: this.options.getCurrentVideoPath(), + }; + + this.options.store.setProjectContext(context); + return context; + } + + async saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) { + const result = await this.options.saveProjectFile( + projectData, + suggestedName, + existingProjectPath, + ); + this.getCurrentContext(); + return result; + } + + async loadProjectFile() { + const result = await this.options.loadProjectFile(); + this.getCurrentContext(); + return result; + } + + async loadCurrentProjectFile() { + const result = await this.options.loadCurrentProjectFile(); + this.getCurrentContext(); + return result; + } + + async setCurrentVideoPath(path: string) { + const result = await this.options.setCurrentVideoPath(path); + this.getCurrentContext(); + return result; + } + + getCurrentVideoPath() { + const result = this.options.getCurrentVideoPathResult(); + this.getCurrentContext(); + return result; + } + + clearCurrentVideoPath() { + const result = this.options.clearCurrentVideoPath(); + this.getCurrentContext(); + return result; + } +} diff --git a/electron/native-bridge/services/systemService.ts b/electron/native-bridge/services/systemService.ts new file mode 100644 index 000000000..50eff2838 --- /dev/null +++ b/electron/native-bridge/services/systemService.ts @@ -0,0 +1,43 @@ +import type { + CursorCapabilities, + NativePlatform, + SystemCapabilities, +} from "../../../src/native/contracts"; +import { NATIVE_BRIDGE_VERSION } from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface SystemServiceOptions { + store: NativeBridgeStateStore; + getPlatform: () => NativePlatform; + getAssetBasePath: () => string | null; + getCursorCapabilities: () => Promise; +} + +export class SystemService { + constructor(private readonly options: SystemServiceOptions) {} + + getPlatform() { + return this.options.getPlatform(); + } + + getAssetBasePath() { + return this.options.getAssetBasePath(); + } + + async getCapabilities(): Promise { + const platform = this.getPlatform(); + const cursorCapabilities = await this.options.getCursorCapabilities(); + + const capabilities: SystemCapabilities = { + bridgeVersion: NATIVE_BRIDGE_VERSION, + platform, + cursor: cursorCapabilities, + project: { + currentContext: true, + }, + }; + + this.options.store.setSystemCapabilities(capabilities); + return capabilities; + } +} diff --git a/electron/native-bridge/store.ts b/electron/native-bridge/store.ts new file mode 100644 index 000000000..dcdbed154 --- /dev/null +++ b/electron/native-bridge/store.ts @@ -0,0 +1,88 @@ +import type { + CursorCapabilities, + NativePlatform, + ProjectContext, + SystemCapabilities, +} from "../../src/native/contracts"; + +export interface NativeBridgeState { + system: { + platform: NativePlatform; + capabilities: SystemCapabilities | null; + }; + project: ProjectContext; + cursor: { + capabilities: CursorCapabilities | null; + lastTelemetryLoad: { + videoPath: string; + sampleCount: number; + loadedAt: number; + } | null; + }; +} + +export class NativeBridgeStateStore { + private state: NativeBridgeState; + + constructor(platform: NativePlatform) { + this.state = { + system: { + platform, + capabilities: null, + }, + project: { + currentProjectPath: null, + currentVideoPath: null, + }, + cursor: { + capabilities: null, + lastTelemetryLoad: null, + }, + }; + } + + getState() { + return this.state; + } + + setProjectContext(project: ProjectContext) { + this.state = { + ...this.state, + project, + }; + } + + setSystemCapabilities(capabilities: SystemCapabilities) { + this.state = { + ...this.state, + system: { + ...this.state.system, + capabilities, + }, + }; + } + + setCursorCapabilities(capabilities: CursorCapabilities) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + capabilities, + }, + }; + } + + markCursorTelemetryLoaded(videoPath: string, sampleCount: number) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + lastTelemetryLoad: { + videoPath, + sampleCount, + loadedAt: Date.now(), + }, + }, + }; + } +} diff --git a/electron/native/README.md b/electron/native/README.md new file mode 100644 index 000000000..659829cd9 --- /dev/null +++ b/electron/native/README.md @@ -0,0 +1,78 @@ +# Native capture helpers + +Windows native recording is resolved from one of these locations: + +1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/wgc-capture/build/wgc-capture.exe`, for a locally built Ninja helper. +3. `electron/native/wgc-capture/build/Release/wgc-capture.exe`, for a locally built multi-config helper. +4. `electron/native/bin/win32-x64/wgc-capture.exe` or `electron/native/bin/win32-arm64/wgc-capture.exe`, for packaged prebuilt helpers. + +Build the Windows helper with: + +```powershell +npm run build:native:win +``` + +The build writes the CMake output to `electron/native/wgc-capture/build/wgc-capture.exe` and copies the redistributable binary to `electron/native/bin/win32-x64/wgc-capture.exe`. + +The helper contract is process-based: the app starts the process with one JSON argument and sends commands on stdin. `stop\n` finalizes the recording. During migration the helper prints both newline-delimited JSON events and the legacy text messages `Recording started` / `Recording stopped. Output path: `. + +Current V2 JSON shape: + +```json +{ + "schemaVersion": 2, + "recordingId": 123, + "sourceType": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowHandle": null, + "outputPath": "C:\\path\\recording-123.mp4", + "videoWidth": 1920, + "videoHeight": 1080, + "fps": 60, + "captureSystemAudio": false, + "captureMic": false, + "microphoneDeviceId": "default", + "microphoneDeviceName": "Microphone (NVIDIA Broadcast)", + "microphoneGain": 1.4, + "webcamEnabled": true, + "webcamDeviceId": "default", + "webcamDeviceName": "Camera (NVIDIA Broadcast)", + "webcamWidth": 1280, + "webcamHeight": 720, + "webcamFps": 30, + "outputs": { + "screenPath": "C:\\path\\recording-123.mp4" + } +} +``` + +The current helper implementation supports display/window video capture, system audio loopback, selected-microphone capture, Media Foundation webcam capture, and a DirectShow webcam fallback for virtual cameras that are not exposed through Media Foundation. Webcam frames are currently composed into the primary MP4 as a bottom-right picture-in-picture overlay. Browser `deviceId` values do not always map to Media Foundation symbolic links or WASAPI endpoint IDs, so the renderer passes both browser IDs and user-visible device names. For microphones, the helper tries the requested WASAPI endpoint ID first, then resolves an active capture endpoint by `microphoneDeviceName`, then falls back to the default endpoint. For webcams, Electron resolves a matching DirectShow filter CLSID for the selected label; the helper uses Media Foundation first, then that exact DirectShow filter when the requested camera is absent from Media Foundation. + +Smoke-test the helper with: + +```powershell +npm run test:wgc-helper:win +npm run test:wgc-window:win +npm run test:wgc-audio:win +npm run test:wgc-mic:win +npm run test:wgc-mixed-audio:win +npm run test:wgc-webcam:win +``` + +To validate a specific native webcam manually: + +```powershell +$env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME = "NVIDIA Broadcast" +npm run test:wgc-webcam:win +Remove-Item Env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME +``` + +To validate a specific native microphone manually: + +```powershell +$env:OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME = "Microphone (NVIDIA Broadcast)" +npm run test:wgc-mic:win +Remove-Item Env:OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME +``` diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt new file mode 100644 index 000000000..750365816 --- /dev/null +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.20) + +# The local Windows SDK image used by some contributors can miss gdi32.lib, +# while CMake's default MSVC console template links it unconditionally. This +# helper does not use GDI, so keep the standard library set minimal and explicit. +set(CMAKE_CXX_STANDARD_LIBRARIES + "kernel32.lib user32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib" + CACHE STRING "" FORCE) + +project(openscreen-wgc-capture LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +add_executable(wgc-capture + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/dshow_webcam_capture.cpp + src/dshow_webcam_capture.h + src/main.cpp + src/mf_encoder.cpp + src/mf_encoder.h + src/monitor_utils.cpp + src/monitor_utils.h + src/wasapi_loopback_capture.cpp + src/wasapi_loopback_capture.h + src/webcam_capture.cpp + src/webcam_capture.h + src/wgc_session.cpp + src/wgc_session.h +) + +target_compile_definitions(wgc-capture PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(wgc-capture PRIVATE + d3d11 + dxgi + mf + mfplat + mfreadwrite + mfuuid + runtimeobject + windowsapp +) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp new file mode 100644 index 000000000..a8058c9f6 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -0,0 +1,409 @@ +#include "audio_sample_utils.h" + +#include + +#include +#include +#include +#include +#include + +namespace { + +bool isFloatFormat(const AudioInputFormat& format) { + return format.subtype == MFAudioFormat_Float && format.bitsPerSample == 32; +} + +bool isPcmFormat(const AudioInputFormat& format, UINT32 bitsPerSample) { + return format.subtype == MFAudioFormat_PCM && format.bitsPerSample == bitsPerSample; +} + +template +T clampTo(double value) { + const double minValue = static_cast(std::numeric_limits::min()); + const double maxValue = static_cast(std::numeric_limits::max()); + return static_cast(std::clamp(std::round(value), minValue, maxValue)); +} + +size_t bytesPerSample(const AudioInputFormat& format) { + return format.bitsPerSample / 8; +} + +double readSampleAsDouble(const BYTE* source, const AudioInputFormat& format, size_t frameIndex, UINT32 channelIndex) { + if (!source || format.blockAlign == 0 || channelIndex >= format.channels) { + return 0.0; + } + + const size_t offset = frameIndex * format.blockAlign + channelIndex * bytesPerSample(format); + if (isFloatFormat(format)) { + return static_cast(*reinterpret_cast(source + offset)); + } + if (isPcmFormat(format, 16)) { + return static_cast(*reinterpret_cast(source + offset)) / 32768.0; + } + if (isPcmFormat(format, 32)) { + return static_cast(*reinterpret_cast(source + offset)) / 2147483648.0; + } + return 0.0; +} + +void writeSampleFromDouble(BYTE* destination, const AudioInputFormat& format, size_t frameIndex, UINT32 channelIndex, double value) { + if (!destination || format.blockAlign == 0 || channelIndex >= format.channels) { + return; + } + + const double clamped = std::clamp(value, -1.0, 1.0); + const size_t offset = frameIndex * format.blockAlign + channelIndex * bytesPerSample(format); + if (isFloatFormat(format)) { + *reinterpret_cast(destination + offset) = static_cast(clamped); + return; + } + if (isPcmFormat(format, 16)) { + *reinterpret_cast(destination + offset) = clampTo(clamped * 32767.0); + return; + } + if (isPcmFormat(format, 32)) { + *reinterpret_cast(destination + offset) = clampTo(clamped * 2147483647.0); + } +} + +double readMappedChannel(const BYTE* source, const AudioInputFormat& format, size_t frameIndex, UINT32 targetChannel, UINT32 targetChannels) { + if (format.channels == 0) { + return 0.0; + } + if (format.channels == targetChannels && targetChannel < format.channels) { + return readSampleAsDouble(source, format, frameIndex, targetChannel); + } + if (format.channels == 1) { + return readSampleAsDouble(source, format, frameIndex, 0); + } + if (targetChannels == 1) { + double sum = 0.0; + for (UINT32 channel = 0; channel < format.channels; ++channel) { + sum += readSampleAsDouble(source, format, frameIndex, channel); + } + return sum / static_cast(format.channels); + } + return readSampleAsDouble(source, format, frameIndex, std::min(targetChannel, format.channels - 1)); +} + +} // namespace + +constexpr int64_t HnsPerSecond = 10'000'000; + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right) { + return left.subtype == right.subtype && + left.sampleRate == right.sampleRate && + left.channels == right.channels && + left.bitsPerSample == right.bitsPerSample && + left.blockAlign == right.blockAlign && + left.avgBytesPerSec == right.avgBytesPerSec; +} + +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination) { + destination.resize(byteCount); + if (!source || byteCount == 0) { + std::fill(destination.begin(), destination.end(), static_cast(0)); + return; + } + + if (std::abs(gain - 1.0) < 0.0001) { + std::memcpy(destination.data(), source, byteCount); + return; + } + + if (isFloatFormat(format)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(input[index] * gain, -1.0, 1.0)); + } + return; + } + + if (isPcmFormat(format, 16)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + if (isPcmFormat(format, 32)) { + const auto* input = reinterpret_cast(source); + auto* output = reinterpret_cast(destination.data()); + const size_t sampleCount = byteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo(static_cast(input[index]) * gain); + } + return; + } + + std::memcpy(destination.data(), source, byteCount); +} + +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination) { + if (!source || byteCount == 0 || sourceFormat.blockAlign == 0 || targetFormat.blockAlign == 0 || + sourceFormat.sampleRate == 0 || targetFormat.sampleRate == 0 || sourceFormat.channels == 0 || + targetFormat.channels == 0) { + destination.clear(); + return; + } + + if (sameAudioFormatForMixing(sourceFormat, targetFormat)) { + copyAudioWithGain(source, byteCount, targetFormat, gain, destination); + return; + } + + const size_t sourceFrames = byteCount / sourceFormat.blockAlign; + if (sourceFrames == 0) { + destination.clear(); + return; + } + + const double rateRatio = static_cast(targetFormat.sampleRate) / + static_cast(sourceFormat.sampleRate); + const size_t targetFrames = std::max(1, static_cast(std::llround(sourceFrames * rateRatio))); + destination.assign(targetFrames * targetFormat.blockAlign, 0); + + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + const double sourcePosition = static_cast(targetFrame) / rateRatio; + const size_t sourceFrame = std::min( + sourceFrames - 1, + static_cast(std::llround(sourcePosition))); + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + const double sample = readMappedChannel( + source, + sourceFormat, + sourceFrame, + channel, + targetFormat.channels); + writeSampleFromDouble(destination.data(), targetFormat, targetFrame, channel, sample * gain); + } + } +} + +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format) { + if (!source || byteCount == 0 || destination.empty()) { + return; + } + + const size_t mixByteCount = std::min(destination.size(), static_cast(byteCount)); + + if (isFloatFormat(format)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(float); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = static_cast(std::clamp(output[index] + input[index], -1.0f, 1.0f)); + } + return; + } + + if (isPcmFormat(format, 16)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int16_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + return; + } + + if (isPcmFormat(format, 32)) { + auto* output = reinterpret_cast(destination.data()); + const auto* input = reinterpret_cast(source); + const size_t sampleCount = mixByteCount / sizeof(int32_t); + for (size_t index = 0; index < sampleCount; index += 1) { + output[index] = clampTo( + static_cast(output[index]) + static_cast(input[index])); + } + } +} + +AudioMixer::AudioMixer( + const AudioInputFormat& format, + const AudioInputFormat& systemFormat, + const AudioInputFormat& microphoneFormat, + bool includeSystem, + bool includeMicrophone, + double microphoneGain, + OutputCallback output) + : format_(format), + systemFormat_(systemFormat), + microphoneFormat_(microphoneFormat), + includeSystem_(includeSystem), + includeMicrophone_(includeMicrophone), + microphoneGain_(microphoneGain), + output_(std::move(output)) {} + +AudioMixer::~AudioMixer() { + stop(); +} + +bool AudioMixer::start() { + if (!output_ || format_.sampleRate == 0 || format_.blockAlign == 0) { + return false; + } + + stopRequested_ = false; + emittedFrames_ = 0; + timelineStarted_ = false; + thread_ = std::thread([this] { + mixLoop(); + }); + return true; +} + +void AudioMixer::beginTimeline() { + { + std::scoped_lock lock(mutex_); + systemQueue_.clear(); + microphoneQueue_.clear(); + emittedFrames_ = 0; + timelineStarted_ = true; + } + cv_.notify_all(); +} + +void AudioMixer::stop() { + stopRequested_ = true; + cv_.notify_all(); + if (thread_.joinable()) { + thread_.join(); + } +} + +void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { + if (!includeSystem_ || stopRequested_) { + return; + } + + { + std::scoped_lock lock(mutex_); + append(systemQueue_, data, byteCount, systemFormat_, 1.0); + } + cv_.notify_all(); +} + +void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { + if (!includeMicrophone_ || stopRequested_) { + return; + } + + { + std::scoped_lock lock(mutex_); + append(microphoneQueue_, data, byteCount, microphoneFormat_, microphoneGain_); + } + cv_.notify_all(); +} + +void AudioMixer::append( + std::vector& queue, + const BYTE* data, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + double gain) { + if (!data || byteCount == 0) { + return; + } + + convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_); + queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); +} + +bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t byteCount) { + if (queue.empty()) { + chunk.assign(byteCount, 0); + return false; + } + + chunk.assign(byteCount, 0); + const size_t copiedBytes = std::min(byteCount, queue.size()); + std::memcpy(chunk.data(), queue.data(), copiedBytes); + queue.erase(queue.begin(), queue.begin() + static_cast(copiedBytes)); + return copiedBytes > 0; +} + +void AudioMixer::mixLoop() { + const uint32_t chunkFrames = std::max(1, format_.sampleRate / 100); + const size_t chunkBytes = static_cast(chunkFrames) * format_.blockAlign; + std::vector mixedChunk; + std::vector sourceChunk; + std::chrono::steady_clock::time_point audioClockStart; + bool audioClockStarted = false; + + while (true) { + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { + const bool hasSystem = !includeSystem_ || systemQueue_.size() >= chunkBytes; + const bool hasMicrophone = !includeMicrophone_ || microphoneQueue_.size() >= chunkBytes; + const bool hasAnySource = !systemQueue_.empty() || !microphoneQueue_.empty(); + return stopRequested_.load() || + (timelineStarted_ && (hasSystem || hasMicrophone) && hasAnySource); + }); + + if (stopRequested_) { + break; + } + if (!timelineStarted_) { + continue; + } + + const bool hasAnyQueuedAudio = !systemQueue_.empty() || !microphoneQueue_.empty(); + if (!hasAnyQueuedAudio) { + continue; + } + + mixedChunk.assign(chunkBytes, 0); + if (includeSystem_) { + pop(systemQueue_, sourceChunk, chunkBytes); + mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); + } + if (includeMicrophone_) { + pop(microphoneQueue_, sourceChunk, chunkBytes); + mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); + } + } + + if (!audioClockStarted) { + audioClockStart = std::chrono::steady_clock::now(); + audioClockStarted = true; + } + + const int64_t timestampHns = + static_cast((emittedFrames_ * HnsPerSecond) / format_.sampleRate); + const int64_t durationHns = + static_cast((static_cast(chunkFrames) * HnsPerSecond) / format_.sampleRate); + if (!output_(mixedChunk.data(), static_cast(mixedChunk.size()), timestampHns, durationHns)) { + stopRequested_ = true; + break; + } + emittedFrames_ += chunkFrames; + + const auto nextDeadline = audioClockStart + + std::chrono::duration_cast( + std::chrono::duration(static_cast(emittedFrames_) / format_.sampleRate)); + std::this_thread::sleep_until(nextDeadline); + } +} diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h new file mode 100644 index 000000000..81fc62dda --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -0,0 +1,84 @@ +#pragma once + +#include "mf_encoder.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputFormat& right); +void copyAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format, + double gain, + std::vector& destination); +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination); +void mixAudioInPlace( + std::vector& destination, + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& format); + +class AudioMixer { +public: + using OutputCallback = std::function; + + AudioMixer( + const AudioInputFormat& format, + const AudioInputFormat& systemFormat, + const AudioInputFormat& microphoneFormat, + bool includeSystem, + bool includeMicrophone, + double microphoneGain, + OutputCallback output); + ~AudioMixer(); + + AudioMixer(const AudioMixer&) = delete; + AudioMixer& operator=(const AudioMixer&) = delete; + + bool start(); + void beginTimeline(); + void stop(); + void pushSystem(const BYTE* data, DWORD byteCount); + void pushMicrophone(const BYTE* data, DWORD byteCount); + +private: + void append( + std::vector& queue, + const BYTE* data, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + double gain); + bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); + void mixLoop(); + + AudioInputFormat format_{}; + AudioInputFormat systemFormat_{}; + AudioInputFormat microphoneFormat_{}; + bool includeSystem_ = false; + bool includeMicrophone_ = false; + double microphoneGain_ = 1.0; + OutputCallback output_; + std::mutex mutex_; + std::condition_variable cv_; + std::vector systemQueue_; + std::vector microphoneQueue_; + std::vector gainBuffer_; + std::thread thread_; + std::atomic stopRequested_ = false; + bool timelineStarted_ = false; + uint64_t emittedFrames_ = 0; +}; diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp new file mode 100644 index 000000000..14cb888c9 --- /dev/null +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -0,0 +1,312 @@ +#include "dshow_webcam_capture.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +const CLSID CLSID_SampleGrabberLocal = {0xC1F400A0, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; +const CLSID CLSID_NullRendererLocal = {0xC1F400A4, 0x3F08, 0x11D3, {0x9F, 0x0B, 0x00, 0x60, 0x08, 0x03, 0x9E, 0x37}}; + +MIDL_INTERFACE("0579154A-2B53-4994-B0D0-E773148EFF85") +ISampleGrabberCB : public IUnknown { +public: + virtual HRESULT STDMETHODCALLTYPE SampleCB(double sampleTime, IMediaSample* sample) = 0; + virtual HRESULT STDMETHODCALLTYPE BufferCB(double sampleTime, BYTE* buffer, long bufferLength) = 0; +}; + +MIDL_INTERFACE("6B652FFF-11FE-4FCE-92AD-0266B5D7C78F") +ISampleGrabber : public IUnknown { +public: + virtual HRESULT STDMETHODCALLTYPE SetOneShot(BOOL oneShot) = 0; + virtual HRESULT STDMETHODCALLTYPE SetMediaType(const AM_MEDIA_TYPE* type) = 0; + virtual HRESULT STDMETHODCALLTYPE GetConnectedMediaType(AM_MEDIA_TYPE* type) = 0; + virtual HRESULT STDMETHODCALLTYPE SetBufferSamples(BOOL bufferThem) = 0; + virtual HRESULT STDMETHODCALLTYPE GetCurrentBuffer(long* bufferSize, long* buffer) = 0; + virtual HRESULT STDMETHODCALLTYPE GetCurrentSample(IMediaSample** sample) = 0; + virtual HRESULT STDMETHODCALLTYPE SetCallback(ISampleGrabberCB* callback, long whichMethodToCallback) = 0; +}; + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +void freeMediaType(AM_MEDIA_TYPE& type) { + if (type.cbFormat != 0) { + CoTaskMemFree(type.pbFormat); + type.cbFormat = 0; + type.pbFormat = nullptr; + } + if (type.pUnk) { + type.pUnk->Release(); + type.pUnk = nullptr; + } +} + +} // namespace + +struct DirectShowWebcamCapture::Impl { + Microsoft::WRL::ComPtr graph; + Microsoft::WRL::ComPtr captureGraph; + Microsoft::WRL::ComPtr captureFilter; + Microsoft::WRL::ComPtr sampleGrabberFilter; + Microsoft::WRL::ComPtr sampleGrabber; + Microsoft::WRL::ComPtr nullRenderer; + Microsoft::WRL::ComPtr mediaControl; + bool comInitialized = false; + bool running = false; +}; + +DirectShowWebcamCapture::~DirectShowWebcamCapture() { + stop(); + delete impl_; +} + +bool DirectShowWebcamCapture::initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps) { + (void)deviceId; + stop(); + delete impl_; + impl_ = nullptr; + impl_ = new Impl(); + fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); + + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (SUCCEEDED(hr)) { + impl_->comInitialized = true; + } else if (hr != RPC_E_CHANGED_MODE) { + return succeeded(hr, "CoInitializeEx(DirectShow webcam)"); + } + + if (directShowClsid.empty()) { + std::cerr << "ERROR: DirectShow webcam fallback requires a resolved filter CLSID" << std::endl; + return false; + } + + CLSID selectedClsid{}; + if (FAILED(CLSIDFromString(directShowClsid.c_str(), &selectedClsid))) { + std::cerr << "ERROR: DirectShow webcam fallback received an invalid filter CLSID" << std::endl; + return false; + } + selectedDeviceName_ = deviceName.empty() ? directShowClsid : deviceName; + + if (!succeeded(CoCreateInstance(selectedClsid, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->captureFilter)), + "CoCreateInstance(DirectShow webcam filter)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_FilterGraph, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->graph)), + "CoCreateInstance(FilterGraph)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_CaptureGraphBuilder2, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->captureGraph)), + "CoCreateInstance(CaptureGraphBuilder2)")) { + return false; + } + if (!succeeded(impl_->captureGraph->SetFiltergraph(impl_->graph.Get()), "SetFiltergraph(DirectShow webcam)")) { + return false; + } + if (!succeeded(impl_->graph->AddFilter(impl_->captureFilter.Get(), L"OpenScreen Webcam Source"), + "AddFilter(DirectShow webcam source)")) { + return false; + } + + if (!succeeded(CoCreateInstance(CLSID_SampleGrabberLocal, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->sampleGrabberFilter)), + "CoCreateInstance(SampleGrabber)")) { + return false; + } + if (!succeeded(impl_->sampleGrabberFilter.As(&impl_->sampleGrabber), "QueryInterface(ISampleGrabber)")) { + return false; + } + + AM_MEDIA_TYPE requestedType{}; + requestedType.majortype = MEDIATYPE_Video; + requestedType.subtype = MEDIASUBTYPE_RGB32; + requestedType.formattype = FORMAT_VideoInfo; + if (!succeeded(impl_->sampleGrabber->SetMediaType(&requestedType), "SetMediaType(DirectShow RGB32)")) { + return false; + } + + if (!succeeded(impl_->graph->AddFilter(impl_->sampleGrabberFilter.Get(), L"OpenScreen Webcam Sample Grabber"), + "AddFilter(SampleGrabber)")) { + return false; + } + if (!succeeded(CoCreateInstance(CLSID_NullRendererLocal, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&impl_->nullRenderer)), + "CoCreateInstance(NullRenderer)")) { + return false; + } + if (!succeeded(impl_->graph->AddFilter(impl_->nullRenderer.Get(), L"OpenScreen Webcam Null Renderer"), + "AddFilter(NullRenderer)")) { + return false; + } + + if (!succeeded(impl_->captureGraph->RenderStream( + &PIN_CATEGORY_CAPTURE, + &MEDIATYPE_Video, + impl_->captureFilter.Get(), + impl_->sampleGrabberFilter.Get(), + impl_->nullRenderer.Get()), + "RenderStream(DirectShow webcam)")) { + return false; + } + + AM_MEDIA_TYPE connectedType{}; + if (!succeeded(impl_->sampleGrabber->GetConnectedMediaType(&connectedType), "GetConnectedMediaType(DirectShow webcam)")) { + return false; + } + if (connectedType.formattype == FORMAT_VideoInfo && connectedType.pbFormat) { + const auto* videoInfo = reinterpret_cast(connectedType.pbFormat); + width_ = std::abs(videoInfo->bmiHeader.biWidth); + height_ = std::abs(videoInfo->bmiHeader.biHeight); + sourceTopDown_ = videoInfo->bmiHeader.biHeight < 0; + } + freeMediaType(connectedType); + if (width_ <= 0 || height_ <= 0) { + width_ = requestedWidth > 0 ? requestedWidth : 1280; + height_ = requestedHeight > 0 ? requestedHeight : 720; + } + + impl_->sampleGrabber->SetBufferSamples(TRUE); + impl_->sampleGrabber->SetOneShot(FALSE); + if (!succeeded(impl_->graph.As(&impl_->mediaControl), "QueryInterface(IMediaControl)")) { + return false; + } + + return true; +} + +bool DirectShowWebcamCapture::start() { + if (!impl_ || !impl_->mediaControl || impl_->running) { + return false; + } + HRESULT hr = impl_->mediaControl->Run(); + if (!succeeded(hr, "Run(DirectShow webcam)")) { + return false; + } + stopRequested_ = false; + try { + thread_ = std::thread(&DirectShowWebcamCapture::captureLoop, this); + } catch (const std::exception& error) { + stopRequested_ = true; + impl_->mediaControl->Stop(); + std::cerr << "ERROR: Failed to start DirectShow webcam capture thread: " << error.what() << std::endl; + return false; + } catch (...) { + stopRequested_ = true; + impl_->mediaControl->Stop(); + std::cerr << "ERROR: Failed to start DirectShow webcam capture thread" << std::endl; + return false; + } + impl_->running = true; + return true; +} + +void DirectShowWebcamCapture::stop() { + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (!impl_) { + return; + } + if (impl_->mediaControl && impl_->running) { + impl_->mediaControl->Stop(); + } + impl_->running = false; + impl_->mediaControl.Reset(); + impl_->nullRenderer.Reset(); + impl_->sampleGrabber.Reset(); + impl_->sampleGrabberFilter.Reset(); + impl_->captureFilter.Reset(); + impl_->captureGraph.Reset(); + impl_->graph.Reset(); + if (impl_->comInitialized) { + CoUninitialize(); + impl_->comInitialized = false; + } +} + +void DirectShowWebcamCapture::captureLoop() { + const HRESULT coinitHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + while (!stopRequested_ && impl_ && impl_->sampleGrabber) { + long bufferSize = 0; + HRESULT hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, nullptr); + if (SUCCEEDED(hr) && bufferSize > 0) { + std::vector buffer(static_cast(bufferSize)); + hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, reinterpret_cast(buffer.data())); + if (SUCCEEDED(hr)) { + storeFrame(buffer.data(), bufferSize); + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1000 / std::max(1, fps_))); + } + if (SUCCEEDED(coinitHr)) { + CoUninitialize(); + } +} + +void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { + const int stride = width_ * 4; + const int expectedLength = stride * height_; + if (!buffer || length < expectedLength || width_ <= 0 || height_ <= 0) { + return; + } + + std::vector frame(static_cast(expectedLength)); + for (int y = 0; y < height_; y += 1) { + const int sourceY = sourceTopDown_ ? y : height_ - 1 - y; + const BYTE* source = buffer + sourceY * stride; + BYTE* destination = frame.data() + y * stride; + std::copy(source, source + stride, destination); + for (int x = 0; x < width_; x += 1) { + destination[x * 4 + 3] = 255; + } + } + + std::scoped_lock lock(frameMutex_); + latestFrame_ = std::move(frame); +} + +bool DirectShowWebcamCapture::copyLatestFrame(std::vector& destination, int& width, int& height) { + std::scoped_lock lock(frameMutex_); + if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { + return false; + } + + destination = latestFrame_; + width = width_; + height = height_; + return true; +} + +int DirectShowWebcamCapture::width() const { + return width_; +} + +int DirectShowWebcamCapture::height() const { + return height_; +} + +int DirectShowWebcamCapture::fps() const { + return fps_; +} + +const std::wstring& DirectShowWebcamCapture::selectedDeviceName() const { + return selectedDeviceName_; +} diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h new file mode 100644 index 000000000..906da8fa8 --- /dev/null +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +class DirectShowWebcamCapture { +public: + DirectShowWebcamCapture() = default; + ~DirectShowWebcamCapture(); + + DirectShowWebcamCapture(const DirectShowWebcamCapture&) = delete; + DirectShowWebcamCapture& operator=(const DirectShowWebcamCapture&) = delete; + + bool initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps); + bool start(); + void stop(); + bool copyLatestFrame(std::vector& destination, int& width, int& height); + + int width() const; + int height() const; + int fps() const; + const std::wstring& selectedDeviceName() const; + void storeFrame(const BYTE* buffer, long length); + +private: + struct Impl; + void captureLoop(); + + Impl* impl_ = nullptr; + std::thread thread_; + std::atomic stopRequested_ = false; + std::mutex frameMutex_; + std::vector latestFrame_; + int width_ = 0; + int height_ = 0; + int fps_ = 30; + bool sourceTopDown_ = false; + std::wstring selectedDeviceName_; +}; diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp new file mode 100644 index 000000000..ad46837e4 --- /dev/null +++ b/electron/native/wgc-capture/src/main.cpp @@ -0,0 +1,751 @@ +#include "audio_sample_utils.h" +#include "mf_encoder.h" +#include "monitor_utils.h" +#include "wasapi_loopback_capture.h" +#include "webcam_capture.h" +#include "wgc_session.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct CaptureConfig { + int schemaVersion = 1; + int64_t displayId = 0; + int64_t recordingId = 0; + std::string sourceType = "display"; + std::string sourceId; + std::string windowHandle; + std::string outputPath; + int fps = 60; + int width = 0; + int height = 0; + MonitorBounds bounds{}; + bool hasDisplayBounds = false; + bool captureSystemAudio = false; + bool captureMic = false; + bool captureCursor = false; + bool webcamEnabled = false; + std::string microphoneDeviceId; + std::string microphoneDeviceName; + double microphoneGain = 1.0; + std::string webcamDeviceId; + std::string webcamDeviceName; + std::string webcamDirectShowClsid; + int webcamWidth = 0; + int webcamHeight = 0; + int webcamFps = 0; +}; + +std::wstring utf8ToWide(const std::string& value) { + if (value.empty()) { + return {}; + } + + const int size = MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0); + std::wstring result(static_cast(size), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size); + return result; +} + +std::string wideToUtf8(const std::wstring& value) { + if (value.empty()) { + return {}; + } + + const int size = WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0, nullptr, nullptr); + std::string result(static_cast(size), '\0'); + WideCharToMultiByte(CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size, nullptr, nullptr); + return result; +} + +std::string jsonEscape(const std::string& value) { + std::string result; + result.reserve(value.size()); + for (const char c : value) { + switch (c) { + case '\\': + result += "\\\\"; + break; + case '"': + result += "\\\""; + break; + case '\n': + result += "\\n"; + break; + case '\r': + result += "\\r"; + break; + case '\t': + result += "\\t"; + break; + default: + result.push_back(c); + break; + } + } + return result; +} + +bool hasVisibleBgraContent(const std::vector& frame) { + if (frame.size() < 4) { + return false; + } + + uint64_t lumaTotal = 0; + BYTE maxLuma = 0; + const size_t pixelCount = frame.size() / 4; + const size_t step = std::max(1, pixelCount / 4096); + size_t sampledPixels = 0; + for (size_t pixel = 0; pixel < pixelCount; pixel += step) { + const size_t offset = pixel * 4; + const BYTE b = frame[offset + 0]; + const BYTE g = frame[offset + 1]; + const BYTE r = frame[offset + 2]; + const BYTE luma = static_cast((static_cast(r) * 54 + static_cast(g) * 183 + static_cast(b) * 19) >> 8); + lumaTotal += luma; + maxLuma = std::max(maxLuma, luma); + sampledPixels += 1; + } + + const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0; + return maxLuma > 24 || averageLuma > 4; +} + +bool findBool(const std::string& json, const std::string& key, bool fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (json.compare(pos, 4, "true") == 0) { + return true; + } + if (json.compare(pos, 5, "false") == 0) { + return false; + } + return fallback; +} + +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +int findInt(const std::string& json, const std::string& key, int fallback) { + return static_cast(findInt64(json, key, fallback)); +} + +double findDouble(const std::string& json, const std::string& key, double fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stod(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +std::string findString(const std::string& json, const std::string& key) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return {}; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return {}; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (pos >= json.size() || json[pos] != '"') { + return {}; + } + pos += 1; + + std::string result; + while (pos < json.size()) { + const char c = json[pos++]; + if (c == '"') { + break; + } + if (c == '\\' && pos < json.size()) { + const char escaped = json[pos++]; + switch (escaped) { + case '\\': + case '"': + case '/': + result.push_back(escaped); + break; + case 'n': + result.push_back('\n'); + break; + case 'r': + result.push_back('\r'); + break; + case 't': + result.push_back('\t'); + break; + default: + result.push_back(escaped); + break; + } + continue; + } + result.push_back(c); + } + return result; +} + +std::string parseWindowHandleFromSourceId(const std::string& sourceId) { + constexpr char prefix[] = "window:"; + if (sourceId.rfind(prefix, 0) != 0) { + return {}; + } + + const size_t start = sizeof(prefix) - 1; + const size_t end = sourceId.find(':', start); + const std::string handle = sourceId.substr(start, end == std::string::npos ? std::string::npos : end - start); + return handle.empty() ? std::string{} : handle; +} + +HWND parseWindowHandle(const std::string& value) { + if (value.empty()) { + return nullptr; + } + + try { + size_t parsed = 0; + const int base = value.rfind("0x", 0) == 0 || value.rfind("0X", 0) == 0 ? 16 : 10; + const uint64_t handleValue = std::stoull(value, &parsed, base); + if (parsed != value.size() || handleValue == 0) { + return nullptr; + } + return reinterpret_cast(static_cast(handleValue)); + } catch (...) { + return nullptr; + } +} + +bool parseConfig(const std::string& json, CaptureConfig& config) { + config.schemaVersion = findInt(json, "schemaVersion", 1); + config.outputPath = findString(json, "screenPath"); + if (config.outputPath.empty()) { + config.outputPath = findString(json, "outputPath"); + } + if (config.outputPath.empty()) { + return false; + } + + config.recordingId = findInt64(json, "recordingId", 0); + config.sourceType = findString(json, "sourceType"); + if (config.sourceType.empty()) { + config.sourceType = "display"; + } + config.sourceId = findString(json, "sourceId"); + config.windowHandle = findString(json, "windowHandle"); + if (config.windowHandle.empty()) { + config.windowHandle = parseWindowHandleFromSourceId(config.sourceId); + } + config.displayId = findInt64(json, "displayId", 0); + config.fps = std::clamp(findInt(json, "fps", 60), 1, 120); + config.width = findInt(json, "videoWidth", findInt(json, "width", 0)); + config.height = findInt(json, "videoHeight", findInt(json, "height", 0)); + config.bounds.x = findInt(json, "displayX", 0); + config.bounds.y = findInt(json, "displayY", 0); + config.bounds.width = findInt(json, "displayW", 0); + config.bounds.height = findInt(json, "displayH", 0); + config.hasDisplayBounds = findBool(json, "hasDisplayBounds", false); + config.captureSystemAudio = findBool(json, "captureSystemAudio", false); + config.captureMic = findBool(json, "captureMic", false); + config.captureCursor = findBool(json, "captureCursor", false); + config.webcamEnabled = findBool(json, "webcamEnabled", false); + config.microphoneDeviceId = findString(json, "microphoneDeviceId"); + config.microphoneDeviceName = findString(json, "microphoneDeviceName"); + config.microphoneGain = findDouble(json, "microphoneGain", 1.0); + config.webcamDeviceId = findString(json, "webcamDeviceId"); + config.webcamDeviceName = findString(json, "webcamDeviceName"); + config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); + config.webcamWidth = findInt(json, "webcamWidth", 0); + config.webcamHeight = findInt(json, "webcamHeight", 0); + config.webcamFps = findInt(json, "webcamFps", 0); + return true; +} + +void readStopCommands(std::atomic& stopRequested, std::condition_variable& cv) { + std::string line; + while (std::getline(std::cin, line)) { + if (line == "stop" || line == "q" || line == "quit") { + stopRequested = true; + cv.notify_all(); + return; + } + } + stopRequested = true; + cv.notify_all(); +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "ERROR: Missing JSON config argument" << std::endl; + return 1; + } + + winrt::init_apartment(winrt::apartment_type::multi_threaded); + + CaptureConfig config; + if (!parseConfig(argv[1], config)) { + std::cerr << "ERROR: Failed to parse config JSON" << std::endl; + return 1; + } + + std::cout << "{\"event\":\"ready\",\"schemaVersion\":2}" << std::endl; + + WgcSession session; + if (config.sourceType == "display") { + HMONITOR monitor = findMonitorForCapture( + config.displayId, + config.hasDisplayBounds ? &config.bounds : nullptr); + if (!monitor) { + std::cerr << "ERROR: Could not resolve monitor" << std::endl; + return 1; + } + if (!session.initialize(monitor, config.fps, config.captureCursor)) { + std::cerr << "ERROR: Failed to initialize WGC display session" << std::endl; + return 1; + } + } else if (config.sourceType == "window") { + HWND window = parseWindowHandle(config.windowHandle); + if (!window || !IsWindow(window)) { + std::cerr << "ERROR: Native window capture requires a valid HWND" << std::endl; + return 1; + } + if (!session.initialize(window, config.fps, config.captureCursor)) { + std::cerr << "ERROR: Failed to initialize WGC window session" << std::endl; + return 1; + } + } else { + std::cerr << "ERROR: Unsupported native capture source type: " << config.sourceType << std::endl; + return 1; + } + + // WGC owns the captured texture size. Encoding must use that exact size + // until a dedicated GPU scaling pass is introduced; CopyResource requires + // matching resource dimensions. + int width = session.captureWidth(); + int height = session.captureHeight(); + width = (std::max(2, width) / 2) * 2; + height = (std::max(2, height) / 2) * 2; + + const int pixels = width * height; + const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000; + + WebcamCapture webcamCapture; + bool webcamActive = false; + if (config.webcamEnabled) { + if (!webcamCapture.initialize( + utf8ToWide(config.webcamDeviceId), + utf8ToWide(config.webcamDeviceName), + utf8ToWide(config.webcamDirectShowClsid), + config.webcamWidth, + config.webcamHeight, + config.webcamFps > 0 ? config.webcamFps : config.fps)) { + std::cerr << "ERROR: Failed to initialize native webcam capture" << std::endl; + return 1; + } + std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"width\":" << webcamCapture.width() + << ",\"height\":" << webcamCapture.height() + << ",\"fps\":" << webcamCapture.fps() + << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) + << "\"}" << std::endl; + } + + WasapiLoopbackCapture loopbackCapture; + WasapiLoopbackCapture microphoneCapture; + const AudioInputFormat* audioFormat = nullptr; + AudioInputFormat systemAudioFormat{}; + AudioInputFormat microphoneAudioFormat{}; + if (config.captureSystemAudio) { + if (!loopbackCapture.initializeSystemLoopback()) { + std::cerr << "ERROR: Failed to initialize WASAPI loopback capture" << std::endl; + return 1; + } + systemAudioFormat = loopbackCapture.inputFormat(); + audioFormat = &loopbackCapture.inputFormat(); + } + if (config.captureMic) { + if (!microphoneCapture.initializeMicrophone( + utf8ToWide(config.microphoneDeviceId), + utf8ToWide(config.microphoneDeviceName))) { + std::cerr << "ERROR: Failed to initialize WASAPI microphone capture" << std::endl; + return 1; + } + microphoneAudioFormat = microphoneCapture.inputFormat(); + if (!audioFormat) { + audioFormat = µphoneCapture.inputFormat(); + } + } + if (audioFormat) { + std::cout << "{\"event\":\"audio-format\",\"schemaVersion\":2,\"sampleRate\":" << audioFormat->sampleRate + << ",\"channels\":" << audioFormat->channels + << ",\"bitsPerSample\":" << audioFormat->bitsPerSample + << ",\"system\":" << (config.captureSystemAudio ? "true" : "false") + << ",\"microphone\":" << (config.captureMic ? "true" : "false"); + if (config.captureMic) { + std::cout << ",\"microphoneDeviceName\":\"" + << jsonEscape(wideToUtf8(microphoneCapture.selectedDeviceName())) << "\""; + } + std::cout << "}" << std::endl; + } + + MFEncoder encoder; + if (!encoder.initialize( + utf8ToWide(config.outputPath), + width, + height, + config.fps, + bitrate, + session.device(), + session.context(), + audioFormat)) { + std::cerr << "ERROR: Failed to initialize Media Foundation encoder" << std::endl; + return 1; + } + + std::mutex mutex; + std::condition_variable cv; + std::atomic stopRequested = false; + std::atomic firstFrameWritten = false; + std::atomic encodeFailed = false; + Microsoft::WRL::ComPtr latestFrameTexture; + int64_t latestFrameTimestampHns = 0; + int64_t firstFrameTimestampHns = -1; + std::vector latestWebcamFrame; + int latestWebcamWidth = 0; + int latestWebcamHeight = 0; + bool hasVisibleWebcamFrame = false; + + session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { + if (stopRequested) { + return; + } + + std::scoped_lock lock(mutex); + if (!latestFrameTexture) { + D3D11_TEXTURE2D_DESC desc{}; + texture->GetDesc(&desc); + desc.BindFlags = 0; + desc.CPUAccessFlags = 0; + desc.MiscFlags = 0; + if (FAILED(session.device()->CreateTexture2D(&desc, nullptr, &latestFrameTexture))) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + return; + } + } + + session.context()->CopyResource(latestFrameTexture.Get(), texture); + latestFrameTimestampHns = timestampHns; + if (!firstFrameWritten.exchange(true)) { + cv.notify_all(); + } + }); + + auto writeVideoFrames = [&]() { + const auto startedAt = std::chrono::steady_clock::now(); + uint64_t frameIndex = 0; + int64_t lastEncodedVideoTimestampHns = -1; + + while (!stopRequested && !encodeFailed) { + { + std::scoped_lock lock(mutex); + if (webcamActive) { + std::vector candidateWebcamFrame; + int candidateWebcamWidth = 0; + int candidateWebcamHeight = 0; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && + hasVisibleBgraContent(candidateWebcamFrame)) { + latestWebcamFrame = std::move(candidateWebcamFrame); + latestWebcamWidth = candidateWebcamWidth; + latestWebcamHeight = candidateWebcamHeight; + hasVisibleWebcamFrame = true; + } + } + const BgraFrameView webcamFrame{ + hasVisibleWebcamFrame && !latestWebcamFrame.empty() ? latestWebcamFrame.data() : nullptr, + latestWebcamWidth, + latestWebcamHeight, + }; + const int64_t syntheticTimestampHns = + static_cast((frameIndex * 10'000'000ULL) / config.fps); + const int64_t sourceTimestampHns = + latestFrameTimestampHns > 0 ? latestFrameTimestampHns : syntheticTimestampHns; + if (firstFrameTimestampHns < 0) { + firstFrameTimestampHns = sourceTimestampHns; + } + int64_t frameTimestampHns = + std::max(0, sourceTimestampHns - firstFrameTimestampHns); + if (lastEncodedVideoTimestampHns >= 0 && + frameTimestampHns <= lastEncodedVideoTimestampHns) { + frameTimestampHns = + lastEncodedVideoTimestampHns + static_cast(10'000'000ULL / config.fps); + } + if (latestFrameTexture && !encoder.writeFrame( + latestFrameTexture.Get(), + frameTimestampHns, + webcamFrame.data ? &webcamFrame : nullptr)) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + return; + } + if (latestFrameTexture) { + lastEncodedVideoTimestampHns = frameTimestampHns; + } + } + + frameIndex += 1; + const auto nextDeadline = startedAt + + std::chrono::duration_cast( + std::chrono::duration(static_cast(frameIndex) / config.fps)); + std::this_thread::sleep_until(nextDeadline); + } + }; + + std::thread videoWriterThread; + + auto stopVideoWriter = [&]() { + if (videoWriterThread.joinable()) { + videoWriterThread.join(); + } + }; + + auto startVideoWriter = [&]() { + videoWriterThread = std::thread(writeVideoFrames); + }; + + std::unique_ptr audioMixer; + auto startAudioCaptures = [&]() -> bool { + if (!audioFormat) { + return true; + } + + audioMixer = std::make_unique( + *audioFormat, + config.captureSystemAudio ? systemAudioFormat : *audioFormat, + config.captureMic ? microphoneAudioFormat : *audioFormat, + config.captureSystemAudio, + config.captureMic, + config.microphoneGain, + [&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + if (!encoder.writeAudio(data, byteCount, timestampHns, durationHns)) { + encodeFailed = true; + stopRequested = true; + cv.notify_all(); + return false; + } + return true; + }); + + if (!audioMixer->start()) { + std::cerr << "ERROR: Failed to start native audio mixer" << std::endl; + return false; + } + + if (config.captureMic) { + if (!microphoneCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + (void)timestampHns; + (void)durationHns; + if (stopRequested || !audioMixer) { + return; + } + + audioMixer->pushMicrophone(data, byteCount); + })) { + std::cerr << "ERROR: Failed to start WASAPI microphone capture" << std::endl; + audioMixer->stop(); + return false; + } + } + + if (config.captureSystemAudio) { + if (!loopbackCapture.start([&](const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + (void)timestampHns; + (void)durationHns; + if (stopRequested || !audioMixer) { + return; + } + + audioMixer->pushSystem(data, byteCount); + })) { + std::cerr << "ERROR: Failed to start WASAPI loopback capture" << std::endl; + microphoneCapture.stop(); + audioMixer->stop(); + return false; + } + } + + return true; + }; + + if (!startAudioCaptures()) { + return 1; + } + if (config.webcamEnabled) { + if (!webcamCapture.start()) { + microphoneCapture.stop(); + loopbackCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + std::cerr << "ERROR: Failed to start native webcam capture" << std::endl; + return 1; + } + webcamActive = true; + const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { + std::vector candidateWebcamFrame; + int candidateWebcamWidth = 0; + int candidateWebcamHeight = 0; + if (webcamCapture.copyLatestFrame(candidateWebcamFrame, candidateWebcamWidth, candidateWebcamHeight) && + hasVisibleBgraContent(candidateWebcamFrame)) { + latestWebcamFrame = std::move(candidateWebcamFrame); + latestWebcamWidth = candidateWebcamWidth; + latestWebcamHeight = candidateWebcamHeight; + hasVisibleWebcamFrame = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + if (!hasVisibleWebcamFrame) { + std::cerr << "WARNING: Native webcam started but no visible frame was available before screen capture" + << std::endl; + } + } + + if (!session.start()) { + webcamCapture.stop(); + microphoneCapture.stop(); + loopbackCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + std::cerr << "ERROR: Failed to start WGC session" << std::endl; + return 1; + } + + std::thread stdinThread(readStopCommands, std::ref(stopRequested), std::ref(cv)); + + { + std::unique_lock lock(mutex); + const bool started = cv.wait_for(lock, std::chrono::seconds(10), [&] { + return firstFrameWritten.load() || stopRequested.load(); + }); + if (!started || !firstFrameWritten) { + stopRequested = true; + cv.notify_all(); + if (stdinThread.joinable()) { + stdinThread.detach(); + } + microphoneCapture.stop(); + loopbackCapture.stop(); + webcamCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + session.stop(); + std::cerr << "ERROR: Timed out waiting for first WGC frame" << std::endl; + return 1; + } + } + + if (audioMixer) { + audioMixer->beginTimeline(); + } + startVideoWriter(); + + std::cout << "{\"event\":\"recording-started\",\"schemaVersion\":2}" << std::endl; + std::cout << "Recording started" << std::endl; + + { + std::unique_lock lock(mutex); + cv.wait(lock, [&] { + return stopRequested.load(); + }); + } + + microphoneCapture.stop(); + loopbackCapture.stop(); + webcamCapture.stop(); + if (audioMixer) { + audioMixer->stop(); + } + stopVideoWriter(); + session.stop(); + { + std::scoped_lock lock(mutex); + encoder.finalize(); + } + + if (stdinThread.joinable()) { + stdinThread.detach(); + } + + if (encodeFailed) { + std::cerr << "ERROR: Failed to encode WGC frame" << std::endl; + return 1; + } + + std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" + << jsonEscape(config.outputPath) << "\"}" << std::endl; + std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; + return 0; +} diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp new file mode 100644 index 000000000..de9220f17 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -0,0 +1,361 @@ +#include "mf_encoder.h" + +#include +#include +#include + +#include +#include +#include + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +void setFrameSize(IMFMediaType* type, UINT32 width, UINT32 height) { + MFSetAttributeSize(type, MF_MT_FRAME_SIZE, width, height); +} + +void setFrameRate(IMFMediaType* type, UINT32 fps) { + MFSetAttributeRatio(type, MF_MT_FRAME_RATE, fps, 1); +} + +void setPixelAspectRatio(IMFMediaType* type) { + MFSetAttributeRatio(type, MF_MT_PIXEL_ASPECT_RATIO, 1, 1); +} + +void setAudioFormat(IMFMediaType* type, UINT32 channels, UINT32 sampleRate, UINT32 bitsPerSample) { + type->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, channels); + type->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + type->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, bitsPerSample); +} + +void compositeWebcam(BYTE* destination, int width, int height, const BgraFrameView& webcamFrame) { + if (!webcamFrame.data || webcamFrame.width <= 0 || webcamFrame.height <= 0 || width <= 0 || height <= 0) { + return; + } + + const int margin = std::max(16, std::min(width, height) / 60); + const int maxOverlayWidth = std::max(2, width / 4); + int overlayWidth = maxOverlayWidth; + int overlayHeight = static_cast( + (static_cast(overlayWidth) * webcamFrame.height) / std::max(1, webcamFrame.width)); + const int maxOverlayHeight = std::max(2, height / 3); + if (overlayHeight > maxOverlayHeight) { + overlayHeight = maxOverlayHeight; + overlayWidth = static_cast( + (static_cast(overlayHeight) * webcamFrame.width) / std::max(1, webcamFrame.height)); + } + + overlayWidth = std::max(2, std::min(overlayWidth, width - margin * 2)); + overlayHeight = std::max(2, std::min(overlayHeight, height - margin * 2)); + const int originX = std::max(0, width - overlayWidth - margin); + const int originY = std::max(0, height - overlayHeight - margin); + + for (int y = 0; y < overlayHeight; y += 1) { + const int sourceY = static_cast((static_cast(y) * webcamFrame.height) / overlayHeight); + BYTE* destinationRow = destination + ((originY + y) * width + originX) * 4; + for (int x = 0; x < overlayWidth; x += 1) { + const int sourceX = static_cast((static_cast(x) * webcamFrame.width) / overlayWidth); + const BYTE* source = webcamFrame.data + (sourceY * webcamFrame.width + sourceX) * 4; + BYTE* target = destinationRow + x * 4; + target[0] = source[0]; + target[1] = source[1]; + target[2] = source[2]; + target[3] = 255; + } + } +} + +} // namespace + +MFEncoder::~MFEncoder() { + finalize(); +} + +bool MFEncoder::initialize( + const std::wstring& outputPath, + int width, + int height, + int fps, + int bitrate, + ID3D11Device* device, + ID3D11DeviceContext* context, + const AudioInputFormat* audioFormat) { + width_ = (std::max(2, width) / 2) * 2; + height_ = (std::max(2, height) / 2) * 2; + fps_ = std::max(1, fps); + device_ = device; + context_ = context; + + if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { + return false; + } + + Microsoft::WRL::ComPtr outputType; + if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(output)")) { + return false; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, static_cast(std::max(1, bitrate))); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + setFrameSize(outputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(outputType.Get(), static_cast(fps_)); + setPixelAspectRatio(outputType.Get()); + + if (!succeeded(MFCreateSinkWriterFromURL(outputPath.c_str(), nullptr, nullptr, &sinkWriter_), + "MFCreateSinkWriterFromURL")) { + return false; + } + if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &videoStreamIndex_), "AddStream")) { + return false; + } + + if (audioFormat && !configureAudioStream(*audioFormat)) { + return false; + } + + Microsoft::WRL::ComPtr inputType; + if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(input)")) { + return false; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_ * 4)); + setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(inputType.Get(), static_cast(fps_)); + setPixelAspectRatio(inputType.Get()); + + if (!succeeded(sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), nullptr), + "SetInputMediaType")) { + return false; + } + if (!succeeded(sinkWriter_->BeginWriting(), "BeginWriting")) { + return false; + } + + return true; +} + +bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) { + if (!sinkWriter_) { + return false; + } + if (audioFormat.sampleRate == 0 || audioFormat.channels == 0 || audioFormat.blockAlign == 0) { + std::cerr << "ERROR: Invalid audio input format" << std::endl; + return false; + } + + const UINT32 bitsPerSample = std::max(8, audioFormat.bitsPerSample); + const UINT32 aacBytesPerSecond = 24'000; + + Microsoft::WRL::ComPtr outputType; + if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(audio output)")) { + return false; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + setAudioFormat(outputType.Get(), audioFormat.channels, audioFormat.sampleRate, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, aacBytesPerSecond); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + if (!succeeded(sinkWriter_->AddStream(outputType.Get(), &audioStreamIndex_), "AddStream(audio)")) { + return false; + } + + Microsoft::WRL::ComPtr inputType; + if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(audio input)")) { + return false; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, audioFormat.subtype); + setAudioFormat(inputType.Get(), audioFormat.channels, audioFormat.sampleRate, bitsPerSample); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, audioFormat.blockAlign); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, audioFormat.avgBytesPerSec); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + + if (!succeeded(sinkWriter_->SetInputMediaType(audioStreamIndex_, inputType.Get(), nullptr), + "SetInputMediaType(audio)")) { + return false; + } + + hasAudioStream_ = true; + return true; +} + +bool MFEncoder::ensureStagingTexture(ID3D11Texture2D* texture) { + if (stagingTexture_) { + return true; + } + + D3D11_TEXTURE2D_DESC desc{}; + texture->GetDesc(&desc); + desc.Width = static_cast(width_); + desc.Height = static_cast(height_); + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + desc.MiscFlags = 0; + + return succeeded(device_->CreateTexture2D(&desc, nullptr, &stagingTexture_), + "CreateTexture2D(staging)"); +} + +bool MFEncoder::copyFrameToBuffer( + ID3D11Texture2D* texture, + BYTE* destination, + DWORD destinationSize, + const BgraFrameView* webcamFrame) { + if (!ensureStagingTexture(texture)) { + return false; + } + + context_->CopyResource(stagingTexture_.Get(), texture); + + D3D11_MAPPED_SUBRESOURCE mapped{}; + if (!succeeded(context_->Map(stagingTexture_.Get(), 0, D3D11_MAP_READ, 0, &mapped), "Map")) { + return false; + } + + const DWORD rowBytes = static_cast(width_ * 4); + const DWORD requiredBytes = rowBytes * static_cast(height_); + if (destinationSize < requiredBytes) { + context_->Unmap(stagingTexture_.Get(), 0); + std::cerr << "ERROR: Media Foundation buffer is too small" << std::endl; + return false; + } + + auto* source = static_cast(mapped.pData); + for (int y = 0; y < height_; y += 1) { + std::memcpy(destination + rowBytes * y, source + mapped.RowPitch * y, rowBytes); + } + if (webcamFrame) { + compositeWebcam(destination, width_, height_, *webcamFrame); + } + + context_->Unmap(stagingTexture_.Get(), 0); + return true; +} + +bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_) { + return false; + } + + if (firstTimestampHns_ < 0) { + firstTimestampHns_ = timestampHns; + } + + int64_t sampleTime = timestampHns - firstTimestampHns_; + if (sampleTime <= lastTimestampHns_) { + sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + } + const int64_t sampleDuration = 10'000'000LL / fps_; + lastTimestampHns_ = sampleTime; + + Microsoft::WRL::ComPtr buffer; + const DWORD frameBytes = static_cast(width_ * height_ * 4); + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock")) { + return false; + } + + const bool copied = copyFrameToBuffer(texture, data, maxLength, webcamFrame); + buffer->Unlock(); + if (!copied) { + return false; + } + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample"); +} + +bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_ || !hasAudioStream_) { + return false; + } + if (!data || byteCount == 0 || durationHns <= 0) { + return true; + } + + Microsoft::WRL::ComPtr buffer; + if (!succeeded(MFCreateMemoryBuffer(byteCount, &buffer), "MFCreateMemoryBuffer(audio)")) { + return false; + } + + BYTE* destination = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&destination, &maxLength, ¤tLength), + "IMFMediaBuffer::Lock(audio)")) { + return false; + } + if (maxLength < byteCount) { + buffer->Unlock(); + std::cerr << "ERROR: Media Foundation audio buffer is too small" << std::endl; + return false; + } + std::memcpy(destination, data, byteCount); + buffer->Unlock(); + buffer->SetCurrentLength(byteCount); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(audio)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(std::max(0, timestampHns)); + sample->SetSampleDuration(durationHns); + + return succeeded(sinkWriter_->WriteSample(audioStreamIndex_, sample.Get()), "WriteSample(audio)"); +} + +bool MFEncoder::finalize() { + std::scoped_lock writerLock(writerMutex_); + if (finalized_) { + return true; + } + + finalized_ = true; + bool ok = true; + if (sinkWriter_) { + ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize"); + sinkWriter_.Reset(); + } + stagingTexture_.Reset(); + context_.Reset(); + device_.Reset(); + MFShutdown(); + return ok; +} diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h new file mode 100644 index 000000000..a82a94023 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct BgraFrameView { + const BYTE* data = nullptr; + int width = 0; + int height = 0; +}; + +struct AudioInputFormat { + GUID subtype = MFAudioFormat_PCM; + UINT32 sampleRate = 0; + UINT32 channels = 0; + UINT32 bitsPerSample = 0; + UINT32 blockAlign = 0; + UINT32 avgBytesPerSec = 0; +}; + +class MFEncoder { +public: + MFEncoder() = default; + ~MFEncoder(); + + MFEncoder(const MFEncoder&) = delete; + MFEncoder& operator=(const MFEncoder&) = delete; + + bool initialize( + const std::wstring& outputPath, + int width, + int height, + int fps, + int bitrate, + ID3D11Device* device, + ID3D11DeviceContext* context, + const AudioInputFormat* audioFormat = nullptr); + bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame = nullptr); + bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); + bool finalize(); + +private: + bool ensureStagingTexture(ID3D11Texture2D* texture); + bool copyFrameToBuffer( + ID3D11Texture2D* texture, + BYTE* destination, + DWORD destinationSize, + const BgraFrameView* webcamFrame); + bool configureAudioStream(const AudioInputFormat& audioFormat); + + Microsoft::WRL::ComPtr sinkWriter_; + Microsoft::WRL::ComPtr device_; + Microsoft::WRL::ComPtr context_; + Microsoft::WRL::ComPtr stagingTexture_; + std::mutex writerMutex_; + DWORD videoStreamIndex_ = 0; + DWORD audioStreamIndex_ = 0; + bool hasAudioStream_ = false; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + int64_t firstTimestampHns_ = -1; + int64_t lastTimestampHns_ = -1; + bool finalized_ = false; +}; diff --git a/electron/native/wgc-capture/src/monitor_utils.cpp b/electron/native/wgc-capture/src/monitor_utils.cpp new file mode 100644 index 000000000..f83e77d1b --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.cpp @@ -0,0 +1,88 @@ +#include "monitor_utils.h" + +#include +#include +#include + +namespace { + +struct MonitorCandidate { + HMONITOR monitor = nullptr; + RECT rect{}; +}; + +std::vector enumerateMonitors() { + std::vector monitors; + EnumDisplayMonitors( + nullptr, + nullptr, + [](HMONITOR monitor, HDC, LPRECT rect, LPARAM userData) -> BOOL { + auto* result = reinterpret_cast*>(userData); + result->push_back({monitor, *rect}); + return TRUE; + }, + reinterpret_cast(&monitors)); + return monitors; +} + +bool rectMatchesBounds(const RECT& rect, const MonitorBounds& bounds) { + return rect.left == bounds.x && + rect.top == bounds.y && + (rect.right - rect.left) == bounds.width && + (rect.bottom - rect.top) == bounds.height; +} + +int64_t overlapArea(const RECT& rect, const MonitorBounds& bounds) { + const LONG left = std::max(rect.left, bounds.x); + const LONG top = std::max(rect.top, bounds.y); + const LONG right = std::min(rect.right, bounds.x + bounds.width); + const LONG bottom = std::min(rect.bottom, bounds.y + bounds.height); + if (right <= left || bottom <= top) { + return 0; + } + return static_cast(right - left) * static_cast(bottom - top); +} + +} // namespace + +HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds) { + const auto monitors = enumerateMonitors(); + if (monitors.empty()) { + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); + } + + // Electron's display_id is not stable across all Windows capture backends. + // Bounds are the most reliable contract because they come from Electron's + // selected display and match the WGC monitor coordinate space. + if (bounds && bounds->width > 0 && bounds->height > 0) { + for (const auto& candidate : monitors) { + if (rectMatchesBounds(candidate.rect, *bounds)) { + return candidate.monitor; + } + } + + HMONITOR bestMonitor = nullptr; + int64_t bestArea = 0; + for (const auto& candidate : monitors) { + const int64_t area = overlapArea(candidate.rect, *bounds); + if (area > bestArea) { + bestArea = area; + bestMonitor = candidate.monitor; + } + } + if (bestMonitor) { + return bestMonitor; + } + } + + // Best-effort fallback for helpers invoked without bounds. Some callers pass + // zero-based ids while Win32 monitor handles are pointer values, so only use + // this when it exactly matches the HMONITOR value. + for (const auto& candidate : monitors) { + if (reinterpret_cast(candidate.monitor) == displayId) { + return candidate.monitor; + } + } + + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); +} diff --git a/electron/native/wgc-capture/src/monitor_utils.h b/electron/native/wgc-capture/src/monitor_utils.h new file mode 100644 index 000000000..11d5d8375 --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +#include + +struct MonitorBounds { + int x = 0; + int y = 0; + int width = 0; + int height = 0; +}; + +HMONITOR findMonitorForCapture(int64_t displayId, const MonitorBounds* bounds); diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp new file mode 100644 index 000000000..0256b0425 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp @@ -0,0 +1,411 @@ +#include "wasapi_loopback_capture.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +constexpr REFERENCE_TIME BufferDurationHns = 10'000'000; +constexpr int64_t HnsPerSecond = 10'000'000; + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +GUID audioSubtypeFromFormat(WAVEFORMATEX* format) { + if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) { + return MFAudioFormat_Float; + } + if (format->wFormatTag == WAVE_FORMAT_PCM) { + return MFAudioFormat_PCM; + } + if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + format->cbSize >= sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX)) { + auto* extensible = reinterpret_cast(format); + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT) { + return MFAudioFormat_Float; + } + if (extensible->SubFormat == KSDATAFORMAT_SUBTYPE_PCM) { + return MFAudioFormat_PCM; + } + } + return GUID_NULL; +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring result; + result.reserve(value.size()); + bool lastWasSpace = true; + + for (const wchar_t c : value) { + if (std::iswalnum(c)) { + result.push_back(static_cast(std::towlower(c))); + lastWasSpace = false; + } else if (!lastWasSpace) { + result.push_back(L' '); + lastWasSpace = true; + } + } + + if (!result.empty() && result.back() == L' ') { + result.pop_back(); + } + return result; +} + +int scoreDeviceName(const std::wstring& candidateName, const std::wstring& candidateId, const std::wstring& requestedName) { + const std::wstring candidate = normalizeDeviceName(candidateName); + const std::wstring id = normalizeDeviceName(candidateId); + const std::wstring requested = normalizeDeviceName(requestedName); + if (requested.empty()) { + return 0; + } + if (candidate == requested) { + return 1000; + } + if (!candidate.empty() && (candidate.find(requested) != std::wstring::npos || requested.find(candidate) != std::wstring::npos)) { + return 900; + } + if (!id.empty() && (id.find(requested) != std::wstring::npos || requested.find(id) != std::wstring::npos)) { + return 800; + } + + int score = 0; + size_t pos = 0; + while (pos < requested.size()) { + const size_t end = requested.find(L' ', pos); + const std::wstring word = requested.substr(pos, end == std::wstring::npos ? std::wstring::npos : end - pos); + if (word.size() > 1 && word != L"microphone" && word != L"mic" && word != L"audio" && word != L"input") { + if (candidate.find(word) != std::wstring::npos) { + score += 100; + } else if (id.find(word) != std::wstring::npos) { + score += 50; + } + } + if (end == std::wstring::npos) { + break; + } + pos = end + 1; + } + return score; +} + +std::wstring getDeviceFriendlyName(IMMDevice* device) { + if (!device) { + return {}; + } + + Microsoft::WRL::ComPtr properties; + HRESULT hr = device->OpenPropertyStore(STGM_READ, &properties); + if (FAILED(hr) || !properties) { + return {}; + } + + PROPVARIANT value; + PropVariantInit(&value); + hr = properties->GetValue(PKEY_Device_FriendlyName, &value); + std::wstring name; + if (SUCCEEDED(hr) && value.vt == VT_LPWSTR && value.pwszVal) { + name = value.pwszVal; + } + PropVariantClear(&value); + return name; +} + +} // namespace + +WasapiLoopbackCapture::~WasapiLoopbackCapture() { + stop(); + if (mixFormat_) { + CoTaskMemFree(mixFormat_); + mixFormat_ = nullptr; + } +} + +bool WasapiLoopbackCapture::initializeSystemLoopback() { + return initialize(WasapiCaptureEndpoint::SystemLoopback, {}, {}); +} + +bool WasapiLoopbackCapture::initializeMicrophone(const std::wstring& deviceId, const std::wstring& deviceName) { + return initialize(WasapiCaptureEndpoint::Microphone, deviceId, deviceName); +} + +bool WasapiLoopbackCapture::initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId, const std::wstring& deviceName) { + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), + nullptr, + CLSCTX_ALL, + IID_PPV_ARGS(&deviceEnumerator_)); + if (!succeeded(hr, "CoCreateInstance(MMDeviceEnumerator)")) { + return false; + } + + if (endpoint == WasapiCaptureEndpoint::Microphone && !deviceId.empty() && deviceId != L"default") { + hr = deviceEnumerator_->GetDevice(deviceId.c_str(), &device_); + if (FAILED(hr)) { + std::wcerr << L"WARNING: Could not resolve microphone device id directly" + << std::endl; + device_.Reset(); + } + } + + if (endpoint == WasapiCaptureEndpoint::Microphone && !device_ && !deviceName.empty()) { + if (!resolveMicrophoneByName(deviceName)) { + std::wcerr << L"WARNING: Could not resolve microphone by name; using default capture endpoint" + << std::endl; + } + } + + if (!device_) { + const EDataFlow flow = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? eRender : eCapture; + hr = deviceEnumerator_->GetDefaultAudioEndpoint(flow, eConsole, &device_); + if (!succeeded(hr, "GetDefaultAudioEndpoint")) { + return false; + } + } + + selectedDeviceName_ = getDeviceFriendlyName(device_.Get()); + + hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_); + if (!succeeded(hr, "IMMDevice::Activate(IAudioClient)")) { + return false; + } + + hr = audioClient_->GetMixFormat(&mixFormat_); + if (!succeeded(hr, "IAudioClient::GetMixFormat") || !mixFormat_) { + return false; + } + + if (!resolveInputFormat(mixFormat_)) { + std::cerr << "ERROR: Unsupported WASAPI loopback mix format" << std::endl; + return false; + } + + const DWORD streamFlags = + endpoint == WasapiCaptureEndpoint::SystemLoopback ? AUDCLNT_STREAMFLAGS_LOOPBACK : 0; + hr = audioClient_->Initialize( + AUDCLNT_SHAREMODE_SHARED, + streamFlags, + BufferDurationHns, + 0, + mixFormat_, + nullptr); + if (!succeeded(hr, "IAudioClient::Initialize(loopback)")) { + return false; + } + + hr = audioClient_->GetService(IID_PPV_ARGS(&captureClient_)); + if (!succeeded(hr, "IAudioClient::GetService(IAudioCaptureClient)")) { + return false; + } + + return true; +} + +bool WasapiLoopbackCapture::resolveMicrophoneByName(const std::wstring& deviceName) { + if (!deviceEnumerator_ || deviceName.empty()) { + return false; + } + + Microsoft::WRL::ComPtr devices; + HRESULT hr = deviceEnumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &devices); + if (!succeeded(hr, "IMMDeviceEnumerator::EnumAudioEndpoints(eCapture)")) { + return false; + } + + UINT count = 0; + hr = devices->GetCount(&count); + if (!succeeded(hr, "IMMDeviceCollection::GetCount")) { + return false; + } + + Microsoft::WRL::ComPtr bestDevice; + std::wstring bestId; + std::wstring bestName; + int bestScore = 0; + for (UINT i = 0; i < count; ++i) { + Microsoft::WRL::ComPtr candidate; + hr = devices->Item(i, &candidate); + if (FAILED(hr) || !candidate) { + continue; + } + + LPWSTR rawId = nullptr; + std::wstring candidateId; + if (SUCCEEDED(candidate->GetId(&rawId)) && rawId) { + candidateId = rawId; + CoTaskMemFree(rawId); + } + + const std::wstring candidateName = getDeviceFriendlyName(candidate.Get()); + const int score = scoreDeviceName(candidateName, candidateId, deviceName); + std::wcerr << L"Native microphone candidate: " << candidateName << L" score=" << score << std::endl; + if (score > bestScore) { + bestScore = score; + bestDevice = candidate; + bestId = candidateId; + bestName = candidateName; + } + } + + if (!bestDevice || bestScore <= 0) { + return false; + } + + device_ = bestDevice; + std::wcerr << L"Selected native microphone endpoint: " << bestName << L" id=" << bestId << std::endl; + return true; +} + +bool WasapiLoopbackCapture::resolveInputFormat(WAVEFORMATEX* mixFormat) { + const GUID subtype = audioSubtypeFromFormat(mixFormat); + if (subtype == GUID_NULL) { + return false; + } + + inputFormat_.subtype = subtype; + inputFormat_.sampleRate = mixFormat->nSamplesPerSec; + inputFormat_.channels = mixFormat->nChannels; + inputFormat_.bitsPerSample = mixFormat->wBitsPerSample; + inputFormat_.blockAlign = mixFormat->nBlockAlign; + inputFormat_.avgBytesPerSec = mixFormat->nAvgBytesPerSec; + return inputFormat_.sampleRate > 0 && inputFormat_.channels > 0 && inputFormat_.blockAlign > 0; +} + +bool WasapiLoopbackCapture::start(AudioCallback callback) { + if (!audioClient_ || !captureClient_ || !callback) { + return false; + } + + callback_ = std::move(callback); + stopRequested_ = false; + writtenFrames_ = 0; + lastDevicePositionEnd_ = 0; + hasLastDevicePosition_ = false; + + HRESULT hr = audioClient_->Start(); + if (!succeeded(hr, "IAudioClient::Start")) { + return false; + } + + thread_ = std::thread([this] { + captureLoop(); + }); + return true; +} + +void WasapiLoopbackCapture::stop() { + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (audioClient_) { + audioClient_->Stop(); + } +} + +const AudioInputFormat& WasapiLoopbackCapture::inputFormat() const { + return inputFormat_; +} + +const std::wstring& WasapiLoopbackCapture::selectedDeviceName() const { + return selectedDeviceName_; +} + +void WasapiLoopbackCapture::captureLoop() { + auto emitSilenceFrames = [&](uint64_t frames, int64_t timestampHns) { + constexpr uint64_t MaxSilenceChunkFrames = 4800; + uint64_t remainingFrames = frames; + int64_t currentTimestampHns = timestampHns; + while (remainingFrames > 0 && !stopRequested_) { + const uint64_t chunkFrames = std::min(remainingFrames, MaxSilenceChunkFrames); + const DWORD chunkBytes = static_cast(chunkFrames * inputFormat_.blockAlign); + const int64_t chunkDurationHns = + static_cast((chunkFrames * HnsPerSecond) / inputFormat_.sampleRate); + silenceBuffer_.assign(chunkBytes, 0); + callback_(silenceBuffer_.data(), chunkBytes, currentTimestampHns, chunkDurationHns); + remainingFrames -= chunkFrames; + currentTimestampHns += chunkDurationHns; + } + }; + + while (!stopRequested_) { + UINT32 packetFrames = 0; + HRESULT hr = captureClient_->GetNextPacketSize(&packetFrames); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x" << std::hex + << hr << std::dec << ")" << std::endl; + break; + } + + while (packetFrames > 0 && !stopRequested_) { + BYTE* data = nullptr; + UINT32 framesAvailable = 0; + DWORD flags = 0; + UINT64 devicePosition = 0; + UINT64 qpcPosition = 0; + + hr = captureClient_->GetBuffer(&data, &framesAvailable, &flags, &devicePosition, &qpcPosition); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetBuffer failed (hr=0x" << std::hex + << hr << std::dec << ")" << std::endl; + break; + } + + (void)qpcPosition; + if (hasLastDevicePosition_ && devicePosition > lastDevicePositionEnd_) { + const uint64_t gapFrames = devicePosition - lastDevicePositionEnd_; + if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0 || gapFrames > framesAvailable) { + const int64_t gapTimestampHns = + static_cast((lastDevicePositionEnd_ * HnsPerSecond) / inputFormat_.sampleRate); + emitSilenceFrames(gapFrames, gapTimestampHns); + } + } + + const DWORD byteCount = framesAvailable * inputFormat_.blockAlign; + const int64_t timestampHns = + static_cast((devicePosition * HnsPerSecond) / inputFormat_.sampleRate); + const int64_t durationHns = + static_cast((static_cast(framesAvailable) * HnsPerSecond) / + inputFormat_.sampleRate); + + if (byteCount > 0) { + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0 || !data) { + silenceBuffer_.assign(byteCount, 0); + callback_(silenceBuffer_.data(), byteCount, timestampHns, durationHns); + } else { + callback_(data, byteCount, timestampHns, durationHns); + } + } + + writtenFrames_ += framesAvailable; + lastDevicePositionEnd_ = devicePosition + framesAvailable; + hasLastDevicePosition_ = true; + captureClient_->ReleaseBuffer(framesAvailable); + + hr = captureClient_->GetNextPacketSize(&packetFrames); + if (FAILED(hr)) { + std::cerr << "ERROR: IAudioCaptureClient::GetNextPacketSize failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + packetFrames = 0; + break; + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + +} diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.h b/electron/native/wgc-capture/src/wasapi_loopback_capture.h new file mode 100644 index 000000000..5c5f2b7c0 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.h @@ -0,0 +1,60 @@ +#pragma once + +#include "mf_encoder.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +enum class WasapiCaptureEndpoint { + SystemLoopback, + Microphone, +}; + +class WasapiLoopbackCapture { +public: + using AudioCallback = std::function; + + WasapiLoopbackCapture() = default; + ~WasapiLoopbackCapture(); + + WasapiLoopbackCapture(const WasapiLoopbackCapture&) = delete; + WasapiLoopbackCapture& operator=(const WasapiLoopbackCapture&) = delete; + + bool initializeSystemLoopback(); + bool initializeMicrophone(const std::wstring& deviceId, const std::wstring& deviceName); + bool start(AudioCallback callback); + void stop(); + + const AudioInputFormat& inputFormat() const; + const std::wstring& selectedDeviceName() const; + +private: + bool initialize(WasapiCaptureEndpoint endpoint, const std::wstring& deviceId, const std::wstring& deviceName); + bool resolveMicrophoneByName(const std::wstring& deviceName); + void captureLoop(); + bool resolveInputFormat(WAVEFORMATEX* mixFormat); + + Microsoft::WRL::ComPtr deviceEnumerator_; + Microsoft::WRL::ComPtr device_; + Microsoft::WRL::ComPtr audioClient_; + Microsoft::WRL::ComPtr captureClient_; + WAVEFORMATEX* mixFormat_ = nullptr; + AudioInputFormat inputFormat_{}; + std::wstring selectedDeviceName_; + AudioCallback callback_; + std::thread thread_; + std::atomic stopRequested_ = false; + std::vector silenceBuffer_; + uint64_t writtenFrames_ = 0; + uint64_t lastDevicePositionEnd_ = 0; + bool hasLastDevicePosition_ = false; +}; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp new file mode 100644 index 000000000..aff9fdbfa --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -0,0 +1,417 @@ +#include "webcam_capture.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +std::wstring readAllocatedString(IMFActivate* activate, REFGUID key) { + WCHAR* value = nullptr; + UINT32 length = 0; + if (FAILED(activate->GetAllocatedString(key, &value, &length)) || !value) { + return {}; + } + + std::wstring result(value, value + length); + CoTaskMemFree(value); + return result; +} + +bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { + if (haystack.empty() || needle.empty()) { + return false; + } + + std::wstring lowerHaystack = haystack; + std::wstring lowerNeedle = needle; + std::transform(lowerHaystack.begin(), lowerHaystack.end(), lowerHaystack.begin(), ::towlower); + std::transform(lowerNeedle.begin(), lowerNeedle.end(), lowerNeedle.begin(), ::towlower); + return lowerHaystack.find(lowerNeedle) != std::wstring::npos || + lowerNeedle.find(lowerHaystack) != std::wstring::npos; +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring normalized; + normalized.reserve(value.size()); + bool lastWasSpace = true; + for (const wchar_t ch : value) { + if (std::iswalnum(ch)) { + normalized.push_back(static_cast(std::towlower(ch))); + lastWasSpace = false; + continue; + } + if (!lastWasSpace) { + normalized.push_back(L' '); + lastWasSpace = true; + } + } + while (!normalized.empty() && normalized.back() == L' ') { + normalized.pop_back(); + } + return normalized; +} + +std::vector splitWords(const std::wstring& value) { + std::vector words; + size_t start = 0; + while (start < value.size()) { + const size_t end = value.find(L' ', start); + const auto word = value.substr(start, end == std::wstring::npos ? std::wstring::npos : end - start); + if (word.size() > 1 && word != L"camera" && word != L"webcam" && word != L"video" && word != L"input") { + words.push_back(word); + } + if (end == std::wstring::npos) { + break; + } + start = end + 1; + } + return words; +} + +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId) { + int score = 0; + const auto normalizedName = normalizeDeviceName(candidateName); + const auto normalizedLink = normalizeDeviceName(candidateLink); + const auto normalizedRequestedName = normalizeDeviceName(requestedName); + const auto normalizedRequestedId = normalizeDeviceName(requestedId); + + if (!normalizedRequestedName.empty()) { + if (normalizedName == normalizedRequestedName) { + score = std::max(score, 1000); + } + if (containsInsensitive(normalizedName, normalizedRequestedName)) { + score = std::max(score, 900); + } + if (containsInsensitive(normalizedLink, normalizedRequestedName)) { + score = std::max(score, 800); + } + + int wordScore = 0; + for (const auto& word : splitWords(normalizedRequestedName)) { + if (normalizedName.find(word) != std::wstring::npos) { + wordScore += 100; + } else if (normalizedLink.find(word) != std::wstring::npos) { + wordScore += 50; + } + } + score = std::max(score, wordScore); + } + + if (!normalizedRequestedId.empty()) { + if (containsInsensitive(normalizedLink, normalizedRequestedId)) { + score = std::max(score, 700); + } + if (containsInsensitive(normalizedName, normalizedRequestedId)) { + score = std::max(score, 600); + } + } + + return score; +} + +} // namespace + +WebcamCapture::~WebcamCapture() { + stop(); +} + +bool WebcamCapture::initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps) { + fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); + usingDirectShow_ = false; + selectedMatchScore_ = 0; + if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + return false; + } + mfStarted_ = true; + if (!selectDevice(deviceId, deviceName)) { + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + return false; + } + + if ((!deviceId.empty() || !deviceName.empty()) && selectedMatchScore_ <= 0) { + if (mediaSource_) { + mediaSource_->Shutdown(); + } + sourceReader_.Reset(); + mediaSource_.Reset(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } + if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { + usingDirectShow_ = true; + return true; + } + std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" + << std::endl; + return false; + } + + return configureReader(requestedWidth, requestedHeight, fps_); +} + +bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstring& deviceName) { + Microsoft::WRL::ComPtr attributes; + if (!succeeded(MFCreateAttributes(&attributes, 1), "MFCreateAttributes(webcam enumeration)")) { + return false; + } + if (!succeeded(attributes->SetGUID( + MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, + MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID), + "SetGUID(webcam source type)")) { + return false; + } + + IMFActivate** devices = nullptr; + UINT32 deviceCount = 0; + HRESULT hr = MFEnumDeviceSources(attributes.Get(), &devices, &deviceCount); + if (!succeeded(hr, "MFEnumDeviceSources") || deviceCount == 0) { + if (devices) { + CoTaskMemFree(devices); + } + std::cerr << "ERROR: No native Windows webcam devices were found" << std::endl; + return false; + } + + UINT32 selectedIndex = 0; + int bestScore = 0; + for (UINT32 index = 0; index < deviceCount; index += 1) { + const std::wstring name = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); + const std::wstring symbolicLink = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK); + const int score = deviceMatchScore(name, symbolicLink, deviceName, deviceId); + std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << name << L"\" score=" << score << std::endl; + if (score > bestScore) { + selectedIndex = index; + bestScore = score; + } + } + + if ((!deviceId.empty() || !deviceName.empty()) && bestScore <= 0) { + std::cerr << "WARNING: Requested webcam device was not found by Media Foundation; trying DirectShow" + << std::endl; + } + + selectedMatchScore_ = bestScore; + selectedDeviceName_ = readAllocatedString(devices[selectedIndex], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); + hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); + + for (UINT32 index = 0; index < deviceCount; index += 1) { + devices[index]->Release(); + } + CoTaskMemFree(devices); + + return succeeded(hr, "ActivateObject(webcam)"); +} + +bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int requestedFps) { + Microsoft::WRL::ComPtr attributes; + if (!succeeded(MFCreateAttributes(&attributes, 2), "MFCreateAttributes(webcam reader)")) { + return false; + } + attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + attributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, FALSE); + + if (!succeeded(MFCreateSourceReaderFromMediaSource(mediaSource_.Get(), attributes.Get(), &sourceReader_), + "MFCreateSourceReaderFromMediaSource(webcam)")) { + return false; + } + + Microsoft::WRL::ComPtr mediaType; + if (!succeeded(MFCreateMediaType(&mediaType), "MFCreateMediaType(webcam output)")) { + return false; + } + mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + if (requestedWidth > 0 && requestedHeight > 0) { + MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(requestedWidth), static_cast(requestedHeight)); + } + MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast(std::max(1, requestedFps)), 1); + + if (!succeeded(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()), + "SetCurrentMediaType(webcam RGB32)")) { + return false; + } + sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE); + sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE); + + Microsoft::WRL::ComPtr currentType; + if (!succeeded(sourceReader_->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, ¤tType), + "GetCurrentMediaType(webcam)")) { + return false; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(currentType.Get(), MF_MT_FRAME_SIZE, &width, &height)) || width == 0 || height == 0) { + width = static_cast(requestedWidth > 0 ? requestedWidth : 1280); + height = static_cast(requestedHeight > 0 ? requestedHeight : 720); + } + width_ = static_cast(width); + height_ = static_cast(height); + return true; +} + +bool WebcamCapture::start() { + if (usingDirectShow_) { + return directShowCapture_.start(); + } + if (!sourceReader_ || thread_.joinable()) { + return false; + } + + stopRequested_ = false; + thread_ = std::thread(&WebcamCapture::captureLoop, this); + return true; +} + +void WebcamCapture::stop() { + directShowCapture_.stop(); + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (mediaSource_) { + mediaSource_->Shutdown(); + } + sourceReader_.Reset(); + mediaSource_.Reset(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } +} + +void WebcamCapture::captureLoop() { + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + + while (!stopRequested_) { + DWORD streamIndex = 0; + DWORD flags = 0; + LONGLONG timestamp = 0; + Microsoft::WRL::ComPtr sample; + HRESULT hr = sourceReader_->ReadSample( + MF_SOURCE_READER_FIRST_VIDEO_STREAM, + 0, + &streamIndex, + &flags, + ×tamp, + &sample); + (void)streamIndex; + (void)timestamp; + + if (FAILED(hr)) { + std::cerr << "WARNING: Failed to read webcam sample (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + continue; + } + if ((flags & MF_SOURCE_READERF_ENDOFSTREAM) != 0) { + break; + } + if (!sample) { + continue; + } + + Microsoft::WRL::ComPtr buffer; + if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || !buffer) { + continue; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (FAILED(buffer->Lock(&data, &maxLength, ¤tLength)) || !data) { + continue; + } + + const DWORD expectedLength = static_cast(std::max(0, width_) * std::max(0, height_) * 4); + if (currentLength >= expectedLength && expectedLength > 0) { + std::scoped_lock lock(frameMutex_); + latestFrame_.assign(data, data + expectedLength); + } + + buffer->Unlock(); + } + + CoUninitialize(); +} + +bool WebcamCapture::copyLatestFrame(std::vector& destination, int& width, int& height) { + if (usingDirectShow_) { + return directShowCapture_.copyLatestFrame(destination, width, height); + } + std::scoped_lock lock(frameMutex_); + if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { + return false; + } + + destination = latestFrame_; + width = width_; + height = height_; + return true; +} + +int WebcamCapture::width() const { + if (usingDirectShow_) { + return directShowCapture_.width(); + } + return width_; +} + +int WebcamCapture::height() const { + if (usingDirectShow_) { + return directShowCapture_.height(); + } + return height_; +} + +int WebcamCapture::fps() const { + if (usingDirectShow_) { + return directShowCapture_.fps(); + } + return fps_; +} + +const std::wstring& WebcamCapture::selectedDeviceName() const { + if (usingDirectShow_) { + return directShowCapture_.selectedDeviceName(); + } + return selectedDeviceName_; +} diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h new file mode 100644 index 000000000..c539d025c --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -0,0 +1,60 @@ +#pragma once + +#include "dshow_webcam_capture.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +class WebcamCapture { +public: + WebcamCapture() = default; + ~WebcamCapture(); + + WebcamCapture(const WebcamCapture&) = delete; + WebcamCapture& operator=(const WebcamCapture&) = delete; + + bool initialize( + const std::wstring& deviceId, + const std::wstring& deviceName, + const std::wstring& directShowClsid, + int requestedWidth, + int requestedHeight, + int requestedFps); + bool start(); + void stop(); + bool copyLatestFrame(std::vector& destination, int& width, int& height); + + int width() const; + int height() const; + int fps() const; + const std::wstring& selectedDeviceName() const; + +private: + bool selectDevice(const std::wstring& deviceId, const std::wstring& deviceName); + bool configureReader(int requestedWidth, int requestedHeight, int requestedFps); + void captureLoop(); + + Microsoft::WRL::ComPtr mediaSource_; + Microsoft::WRL::ComPtr sourceReader_; + DirectShowWebcamCapture directShowCapture_; + std::thread thread_; + std::atomic stopRequested_ = false; + std::mutex frameMutex_; + std::vector latestFrame_; + int width_ = 0; + int height_ = 0; + int fps_ = 30; + bool mfStarted_ = false; + bool usingDirectShow_ = false; + int selectedMatchScore_ = 0; + std::wstring selectedDeviceName_; +}; diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp new file mode 100644 index 000000000..fbdc5a576 --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -0,0 +1,276 @@ +#include "wgc_session.h" + +#include +#include +#include +#include + +#include + +namespace wf = winrt::Windows::Foundation; +namespace wgcap = winrt::Windows::Graphics::Capture; +namespace wgdx = winrt::Windows::Graphics::DirectX; +namespace wgd3d = winrt::Windows::Graphics::DirectX::Direct3D11; + +extern "C" HRESULT __stdcall CreateDirect3D11DeviceFromDXGIDevice( + ::IDXGIDevice* dxgiDevice, + ::IInspectable** graphicsDevice); + +namespace { + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + + std::cerr << "ERROR: " << label << " failed (hr=0x" << std::hex << hr << std::dec << ")" + << std::endl; + return false; +} + +int64_t timeSpanToHns(wf::TimeSpan const& value) { + return value.count(); +} + +} // namespace + +WgcSession::~WgcSession() { + stop(); +} + +bool WgcSession::createD3DDevice() { + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; +#if defined(_DEBUG) + flags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + }; + D3D_FEATURE_LEVEL featureLevel{}; + + HRESULT hr = D3D11CreateDevice( + nullptr, + D3D_DRIVER_TYPE_HARDWARE, + nullptr, + flags, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + &featureLevel, + &d3dContext_); + +#if defined(_DEBUG) + if (FAILED(hr)) { + flags &= ~D3D11_CREATE_DEVICE_DEBUG; + hr = D3D11CreateDevice( + nullptr, + D3D_DRIVER_TYPE_HARDWARE, + nullptr, + flags, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + &featureLevel, + &d3dContext_); + } +#endif + + if (!succeeded(hr, "D3D11CreateDevice")) { + return false; + } + + Microsoft::WRL::ComPtr dxgiDevice; + if (!succeeded(d3dDevice_.As(&dxgiDevice), "Query IDXGIDevice")) { + return false; + } + + winrt::com_ptr<::IInspectable> inspectableDevice; + if (!succeeded(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.Get(), inspectableDevice.put()), + "CreateDirect3D11DeviceFromDXGIDevice")) { + return false; + } + + winrtDevice_ = inspectableDevice.as(); + return true; +} + +bool WgcSession::createCaptureItem(HMONITOR monitor) { + auto factory = winrt::get_activation_factory(); + auto interop = factory.as(); + + wgcap::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForMonitor( + monitor, + winrt::guid_of(), + reinterpret_cast(winrt::put_abi(item))); + if (!succeeded(hr, "CreateForMonitor")) { + return false; + } + + item_ = item; + const auto size = item_.Size(); + width_ = static_cast(size.Width); + height_ = static_cast(size.Height); + return width_ > 0 && height_ > 0; +} + +bool WgcSession::createCaptureItem(HWND window) { + auto factory = winrt::get_activation_factory(); + auto interop = factory.as(); + + wgcap::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForWindow( + window, + winrt::guid_of(), + reinterpret_cast(winrt::put_abi(item))); + if (!succeeded(hr, "CreateForWindow")) { + return false; + } + + item_ = item; + const auto size = item_.Size(); + width_ = static_cast(size.Width); + height_ = static_cast(size.Height); + return width_ > 0 && height_ > 0; +} + +void WgcSession::applySessionOptions(bool captureCursor) { + try { + session_.IsCursorCaptureEnabled(captureCursor); + } catch (...) { + // Older WGC builds can omit this property. They will keep the OS default. + } + + try { + session_.IsBorderRequired(false); + } catch (...) { + // IsBorderRequired is Windows 11-only. Ignore it on older builds. + } +} + +bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) { + fps_ = fps > 0 ? fps : 60; + if (!createD3DDevice()) { + return false; + } + if (!createCaptureItem(monitor)) { + return false; + } + + framePool_ = wgcap::Direct3D11CaptureFramePool::CreateFreeThreaded( + winrtDevice_, + wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized, + 2, + item_.Size()); + session_ = framePool_.CreateCaptureSession(item_); + + applySessionOptions(captureCursor); + + frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); + return true; +} + +bool WgcSession::initialize(HWND window, int fps, bool captureCursor) { + fps_ = fps > 0 ? fps : 60; + if (!createD3DDevice()) { + return false; + } + if (!createCaptureItem(window)) { + return false; + } + + framePool_ = wgcap::Direct3D11CaptureFramePool::CreateFreeThreaded( + winrtDevice_, + wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized, + 2, + item_.Size()); + session_ = framePool_.CreateCaptureSession(item_); + + applySessionOptions(captureCursor); + + frameArrivedToken_ = framePool_.FrameArrived({this, &WgcSession::onFrameArrived}); + return true; +} + +void WgcSession::setFrameCallback(FrameCallback callback) { + std::scoped_lock lock(callbackMutex_); + frameCallback_ = std::move(callback); +} + +bool WgcSession::start() { + if (!session_) { + return false; + } + session_.StartCapture(); + started_ = true; + return true; +} + +void WgcSession::stop() { + if (framePool_) { + framePool_.FrameArrived(frameArrivedToken_); + } + if (session_) { + session_.Close(); + session_ = nullptr; + } + if (framePool_) { + framePool_.Close(); + framePool_ = nullptr; + } + item_ = nullptr; + winrtDevice_ = nullptr; + d3dContext_.Reset(); + d3dDevice_.Reset(); + started_ = false; +} + +void WgcSession::onFrameArrived( + wgcap::Direct3D11CaptureFramePool const& sender, + wf::IInspectable const&) { + auto frame = sender.TryGetNextFrame(); + if (!frame) { + return; + } + + auto surface = frame.Surface(); + auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + Microsoft::WRL::ComPtr texture; + HRESULT hr = access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); + if (FAILED(hr) || !texture) { + return; + } + + FrameCallback callback; + { + std::scoped_lock lock(callbackMutex_); + callback = frameCallback_; + } + + if (callback) { + callback(texture.Get(), timeSpanToHns(frame.SystemRelativeTime())); + } + frame.Close(); +} + +int WgcSession::captureWidth() const { + return width_; +} + +int WgcSession::captureHeight() const { + return height_; +} + +ID3D11Device* WgcSession::device() const { + return d3dDevice_.Get(); +} + +ID3D11DeviceContext* WgcSession::context() const { + return d3dContext_.Get(); +} diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h new file mode 100644 index 000000000..4b7a0c4b7 --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +class WgcSession { +public: + using FrameCallback = std::function; + + WgcSession() = default; + ~WgcSession(); + + WgcSession(const WgcSession&) = delete; + WgcSession& operator=(const WgcSession&) = delete; + + bool initialize(HMONITOR monitor, int fps, bool captureCursor); + bool initialize(HWND window, int fps, bool captureCursor); + void setFrameCallback(FrameCallback callback); + bool start(); + void stop(); + + int captureWidth() const; + int captureHeight() const; + ID3D11Device* device() const; + ID3D11DeviceContext* context() const; + +private: + bool createD3DDevice(); + bool createCaptureItem(HMONITOR monitor); + bool createCaptureItem(HWND window); + void applySessionOptions(bool captureCursor); + void onFrameArrived( + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, + winrt::Windows::Foundation::IInspectable const&); + + Microsoft::WRL::ComPtr d3dDevice_; + Microsoft::WRL::ComPtr d3dContext_; + winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice winrtDevice_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureItem item_{nullptr}; + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool framePool_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr}; + winrt::event_token frameArrivedToken_{}; + FrameCallback frameCallback_; + std::mutex callbackMutex_; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + bool started_ = false; +}; diff --git a/electron/preload.ts b/electron/preload.ts index 6c705d7b8..74893809a 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,7 @@ import { contextBridge, ipcRenderer } from "electron"; +import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; +import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; // Asset base URL is passed from the main process via webPreferences.additionalArguments // (see windows.ts). Sandboxed preloads cannot import node:path / node:url, so we @@ -10,6 +12,9 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_ contextBridge.exposeInMainWorld("electronAPI", { assetBaseUrl, + invokeNativeBridge: (request: NativeBridgeRequest) => { + return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; + }, hudOverlayHide: () => { ipcRenderer.send("hud-overlay-hide"); }, @@ -54,8 +59,21 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, - setRecordingState: (recording: boolean, recordingId?: number) => { - return ipcRenderer.invoke("set-recording-state", recording, recordingId); + setRecordingState: ( + recording: boolean, + recordingId?: number, + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode, + ) => { + return ipcRenderer.invoke("set-recording-state", recording, recordingId, cursorCaptureMode); + }, + isNativeWindowsCaptureAvailable: () => { + return ipcRenderer.invoke("is-native-windows-capture-available"); + }, + startNativeWindowsRecording: (request: NativeWindowsRecordingRequest) => { + return ipcRenderer.invoke("start-native-windows-recording", request); + }, + stopNativeWindowsRecording: (discard?: boolean) => { + return ipcRenderer.invoke("stop-native-windows-recording", discard); }, getCursorTelemetry: (videoPath?: string) => { return ipcRenderer.invoke("get-cursor-telemetry", videoPath); diff --git a/package.json b/package.json index 546108c8b..0bd0d576b 100644 --- a/package.json +++ b/package.json @@ -21,14 +21,26 @@ "i18n:check": "node scripts/i18n-check.mjs", "preview": "vite preview", "build:mac": "tsc && vite build && electron-builder --mac", - "build:win": "tsc && vite build && electron-builder --win --config.npmRebuild=false", + "build:native:win": "node scripts/build-windows-wgc-helper.mjs", + "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false", "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", "test": "vitest --run", "test:watch": "vitest", + "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", + "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs", + "test:wgc-window:win": "node scripts/test-windows-wgc-helper.mjs --window", + "test:wgc-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio", + "test:wgc-mic:win": "node scripts/test-windows-wgc-helper.mjs --microphone", + "test:wgc-mixed-audio:win": "node scripts/test-windows-wgc-helper.mjs --system-audio --microphone", + "test:wgc-webcam:win": "node scripts/test-windows-wgc-helper.mjs --webcam", + "test:wgc-full:win": "node scripts/test-windows-wgc-helper.mjs --webcam --system-audio --microphone", + "capture:openscreen-preview": "node scripts/capture-openscreen-preview.mjs", + "inspect:cursor-click-bounce": "node scripts/inspect-native-cursor-click-bounce.mjs", "build-vite": "tsc && vite build", "test:browser": "vitest --config vitest.browser.config.ts --run", "test:browser:install": "playwright install --with-deps chromium-headless-shell", "test:e2e": "playwright test", + "test:e2e:windows-native-checklist": "playwright test tests/e2e/windows-native-checklist.spec.ts", "prepare": "husky", "rebuild:native": "node ./scripts/rebuild-native.mjs", "postinstall": "npm run rebuild:native" diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs new file mode 100644 index 000000000..85da01e57 --- /dev/null +++ b/scripts/build-windows-wgc-helper.mjs @@ -0,0 +1,112 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const SOURCE_DIR = path.join(ROOT, "electron", "native", "wgc-capture"); +const BUILD_DIR = path.join(SOURCE_DIR, "build"); +const COMPAT_LIB_DIR = path.join(BUILD_DIR, "compat-libs"); +const BIN_DIR = path.join(ROOT, "electron", "native", "bin", "win32-x64"); +const CMAKE = process.env.CMAKE_EXE ?? "cmake"; + +function findVcVarsAll() { + const explicit = process.env.VCVARSALL; + if (explicit && fs.existsSync(explicit)) { + return explicit; + } + + const roots = [ + process.env.VSINSTALLDIR, + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community", + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Professional", + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise", + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools", + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Community", + ]; + + for (const root of roots.filter(Boolean)) { + const candidate = path.join(root, "VC", "Auxiliary", "Build", "vcvarsall.bat"); + if (fs.existsSync(candidate)) { + return candidate; + } + } + + return null; +} + +function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: ROOT, + stdio: "inherit", + windowsHide: true, + ...options, + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`${command} ${args.join(" ")} failed with code ${code}`)); + } + }); + }); +} + +async function runInVsEnv(command) { + const vcvarsAll = findVcVarsAll(); + if (!vcvarsAll) { + throw new Error( + "Could not find Visual Studio vcvarsall.bat. Install Visual Studio Build Tools with C++.", + ); + } + + const cmdPath = path.join(os.tmpdir(), `openscreen-build-wgc-${process.pid}-${Date.now()}.cmd`); + fs.writeFileSync( + cmdPath, + [ + "@echo off", + `call "${vcvarsAll}" x64`, + "if errorlevel 1 exit /b %errorlevel%", + `if not exist "${COMPAT_LIB_DIR}" mkdir "${COMPAT_LIB_DIR}"`, + `for %%L in (gdi32.lib winspool.lib shell32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib) do if not exist "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\%%L" copy /Y "%WindowsSdkDir%Lib\\%WindowsSDKLibVersion%um\\x64\\kernel32.Lib" "${COMPAT_LIB_DIR}\\%%L" >nul`, + "if errorlevel 1 exit /b %errorlevel%", + `set "LIB=${COMPAT_LIB_DIR};%LIB%"`, + command, + "exit /b %errorlevel%", + "", + ].join("\r\n"), + ); + try { + await run("cmd.exe", ["/d", "/c", cmdPath]); + } finally { + fs.rmSync(cmdPath, { force: true }); + } +} + +if (process.platform !== "win32") { + console.log("Skipping WGC helper build: Windows-only."); + process.exit(0); +} + +fs.mkdirSync(BUILD_DIR, { recursive: true }); + +await runInVsEnv( + `"${CMAKE}" -S "${SOURCE_DIR}" -B "${BUILD_DIR}" -G Ninja -DCMAKE_BUILD_TYPE=Release`, +); +await runInVsEnv(`"${CMAKE}" --build "${BUILD_DIR}" --config Release`); + +const outputPath = path.join(BUILD_DIR, "wgc-capture.exe"); +if (!fs.existsSync(outputPath)) { + throw new Error(`WGC helper build completed but ${outputPath} was not found.`); +} + +fs.mkdirSync(BIN_DIR, { recursive: true }); +const distributablePath = path.join(BIN_DIR, "wgc-capture.exe"); +fs.copyFileSync(outputPath, distributablePath); + +console.log(`Built ${outputPath}`); +console.log(`Copied ${distributablePath}`); diff --git a/scripts/capture-openscreen-preview.mjs b/scripts/capture-openscreen-preview.mjs new file mode 100644 index 000000000..25f86db59 --- /dev/null +++ b/scripts/capture-openscreen-preview.mjs @@ -0,0 +1,258 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium, _electron as electron } from "@playwright/test"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const MAIN_JS = path.join(ROOT, "dist-electron", "main.js"); +const TEST_VIDEO = path.join(ROOT, "tests", "fixtures", "sample.webm"); +const OUTPUT_DIR = + process.env.OPENSCREEN_PREVIEW_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-real-preview-${Date.now()}`); +const FRAME_COUNT = Number(process.env.OPENSCREEN_PREVIEW_FRAME_COUNT ?? 90); +const FPS = Number(process.env.OPENSCREEN_PREVIEW_FPS ?? 30); + +function findLatestCursorRecordingData() { + const explicit = process.env.CURSOR_RECORDING_DATA_PATH; + if (explicit) { + if (!fs.existsSync(explicit)) { + throw new Error(`CURSOR_RECORDING_DATA_PATH does not exist: ${explicit}`); + } + return explicit; + } + + const tempDir = os.tmpdir(); + const candidates = fs + .readdirSync(tempDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("openscreen-cursor-native-")) + .map((entry) => path.join(tempDir, entry.name, "cursor-recording-data.json")) + .filter((candidate) => fs.existsSync(candidate)) + .map((candidate) => ({ path: candidate, mtimeMs: fs.statSync(candidate).mtimeMs })) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + + if (!candidates[0]) { + throw new Error( + "No cursor-recording-data.json found. Run npm run test:cursor-native:win first.", + ); + } + + return candidates[0].path; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => path.join(baseDir, entry.name, "chrome-win64", "chrome.exe")) + .filter((candidate) => fs.existsSync(candidate)) + .sort() + .reverse(); + + return candidates[0] ?? defaultPath; +} + +function ensureBuildExists() { + if (!fs.existsSync(MAIN_JS)) { + throw new Error(`Missing ${MAIN_JS}. Run npm run build-vite first.`); + } + if (!fs.existsSync(path.join(ROOT, "dist", "index.html"))) { + throw new Error(`Missing renderer build. Run npm run build-vite first.`); + } +} + +function runNpmBuildViteIfRequested() { + if (process.env.OPENSCREEN_PREVIEW_SKIP_BUILD === "true") { + ensureBuildExists(); + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const child = spawn("cmd.exe", ["/d", "/s", "/c", "npm run build-vite"], { + cwd: ROOT, + stdio: "inherit", + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`npm run build-vite failed with code ${code}`)); + }); + }); +} + +async function encodeFramesToWebm(framePaths, outputPath) { + const frameData = framePaths.map((framePath) => ({ + src: `data:image/png;base64,${fs.readFileSync(framePath).toString("base64")}`, + })); + const html = ` + + + + + +`; + + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage(); + await page.setContent(html); + const base64 = await page.evaluate(() => window.__encode()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); +const cursorRecordingDataPath = findLatestCursorRecordingData(); +const fixtureVideoPath = path.join(OUTPUT_DIR, "openscreen-preview-fixture.webm"); +const outputVideoPath = path.join(OUTPUT_DIR, "openscreen-preview.webm"); +fs.copyFileSync(TEST_VIDEO, fixtureVideoPath); +fs.copyFileSync(cursorRecordingDataPath, `${fixtureVideoPath}.cursor.json`); + +await runNpmBuildViteIfRequested(); + +const app = await electron.launch({ + args: [MAIN_JS, "--no-sandbox", "--enable-unsafe-swiftshader"], + env: { + ...process.env, + HEADLESS: "false", + }, +}); + +app.process().stdout?.on("data", (data) => process.stdout.write(`[electron] ${data}`)); +app.process().stderr?.on("data", (data) => process.stderr.write(`[electron] ${data}`)); + +const framesDir = path.join(OUTPUT_DIR, "frames"); +fs.mkdirSync(framesDir, { recursive: true }); + +try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await hudWindow.evaluate(async () => { + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await window.electronAPI.getCurrentRecordingSession(); + await window.electronAPI.getCurrentVideoPath(); + return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + throw new Error("Timed out waiting for OpenScreen IPC handlers."); + }); + + try { + await hudWindow.evaluate(async (videoPath) => { + await window.electronAPI.setCurrentVideoPath(videoPath); + await window.electronAPI.switchToEditor(); + }, fixtureVideoPath); + } catch { + // switchToEditor closes the HUD page before the evaluate promise can always resolve. + } + + const editorWindow = await app.waitForEvent("window", { + predicate: (window) => window.url().includes("windowType=editor"), + timeout: 30_000, + }); + await editorWindow.waitForLoadState("domcontentloaded"); + await editorWindow.waitForSelector("video", { state: "attached", timeout: 30_000 }); + await editorWindow.waitForSelector("canvas", { state: "attached", timeout: 30_000 }); + + await editorWindow.setViewportSize({ width: 1280, height: 800 }); + await editorWindow.evaluate(async () => { + await document.fonts.ready; + for (const video of [...document.querySelectorAll("video")]) { + video.muted = true; + video.currentTime = 0; + video.dispatchEvent(new Event("timeupdate")); + } + }); + await editorWindow.waitForTimeout(1000); + + const framePaths = []; + for (let index = 0; index < FRAME_COUNT; index += 1) { + const timeSec = index / FPS; + await editorWindow.evaluate((time) => { + for (const video of [...document.querySelectorAll("video")]) { + video.currentTime = Math.min(time, Math.max(0, video.duration || time)); + video.dispatchEvent(new Event("timeupdate")); + } + }, timeSec); + await editorWindow.waitForTimeout(40); + const framePath = path.join(framesDir, `frame-${String(index).padStart(4, "0")}.png`); + await editorWindow.screenshot({ path: framePath }); + framePaths.push(framePath); + } + + await encodeFramesToWebm(framePaths, outputVideoPath); + + const report = { + outputDir: OUTPUT_DIR, + sourceCursorRecordingDataPath: cursorRecordingDataPath, + fixtureVideoPath, + outputVideoPath, + frameCount: framePaths.length, + fps: FPS, + }; + fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report, null, 2)); +} finally { + await app.close(); +} diff --git a/scripts/inspect-native-cursor-click-bounce.mjs b/scripts/inspect-native-cursor-click-bounce.mjs new file mode 100644 index 000000000..870ee8d43 --- /dev/null +++ b/scripts/inspect-native-cursor-click-bounce.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; + +const CLICK_ANIMATION_MS = 260; + +function usage() { + console.error( + "Usage: node scripts/inspect-native-cursor-click-bounce.mjs [--bounce=5]", + ); + process.exit(1); +} + +function getCursorJsonPath(inputPath) { + if (!inputPath) { + usage(); + } + + const resolved = path.resolve(inputPath); + if (resolved.endsWith(".cursor.json")) { + return resolved; + } + return `${resolved}.cursor.json`; +} + +function getBounceValue() { + const arg = process.argv.find((value) => value.startsWith("--bounce=")); + const parsed = Number(arg?.slice("--bounce=".length) ?? 5); + return Number.isFinite(parsed) ? Math.min(5, Math.max(0, parsed)) : 5; +} + +function clickBounceProgress(samples, timeMs) { + for (let index = samples.length - 1; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + const ageMs = timeMs - sample.timeMs; + if (ageMs > CLICK_ANIMATION_MS) { + return 0; + } + + if (sample.interactionType === "click") { + return 1 - ageMs / CLICK_ANIMATION_MS; + } + } + + return 0; +} + +function clickBounceScale(clickBounce, progress) { + if (progress <= 0 || clickBounce <= 0) { + return 1; + } + + const intensity = Math.min(5, Math.max(0, clickBounce)) / 5; + const elapsed = 1 - Math.min(1, Math.max(0, progress)); + if (elapsed < 0.38) { + const pressProgress = Math.sin((elapsed / 0.38) * Math.PI); + return 1 - pressProgress * intensity * 0.24; + } + + const reboundProgress = Math.sin(((elapsed - 0.38) / 0.62) * Math.PI); + return 1 + reboundProgress * intensity * 0.16; +} + +const cursorJsonPath = getCursorJsonPath(process.argv[2]); +const clickBounce = getBounceValue(); +const parsed = JSON.parse(fs.readFileSync(cursorJsonPath, "utf8")); +const samples = (Array.isArray(parsed) ? parsed : (parsed.samples ?? [])).sort( + (a, b) => (a.timeMs ?? 0) - (b.timeMs ?? 0), +); +const clicks = samples.filter((sample) => sample.interactionType === "click"); + +const windows = clicks.slice(0, 8).map((click) => { + const times = [0, 33, 66, 100, 133, 166, 200, 233, 260].map( + (offsetMs) => click.timeMs + offsetMs, + ); + return { + clickTimeMs: click.timeMs, + cursorType: click.cursorType ?? null, + assetId: click.assetId ?? null, + scales: times.map((timeMs) => ({ + timeMs, + progress: Number(clickBounceProgress(samples, timeMs).toFixed(3)), + scale: Number(clickBounceScale(clickBounce, clickBounceProgress(samples, timeMs)).toFixed(3)), + })), + }; +}); + +const report = { + cursorJsonPath, + provider: parsed.provider ?? (Array.isArray(parsed) ? "legacy-array" : null), + sampleCount: samples.length, + assetCount: Array.isArray(parsed.assets) ? parsed.assets.length : 0, + clickCount: clicks.length, + interactionCounts: samples.reduce((counts, sample) => { + const key = sample.interactionType ?? "missing"; + counts[key] = (counts[key] ?? 0) + 1; + return counts; + }, {}), + clickBounce, + windows, +}; + +console.log(JSON.stringify(report, null, 2)); +if (clicks.length === 0) { + process.exitCode = 2; +} diff --git a/scripts/test-windows-native-cursor.mjs b/scripts/test-windows-native-cursor.mjs new file mode 100644 index 000000000..44cabbe2b --- /dev/null +++ b/scripts/test-windows-native-cursor.mjs @@ -0,0 +1,1342 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +function readPositiveIntEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined) { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + console.warn(`[cursor-native-test] ignoring invalid ${name}=${raw}; using ${fallback}`); + return fallback; + } + + return Math.floor(parsed); +} + +const SAMPLE_INTERVAL_MS = readPositiveIntEnv("CURSOR_TEST_SAMPLE_INTERVAL_MS", 25); +const DURATION_MS = readPositiveIntEnv("CURSOR_TEST_DURATION_MS", 1800); +const SCREEN_FRAME_INTERVAL_MS = readPositiveIntEnv("CURSOR_TEST_SCREEN_FRAME_INTERVAL_MS", 100); +const READY_TIMEOUT_MS = readPositiveIntEnv("CURSOR_TEST_READY_TIMEOUT_MS", 5000); +const OUTPUT_DIR = + process.env.CURSOR_TEST_OUTPUT_DIR ?? + path.join(os.tmpdir(), `openscreen-cursor-native-${Date.now()}`); + +if (process.platform !== "win32") { + console.error("This diagnostic is Windows-only."); + process.exit(1); +} + +function encodePowerShell(script) { + return Buffer.from(script, "utf16le").toString("base64"); +} + +function quotePowerShellString(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function runPowerShell(script) { + return new Promise((resolve, reject) => { + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(script), + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0) { + resolve(stdout); + return; + } + + reject( + new Error(`PowerShell command failed (code=${code}, signal=${signal}): ${stderr.trim()}`), + ); + }); + }); +} + +function spawnPowerShell(script, { onStdout, onStderr } = {}) { + const scriptPath = path.join( + os.tmpdir(), + `openscreen-powershell-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.ps1`, + ); + fs.writeFileSync(scriptPath, script, "utf8"); + const child = spawn( + "powershell.exe", + ["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => onStdout?.(chunk)); + child.stderr.on("data", (chunk) => onStderr?.(chunk)); + + const done = new Promise((resolve, reject) => { + const cleanup = () => { + fs.rmSync(scriptPath, { force: true }); + }; + child.once("error", (error) => { + cleanup(); + reject(error); + }); + child.once("exit", (code, signal) => { + cleanup(); + if (code === 0 || child.killed) { + resolve({ code, signal }); + return; + } + + reject(new Error(`PowerShell process failed (code=${code}, signal=${signal})`)); + }); + }); + + return { child, done }; +} + +function buildSamplerScript() { + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorDiagnosticInterop { + private const int WH_MOUSE_LL = 14; + private const int WM_LBUTTONDOWN = 0x0201; + private const int WM_LBUTTONUP = 0x0202; + private static readonly object MouseSync = new object(); + private static int LeftDownCount = 0; + private static int LeftUpCount = 0; + private static IntPtr MouseHook = IntPtr.Zero; + private static LowLevelMouseProc MouseProcDelegate = MouseHookCallback; + + public delegate IntPtr LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam); + + public struct MouseButtonEvents { + public int LeftDownCount; + public int LeftUpCount; + } + + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + public static bool InstallMouseHook() { + if (MouseHook != IntPtr.Zero) { + return true; + } + + using (Process process = Process.GetCurrentProcess()) + using (ProcessModule module = process.MainModule) { + MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseProcDelegate, GetModuleHandle(module.ModuleName), 0); + } + + return MouseHook != IntPtr.Zero; + } + + public static MouseButtonEvents ConsumeMouseButtonEvents() { + lock (MouseSync) { + MouseButtonEvents events = new MouseButtonEvents { + LeftDownCount = LeftDownCount, + LeftUpCount = LeftUpCount + }; + LeftDownCount = 0; + LeftUpCount = 0; + return events; + } + } + + private static IntPtr MouseHookCallback(int nCode, IntPtr wParam, IntPtr lParam) { + if (nCode >= 0) { + int message = wParam.ToInt32(); + if (message == WM_LBUTTONDOWN || message == WM_LBUTTONUP) { + lock (MouseSync) { + if (message == WM_LBUTTONDOWN) { + LeftDownCount += 1; + } else { + LeftUpCount += 1; + } + } + } + } + + return CallNextHookEx(MouseHook, nCode, wParam, lParam); + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll")] + public static extern short GetAsyncKeyState(int vKey); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr LoadCursor(IntPtr hInstance, IntPtr lpCursorName); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr SetWindowsHookEx(int idHook, LowLevelMouseProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", SetLastError = true)] + private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + private static extern IntPtr GetModuleHandle(string lpModuleName); +} +"@ + +Add-Type -TypeDefinition $source + +$standardCursors = @{ + arrow = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32512)) + text = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32513)) + wait = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32514)) + crosshair = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32515)) + 'up-arrow' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32516)) + 'resize-nwse' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32642)) + 'resize-nesw' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32643)) + 'resize-ew' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32644)) + 'resize-ns' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32645)) + move = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32646)) + 'not-allowed' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32648)) + pointer = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32649)) + 'app-starting' = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32650)) + help = [OpenScreenCursorDiagnosticInterop]::LoadCursor([IntPtr]::Zero, [IntPtr]::new(32651)) +} + +function Get-StandardCursorType($cursorHandle) { + if ($cursorHandle -eq [IntPtr]::Zero) { + return $null + } + + foreach ($entry in $standardCursors.GetEnumerator()) { + if ($entry.Value -eq $cursorHandle) { + return $entry.Key + } + } + + return $null +} + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CustomCursorType($bitmap, $hotspotX, $hotspotY) { + if ($bitmap.Width -lt 24 -or $bitmap.Height -lt 24 -or $bitmap.Width -gt 64 -or $bitmap.Height -gt 64) { + return $null + } + + if ($hotspotX -lt ($bitmap.Width * 0.25) -or $hotspotX -gt ($bitmap.Width * 0.75) -or + $hotspotY -lt ($bitmap.Height * 0.15) -or $hotspotY -gt ($bitmap.Height * 0.55)) { + return $null + } + + $opaquePixels = 0 + $topHalfOpaquePixels = 0 + $left = $bitmap.Width + $top = $bitmap.Height + $right = -1 + $bottom = -1 + + for ($y = 0; $y -lt $bitmap.Height; $y++) { + for ($x = 0; $x -lt $bitmap.Width; $x++) { + if ($bitmap.GetPixel($x, $y).A -le 32) { + continue + } + + $opaquePixels += 1 + if ($y -lt ($bitmap.Height / 2)) { + $topHalfOpaquePixels += 1 + } + if ($x -lt $left) { $left = $x } + if ($x -gt $right) { $right = $x } + if ($y -lt $top) { $top = $y } + if ($y -gt $bottom) { $bottom = $y } + } + } + + if ($opaquePixels -lt 90 -or $right -lt $left -or $bottom -lt $top) { + return $null + } + + $opaqueWidth = $right - $left + 1 + $opaqueHeight = $bottom - $top + 1 + if ($opaqueWidth -lt ($bitmap.Width * 0.35) -or $opaqueWidth -gt ($bitmap.Width * 0.9) -or + $opaqueHeight -lt ($bitmap.Height * 0.45) -or $opaqueHeight -gt $bitmap.Height) { + return $null + } + + if ($top -gt ($bitmap.Height * 0.45) -or $bottom -lt ($bitmap.Height * 0.65)) { + return $null + } + + if ($topHalfOpaquePixels -gt ($opaquePixels * 0.55)) { + return 'closed-hand' + } + + return 'open-hand' +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorDiagnosticInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorDiagnosticInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorDiagnosticInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + $hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + $customCursorType = Get-CustomCursorType -bitmap $bitmap -hotspotX $hotspotX -hotspotY $hotspotY + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = $hotspotX + hotspotY = $hotspotY + cursorType = $customCursorType + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorDiagnosticInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorDiagnosticInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +[OpenScreenCursorDiagnosticInterop]::InstallMouseHook() | Out-Null +[OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) | Out-Null +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +$screenBounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +while ($true) { + [System.Windows.Forms.Application]::DoEvents() + $mouseEvents = [OpenScreenCursorDiagnosticInterop]::ConsumeMouseButtonEvents() + $cursorInfo = New-Object OpenScreenCursorDiagnosticInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorDiagnosticInterop+CURSORINFO]) + + if (-not [OpenScreenCursorDiagnosticInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $cursorType = Get-StandardCursorType $cursorInfo.hCursor + $leftButtonState = [OpenScreenCursorDiagnosticInterop]::GetAsyncKeyState(0x01) + $leftButtonDown = ($leftButtonState -band 0x8000) -ne 0 + $leftButtonPressed = ($mouseEvents.LeftDownCount -gt 0) -or (($leftButtonState -band 0x0001) -ne 0) + $leftButtonReleased = $mouseEvents.LeftUpCount -gt 0 + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + if ($asset -and $cursorType) { + $asset.cursorType = $cursorType + } elseif ($asset -and $asset.cursorType) { + $cursorType = $asset.cursorType + } + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + cursorType = $cursorType + leftButtonDown = $leftButtonDown + leftButtonPressed = $leftButtonPressed + leftButtonReleased = $leftButtonReleased + bounds = @{ + x = $screenBounds.Left + y = $screenBounds.Top + width = $screenBounds.Width + height = $screenBounds.Height + } + asset = $asset + } + + Start-Sleep -Milliseconds ${SAMPLE_INTERVAL_MS} +} +`; +} + +function buildMousePathScript(durationMs) { + const stepMs = 120; + const steps = Math.max(8, Math.floor(durationMs / stepMs)); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Windows.Forms + +$source = @" +using System.Runtime.InteropServices; +using System; + +public static class OpenScreenMouseDiagnosticInterop { + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool SetCursorPos(int X, int Y); + + [DllImport("user32.dll")] + public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo); +} +"@ + +Add-Type -TypeDefinition $source + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$points = @() +for ($i = 0; $i -lt ${steps}; $i++) { + $t = if (${steps} -le 1) { 0 } else { $i / (${steps} - 1) } + $x = [int]($bounds.Left + 80 + (($bounds.Width - 160) * $t)) + $wave = [Math]::Sin($t * [Math]::PI * 2) + $y = [int]($bounds.Top + ($bounds.Height / 2) + ($wave * [Math]::Min(180, $bounds.Height / 4))) + $points += @{ x = $x; y = $y } +} + +for ($i = 0; $i -lt $points.Count; $i++) { + $point = $points[$i] + [OpenScreenMouseDiagnosticInterop]::SetCursorPos($point.x, $point.y) | Out-Null + if ($i -eq [int]([Math]::Floor($points.Count / 2))) { + [OpenScreenMouseDiagnosticInterop]::mouse_event(0x0002, 0, 0, 0, [UIntPtr]::Zero) + Start-Sleep -Milliseconds 12 + [OpenScreenMouseDiagnosticInterop]::mouse_event(0x0004, 0, 0, 0, [UIntPtr]::Zero) + } + Start-Sleep -Milliseconds ${stepMs} +} +`; +} + +function buildScreenRecorderScript(outputDir, durationMs) { + const framesDir = path.join(outputDir, "screen-frames"); + + return String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +Add-Type -AssemblyName System.Windows.Forms + +$framesDir = ${quotePowerShellString(framesDir)} +New-Item -ItemType Directory -Force -Path $framesDir | Out-Null + +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +$targetWidth = 960 +$targetHeight = [int]([Math]::Round($targetWidth * ($bounds.Height / $bounds.Width))) +$frames = New-Object System.Collections.Generic.List[object] +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +$index = 0 + +while ($stopwatch.ElapsedMilliseconds -le ${durationMs + 700}) { + $sourceBitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($sourceBitmap) + $scaledBitmap = New-Object System.Drawing.Bitmap $targetWidth, $targetHeight, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $scaledGraphics = [System.Drawing.Graphics]::FromImage($scaledBitmap) + $timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + $fileName = ('frame_{0:D4}.png' -f $index) + $path = Join-Path $framesDir $fileName + + try { + $graphics.CopyFromScreen($bounds.Left, $bounds.Top, 0, 0, $bounds.Size) + $scaledGraphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic + $scaledGraphics.DrawImage($sourceBitmap, 0, 0, $targetWidth, $targetHeight) + $scaledBitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + $frames.Add(@{ + index = $index + timestampMs = $timestampMs + path = $path + width = $targetWidth + height = $targetHeight + bounds = @{ + x = $bounds.Left + y = $bounds.Top + width = $bounds.Width + height = $bounds.Height + } + }) | Out-Null + } + finally { + $scaledGraphics.Dispose() + $scaledBitmap.Dispose() + $graphics.Dispose() + $sourceBitmap.Dispose() + } + + $index += 1 + Start-Sleep -Milliseconds ${SCREEN_FRAME_INTERVAL_MS} +} + +($frames | ConvertTo-Json -Depth 6) | Set-Content -Path (Join-Path $framesDir 'frames.json') -Encoding UTF8 +`; +} + +function createReadyWaiter() { + let settled = false; + let resolveReady = null; + const promise = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + reject(new Error("Timed out waiting for cursor sampler readiness.")); + }, READY_TIMEOUT_MS); + + resolveReady = () => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve(); + }; + }); + + return { + promise, + resolve: () => resolveReady?.(), + }; +} + +function writeAssets(assets, outputDir) { + const assetDir = path.join(outputDir, "assets"); + fs.mkdirSync(assetDir, { recursive: true }); + + for (const asset of assets.values()) { + const base64 = asset.imageDataUrl?.replace(/^data:image\/png;base64,/, ""); + if (!base64) { + continue; + } + + const safeId = String(asset.id).replace(/[^a-zA-Z0-9_-]/g, "_"); + fs.writeFileSync(path.join(assetDir, `${safeId}.png`), Buffer.from(base64, "base64")); + } +} + +function toRecordingData(samples, assets) { + const firstTimestampMs = samples[0]?.timestampMs ?? Date.now(); + let previousLeftButtonDown = false; + const normalizedSamples = samples.flatMap((sample) => { + const bounds = sample.bounds; + if (!bounds || bounds.width <= 0 || bounds.height <= 0) { + return []; + } + + const leftButtonDown = sample.leftButtonDown === true; + const leftButtonPressed = sample.leftButtonPressed === true; + const leftButtonReleased = sample.leftButtonReleased === true; + const interactionType = + leftButtonPressed || (leftButtonDown && !previousLeftButtonDown) + ? "click" + : leftButtonReleased || (!leftButtonDown && previousLeftButtonDown) + ? "mouseup" + : "move"; + previousLeftButtonDown = leftButtonDown; + + return [ + { + timeMs: Math.max(0, sample.timestampMs - firstTimestampMs), + cx: (sample.x - bounds.x) / bounds.width, + cy: (sample.y - bounds.y) / bounds.height, + assetId: sample.handle, + visible: Boolean(sample.visible), + cursorType: sample.cursorType ?? null, + interactionType, + }, + ]; + }); + + return { + version: 2, + provider: assets.size > 0 ? "native" : "none", + samples: normalizedSamples, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + platform: "win32", + imageDataUrl: asset.imageDataUrl, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + scaleFactor: 1, + cursorType: asset.cursorType ?? null, + })), + }; +} + +function escapeScriptJson(value) { + return JSON.stringify(value).replace(/ + + + + +OpenScreen native cursor diagnostic + + + +
+

OpenScreen native cursor diagnostic

+
+
${report.sampleCount}samples
+
${report.assetCount}assets
+
${report.uniquePositionCount}positions
+
${report.errorCount}errors
+
+

The red cross is the captured native hotspot. Native bitmaps are drawn at 1x, 2x, and 3x. The last cursor is a crisp vector 3x replacement anchored on the same hotspot.

+ +
+
+ + + +`; +} + +function readScreenFrames(outputDir, recordingStartTimestampMs) { + const framesJsonPath = path.join(outputDir, "screen-frames", "frames.json"); + if (!fs.existsSync(framesJsonPath)) { + return []; + } + + const rawFrames = JSON.parse(fs.readFileSync(framesJsonPath, "utf8").replace(/^\uFEFF/, "")); + const frames = Array.isArray(rawFrames) ? rawFrames : [rawFrames]; + + return frames + .filter((frame) => frame?.path && fs.existsSync(frame.path)) + .map((frame) => ({ + ...frame, + timeMs: Math.max(0, frame.timestampMs - recordingStartTimestampMs), + imageDataUrl: `data:image/png;base64,${fs.readFileSync(frame.path).toString("base64")}`, + })); +} + +function buildRealCaptureHtml(report, recordingData, screenFrames) { + return ` + + + + +OpenScreen native cursor real capture diagnostic + + + +
+

Real screen capture + reconstructed native cursor

+

Background frames are real Windows screenshots. Native bitmaps are reconstructed at 1x, 2x, and 3x; the last cursor is a crisp vector 3x replacement. The red cross marks the recorded hotspot.

+ +
+ + + + +`; +} + +function findPlaywrightChromiumExecutable(defaultPath) { + if (fs.existsSync(defaultPath)) { + return defaultPath; + } + + const baseDir = path.join(process.env.LOCALAPPDATA ?? "", "ms-playwright"); + if (!baseDir || !fs.existsSync(baseDir)) { + return defaultPath; + } + + const candidates = fs + .readdirSync(baseDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith("chromium-")) + .map((entry) => ({ + executablePath: path.join(baseDir, entry.name, "chrome-win64", "chrome.exe"), + revision: Number.parseInt(entry.name.slice("chromium-".length), 10), + })) + .filter( + (candidate) => Number.isFinite(candidate.revision) && fs.existsSync(candidate.executablePath), + ) + .sort((a, b) => b.revision - a.revision) + .map((candidate) => candidate.executablePath); + + return candidates[0] ?? defaultPath; +} + +async function writePreviewVideo(reportPath, outputPath) { + const { chromium } = await import("playwright"); + const browser = await chromium.launch({ + executablePath: findPlaywrightChromiumExecutable(chromium.executablePath()), + headless: true, + }); + try { + const page = await browser.newPage({ viewport: { width: 1180, height: 760 } }); + await page.goto(`file:///${reportPath.replaceAll("\\", "/")}`); + const base64 = await page.evaluate(() => window.__exportWebm()); + fs.writeFileSync(outputPath, Buffer.from(base64, "base64")); + } finally { + await browser.close(); + } +} + +function assertReport(report) { + const failures = []; + if (report.sampleCount < Math.floor(DURATION_MS / SAMPLE_INTERVAL_MS / 3)) { + failures.push(`Too few samples: ${report.sampleCount}.`); + } + if (report.visibleSampleCount === 0) { + failures.push("No visible cursor samples were captured."); + } + if (report.assetCount === 0) { + failures.push("No cursor asset PNG was captured."); + } + if (report.uniquePositionCount < 4) { + failures.push(`Cursor movement was not observed enough times: ${report.uniquePositionCount}.`); + } + if (report.errorCount > 0) { + failures.push(`Sampler reported ${report.errorCount} error event(s).`); + } + if (report.leftButtonPressedSampleCount === 0 || report.clickSampleCount === 0) { + failures.push("Left button click interaction was not observed."); + } + + if (failures.length > 0) { + throw new Error(failures.join(" ")); + } +} + +fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + +const events = []; +const assets = new Map(); +let lineBuffer = ""; +let stoppingSampler = false; +const readyWaiter = createReadyWaiter(); +const sampler = spawnPowerShell(buildSamplerScript(), { + onStdout: (chunk) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + let event; + try { + event = JSON.parse(trimmed); + } catch { + process.stderr.write(`[cursor-native-test] dropping non-JSON line: ${trimmed}\n`); + continue; + } + events.push(event); + if (event.type === "ready") { + readyWaiter.resolve(); + } + if (event.asset?.id && !assets.has(event.asset.id)) { + assets.set(event.asset.id, event.asset); + } + } + }, + onStderr: (chunk) => { + if (!stoppingSampler && !chunk.startsWith("#< CLIXML")) { + process.stderr.write(`[cursor-native-test] ${chunk}`); + } + }, +}); +let screenRecorder = null; + +try { + await readyWaiter.promise; + screenRecorder = spawnPowerShell(buildScreenRecorderScript(OUTPUT_DIR, DURATION_MS), { + onStderr: (chunk) => { + if (!chunk.startsWith("#< CLIXML") && !chunk.startsWith(" setTimeout(resolve, 150)); + await runPowerShell(buildMousePathScript(DURATION_MS)); + await new Promise((resolve) => setTimeout(resolve, Math.max(250, SAMPLE_INTERVAL_MS * 3))); + await screenRecorder.done; +} finally { + if (!sampler.child.killed) { + stoppingSampler = true; + sampler.child.kill(); + } + if (screenRecorder && !screenRecorder.child.killed) { + screenRecorder.child.kill(); + } +} + +const samples = events.filter((event) => event.type === "sample"); +const errors = events.filter((event) => event.type === "error"); +const recordingStartTimestampMs = samples[0]?.timestampMs ?? Date.now(); +const uniquePositions = new Set(samples.map((sample) => `${sample.x},${sample.y}`)); +let previousLeftButtonDown = false; +let clickSampleCount = 0; +for (const sample of samples) { + const leftButtonDown = sample.leftButtonDown === true; + const leftButtonPressed = sample.leftButtonPressed === true; + if (leftButtonPressed || (leftButtonDown && !previousLeftButtonDown)) { + clickSampleCount += 1; + } + previousLeftButtonDown = leftButtonDown; +} +const report = { + outputDir: OUTPUT_DIR, + sampleIntervalMs: SAMPLE_INTERVAL_MS, + durationMs: DURATION_MS, + eventCount: events.length, + sampleCount: samples.length, + visibleSampleCount: samples.filter((sample) => sample.visible).length, + assetCount: assets.size, + uniqueCursorHandleCount: new Set(samples.map((sample) => sample.handle).filter(Boolean)).size, + uniquePositionCount: uniquePositions.size, + leftButtonDownSampleCount: samples.filter((sample) => sample.leftButtonDown === true).length, + leftButtonPressedSampleCount: samples.filter((sample) => sample.leftButtonPressed === true) + .length, + clickSampleCount, + errorCount: errors.length, + firstSample: samples[0] ?? null, + lastSample: samples.at(-1) ?? null, + assets: [...assets.values()].map((asset) => ({ + id: asset.id, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + cursorType: asset.cursorType ?? null, + })), +}; +const recordingData = toRecordingData(samples, assets); +const screenFrames = readScreenFrames(OUTPUT_DIR, recordingStartTimestampMs); +const reportHtmlPath = path.join(OUTPUT_DIR, "report.html"); +const previewVideoPath = path.join(OUTPUT_DIR, "preview.webm"); +const realCaptureHtmlPath = path.join(OUTPUT_DIR, "real-capture-report.html"); +const realCaptureVideoPath = path.join(OUTPUT_DIR, "real-capture-preview.webm"); + +writeAssets(assets, OUTPUT_DIR); +fs.writeFileSync(path.join(OUTPUT_DIR, "events.json"), JSON.stringify(events, null, 2)); +fs.writeFileSync( + path.join(OUTPUT_DIR, "cursor-recording-data.json"), + JSON.stringify(recordingData, null, 2), +); +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); +fs.writeFileSync(reportHtmlPath, buildVisualReportHtml(report, recordingData)); +if (screenFrames.length > 0) { + fs.writeFileSync(realCaptureHtmlPath, buildRealCaptureHtml(report, recordingData, screenFrames)); + report.screenFrameCount = screenFrames.length; +} + +try { + await writePreviewVideo(reportHtmlPath, previewVideoPath); + report.previewVideoPath = previewVideoPath; +} catch (error) { + report.previewVideoError = error instanceof Error ? error.message : String(error); +} + +if (screenFrames.length > 0) { + try { + await writePreviewVideo(realCaptureHtmlPath, realCaptureVideoPath); + report.realCaptureVideoPath = realCaptureVideoPath; + } catch (error) { + report.realCaptureVideoError = error instanceof Error ? error.message : String(error); + } +} + +fs.writeFileSync(path.join(OUTPUT_DIR, "report.json"), JSON.stringify(report, null, 2)); + +assertReport(report); + +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs new file mode 100644 index 000000000..82a0a8dac --- /dev/null +++ b/scripts/test-windows-wgc-helper.mjs @@ -0,0 +1,344 @@ +import { spawn, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, ".."); +const HELPER_PATH = + process.env.OPENSCREEN_WGC_CAPTURE_EXE ?? + path.join(ROOT, "electron", "native", "bin", "win32-x64", "wgc-capture.exe"); + +const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000); +const WITH_SYSTEM_AUDIO = + process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" || + process.argv.includes("--system-audio"); +const WITH_MICROPHONE = + process.env.OPENSCREEN_WGC_TEST_MICROPHONE === "true" || + process.argv.includes("--microphone") || + process.argv.includes("--mic"); +const WITH_WINDOW = + process.env.OPENSCREEN_WGC_TEST_WINDOW === "true" || process.argv.includes("--window"); +const WITH_WEBCAM = + process.env.OPENSCREEN_WGC_TEST_WEBCAM === "true" || process.argv.includes("--webcam"); + +function runHelper(config) { + return new Promise((resolve, reject) => { + const child = spawn(HELPER_PATH, [JSON.stringify(config)], { + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + + let stdout = ""; + let stderr = ""; + let stopTimer = null; + const scheduleStop = () => { + if (stopTimer) { + return; + } + stopTimer = setTimeout(() => { + child.stdin.write("stop\n"); + }, DURATION_MS); + }; + const fallbackTimer = setTimeout(scheduleStop, 15_000); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + if (stdout.includes('"recording-started"') || stdout.includes("Recording started")) { + scheduleStop(); + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("error", reject); + child.once("exit", (code) => { + clearTimeout(fallbackTimer); + if (stopTimer) { + clearTimeout(stopTimer); + } + resolve({ code, stdout, stderr }); + }); + }); +} + +function startFixtureWindow() { + return new Promise((resolve, reject) => { + const child = spawn("mspaint.exe", [], { + stdio: ["ignore", "ignore", "ignore"], + windowsHide: false, + }); + + const poll = setInterval(() => { + const lookup = spawnSync( + "powershell", + [ + "-NoProfile", + "-Command", + `(Get-Process -Id ${child.pid} -ErrorAction SilentlyContinue).MainWindowHandle`, + ], + { encoding: "utf8", windowsHide: true }, + ); + const handle = lookup.stdout + .trim() + .split(/\r?\n/) + .find((line) => /^\d+$/.test(line.trim())); + if (handle && handle !== "0") { + clearInterval(poll); + clearTimeout(timer); + resolve({ child, sourceId: `window:${handle.trim()}:0` }); + } + }, 250); + + const timer = setTimeout(() => { + clearInterval(poll); + child.kill(); + reject(new Error("Timed out waiting for fixture window handle")); + }, 10_000); + child.once("error", (error) => { + clearInterval(poll); + clearTimeout(timer); + reject(error); + }); + }); +} + +function normalizeDeviceName(value) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function scoreDeviceName(candidateName, candidateId, requestedName) { + const candidate = normalizeDeviceName(candidateName ?? ""); + const id = normalizeDeviceName(candidateId ?? ""); + const requested = normalizeDeviceName(requestedName ?? ""); + if (!requested) return 0; + if (candidate === requested) return 1000; + if (candidate.includes(requested) || requested.includes(candidate)) return 900; + if (id.includes(requested) || requested.includes(id)) return 800; + return requested + .split(/\s+/) + .filter((word) => word.length > 1 && !["camera", "webcam", "video", "input"].includes(word)) + .reduce((score, word) => { + if (candidate.includes(word)) return score + 100; + if (id.includes(word)) return score + 50; + return score; + }, 0); +} + +function resolveDirectShowWebcamClsid(requestedName) { + if (!requestedName) return ""; + const query = spawnSync( + "reg.exe", + ["query", "HKCR\\CLSID\\{860BB310-5D01-11D0-BD3B-00A0C911CE86}\\Instance", "/s"], + { encoding: "utf8", windowsHide: true }, + ); + if (query.status !== 0) return ""; + const entries = []; + let current = {}; + for (const rawLine of query.stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + if (/^HKEY_/i.test(line)) { + if (current.friendlyName || current.clsid) entries.push(current); + current = {}; + continue; + } + const match = line.match(/^(\S+)\s+REG_SZ\s+(.+)$/); + if (!match) continue; + if (match[1] === "FriendlyName") current.friendlyName = match[2].trim(); + if (match[1] === "CLSID") current.clsid = match[2].trim(); + } + if (current.friendlyName || current.clsid) entries.push(current); + + let best = null; + for (const entry of entries) { + if (!entry.clsid) continue; + const score = scoreDeviceName(entry.friendlyName, entry.clsid, requestedName); + if (!best || score > best.score) { + best = { ...entry, score }; + } + } + return best && best.score > 0 ? best.clsid : ""; +} + +function probeStreams(outputPath) { + const ffprobe = spawnSync( + "ffprobe", + ["-v", "error", "-show_streams", "-of", "json", outputPath], + { encoding: "utf8", windowsHide: true }, + ); + if (ffprobe.status !== 0) { + throw new Error(`ffprobe failed: ${ffprobe.stderr || ffprobe.stdout}`); + } + return JSON.parse(ffprobe.stdout).streams ?? []; +} + +function measureFirstFrameLuma(outputPath) { + const ffmpeg = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + outputPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "pipe:1", + ], + { windowsHide: true, maxBuffer: 64 * 1024 * 1024 }, + ); + if (ffmpeg.status !== 0) { + throw new Error(`ffmpeg frame extraction failed: ${ffmpeg.stderr?.toString() ?? ""}`); + } + const data = ffmpeg.stdout; + if (!data || data.length === 0) { + throw new Error(`ffmpeg did not return frame data for ${outputPath}`); + } + let sum = 0; + let max = 0; + for (const value of data) { + sum += value; + if (value > max) { + max = value; + } + } + return { average: sum / data.length, max }; +} + +if (process.platform !== "win32") { + console.log("Skipping WGC helper smoke test: Windows-only."); + process.exit(0); +} + +if (!fs.existsSync(HELPER_PATH)) { + throw new Error(`WGC helper not found at ${HELPER_PATH}. Run npm run build:native:win first.`); +} + +const outputPath = path.join( + os.tmpdir(), + `openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`, +); + +const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null; + +const config = { + schemaVersion: 2, + recordingId: Date.now(), + outputPath, + sourceType: fixtureWindow ? "window" : "display", + sourceId: fixtureWindow ? fixtureWindow.sourceId : "screen:0:0", + displayId: 0, + fps: 30, + videoWidth: 1280, + videoHeight: 720, + displayX: 0, + displayY: 0, + displayW: 1920, + displayH: 1080, + hasDisplayBounds: true, + captureSystemAudio: WITH_SYSTEM_AUDIO, + captureMic: WITH_MICROPHONE, + microphoneDeviceId: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_ID ?? "default", + microphoneDeviceName: process.env.OPENSCREEN_WGC_TEST_MICROPHONE_DEVICE_NAME ?? "", + microphoneGain: 1.4, + webcamEnabled: WITH_WEBCAM, + webcamDeviceId: process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_ID ?? "", + webcamDeviceName: process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", + webcamDirectShowClsid: resolveDirectShowWebcamClsid( + process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", + ), + webcamWidth: 640, + webcamHeight: 360, + webcamFps: 30, + outputs: { screenPath: outputPath }, +}; + +let result; +try { + result = await runHelper(config); +} finally { + if (fixtureWindow) { + fixtureWindow.child.kill(); + } +} +if (result.code !== 0) { + if ( + WITH_WEBCAM && + /No native Windows webcam devices were found|Failed to initialize native webcam/.test( + result.stderr, + ) + ) { + console.log("Skipping WGC webcam smoke test: no native Windows webcam device is available."); + process.exit(0); + } + throw new Error(`WGC helper exited with ${result.code}\n${result.stdout}\n${result.stderr}`); +} +if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) { + throw new Error(`WGC helper did not produce a video at ${outputPath}`); +} + +const streams = probeStreams(outputPath); +const hasVideo = streams.some((stream) => stream.codec_type === "video"); +const hasAudio = streams.some((stream) => stream.codec_type === "audio"); +const webcamFormatLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"webcam-format"')); +const webcamFormat = webcamFormatLine ? JSON.parse(webcamFormatLine) : null; +const audioFormatLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"audio-format"')); +const audioFormat = audioFormatLine ? JSON.parse(audioFormatLine) : null; +const nativeWebcamDiagnostics = result.stderr + .split(/\r?\n/) + .filter((line) => line.includes("Native webcam candidate")); +const nativeMicrophoneDiagnostics = result.stderr + .split(/\r?\n/) + .filter( + (line) => + line.includes("Native microphone candidate") || + line.includes("Selected native microphone endpoint"), + ); +if (!hasVideo) { + throw new Error(`WGC helper output has no video stream: ${outputPath}`); +} +if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) { + throw new Error(`WGC helper output has no audio stream: ${outputPath}`); +} +const frameLuma = measureFirstFrameLuma(outputPath); +if (frameLuma.average < 1 && frameLuma.max < 5) { + throw new Error( + `WGC helper output first frame is black: ${outputPath}\n${result.stdout}\n${result.stderr}`, + ); +} + +console.log( + JSON.stringify( + { + success: true, + outputPath, + bytes: fs.statSync(outputPath).size, + streams: streams.map((stream) => ({ + index: stream.index, + codecType: stream.codec_type, + codecName: stream.codec_name, + duration: stream.duration, + })), + selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName, + selectedWebcamDeviceName: webcamFormat?.deviceName, + nativeMicrophoneDiagnostics, + nativeWebcamDiagnostics, + firstFrameLuma: frameLuma, + }, + null, + 2, + ), +); diff --git a/src/assets/cursors/Cursor=App-Starting.svg b/src/assets/cursors/Cursor=App-Starting.svg new file mode 100644 index 000000000..7a10d4080 --- /dev/null +++ b/src/assets/cursors/Cursor=App-Starting.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Beachball.svg b/src/assets/cursors/Cursor=Beachball.svg new file mode 100644 index 000000000..30bdbe502 --- /dev/null +++ b/src/assets/cursors/Cursor=Beachball.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Cross.svg b/src/assets/cursors/Cursor=Cross.svg new file mode 100644 index 000000000..b404553da --- /dev/null +++ b/src/assets/cursors/Cursor=Cross.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Default.svg b/src/assets/cursors/Cursor=Default.svg new file mode 100644 index 000000000..f76f31fd7 --- /dev/null +++ b/src/assets/cursors/Cursor=Default.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Grabbing).svg b/src/assets/cursors/Cursor=Hand-(Grabbing).svg new file mode 100644 index 000000000..082786750 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Grabbing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Open).svg b/src/assets/cursors/Cursor=Hand-(Open).svg new file mode 100644 index 000000000..4ceafb0f0 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Open).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Pointing).svg b/src/assets/cursors/Cursor=Hand-(Pointing).svg new file mode 100644 index 000000000..19a70a673 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Pointing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Help.svg b/src/assets/cursors/Cursor=Help.svg new file mode 100644 index 000000000..d187c5227 --- /dev/null +++ b/src/assets/cursors/Cursor=Help.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Menu.svg b/src/assets/cursors/Cursor=Menu.svg new file mode 100644 index 000000000..3489257b1 --- /dev/null +++ b/src/assets/cursors/Cursor=Menu.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Move.svg b/src/assets/cursors/Cursor=Move.svg new file mode 100644 index 000000000..50e56b767 --- /dev/null +++ b/src/assets/cursors/Cursor=Move.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Not-Allowed.svg b/src/assets/cursors/Cursor=Not-Allowed.svg new file mode 100644 index 000000000..8b2c3f8f5 --- /dev/null +++ b/src/assets/cursors/Cursor=Not-Allowed.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Down).svg b/src/assets/cursors/Cursor=Resize-(Down).svg new file mode 100644 index 000000000..fba367294 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left).svg b/src/assets/cursors/Cursor=Resize-(Left).svg new file mode 100644 index 000000000..6e21fb77d --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left-Right).svg b/src/assets/cursors/Cursor=Resize-(Left-Right).svg new file mode 100644 index 000000000..7021d2297 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left-Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Right).svg b/src/assets/cursors/Cursor=Resize-(Right).svg new file mode 100644 index 000000000..1ce801ce1 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up).svg b/src/assets/cursors/Cursor=Resize-(Up).svg new file mode 100644 index 000000000..9c4ac0f00 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up-Down).svg b/src/assets/cursors/Cursor=Resize-(Up-Down).svg new file mode 100644 index 000000000..b01a40e3a --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up-Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-East-South-West.svg b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg new file mode 100644 index 000000000..1185c1fff --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-South.svg b/src/assets/cursors/Cursor=Resize-North-South.svg new file mode 100644 index 000000000..57eaa0563 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-South.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-West-South-East.svg b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg new file mode 100644 index 000000000..f00fc8797 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-West-East.svg b/src/assets/cursors/Cursor=Resize-West-East.svg new file mode 100644 index 000000000..ef1929fbe --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-West-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Text-Cursor.svg b/src/assets/cursors/Cursor=Text-Cursor.svg new file mode 100644 index 000000000..1bfd0809f --- /dev/null +++ b/src/assets/cursors/Cursor=Text-Cursor.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Up-Arrow.svg b/src/assets/cursors/Cursor=Up-Arrow.svg new file mode 100644 index 000000000..b742e7058 --- /dev/null +++ b/src/assets/cursors/Cursor=Up-Arrow.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/assets/cursors/Cursor=Wait.svg b/src/assets/cursors/Cursor=Wait.svg new file mode 100644 index 000000000..2b569340c --- /dev/null +++ b/src/assets/cursors/Cursor=Wait.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-In.svg b/src/assets/cursors/Cursor=Zoom-In.svg new file mode 100644 index 000000000..8ec9b3ce5 --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-In.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-Out.svg b/src/assets/cursors/Cursor=Zoom-Out.svg new file mode 100644 index 000000000..810878bad --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-Out.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index bffbd9c9a..b786e6940 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -10,6 +10,7 @@ import { MdMic, MdMicOff, MdMonitor, + MdMouse, MdRestartAlt, MdVideocam, MdVideocamOff, @@ -20,6 +21,7 @@ import { import { RxDragHandleDots2 } from "react-icons/rx"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; +import { nativeBridgeClient } from "@/native"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useCameraDevices } from "../../hooks/useCameraDevices"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; @@ -42,6 +44,7 @@ const ICON_CONFIG = { micOff: { icon: MdMicOff, size: ICON_SIZE }, webcamOn: { icon: MdVideocam, size: ICON_SIZE }, webcamOff: { icon: MdVideocamOff, size: ICON_SIZE }, + cursor: { icon: MdMouse, size: ICON_SIZE }, pause: { icon: BsPauseCircle, size: ICON_SIZE }, resume: { icon: BsPlayCircle, size: ICON_SIZE }, stop: { icon: FaRegStopCircle, size: ICON_SIZE }, @@ -100,12 +103,16 @@ export function LaunchWindow() { setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId, + setMicrophoneDeviceName, systemAudioEnabled, setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, webcamDeviceId, setWebcamDeviceId, + setWebcamDeviceName, + cursorCaptureMode, + setCursorCaptureMode, } = useScreenRecorder(); const showMicControls = microphoneEnabled && !recording; @@ -119,6 +126,7 @@ export function LaunchWindow() { const [isWebcamFocused, setIsWebcamFocused] = useState(false); const webcamExpanded = isWebcamHovered || isWebcamFocused; const [isLanguageMenuOpen, setIsLanguageMenuOpen] = useState(false); + const [isWindows, setIsWindows] = useState(false); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ @@ -147,14 +155,16 @@ export function LaunchWindow() { const selectedMicLabel = micDevices.find((d) => d.deviceId === (microphoneDeviceId || selectedMicId))?.label || t("audio.defaultMicrophone"); + const selectedCameraDevice = cameraDevices.find( + (d) => d.deviceId === (webcamDeviceId || selectedCameraId), + ); const selectedCameraLabel = isCameraDevicesLoading ? t("webcam.searching") : cameraDevicesError ? t("webcam.unavailable") : cameraDevices.length === 0 ? t("webcam.noneFound") - : cameraDevices.find((d) => d.deviceId === (webcamDeviceId || selectedCameraId))?.label || - t("webcam.defaultCamera"); + : selectedCameraDevice?.label || t("webcam.defaultCamera"); const { level } = useAudioLevelMeter({ enabled: showMicControls, @@ -164,14 +174,36 @@ export function LaunchWindow() { useEffect(() => { if (selectedMicId && selectedMicId !== "default") { setMicrophoneDeviceId(selectedMicId); + setMicrophoneDeviceName(micDevices.find((d) => d.deviceId === selectedMicId)?.label); } - }, [selectedMicId, setMicrophoneDeviceId]); + }, [selectedMicId, micDevices, setMicrophoneDeviceId, setMicrophoneDeviceName]); useEffect(() => { if (selectedCameraId) { setWebcamDeviceId(selectedCameraId); + setWebcamDeviceName(cameraDevices.find((d) => d.deviceId === selectedCameraId)?.label); } - }, [selectedCameraId, setWebcamDeviceId]); + }, [selectedCameraId, cameraDevices, setWebcamDeviceId, setWebcamDeviceName]); + + useEffect(() => { + let cancelled = false; + nativeBridgeClient.system + .getPlatform() + .then((platform) => { + if (!cancelled) { + setIsWindows(platform === "win32"); + } + }) + .catch(() => { + if (!cancelled) { + setIsWindows(false); + } + }); + + return () => { + cancelled = true; + }; + }, []); useEffect(() => { if (!import.meta.env.DEV) { @@ -250,6 +282,8 @@ export function LaunchWindow() { const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); + const [, setHudPointerDownCount] = useState(0); + const [, setRecordPointerDownCount] = useState(0); useEffect(() => { const checkSelectedSource = async () => { @@ -285,13 +319,17 @@ export function LaunchWindow() { } if (result.success && result.path) { - await window.electronAPI.setCurrentVideoPath(result.path); + const setVideoPathResult = await nativeBridgeClient.project.setCurrentVideoPath(result.path); + if (!setVideoPathResult.success) { + console.error("Failed to set current video path:", setVideoPathResult); + return; + } await window.electronAPI.switchToEditor(); } }; const openProjectFile = async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled || !result.success) return; await window.electronAPI.switchToEditor(); }; @@ -320,6 +358,9 @@ export function LaunchWindow() { // recording toolbar widened (issue #305).
{ + setHudPointerDownCount((count) => count + 1); + }} > {systemLocaleSuggestion && (
{ + const selectedDevice = micDevices.find((d) => d.deviceId === e.target.value); setSelectedMicId(e.target.value); setMicrophoneDeviceId(e.target.value); + setMicrophoneDeviceName(selectedDevice?.label); }} className={`w-full appearance-none bg-white/5 text-white text-[11px] rounded-lg pl-2 pr-6 py-1 border border-white/10 outline-none hover:bg-white/10 transition-colors cursor-pointer ${!micExpanded ? "sr-only" : ""}`} > @@ -440,8 +483,12 @@ export function LaunchWindow() { { + const device = cameraDevices.find((item) => item.deviceId === e.target.value); setSelectedCameraId(e.target.value); setWebcamDeviceId(e.target.value); + setWebcamDeviceName(device?.label); }} className="sr-only" > @@ -494,6 +543,7 @@ export function LaunchWindow() { {/* Source selector */} + {isWindows && ( + + )}
{/* Record/Stop group */}
- {cursorHighlight && onCursorHighlightChange && ( + {showCursorHighlightSettings && cursorHighlight && onCursorHighlightChange && (
Cursor highlight
@@ -1218,7 +1230,7 @@ export function SettingsPanel({ )}
- {showCropModal && cropRegion && onCropChange && ( + {showCropDropdown && cropRegion && onCropChange && ( <>
setShowCropDropdown(false)} />
@@ -1395,7 +1499,7 @@ export function SettingsPanel({
@@ -1942,6 +2026,7 @@ export default function VideoEditor() {
0 || hasNativeCursorRecordingData(cursorRecordingData) + } + showCursorSettings={showCursorSettings} + showCursorHighlightSettings={isMac} />
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index ee52bd9f6..10eb8a0ce 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -25,8 +25,24 @@ import { type WebcamLayoutPreset, type WebcamSizePreset, } from "@/lib/compositeLayout"; +import { + createNativeCursorMotionBlurState, + createNativeCursorSmoothingState, + getNativeCursorClickBounceProgress, + getNativeCursorClickBounceScale, + getNativeCursorMotionBlurPx, + hasNativeCursorRecordingData, + projectNativeCursorToLocal, + projectNativeCursorToStage, + resetNativeCursorMotionBlurState, + resetNativeCursorSmoothingState, + resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, + smoothNativeCursorSample, +} from "@/lib/cursor/nativeCursor"; import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; +import type { CursorRecordingData } from "@/native/contracts"; import { type AspectRatio, formatAspectRatioForCSS, @@ -36,7 +52,12 @@ import { AnnotationOverlay } from "./AnnotationOverlay"; import { type AnnotationRegion, type BlurData, + type CursorTelemetryPoint, computeRotation3DContainScale, + DEFAULT_CURSOR_CLICK_BOUNCE, + DEFAULT_CURSOR_MOTION_BLUR, + DEFAULT_CURSOR_SIZE, + DEFAULT_CURSOR_SMOOTHING, DEFAULT_ROTATION_3D, isRotation3DIdentity, lerpRotation3D, @@ -67,6 +88,11 @@ import { DEFAULT_CURSOR_HIGHLIGHT, drawCursorHighlightGraphics, } from "./videoPlayback/cursorHighlight"; +import { + DEFAULT_CURSOR_CONFIG, + PixiCursorOverlay, + preloadCursorAssets, +} from "./videoPlayback/cursorRenderer"; import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils"; import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils"; import { clamp01 } from "./videoPlayback/mathUtils"; @@ -112,6 +138,7 @@ interface VideoPlaybackProps { trimRegions?: TrimRegion[]; speedRegions?: SpeedRegion[]; aspectRatio: AspectRatio; + cursorRecordingData?: CursorRecordingData | null; annotationRegions?: AnnotationRegion[]; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string | null) => void; @@ -124,9 +151,14 @@ interface VideoPlaybackProps { onBlurSizeChange?: (id: string, size: { width: number; height: number }) => void; onBlurDataChange?: (id: string, blurData: BlurData) => void; onBlurDataCommit?: () => void; - cursorTelemetry?: import("./types").CursorTelemetryPoint[]; + cursorTelemetry?: CursorTelemetryPoint[]; cursorHighlight?: CursorHighlightConfig; cursorClickTimestamps?: number[]; + showCursor?: boolean; + cursorSize?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; } export interface VideoPlaybackRef { @@ -139,6 +171,41 @@ export interface VideoPlaybackRef { pause: () => void; } +function getResolvedVideoDuration(video: HTMLVideoElement): number | null { + if (Number.isFinite(video.duration) && video.duration > 0) { + return video.duration; + } + + if (video.seekable.length > 0) { + const lastRangeIndex = video.seekable.length - 1; + const seekableEnd = video.seekable.end(lastRangeIndex); + if (Number.isFinite(seekableEnd) && seekableEnd > 0) { + return seekableEnd; + } + } + + return null; +} + +function getEndedVideoDuration(video: HTMLVideoElement): number | null { + const currentTime = video.currentTime; + if (!Number.isFinite(currentTime) || currentTime <= 0) { + return null; + } + + if (video.ended) { + return currentTime; + } + + const resolvedDuration = getResolvedVideoDuration(video); + const durationEpsilonSeconds = 0.05; + if (resolvedDuration && currentTime >= resolvedDuration - durationEpsilonSeconds) { + return resolvedDuration; + } + + return null; +} + const VideoPlayback = forwardRef( ( { @@ -172,6 +239,7 @@ const VideoPlayback = forwardRef( trimRegions = [], speedRegions = [], aspectRatio, + cursorRecordingData, annotationRegions = [], selectedAnnotationId, onSelectAnnotation, @@ -187,6 +255,11 @@ const VideoPlayback = forwardRef( cursorTelemetry = [], cursorHighlight = DEFAULT_CURSOR_HIGHLIGHT, cursorClickTimestamps = [], + showCursor = false, + cursorSize = DEFAULT_CURSOR_SIZE, + cursorSmoothing = DEFAULT_CURSOR_SMOOTHING, + cursorMotionBlur = DEFAULT_CURSOR_MOTION_BLUR, + cursorClickBounce = DEFAULT_CURSOR_CLICK_BOUNCE, }, ref, ) => { @@ -211,7 +284,7 @@ const VideoPlayback = forwardRef( const [webcamDimensions, setWebcamDimensions] = useState(null); const currentTimeRef = useRef(0); const zoomRegionsRef = useRef([]); - const cursorTelemetryRef = useRef([]); + const cursorTelemetryRef = useRef([]); const cursorHighlightRef = useRef(DEFAULT_CURSOR_HIGHLIGHT); const cursorClickTimestampsRef = useRef([]); const cursorHighlightGraphicsRef = useRef(null); @@ -251,12 +324,139 @@ const VideoPlayback = forwardRef( const trimRegionsRef = useRef([]); const speedRegionsRef = useRef([]); const motionBlurAmountRef = useRef(motionBlurAmount); + const cursorOverlayRef = useRef(null); + const showCursorRef = useRef(showCursor); + const cursorSizeRef = useRef(cursorSize); + const cursorSmoothingRef = useRef(cursorSmoothing); + const cursorMotionBlurRef = useRef(cursorMotionBlur); + const cursorClickBounceRef = useRef(cursorClickBounce); const motionBlurStateRef = useRef(createMotionBlurState()); const onTimeUpdateRef = useRef(onTimeUpdate); const onPlayStateChangeRef = useRef(onPlayStateChange); const videoReadyRafRef = useRef(null); const smoothedAutoFocusRef = useRef(null); const prevTargetProgressRef = useRef(0); + const durationResolutionTimeoutRef = useRef(null); + const lastResolvedDurationRef = useRef(null); + const isResolvingDurationRef = useRef(false); + const hasNativeCursorRecordingRef = useRef(false); + const cursorRecordingDataRef = useRef(cursorRecordingData); + const cropRegionRef = useRef(cropRegion); + const nativeCursorSpriteRef = useRef(null); + const nativeCursorTextureIdRef = useRef(null); + const nativeCursorImageRef = useRef(null); + const nativeCursorImageIdRef = useRef(null); + const nativeCursorSmoothingStateRef = useRef(createNativeCursorSmoothingState()); + const nativeCursorMotionBlurStateRef = useRef(createNativeCursorMotionBlurState()); + + const hasNativeCursorRecording = useMemo( + () => hasNativeCursorRecordingData(cursorRecordingData), + [cursorRecordingData], + ); + + const syncResolvedDuration = useCallback( + (video: HTMLVideoElement) => { + const resolvedDuration = getResolvedVideoDuration(video); + if (!resolvedDuration) { + return false; + } + + const normalizedDuration = Math.round(resolvedDuration * 1000) / 1000; + if (lastResolvedDurationRef.current !== normalizedDuration) { + lastResolvedDurationRef.current = normalizedDuration; + onDurationChange(normalizedDuration); + } + + return true; + }, + [onDurationChange], + ); + + const forceResolveDuration = useCallback( + (video: HTMLVideoElement) => { + if (isResolvingDurationRef.current) { + return; + } + + if (video.readyState < HTMLMediaElement.HAVE_METADATA) { + return; + } + + isResolvingDurationRef.current = true; + const previousMuted = video.muted; + + const finalize = () => { + video.removeEventListener("durationchange", handleProgress); + video.removeEventListener("timeupdate", handleProgress); + video.removeEventListener("loadeddata", handleProgress); + video.removeEventListener("ended", handleProgress); + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } + video.muted = previousMuted; + isResolvingDurationRef.current = false; + }; + + const resolveCurrentDuration = () => { + if (syncResolvedDuration(video)) { + return true; + } + + const endedDuration = getEndedVideoDuration(video); + if (endedDuration) { + lastResolvedDurationRef.current = null; + onDurationChange(Math.round(endedDuration * 1000) / 1000); + return true; + } + + return false; + }; + + const handleProgress = () => { + if (!resolveCurrentDuration()) { + return; + } + + try { + video.pause(); + video.currentTime = 0; + } catch { + // no-op + } + currentTimeRef.current = 0; + finalize(); + }; + + video.addEventListener("durationchange", handleProgress); + video.addEventListener("timeupdate", handleProgress); + video.addEventListener("loadeddata", handleProgress); + video.addEventListener("ended", handleProgress); + durationResolutionTimeoutRef.current = window.setTimeout(() => { + handleProgress(); + finalize(); + }, 1500); + video.muted = true; + + const playAttempt = video.play(); + if (playAttempt && typeof playAttempt.catch === "function") { + playAttempt.catch(() => { + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }); + } + + try { + video.currentTime = Math.max(video.currentTime, 0.1); + } catch { + finalize(); + } + }, + [onDurationChange, syncResolvedDuration], + ); const clampFocusToStage = useCallback((focus: ZoomFocus, depth: ZoomDepth) => { return clampFocusToStageUtil(focus, depth, stageSizeRef.current); @@ -573,6 +773,55 @@ const VideoPlayback = forwardRef( motionBlurAmountRef.current = motionBlurAmount; }, [motionBlurAmount]); + useEffect(() => { + cursorTelemetryRef.current = cursorTelemetry; + }, [cursorTelemetry]); + + useEffect(() => { + showCursorRef.current = showCursor; + }, [showCursor]); + + useEffect(() => { + hasNativeCursorRecordingRef.current = hasNativeCursorRecording; + }, [hasNativeCursorRecording]); + + useEffect(() => { + cursorRecordingDataRef.current = cursorRecordingData; + resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); + resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); + }, [cursorRecordingData]); + + useEffect(() => { + cropRegionRef.current = cropRegion; + }, [cropRegion]); + + useEffect(() => { + cursorSizeRef.current = cursorSize; + }, [cursorSize]); + + useEffect(() => { + cursorSmoothingRef.current = cursorSmoothing; + }, [cursorSmoothing]); + + useEffect(() => { + cursorMotionBlurRef.current = cursorMotionBlur; + }, [cursorMotionBlur]); + + useEffect(() => { + cursorClickBounceRef.current = cursorClickBounce; + }, [cursorClickBounce]); + + // Sync cursor overlay config when settings change + useEffect(() => { + const overlay = cursorOverlayRef.current; + if (!overlay) return; + overlay.setDotRadius(DEFAULT_CURSOR_CONFIG.dotRadius * cursorSize); + overlay.setSmoothingFactor(cursorSmoothing); + overlay.setMotionBlur(cursorMotionBlur); + overlay.setClickBounce(cursorClickBounce); + overlay.reset(); + }, [cursorSize, cursorSmoothing, cursorMotionBlur, cursorClickBounce]); + useEffect(() => { onTimeUpdateRef.current = onTimeUpdate; }, [onTimeUpdate]); @@ -692,6 +941,13 @@ const VideoPlayback = forwardRef( let app: Application | null = null; (async () => { + let cursorOverlayEnabled = true; + try { + await preloadCursorAssets(); + } catch { + cursorOverlayEnabled = false; + } + app = new Application(); await app.init({ @@ -727,12 +983,30 @@ const VideoPlayback = forwardRef( videoContainerRef.current = videoContainer; cameraContainer.addChild(videoContainer); + // Cursor overlay - rendered above the masked video + if (cursorOverlayEnabled) { + const cursorOverlay = new PixiCursorOverlay({ + dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * cursorSizeRef.current, + smoothingFactor: cursorSmoothingRef.current, + motionBlur: cursorMotionBlurRef.current, + clickBounce: cursorClickBounceRef.current, + }); + cursorOverlayRef.current = cursorOverlay; + } + setPixiReady(true); })(); return () => { mounted = false; setPixiReady(false); + if (cursorOverlayRef.current) { + cursorOverlayRef.current.destroy(); + cursorOverlayRef.current = null; + } + nativeCursorSpriteRef.current = null; + nativeCursorTextureIdRef.current = null; + nativeCursorImageIdRef.current = null; if (app && app.renderer) { app.destroy(true, { children: true, @@ -749,6 +1023,8 @@ const VideoPlayback = forwardRef( useEffect(() => { if (!videoPath) { + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; setVideoReady(false); return; } @@ -759,11 +1035,18 @@ const VideoPlayback = forwardRef( video.currentTime = 0; allowPlaybackRef.current = false; lockedVideoDimensionsRef.current = null; + lastResolvedDurationRef.current = null; + isResolvingDurationRef.current = false; + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } setVideoReady(false); if (videoReadyRafRef.current) { cancelAnimationFrame(videoReadyRafRef.current); videoReadyRafRef.current = null; } + video.load(); }, [videoPath]); useEffect(() => { @@ -793,12 +1076,20 @@ const VideoPlayback = forwardRef( videoContainer.addChild(maskGraphics); videoContainer.mask = maskGraphics; maskGraphicsRef.current = maskGraphics; + const nativeCursorSprite = new Sprite(Texture.EMPTY); + nativeCursorSprite.visible = false; + nativeCursorSprite.eventMode = "none"; + nativeCursorSpriteRef.current = nativeCursorSprite; + if (cursorOverlayRef.current) { + videoContainer.addChild(cursorOverlayRef.current.container); + } const cursorHighlightGraphics = new Graphics(); cursorHighlightGraphics.visible = false; videoContainer.addChild(cursorHighlightGraphics); cursorHighlightGraphicsRef.current = cursorHighlightGraphics; drawCursorHighlightGraphics(cursorHighlightGraphics, cursorHighlightRef.current); + videoContainer.addChild(nativeCursorSprite); animationStateRef.current = { scale: 1, @@ -867,6 +1158,12 @@ const VideoPlayback = forwardRef( cursorHighlightGraphicsRef.current.destroy(); cursorHighlightGraphicsRef.current = null; } + if (nativeCursorSpriteRef.current) { + videoContainer.removeChild(nativeCursorSpriteRef.current); + nativeCursorSpriteRef.current.destroy(); + nativeCursorSpriteRef.current = null; + nativeCursorTextureIdRef.current = null; + } videoContainer.mask = null; maskGraphicsRef.current = null; if (blurFilterRef.current) { @@ -1139,6 +1436,130 @@ const VideoPlayback = forwardRef( } } + // Update cursor overlay + const cursorOverlay = cursorOverlayRef.current; + if (cursorOverlay) { + const timeMs = currentTimeRef.current; // already in ms + cursorOverlay.update( + cursorTelemetryRef.current, + timeMs, + baseMaskRef.current, + showCursorRef.current && !hasNativeCursorRecordingRef.current, + !isPlayingRef.current || isSeekingRef.current, + ); + } + + // Keep the native cursor preview in the same transformed coordinate space as PIXI. + const nativeCursorSprite = nativeCursorSpriteRef.current; + const nativeCursorImage = nativeCursorImageRef.current; + const hideNativeCursorPreview = () => { + if (nativeCursorSprite) { + nativeCursorSprite.visible = false; + } + if (nativeCursorImage) { + nativeCursorImage.style.display = "none"; + nativeCursorImage.style.filter = "none"; + } + resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); + resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); + }; + if (nativeCursorImage) { + if (hasNativeCursorRecordingRef.current && showCursorRef.current) { + const timeMs = currentTimeRef.current; // already in ms + const frame = resolveInterpolatedNativeCursorFrame( + cursorRecordingDataRef.current, + timeMs, + ); + if (frame) { + const displaySample = smoothNativeCursorSample({ + forceSnap: !isPlayingRef.current || isSeekingRef.current, + sample: frame.sample, + smoothing: cursorSmoothingRef.current, + state: nativeCursorSmoothingStateRef.current, + timeMs, + }); + const cameraContainer = cameraContainerRef.current; + const videoContainer = videoContainerRef.current; + const cropRegionValue = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }; + const projectedLocalPoint = projectNativeCursorToLocal({ + cropRegion: cropRegionValue, + maskRect: baseMaskRef.current, + sample: displaySample, + }); + const projectedStagePoint = + cameraContainer && videoContainer + ? projectNativeCursorToStage({ + cameraContainer, + cropRegion: cropRegionValue, + maskRect: baseMaskRef.current, + videoContainerPosition: { + x: videoContainer.x, + y: videoContainer.y, + }, + sample: displaySample, + }) + : null; + if (projectedLocalPoint && projectedStagePoint) { + const renderAsset = resolveNativeCursorRenderAsset( + frame.asset, + window.devicePixelRatio || 1, + displaySample, + ); + const bounceProgress = getNativeCursorClickBounceProgress( + cursorRecordingDataRef.current, + timeMs, + ); + const scale = + Math.max(0, cursorSizeRef.current) * + getNativeCursorClickBounceScale(cursorClickBounceRef.current, bounceProgress); + const transformedScale = scale * Math.abs(cameraContainer?.scale.x || 1); + const blurPx = + !isPlayingRef.current || isSeekingRef.current + ? 0 + : getNativeCursorMotionBlurPx({ + motionBlur: cursorMotionBlurRef.current, + point: projectedStagePoint, + state: nativeCursorMotionBlurStateRef.current, + timeMs, + }); + if (nativeCursorImageIdRef.current !== renderAsset.id) { + nativeCursorImage.src = renderAsset.imageDataUrl; + nativeCursorImageIdRef.current = renderAsset.id; + } + nativeCursorImage.style.display = "block"; + nativeCursorImage.style.width = `${renderAsset.width * transformedScale}px`; + nativeCursorImage.style.height = `${renderAsset.height * transformedScale}px`; + nativeCursorImage.style.filter = + blurPx > 0 ? `blur(${blurPx.toFixed(2)}px)` : "none"; + nativeCursorImage.style.transform = `translate3d(${ + projectedStagePoint.x - renderAsset.hotspotX * transformedScale + }px, ${projectedStagePoint.y - renderAsset.hotspotY * transformedScale}px, 0)`; + if (nativeCursorSprite) { + nativeCursorSprite.visible = false; + if (nativeCursorTextureIdRef.current !== renderAsset.id) { + nativeCursorSprite.texture = Texture.from(renderAsset.imageDataUrl); + nativeCursorTextureIdRef.current = renderAsset.id; + } + nativeCursorSprite.position.set( + projectedLocalPoint.x - renderAsset.hotspotX * scale, + projectedLocalPoint.y - renderAsset.hotspotY * scale, + ); + nativeCursorSprite.width = renderAsset.width * scale; + nativeCursorSprite.height = renderAsset.height * scale; + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + } else { + hideNativeCursorPreview(); + } + const composite3D = composite3DRef.current; const outerWrapper = outerWrapperRef.current; if (composite3D && outerWrapper) { @@ -1188,8 +1609,12 @@ const VideoPlayback = forwardRef( const handleLoadedMetadata = (e: React.SyntheticEvent) => { const video = e.currentTarget; - onDurationChange(video.duration); - video.currentTime = 0; + const hasResolvedDuration = syncResolvedDuration(video); + if (!hasResolvedDuration) { + forceResolveDuration(video); + } else { + video.currentTime = 0; + } video.pause(); allowPlaybackRef.current = false; currentTimeRef.current = 0; @@ -1202,6 +1627,9 @@ const VideoPlayback = forwardRef( const waitForRenderableFrame = () => { const hasDimensions = video.videoWidth > 0 && video.videoHeight > 0; const hasData = video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA; + if (!syncResolvedDuration(video)) { + forceResolveDuration(video); + } if (hasDimensions && hasData) { videoReadyRafRef.current = null; setVideoReady(true); @@ -1301,6 +1729,10 @@ const VideoPlayback = forwardRef( window.clearTimeout(scrubEndTimerRef.current); scrubEndTimerRef.current = null; } + if (durationResolutionTimeoutRef.current) { + clearTimeout(durationResolutionTimeoutRef.current); + durationResolutionTimeoutRef.current = null; + } }; }, []); @@ -1359,6 +1791,18 @@ const VideoPlayback = forwardRef( : "none", }} /> + {webcamVideoPath && (() => { const clipPath = getCssClipPath(webcamLayout?.maskShape ?? "rectangle"); @@ -1544,11 +1988,23 @@ const VideoPlayback = forwardRef( ref={videoRef} src={videoPath} className="hidden" - preload="metadata" + preload="auto" playsInline onLoadedMetadata={handleLoadedMetadata} onDurationChange={(e) => { - onDurationChange(e.currentTarget.duration); + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onLoadedData={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } + }} + onCanPlay={(e) => { + if (!syncResolvedDuration(e.currentTarget)) { + forceResolveDuration(e.currentTarget); + } }} onError={() => onError("Failed to load video")} /> diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 8d134282d..3e0f635be 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -118,16 +118,14 @@ function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } -function isFileUrl(value: string): boolean { - return /^file:\/\//i.test(value); -} - function encodePathSegments(pathname: string, keepWindowsDrive = false): string { return pathname .split("/") .map((segment, index) => { - if (!segment) return ""; - if (keepWindowsDrive && index === 1 && /^[a-zA-Z]:$/.test(segment)) { + if (!segment) { + return segment; + } + if (keepWindowsDrive && index === 0 && /^[a-zA-Z]:$/.test(segment)) { return segment; } return encodeURIComponent(segment); @@ -137,31 +135,25 @@ function encodePathSegments(pathname: string, keepWindowsDrive = false): string export function toFileUrl(filePath: string): string { const normalized = filePath.replace(/\\/g, "/"); - - // Windows drive path: C:/Users/... - if (/^[a-zA-Z]:\//.test(normalized)) { - return `file://${encodePathSegments(`/${normalized}`, true)}`; + if (normalized.match(/^[a-zA-Z]:/)) { + return `file:///${encodePathSegments(normalized, true)}`; } - - // UNC path: //server/share/... if (normalized.startsWith("//")) { - const [host, ...pathParts] = normalized.replace(/^\/+/, "").split("/"); - const encodedPath = pathParts.map((part) => encodeURIComponent(part)).join("/"); - return encodedPath ? `file://${host}/${encodedPath}` : `file://${host}/`; + const withoutPrefix = normalized.slice(2); + const [host = "", ...segments] = withoutPrefix.split("/"); + return `file://${host}/${encodePathSegments(segments.join("/"))}`; } - const absolutePath = normalized.startsWith("/") ? normalized : `/${normalized}`; return `file://${encodePathSegments(absolutePath)}`; } export function fromFileUrl(fileUrl: string): string { - const value = fileUrl.trim(); - if (!isFileUrl(value)) { + if (!fileUrl.startsWith("file://")) { return fileUrl; } try { - const url = new URL(value); + const url = new URL(fileUrl); const pathname = decodeURIComponent(url.pathname); if (url.host && url.host !== "localhost") { @@ -174,13 +166,7 @@ export function fromFileUrl(fileUrl: string): string { return pathname; } catch { - const rawFallbackPath = value.replace(/^file:\/\//i, ""); - let fallbackPath = rawFallbackPath; - try { - fallbackPath = decodeURIComponent(rawFallbackPath); - } catch { - // Keep raw best-effort path if percent decoding fails. - } + const fallbackPath = decodeURIComponent(fileUrl.replace(/^file:\/\//, "")); return fallbackPath.replace(/^\/([a-zA-Z]:)/, "$1"); } } diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 6fe3474d8..e407aa984 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -52,6 +52,7 @@ const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; + hasVideoSource?: boolean; currentTime: number; onSeek?: (time: number) => void; cursorTelemetry?: CursorTelemetryPoint[]; @@ -765,6 +766,7 @@ function Timeline({ export default function TimelineEditor({ videoDuration, + hasVideoSource = false, currentTime, onSeek, cursorTelemetry = [], @@ -1437,8 +1439,14 @@ export default function TimelineEditor({
-

{t("emptyState.noVideo")}

-

{t("emptyState.dragAndDrop")}

+

+ {hasVideoSource ? "Loading Timeline" : "No Video Loaded"} +

+

+ {hasVideoSource + ? "Video opened, waiting for duration metadata" + : "Drag and drop a video to start editing"} +

); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index f976efc8c..3785eae3b 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -168,8 +168,32 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; + cursorType?: + | "arrow" + | "text" + | "pointer" + | "crosshair" + | "open-hand" + | "closed-hand" + | "resize-ew" + | "resize-ns" + | "not-allowed"; } +export interface CursorVisualSettings { + size: number; + smoothing: number; + motionBlur: number; + clickBounce: number; +} + +export const DEFAULT_CURSOR_SIZE = 3.0; +export const DEFAULT_CURSOR_SMOOTHING = 0.67; +export const DEFAULT_CURSOR_MOTION_BLUR = 0.35; +export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5; +export const DEFAULT_ZOOM_MOTION_BLUR = 0.35; + export interface TrimRegion { id: string; startMs: number; diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts new file mode 100644 index 000000000..dd3087bd2 --- /dev/null +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -0,0 +1,768 @@ +import { Assets, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js"; +import { MotionBlurFilter } from "pixi-filters/motion-blur"; +import type { CursorTelemetryPoint } from "../types"; +import { + createSpringState, + getCursorSpringConfig, + resetSpringState, + stepSpringValue, +} from "./motionSmoothing"; +import { UPLOADED_CURSOR_SAMPLE_SIZE, uploadedCursorAssets } from "./uploadedCursorAssets"; + +type CursorAssetKey = NonNullable; + +/** System cursor asset from native helper (macOS only). */ +type SystemCursorAsset = { + dataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +}; + +type LoadedCursorAsset = { + texture: Texture; + image: HTMLImageElement; + aspectRatio: number; + anchorX: number; + anchorY: number; +}; + +export interface CursorViewportRect { + x: number; + y: number; + width: number; + height: number; +} + +/** + * Configuration for cursor rendering. + */ +export interface CursorRenderConfig { + /** Base cursor height in pixels (at reference width of 1920px) */ + dotRadius: number; + /** Cursor fill color (hex number for PixiJS) */ + dotColor: number; + /** Cursor opacity (0–1) */ + dotAlpha: number; + /** Unused, kept for interface compatibility */ + trailLength: number; + /** Smoothing factor for cursor interpolation (0–1, lower = smoother/slower) */ + smoothingFactor: number; + /** Directional cursor motion blur amount. */ + motionBlur: number; + /** Click bounce multiplier. */ + clickBounce: number; +} + +export const DEFAULT_CURSOR_CONFIG: CursorRenderConfig = { + dotRadius: 28, + dotColor: 0xffffff, + dotAlpha: 0.95, + trailLength: 0, + smoothingFactor: 0.18, + motionBlur: 0, + clickBounce: 1, +}; + +const REFERENCE_WIDTH = 1920; +const MIN_CURSOR_VIEWPORT_SCALE = 0.55; +const CLICK_ANIMATION_MS = 140; +const CLICK_RING_FADE_MS = 240; +const CURSOR_MOTION_BLUR_BASE_MULTIPLIER = 0.08; +const CURSOR_TIME_DISCONTINUITY_MS = 100; +const CURSOR_SVG_DROP_SHADOW_FILTER = "drop-shadow(0px 2px 3px rgba(0, 0, 0, 0.35))"; +const CURSOR_SHADOW_COLOR = 0x000000; +const CURSOR_SHADOW_ALPHA = 0.35; +const CURSOR_SHADOW_OFFSET_X = 0; +const CURSOR_SHADOW_OFFSET_Y = 2; +const CURSOR_SHADOW_BLUR = 3; +const CURSOR_SHADOW_PADDING = 12; + +let cursorAssetsPromise: Promise | null = null; +let loadedCursorAssets: Partial> = {}; +const SUPPORTED_CURSOR_KEYS: CursorAssetKey[] = [ + "arrow", + "text", + "pointer", + "crosshair", + "open-hand", + "closed-hand", + "resize-ew", + "resize-ns", + "not-allowed", +]; + +function loadImage(dataUrl: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => + reject(new Error(`Failed to load cursor image: ${dataUrl.slice(0, 128)}`)); + image.src = dataUrl; + }); +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function getNormalizedAnchor( + systemAsset: SystemCursorAsset | undefined, + fallbackAnchor: { x: number; y: number }, +) { + if (!systemAsset || systemAsset.width <= 0 || systemAsset.height <= 0) { + return fallbackAnchor; + } + + return { + x: clamp(systemAsset.hotspotX / systemAsset.width, 0, 1), + y: clamp(systemAsset.hotspotY / systemAsset.height, 0, 1), + }; +} + +/** + * Loads an SVG at `sampleSize × sampleSize`, crops the trim region out of it, + * and returns a PNG data-URL of the cropped result. This is required because + * SVG files have their own natural pixel size (e.g. 32×32) which does not + * match the 1024-sample coordinate space used by the trim measurements. + */ +async function rasterizeAndCropSvg( + url: string, + sampleSize: number, + trimX: number, + trimY: number, + trimWidth: number, + trimHeight: number, +): Promise<{ dataUrl: string; width: number; height: number }> { + const img = await loadImage(url); + + // Draw at full sample size + const srcCanvas = document.createElement("canvas"); + srcCanvas.width = sampleSize; + srcCanvas.height = sampleSize; + const srcCtx = srcCanvas.getContext("2d")!; + srcCtx.drawImage(img, 0, 0, sampleSize, sampleSize); + + // Crop to trim bounds + const dstCanvas = document.createElement("canvas"); + dstCanvas.width = trimWidth; + dstCanvas.height = trimHeight; + const dstCtx = dstCanvas.getContext("2d")!; + dstCtx.drawImage(srcCanvas, trimX, trimY, trimWidth, trimHeight, 0, 0, trimWidth, trimHeight); + + return { + dataUrl: dstCanvas.toDataURL("image/png"), + width: dstCanvas.width, + height: dstCanvas.height, + }; +} + +function getCursorAsset(key: CursorAssetKey): LoadedCursorAsset { + const asset = loadedCursorAssets[key]; + if (!asset) { + throw new Error(`Missing cursor asset for ${key}`); + } + + return asset; +} + +function getAvailableCursorKeys(): CursorAssetKey[] { + const loadedKeys = Object.keys(loadedCursorAssets) as CursorAssetKey[]; + return loadedKeys.length > 0 ? loadedKeys : ["arrow"]; +} + +export async function preloadCursorAssets() { + if (!cursorAssetsPromise) { + cursorAssetsPromise = (async () => { + let systemCursors: Record = {}; + + try { + const api = window.electronAPI as Record; + if (typeof api.getSystemCursorAssets === "function") { + const result = await ( + api.getSystemCursorAssets as () => Promise<{ + success: boolean; + cursors?: Record; + }> + )(); + if (result.success && result.cursors) { + systemCursors = result.cursors; + } + } + } catch (error) { + console.warn("[CursorRenderer] Failed to fetch system cursor assets:", error); + } + + const entries = await Promise.all( + SUPPORTED_CURSOR_KEYS.map(async (key) => { + const systemAsset = systemCursors[key]; + const uploadedAsset = uploadedCursorAssets[key]; + const assetUrl = uploadedAsset?.url ?? systemAsset?.dataUrl; + + if (!assetUrl) { + console.warn(`[CursorRenderer] No cursor image for: ${key}`); + return null; + } + + try { + let finalUrl: string; + let width: number; + let height: number; + let normalizedAnchor: { x: number; y: number }; + + if (uploadedAsset) { + const { trim, fallbackAnchor } = uploadedAsset; + const rasterized = await rasterizeAndCropSvg( + assetUrl, + UPLOADED_CURSOR_SAMPLE_SIZE, + trim.x, + trim.y, + trim.width, + trim.height, + ); + finalUrl = rasterized.dataUrl; + width = rasterized.width; + height = rasterized.height; + normalizedAnchor = { + x: clamp((fallbackAnchor.x * trim.width) / width, 0, 1), + y: clamp((fallbackAnchor.y * trim.height) / height, 0, 1), + }; + } else { + finalUrl = assetUrl; + const img = await loadImage(finalUrl); + width = img.naturalWidth; + height = img.naturalHeight; + normalizedAnchor = getNormalizedAnchor(systemAsset, { x: 0, y: 0 }); + } + + await Assets.load(finalUrl); + const image = await loadImage(finalUrl); + const texture = Texture.from(finalUrl); + + return [ + key, + { + texture, + image, + aspectRatio: height > 0 ? width / height : 1, + anchorX: normalizedAnchor.x, + anchorY: normalizedAnchor.y, + } satisfies LoadedCursorAsset, + ] as const; + } catch (error) { + console.warn(`[CursorRenderer] Failed to load cursor image for: ${key}`, error); + return null; + } + }), + ); + + loadedCursorAssets = Object.fromEntries( + entries.filter(Boolean).map((entry) => entry!), + ) as Partial>; + + if (!loadedCursorAssets.arrow) { + throw new Error("Failed to initialize the fallback arrow cursor asset"); + } + })(); + } + + return cursorAssetsPromise; +} + +/** + * Interpolates cursor position from telemetry samples at a given time. + * Uses linear interpolation between the two nearest samples. + */ +export function interpolateCursorPosition( + samples: CursorTelemetryPoint[], + timeMs: number, +): { cx: number; cy: number } | null { + if (!samples || samples.length === 0) return null; + + if (timeMs <= samples[0].timeMs) { + return { cx: samples[0].cx, cy: samples[0].cy }; + } + + if (timeMs >= samples[samples.length - 1].timeMs) { + return { cx: samples[samples.length - 1].cx, cy: samples[samples.length - 1].cy }; + } + + let lo = 0; + let hi = samples.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; + } + } + + const a = samples[lo]; + const b = samples[hi]; + const span = b.timeMs - a.timeMs; + if (span <= 0) return { cx: a.cx, cy: a.cy }; + + const t = (timeMs - a.timeMs) / span; + return { + cx: a.cx + (b.cx - a.cx) * t, + cy: a.cy + (b.cy - a.cy) * t, + }; +} + +function findLatestSample(samples: CursorTelemetryPoint[], timeMs: number) { + if (samples.length === 0) return null; + + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + return samples[lo]?.timeMs <= timeMs ? samples[lo] : null; +} + +function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: number) { + for (let index = samples.length - 1; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if ( + sample.interactionType === "click" || + sample.interactionType === "double-click" || + sample.interactionType === "right-click" || + sample.interactionType === "middle-click" + ) { + return sample; + } + } + + return null; +} + +function findLatestStableCursorType(samples: CursorTelemetryPoint[], timeMs: number) { + // Binary search to find position at timeMs, then scan backwards + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + // Scan backwards from the position to find a sample with cursorType + // Skip click events only (not mouseup) to avoid transient re-type during clicks + for (let index = lo; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample.timeMs > timeMs) { + continue; + } + + if (!sample.cursorType) { + continue; + } + + if ( + sample.interactionType === "click" || + sample.interactionType === "double-click" || + sample.interactionType === "right-click" || + sample.interactionType === "middle-click" + ) { + continue; + } + + return sample.cursorType; + } + + return findLatestSample(samples, timeMs)?.cursorType ?? "arrow"; +} + +function getCursorViewportScale(viewport: CursorViewportRect) { + return Math.max(MIN_CURSOR_VIEWPORT_SCALE, viewport.width / REFERENCE_WIDTH); +} + +function getCursorVisualState(samples: CursorTelemetryPoint[], timeMs: number) { + const latestClick = findLatestInteractionSample(samples, timeMs); + const interactionType = latestClick?.interactionType; + const ageMs = latestClick ? Math.max(0, timeMs - latestClick.timeMs) : Number.POSITIVE_INFINITY; + const isClickEvent = + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click"; + const clickBounceProgress = + latestClick && isClickEvent && ageMs <= CLICK_ANIMATION_MS ? 1 - ageMs / CLICK_ANIMATION_MS : 0; + + return { + cursorType: findLatestStableCursorType(samples, timeMs), + clickBounceProgress, + clickProgress: + latestClick && isClickEvent && ageMs <= CLICK_RING_FADE_MS + ? 1 - ageMs / CLICK_RING_FADE_MS + : 0, + }; +} + +/** + * Manages a smoothed cursor state that chases the interpolated target. + */ +export class SmoothedCursorState { + public x = 0.5; + public y = 0.5; + public trail: Array<{ x: number; y: number }> = []; + private smoothingFactor: number; + private trailLength: number; + private initialized = false; + private lastTimeMs: number | null = null; + private xSpring = createSpringState(0.5); + private ySpring = createSpringState(0.5); + + constructor(config: Pick) { + this.smoothingFactor = config.smoothingFactor; + this.trailLength = config.trailLength; + } + + update(targetX: number, targetY: number, timeMs: number): void { + if (!this.initialized) { + this.x = targetX; + this.y = targetY; + this.initialized = true; + this.lastTimeMs = timeMs; + this.xSpring.value = targetX; + this.ySpring.value = targetY; + this.xSpring.velocity = 0; + this.ySpring.velocity = 0; + this.xSpring.initialized = true; + this.ySpring.initialized = true; + this.trail = []; + return; + } + + if (this.smoothingFactor <= 0 || (this.lastTimeMs !== null && timeMs < this.lastTimeMs)) { + this.snapTo(targetX, targetY, timeMs); + return; + } + + this.trail.unshift({ x: this.x, y: this.y }); + if (this.trail.length > this.trailLength) { + this.trail.length = this.trailLength; + } + + const deltaMs = this.lastTimeMs === null ? 1000 / 60 : Math.max(1, timeMs - this.lastTimeMs); + this.lastTimeMs = timeMs; + + const springConfig = getCursorSpringConfig(this.smoothingFactor); + this.x = stepSpringValue(this.xSpring, targetX, deltaMs, springConfig); + this.y = stepSpringValue(this.ySpring, targetY, deltaMs, springConfig); + } + + setSmoothingFactor(smoothingFactor: number): void { + this.smoothingFactor = smoothingFactor; + } + + snapTo(targetX: number, targetY: number, timeMs: number): void { + this.x = targetX; + this.y = targetY; + this.initialized = true; + this.lastTimeMs = timeMs; + this.xSpring.value = targetX; + this.ySpring.value = targetY; + this.xSpring.velocity = 0; + this.ySpring.velocity = 0; + this.xSpring.initialized = true; + this.ySpring.initialized = true; + this.trail = []; + } + + reset(): void { + this.initialized = false; + this.lastTimeMs = null; + this.trail = []; + resetSpringState(this.xSpring, this.x); + resetSpringState(this.ySpring, this.y); + } +} + +function drawClickRing(graphics: Graphics, px: number, py: number, h: number, progress: number) { + void graphics; + void px; + void py; + void h; + void progress; +} + +export class PixiCursorOverlay { + public readonly container: Container; + private clickRingGraphics: Graphics; + private cursorShadowSprites: Partial>; + private cursorShadowFilters: Partial>; + private cursorSprites: Partial>; + private cursorMotionBlurFilter: MotionBlurFilter; + private state: SmoothedCursorState; + private config: CursorRenderConfig; + private lastRenderedPoint: { px: number; py: number } | null = null; + private lastRenderedTimeMs: number | null = null; + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_CURSOR_CONFIG, ...config }; + this.state = new SmoothedCursorState(this.config); + + this.container = new Container(); + this.container.label = "cursor-overlay"; + + this.clickRingGraphics = new Graphics(); + this.cursorShadowSprites = {}; + this.cursorShadowFilters = {}; + this.cursorSprites = {}; + for (const key of getAvailableCursorKeys()) { + const asset = getCursorAsset(key); + const shadowSprite = new Sprite(asset.texture); + shadowSprite.anchor.set(asset.anchorX, asset.anchorY); + shadowSprite.visible = false; + shadowSprite.tint = CURSOR_SHADOW_COLOR; + shadowSprite.alpha = CURSOR_SHADOW_ALPHA; + const shadowFilter = new BlurFilter(); + shadowFilter.blur = CURSOR_SHADOW_BLUR; + shadowFilter.quality = 4; + shadowFilter.padding = CURSOR_SHADOW_PADDING; + shadowSprite.filters = [shadowFilter]; + this.cursorShadowSprites[key] = shadowSprite; + this.cursorShadowFilters[key] = shadowFilter; + + const sprite = new Sprite(asset.texture); + sprite.anchor.set(asset.anchorX, asset.anchorY); + sprite.visible = false; + this.cursorSprites[key] = sprite; + } + + this.cursorMotionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); + this.container.filters = null; + + this.container.addChild( + this.clickRingGraphics, + ...Object.values(this.cursorShadowSprites), + ...Object.values(this.cursorSprites), + ); + this.setMotionBlur(this.config.motionBlur); + } + + setDotRadius(dotRadius: number) { + this.config.dotRadius = dotRadius; + } + + setSmoothingFactor(smoothingFactor: number) { + this.config.smoothingFactor = smoothingFactor; + this.state.setSmoothingFactor(smoothingFactor); + } + + setMotionBlur(motionBlur: number) { + this.config.motionBlur = Math.max(0, motionBlur); + this.container.filters = this.config.motionBlur > 0 ? [this.cursorMotionBlurFilter] : null; + if (this.config.motionBlur <= 0) { + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + } + } + + setClickBounce(clickBounce: number) { + this.config.clickBounce = Math.max(0, clickBounce); + } + + update( + samples: CursorTelemetryPoint[], + timeMs: number, + viewport: CursorViewportRect, + visible: boolean, + freeze = false, + ): void { + if (!visible || samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) { + this.container.visible = false; + this.lastRenderedPoint = null; + this.lastRenderedTimeMs = null; + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + return; + } + + const target = interpolateCursorPosition(samples, timeMs); + if (!target) { + this.container.visible = false; + return; + } + + const sameFrameTime = + this.lastRenderedTimeMs !== null && Math.abs(this.lastRenderedTimeMs - timeMs) < 0.0001; + const hasTimeDiscontinuity = + this.lastRenderedTimeMs !== null && + Math.abs(timeMs - this.lastRenderedTimeMs) > CURSOR_TIME_DISCONTINUITY_MS; + + if (freeze || hasTimeDiscontinuity) { + if (!sameFrameTime || !this.lastRenderedPoint) { + this.state.snapTo(target.cx, target.cy, timeMs); + } + } else { + this.state.update(target.cx, target.cy, timeMs); + } + this.container.visible = true; + + const px = viewport.x + this.state.x * viewport.width; + const py = viewport.y + this.state.y * viewport.height; + const h = this.config.dotRadius * getCursorViewportScale(viewport); + const { cursorType, clickBounceProgress, clickProgress } = getCursorVisualState( + samples, + timeMs, + ); + const spriteKey = (cursorType in this.cursorSprites ? cursorType : "arrow") as CursorAssetKey; + const asset = getCursorAsset(spriteKey); + const shadowSprite = this.cursorShadowSprites[spriteKey] ?? this.cursorShadowSprites.arrow!; + const sprite = this.cursorSprites[spriteKey] ?? this.cursorSprites.arrow!; + const bounceScale = Math.max( + 0.72, + 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * this.config.clickBounce), + ); + const scaledH = h; + + this.clickRingGraphics.clear(); + drawClickRing(this.clickRingGraphics, px, py, h, clickProgress); + + for (const [key, currentShadowSprite] of Object.entries(this.cursorShadowSprites) as Array< + [CursorAssetKey, Sprite] + >) { + currentShadowSprite.visible = key === spriteKey; + } + + for (const [key, currentSprite] of Object.entries(this.cursorSprites) as Array< + [CursorAssetKey, Sprite] + >) { + currentSprite.visible = key === spriteKey; + } + + if (shadowSprite) { + shadowSprite.height = scaledH * bounceScale; + shadowSprite.width = scaledH * bounceScale * asset.aspectRatio; + shadowSprite.position.set(px + CURSOR_SHADOW_OFFSET_X, py + CURSOR_SHADOW_OFFSET_Y); + } + + if (sprite) { + sprite.alpha = this.config.dotAlpha; + sprite.height = scaledH * bounceScale; + sprite.width = scaledH * bounceScale * asset.aspectRatio; + sprite.position.set(px, py); + } + + this.applyCursorMotionBlur(px, py, timeMs, freeze); + this.lastRenderedPoint = { px, py }; + this.lastRenderedTimeMs = timeMs; + } + + private applyCursorMotionBlur(px: number, py: number, timeMs: number, freeze: boolean) { + if ( + freeze || + this.config.motionBlur <= 0 || + !this.lastRenderedPoint || + this.lastRenderedTimeMs === null + ) { + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + return; + } + + const deltaMs = Math.max(1, timeMs - this.lastRenderedTimeMs); + const dx = px - this.lastRenderedPoint.px; + const dy = py - this.lastRenderedPoint.py; + const velocityScale = + (1000 / deltaMs) * this.config.motionBlur * CURSOR_MOTION_BLUR_BASE_MULTIPLIER; + const velocity = { + x: dx * velocityScale, + y: dy * velocityScale, + }; + const magnitude = Math.hypot(velocity.x, velocity.y); + + this.cursorMotionBlurFilter.velocity = magnitude > 0.05 ? velocity : { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = magnitude > 3 ? 9 : magnitude > 1 ? 7 : 5; + this.cursorMotionBlurFilter.offset = magnitude > 0.5 ? -0.25 : 0; + } + + reset(): void { + this.state.reset(); + this.clickRingGraphics.clear(); + for (const shadowSprite of Object.values(this.cursorShadowSprites)) { + shadowSprite.visible = false; + shadowSprite.scale.set(1); + } + for (const sprite of Object.values(this.cursorSprites)) { + sprite.visible = false; + sprite.scale.set(1); + } + this.container.visible = false; + this.lastRenderedPoint = null; + this.lastRenderedTimeMs = null; + this.cursorMotionBlurFilter.velocity = { x: 0, y: 0 }; + this.cursorMotionBlurFilter.kernelSize = 5; + this.cursorMotionBlurFilter.offset = 0; + } + + destroy(): void { + this.clickRingGraphics.destroy(); + for (const shadowFilter of Object.values(this.cursorShadowFilters)) { + shadowFilter.destroy(); + } + this.cursorMotionBlurFilter.destroy(); + this.container.destroy({ children: true }); + cursorAssetsPromise = null; + loadedCursorAssets = {}; + } +} + +export function drawCursorOnCanvas( + ctx: CanvasRenderingContext2D, + samples: CursorTelemetryPoint[], + timeMs: number, + viewport: CursorViewportRect, + smoothedState: SmoothedCursorState, + config: CursorRenderConfig = DEFAULT_CURSOR_CONFIG, +): void { + if (samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) return; + + const target = interpolateCursorPosition(samples, timeMs); + if (!target) return; + + smoothedState.update(target.cx, target.cy, timeMs); + + const px = viewport.x + smoothedState.x * viewport.width; + const py = viewport.y + smoothedState.y * viewport.height; + const h = config.dotRadius * getCursorViewportScale(viewport); + const { cursorType, clickBounceProgress } = getCursorVisualState(samples, timeMs); + const spriteKey = ( + cursorType && loadedCursorAssets[cursorType] ? cursorType : "arrow" + ) as CursorAssetKey; + const asset = getCursorAsset(spriteKey); + const bounceScale = Math.max( + 0.72, + 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * config.clickBounce), + ); + + ctx.save(); + ctx.filter = CURSOR_SVG_DROP_SHADOW_FILTER; + + const drawHeight = h * bounceScale; + const drawWidth = drawHeight * asset.aspectRatio; + const hotspotX = asset.anchorX * drawWidth; + const hotspotY = asset.anchorY * drawHeight; + ctx.globalAlpha = config.dotAlpha; + ctx.drawImage(asset.image, px - hotspotX, py - hotspotY, drawWidth, drawHeight); + + ctx.restore(); +} diff --git a/src/components/video-editor/videoPlayback/motionSmoothing.ts b/src/components/video-editor/videoPlayback/motionSmoothing.ts new file mode 100644 index 000000000..e2cec6fd1 --- /dev/null +++ b/src/components/video-editor/videoPlayback/motionSmoothing.ts @@ -0,0 +1,149 @@ +import { spring } from "motion"; + +export interface SpringState { + value: number; + velocity: number; + initialized: boolean; +} + +export interface SpringConfig { + stiffness: number; + damping: number; + mass: number; + restDelta?: number; + restSpeed?: number; +} + +const CURSOR_SMOOTHING_MIN = 0; +const CURSOR_SMOOTHING_MAX = 2; +const CURSOR_SMOOTHING_LEGACY_MAX = 0.5; + +export function createSpringState(initialValue = 0): SpringState { + return { + value: initialValue, + velocity: 0, + initialized: false, + }; +} + +export function resetSpringState(state: SpringState, initialValue?: number) { + if (typeof initialValue === "number") { + state.value = initialValue; + } + + state.velocity = 0; + state.initialized = false; +} + +export function clampDeltaMs(deltaMs: number, fallbackMs = 1000 / 60) { + if (!Number.isFinite(deltaMs) || deltaMs <= 0) { + return fallbackMs; + } + + return Math.min(80, Math.max(1, deltaMs)); +} + +export function stepSpringValue( + state: SpringState, + target: number, + deltaMs: number, + config: SpringConfig, +) { + const safeDeltaMs = clampDeltaMs(deltaMs); + + if (!state.initialized || !Number.isFinite(state.value)) { + state.value = target; + state.velocity = 0; + state.initialized = true; + return state.value; + } + + const restDelta = config.restDelta ?? 0.0005; + const restSpeed = config.restSpeed ?? 0.02; + + if (Math.abs(target - state.value) <= restDelta && Math.abs(state.velocity) <= restSpeed) { + state.value = target; + state.velocity = 0; + return state.value; + } + + const previousValue = state.value; + const generator = spring({ + keyframes: [state.value, target], + velocity: state.velocity, + stiffness: config.stiffness, + damping: config.damping, + mass: config.mass, + restDelta, + restSpeed, + }); + + const result = generator.next(safeDeltaMs); + state.value = result.done ? target : result.value; + state.velocity = ((state.value - previousValue) / safeDeltaMs) * 1000; + + if (result.done) { + state.velocity = 0; + } + + return state.value; +} + +export function getCursorSpringConfig(smoothingFactor: number): SpringConfig { + const clamped = Math.min(CURSOR_SMOOTHING_MAX, Math.max(CURSOR_SMOOTHING_MIN, smoothingFactor)); + + if (clamped <= 0) { + return { + stiffness: 1000, + damping: 100, + mass: 1, + restDelta: 0.0001, + restSpeed: 0.001, + }; + } + + if (clamped <= CURSOR_SMOOTHING_LEGACY_MAX) { + const legacyNormalized = Math.min( + 1, + Math.max( + 0, + (clamped - CURSOR_SMOOTHING_MIN) / (CURSOR_SMOOTHING_LEGACY_MAX - CURSOR_SMOOTHING_MIN), + ), + ); + + return { + stiffness: 760 - legacyNormalized * 420, + damping: 34 + legacyNormalized * 24, + mass: 0.55 + legacyNormalized * 0.45, + restDelta: 0.0002, + restSpeed: 0.01, + }; + } + + const extendedNormalized = Math.min( + 1, + Math.max( + 0, + (clamped - CURSOR_SMOOTHING_LEGACY_MAX) / + (CURSOR_SMOOTHING_MAX - CURSOR_SMOOTHING_LEGACY_MAX), + ), + ); + + return { + stiffness: 340 - extendedNormalized * 180, + damping: 58 + extendedNormalized * 22, + mass: 1 + extendedNormalized * 0.35, + restDelta: 0.0002, + restSpeed: 0.01, + }; +} + +export function getZoomSpringConfig(): SpringConfig { + return { + stiffness: 320, + damping: 40, + mass: 0.92, + restDelta: 0.0005, + restSpeed: 0.015, + }; +} diff --git a/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts new file mode 100644 index 000000000..4a0cd29bc --- /dev/null +++ b/src/components/video-editor/videoPlayback/uploadedCursorAssets.ts @@ -0,0 +1,70 @@ +import crosshairUrl from "../../../assets/cursors/Cursor=Cross.svg"; +import arrowUrl from "../../../assets/cursors/Cursor=Default.svg"; +import closedHandUrl from "../../../assets/cursors/Cursor=Hand-(Grabbing).svg"; +import openHandUrl from "../../../assets/cursors/Cursor=Hand-(Open).svg"; +import pointerUrl from "../../../assets/cursors/Cursor=Hand-(Pointing).svg"; +import resizeNsUrl from "../../../assets/cursors/Cursor=Resize-North-South.svg"; +import resizeEwUrl from "../../../assets/cursors/Cursor=Resize-West-East.svg"; +import textUrl from "../../../assets/cursors/Cursor=Text-Cursor.svg"; +import type { CursorTelemetryPoint } from "../types"; + +type CursorAssetKey = NonNullable; + +export type UploadedCursorAsset = { + url: string; + trim: { + x: number; + y: number; + width: number; + height: number; + }; + fallbackAnchor: { + x: number; + y: number; + }; +}; + +export const UPLOADED_CURSOR_SAMPLE_SIZE = 1024; + +export const uploadedCursorAssets: Partial> = { + arrow: { + url: arrowUrl, + trim: { x: 480, y: 435, width: 333, height: 553 }, + fallbackAnchor: { x: 0.18, y: 0.1 }, + }, + text: { + url: textUrl, + trim: { x: 404, y: 192, width: 247, height: 596 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + pointer: { + url: pointerUrl, + trim: { x: 352, y: 441, width: 466, height: 583 }, + fallbackAnchor: { x: 0.37, y: 0.08 }, + }, + crosshair: { + url: crosshairUrl, + trim: { x: 288, y: 288, width: 480, height: 480 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + "open-hand": { + url: openHandUrl, + trim: { x: 288, y: 188, width: 512, height: 580 }, + fallbackAnchor: { x: 0.5, y: 0.28 }, + }, + "closed-hand": { + url: closedHandUrl, + trim: { x: 344, y: 365, width: 432, height: 403 }, + fallbackAnchor: { x: 0.5, y: 0.28 }, + }, + "resize-ew": { + url: resizeEwUrl, + trim: { x: 187, y: 384, width: 669, height: 270 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, + "resize-ns": { + url: resizeNsUrl, + trim: { x: 376, y: 178, width: 271, height: 669 }, + fallbackAnchor: { x: 0.5, y: 0.5 }, + }, +}; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index dc9758fbd..45aa7b3b5 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -2,6 +2,11 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; +import { + type NativeWindowsRecordingRequest, + parseWindowHandleFromSourceId, +} from "@/lib/nativeWindowsRecording"; +import type { CursorCaptureMode } from "@/lib/recordingSession"; import { requestCameraAccess } from "@/lib/requestCameraAccess"; const TARGET_FRAME_RATE = 60; @@ -51,12 +56,18 @@ type UseScreenRecorderReturn = { setMicrophoneEnabled: (enabled: boolean) => void; microphoneDeviceId: string | undefined; setMicrophoneDeviceId: (deviceId: string | undefined) => void; + microphoneDeviceName: string | undefined; + setMicrophoneDeviceName: (deviceName: string | undefined) => void; webcamDeviceId: string | undefined; setWebcamDeviceId: (deviceId: string | undefined) => void; + webcamDeviceName: string | undefined; + setWebcamDeviceName: (deviceName: string | undefined) => void; systemAudioEnabled: boolean; setSystemAudioEnabled: (enabled: boolean) => void; webcamEnabled: boolean; setWebcamEnabled: (enabled: boolean) => Promise; + cursorCaptureMode: CursorCaptureMode; + setCursorCaptureMode: (mode: CursorCaptureMode) => void; }; type RecorderHandle = { @@ -64,6 +75,11 @@ type RecorderHandle = { recordedBlobPromise: Promise; }; +type NativeWindowsRecordingHandle = { + recordingId: number; + finalizing: boolean; +}; + function createRecorderHandle(stream: MediaStream, options: MediaRecorderOptions): RecorderHandle { const recorder = new MediaRecorder(stream, options); const chunks: Blob[] = []; @@ -93,11 +109,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const [elapsedSeconds, setElapsedSeconds] = useState(0); const [microphoneEnabled, setMicrophoneEnabled] = useState(false); const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); + const [microphoneDeviceName, setMicrophoneDeviceName] = useState(undefined); const [webcamDeviceId, setWebcamDeviceId] = useState(undefined); + const [webcamDeviceName, setWebcamDeviceName] = useState(undefined); const [systemAudioEnabled, setSystemAudioEnabled] = useState(false); const [webcamEnabled, setWebcamEnabledState] = useState(false); + const [cursorCaptureMode, setCursorCaptureMode] = useState("editable-overlay"); const screenRecorder = useRef(null); const webcamRecorder = useRef(null); + const nativeWindowsRecording = useRef(null); const stream = useRef(null); const screenStream = useRef(null); const microphoneStream = useRef(null); @@ -174,6 +194,20 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }, []); + const stopWebcamPreviewStream = useCallback(() => { + if (!webcamStream.current) { + return; + } + + webcamAcquireId.current++; + webcamStream.current.getTracks().forEach((track) => { + track.onended = null; + track.stop(); + }); + webcamStream.current = null; + webcamReady.current = true; + }, []); + const setWebcamEnabled = useCallback( async (enabled: boolean) => { if (!enabled) { @@ -338,6 +372,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } : undefined, createdAt: activeRecordingId, + cursorCaptureMode, }); if (!result.success) { @@ -364,10 +399,68 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } })(); }, - [teardownMedia], + [cursorCaptureMode, teardownMedia], ); + const finalizeNativeWindowsRecording = useCallback(async (discard = false) => { + const activeNativeRecording = nativeWindowsRecording.current; + if (!activeNativeRecording || activeNativeRecording.finalizing) { + return false; + } + + activeNativeRecording.finalizing = true; + + const clearNativeRecordingState = () => { + nativeWindowsRecording.current = null; + setRecording(false); + setPaused(false); + setElapsedSeconds(0); + accumulatedDurationMs.current = 0; + segmentStartedAt.current = null; + }; + + try { + const result = await window.electronAPI.stopNativeWindowsRecording(discard); + if (discard || result.discarded) { + clearNativeRecordingState(); + return true; + } + if (!result.success) { + console.error("Failed to stop native Windows recording:", result.error); + toast.error(result.error ?? "Failed to stop native Windows recording"); + activeNativeRecording.finalizing = false; + return true; + } + + clearNativeRecordingState(); + if (result.session) { + await window.electronAPI.setCurrentRecordingSession(result.session); + } else if (result.path) { + await window.electronAPI.setCurrentVideoPath(result.path); + } + + await window.electronAPI.switchToEditor(); + return true; + } catch (error) { + console.error("Error saving native Windows recording:", error); + toast.error( + error instanceof Error ? error.message : "Failed to save native Windows recording", + ); + activeNativeRecording.finalizing = false; + return true; + } finally { + if (discardRecordingId.current === activeNativeRecording.recordingId) { + discardRecordingId.current = null; + } + } + }, []); + const stopRecording = useRef(() => { + if (nativeWindowsRecording.current) { + void finalizeNativeWindowsRecording(false); + return; + } + const activeScreenRecorder = screenRecorder.current; if (!activeScreenRecorder) { return; @@ -408,6 +501,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }); + const safeHideCountdownOverlay = useCallback(async (runId: number) => { + try { + await window.electronAPI.hideCountdownOverlay(runId); + } catch (error) { + console.warn("Failed to hide countdown overlay:", error); + } + }, []); + useEffect(() => { let cleanup: (() => void) | undefined; @@ -425,6 +526,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { allowAutoFinalize.current = false; restarting.current = false; discardRecordingId.current = null; + if (nativeWindowsRecording.current) { + void finalizeNativeWindowsRecording(true); + } if ( screenRecorder.current?.recorder.state === "recording" || @@ -450,7 +554,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = null; teardownMedia(); }; - }, [teardownMedia]); + }, [teardownMedia, safeHideCountdownOverlay, finalizeNativeWindowsRecording]); const safeShowCountdownOverlay = async (value: number, runId: number) => { try { @@ -477,17 +581,102 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }; - const safeHideCountdownOverlay = async (runId: number) => { + const isCountdownRunActive = (runId?: number) => + runId === undefined || countdownRunId.current === runId; + + const startNativeWindowsRecordingIfAvailable = async ( + selectedSource: ProcessedDesktopSource, + countdownRunToken?: number, + ) => { try { - await window.electronAPI.hideCountdownOverlay(runId); + const platform = await window.electronAPI.getPlatform(); + if (platform !== "win32") { + return false; + } + + const availability = await window.electronAPI.isNativeWindowsCaptureAvailable(); + if (!availability.success || !availability.available) { + if (availability.reason === "unsupported-os") { + return false; + } + + throw new Error( + availability.reason === "missing-helper" + ? "Native Windows capture helper is not available." + : (availability.error ?? "Native Windows capture is not available."), + ); + } + + if (!isCountdownRunActive(countdownRunToken)) { + return true; + } + + const activeRecordingId = Date.now(); + const displayId = Number(selectedSource.display_id); + const sourceType = selectedSource.id.startsWith("window:") ? "window" : "display"; + const windowHandle = parseWindowHandleFromSourceId(selectedSource.id); + if (webcamEnabled) { + stopWebcamPreviewStream(); + } + const request: NativeWindowsRecordingRequest = { + recordingId: activeRecordingId, + source: { + type: sourceType, + sourceId: selectedSource.id, + ...(Number.isFinite(displayId) ? { displayId } : {}), + ...(windowHandle ? { windowHandle } : {}), + }, + video: { + fps: TARGET_FRAME_RATE, + width: TARGET_WIDTH, + height: TARGET_HEIGHT, + }, + audio: { + system: { + enabled: systemAudioEnabled, + }, + microphone: { + enabled: microphoneEnabled, + deviceId: microphoneDeviceId, + deviceName: microphoneDeviceName, + gain: MIC_GAIN_BOOST, + }, + }, + webcam: { + enabled: webcamEnabled, + deviceId: webcamDeviceId, + deviceName: webcamDeviceName, + width: WEBCAM_TARGET_WIDTH, + height: WEBCAM_TARGET_HEIGHT, + fps: WEBCAM_TARGET_FRAME_RATE, + }, + cursor: { + mode: cursorCaptureMode, + }, + }; + const result = await window.electronAPI.startNativeWindowsRecording(request); + if (!result.success || !result.recordingId) { + throw new Error(result.error ?? "Native Windows capture failed."); + } + + recordingId.current = result.recordingId; + nativeWindowsRecording.current = { + recordingId: result.recordingId, + finalizing: false, + }; + accumulatedDurationMs.current = 0; + segmentStartedAt.current = Date.now(); + allowAutoFinalize.current = true; + setRecording(true); + setPaused(false); + setElapsedSeconds(0); + return true; } catch (error) { - console.warn("Failed to hide countdown overlay:", error); + console.error("Native Windows capture failed:", error); + throw error; } }; - const isCountdownRunActive = (runId?: number) => - runId === undefined || countdownRunId.current === runId; - const startRecordCountdown = async () => { if (countdownActive || recording) { return; @@ -575,43 +764,63 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - let screenMediaStream: MediaStream; - - const videoConstraints = { - mandatory: { - chromeMediaSource: CHROME_MEDIA_SOURCE, - chromeMediaSourceId: selectedSource.id, - maxWidth: TARGET_WIDTH, - maxHeight: TARGET_HEIGHT, - maxFrameRate: TARGET_FRAME_RATE, - minFrameRate: MIN_FRAME_RATE, - }, - }; + if (await startNativeWindowsRecordingIfAvailable(selectedSource, countdownRunToken)) { + return; + } - if (systemAudioEnabled) { - try { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: { - mandatory: { - chromeMediaSource: CHROME_MEDIA_SOURCE, - chromeMediaSourceId: selectedSource.id, + let screenMediaStream: MediaStream; + const platform = await window.electronAPI.getPlatform(); + + if (platform === "win32") { + // getDisplayMedia + setDisplayMediaRequestHandler (main.ts) supplies the + // pre-selected source. Editable cursor mode excludes the system cursor so + // the editor can render a replacement; system mode bakes it into the video. + screenMediaStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: cursorCaptureMode === "editable-overlay" ? "never" : "always", + width: { max: TARGET_WIDTH }, + height: { max: TARGET_HEIGHT }, + frameRate: { ideal: TARGET_FRAME_RATE }, + } as MediaTrackConstraints, + audio: systemAudioEnabled, + } as DisplayMediaStreamOptions); + } else { + const videoConstraints = { + mandatory: { + chromeMediaSource: CHROME_MEDIA_SOURCE, + chromeMediaSourceId: selectedSource.id, + maxWidth: TARGET_WIDTH, + maxHeight: TARGET_HEIGHT, + maxFrameRate: TARGET_FRAME_RATE, + minFrameRate: MIN_FRAME_RATE, + }, + }; + + if (systemAudioEnabled) { + try { + screenMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + mandatory: { + chromeMediaSource: CHROME_MEDIA_SOURCE, + chromeMediaSourceId: selectedSource.id, + }, }, - }, - video: videoConstraints, - } as unknown as MediaStreamConstraints); - } catch (audioErr) { - console.warn("System audio capture failed, falling back to video-only:", audioErr); - toast.error(t("recording.systemAudioUnavailable")); + video: videoConstraints, + } as unknown as MediaStreamConstraints); + } catch (audioErr) { + console.warn("System audio capture failed, falling back to video-only:", audioErr); + toast.error(t("recording.systemAudioUnavailable")); + screenMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: videoConstraints, + } as unknown as MediaStreamConstraints); + } + } else { screenMediaStream = await navigator.mediaDevices.getUserMedia({ audio: false, video: videoConstraints, } as unknown as MediaStreamConstraints); } - } else { - screenMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: videoConstraints, - } as unknown as MediaStreamConstraints); } screenStream.current = screenMediaStream; @@ -744,6 +953,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } + recordingId.current = Date.now(); + const activeRecordingId = recordingId.current; screenRecorder.current = createRecorderHandle(stream.current, { mimeType, videoBitsPerSecond, @@ -766,18 +977,16 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); } - recordingId.current = Date.now(); accumulatedDurationMs.current = 0; segmentStartedAt.current = Date.now(); allowAutoFinalize.current = true; setRecording(true); setPaused(false); setElapsedSeconds(0); - window.electronAPI?.setRecordingState(true, recordingId.current); + window.electronAPI?.setRecordingState(true, recordingId.current, cursorCaptureMode); const activeScreenRecorder = screenRecorder.current; const activeWebcamRecorder = webcamRecorder.current; - const activeRecordingId = recordingId.current; if (activeScreenRecorder) { activeScreenRecorder.recorder.addEventListener( "stop", @@ -871,6 +1080,19 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const restartRecording = async () => { if (restarting.current) return; + if (nativeWindowsRecording.current) { + const activeRecordingId = recordingId.current; + restarting.current = true; + discardRecordingId.current = activeRecordingId; + try { + await finalizeNativeWindowsRecording(true); + await startRecording(); + } finally { + restarting.current = false; + } + return; + } + const activeScreenRecorder = screenRecorder.current; if (!activeScreenRecorder || activeScreenRecorder.recorder.state === "inactive") return; @@ -928,6 +1150,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, [getRecordingDurationMs, paused, recording]); const cancelRecording = () => { + if (nativeWindowsRecording.current) { + const activeRecordingId = recordingId.current; + discardRecordingId.current = activeRecordingId; + allowAutoFinalize.current = false; + void finalizeNativeWindowsRecording(true); + return; + } + const activeScreenRecorder = screenRecorder.current; if ( activeScreenRecorder?.recorder.state === "recording" || @@ -959,11 +1189,17 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId, + microphoneDeviceName, + setMicrophoneDeviceName, webcamDeviceId, setWebcamDeviceId, + webcamDeviceName, + setWebcamDeviceName, systemAudioEnabled, setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + cursorCaptureMode, + setCursorCaptureMode, }; } diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index e959a5478..133a96127 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -24,6 +24,10 @@ "noneFound": "No camera found", "unavailable": "Camera unavailable" }, + "cursor": { + "useEditableCursor": "Use editable cursor", + "useSystemCursor": "Use system cursor" + }, "sourceSelector": { "loading": "Loading sources...", "screens": "Screens ({{count}})", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 68919aac2..e0d2c3c6c 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -24,6 +24,10 @@ "noneFound": "No se encontró cámara", "unavailable": "Cámara no disponible" }, + "cursor": { + "useEditableCursor": "Usar cursor editable", + "useSystemCursor": "Usar cursor del sistema" + }, "sourceSelector": { "loading": "Cargando fuentes...", "screens": "Pantallas ({{count}})", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index 55521cb44..33e623660 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -24,6 +24,10 @@ "noneFound": "Aucune caméra trouvée", "unavailable": "Caméra non disponible" }, + "cursor": { + "useEditableCursor": "Utiliser le curseur éditable", + "useSystemCursor": "Utiliser le curseur système" + }, "sourceSelector": { "loading": "Chargement des sources...", "screens": "Écrans ({{count}})", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index 51e3833f9..601cf2d8b 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -24,6 +24,10 @@ "noneFound": "カメラが見つかりません", "unavailable": "カメラが利用できません" }, + "cursor": { + "useEditableCursor": "編集可能なカーソルを使う", + "useSystemCursor": "システムカーソルを使う" + }, "sourceSelector": { "loading": "ソースを読み込み中...", "screens": "画面 ({{count}})", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index d9f6d6ab8..07e063283 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -24,6 +24,10 @@ "noneFound": "카메라를 찾을 수 없음", "unavailable": "카메라를 사용할 수 없음" }, + "cursor": { + "useEditableCursor": "편집 가능한 커서 사용", + "useSystemCursor": "시스템 커서 사용" + }, "sourceSelector": { "loading": "소스 불러오는 중...", "screens": "화면 ({{count}}개)", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index 177ba3f52..19039c84b 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -35,6 +35,10 @@ "noneFound": "Kamera bulunamadı", "unavailable": "Kamera kullanılamıyor" }, + "cursor": { + "useEditableCursor": "Düzenlenebilir imleci kullan", + "useSystemCursor": "Sistem imlecini kullan" + }, "sourceSelector": { "loading": "Kaynaklar yükleniyor...", "screens": "Ekranlar ({{count}})", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index a5c2a9d07..ae399e3f5 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -24,6 +24,10 @@ "noneFound": "未找到摄像头", "unavailable": "摄像头不可用" }, + "cursor": { + "useEditableCursor": "使用可编辑光标", + "useSystemCursor": "使用系统光标" + }, "sourceSelector": { "loading": "正在加载源...", "screens": "屏幕 ({{count}})", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index ea7e6251c..9629c86f7 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -24,6 +24,10 @@ "noneFound": "未找到攝影機", "unavailable": "攝影機不可用" }, + "cursor": { + "useEditableCursor": "使用可編輯游標", + "useSystemCursor": "使用系統游標" + }, "sourceSelector": { "loading": "正在載入來源...", "screens": "螢幕 ({{count}})", diff --git a/src/lib/cursor/nativeCursor.test.ts b/src/lib/cursor/nativeCursor.test.ts new file mode 100644 index 000000000..1b919d65d --- /dev/null +++ b/src/lib/cursor/nativeCursor.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + getNativeCursorClickBounceProgress, + getNativeCursorClickBounceScale, +} from "./nativeCursor"; + +describe("native cursor click bounce", () => { + it("keeps click progress visible across several frames", () => { + const recordingData = { + version: 2, + provider: "native" as const, + assets: [], + samples: [ + { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move" as const }, + { timeMs: 100, cx: 0.5, cy: 0.5, interactionType: "click" as const }, + { timeMs: 133, cx: 0.5, cy: 0.5, interactionType: "move" as const }, + { timeMs: 166, cx: 0.5, cy: 0.5, interactionType: "move" as const }, + { timeMs: 200, cx: 0.5, cy: 0.5, interactionType: "move" as const }, + { timeMs: 300, cx: 0.5, cy: 0.5, interactionType: "move" as const }, + ], + }; + + expect(getNativeCursorClickBounceProgress(recordingData, 133)).toBeGreaterThan(0); + expect(getNativeCursorClickBounceProgress(recordingData, 200)).toBeGreaterThan(0); + expect(getNativeCursorClickBounceProgress(recordingData, 400)).toBe(0); + }); + + it("applies a visible press and rebound scale at high intensity", () => { + expect(getNativeCursorClickBounceScale(5, 1)).toBe(1); + expect(getNativeCursorClickBounceScale(5, 0.82)).toBeLessThan(0.9); + expect(getNativeCursorClickBounceScale(5, 0.28)).toBeGreaterThan(1.05); + expect(getNativeCursorClickBounceScale(5, 0)).toBe(1); + }); +}); diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts new file mode 100644 index 000000000..9ce308a80 --- /dev/null +++ b/src/lib/cursor/nativeCursor.ts @@ -0,0 +1,591 @@ +import { type Container, Point } from "pixi.js"; +import appStartingUrl from "@/assets/cursors/Cursor=App-Starting.svg"; +import crosshairUrl from "@/assets/cursors/Cursor=Cross.svg"; +import arrowUrl from "@/assets/cursors/Cursor=Default.svg"; +import closedHandUrl from "@/assets/cursors/Cursor=Hand-(Grabbing).svg"; +import openHandUrl from "@/assets/cursors/Cursor=Hand-(Open).svg"; +import pointerUrl from "@/assets/cursors/Cursor=Hand-(Pointing).svg"; +import helpUrl from "@/assets/cursors/Cursor=Help.svg"; +import moveUrl from "@/assets/cursors/Cursor=Move.svg"; +import notAllowedUrl from "@/assets/cursors/Cursor=Not-Allowed.svg"; +import resizeNeswUrl from "@/assets/cursors/Cursor=Resize-North-East-South-West.svg"; +import resizeNsUrl from "@/assets/cursors/Cursor=Resize-North-South.svg"; +import resizeNwseUrl from "@/assets/cursors/Cursor=Resize-North-West-South-East.svg"; +import resizeEwUrl from "@/assets/cursors/Cursor=Resize-West-East.svg"; +import textUrl from "@/assets/cursors/Cursor=Text-Cursor.svg"; +import upArrowUrl from "@/assets/cursors/Cursor=Up-Arrow.svg"; +import waitUrl from "@/assets/cursors/Cursor=Wait.svg"; +import type { CropRegion } from "@/components/video-editor/types"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + NativeCursorType, +} from "@/native/contracts"; + +export interface ActiveNativeCursorFrame { + asset: NativeCursorAsset; + sample: CursorRecordingSample; +} + +export interface NativeCursorSmoothingState { + cx: number; + cy: number; + lastTimeMs: number | null; + initialized: boolean; +} + +export interface NativeCursorMotionBlurState { + x: number; + y: number; + lastTimeMs: number | null; + initialized: boolean; +} + +interface ProjectNativeCursorOptions { + cropRegion: CropRegion; + maskRect: { x: number; y: number; width: number; height: number }; + sample: CursorRecordingSample; +} + +interface ProjectNativeCursorToStageOptions extends ProjectNativeCursorOptions { + cameraContainer: Container; + videoContainerPosition: { x: number; y: number }; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +const NATIVE_CURSOR_CLICK_ANIMATION_MS = 260; +const NATIVE_CURSOR_MOTION_BLUR_MAX_PX = 6; +const nativeCursorAssetMapCache = new WeakMap< + CursorRecordingData, + Map +>(); + +function findNativeCursorSampleIndexAtOrBefore(samples: CursorRecordingSample[], timeMs: number) { + let low = 0; + let high = samples.length - 1; + let result = -1; + + while (low <= high) { + const middle = low + Math.floor((high - low) / 2); + if (samples[middle].timeMs <= timeMs) { + result = middle; + low = middle + 1; + } else { + high = middle - 1; + } + } + + return result; +} + +function getNativeCursorAssetMap(recordingData: CursorRecordingData) { + const cached = nativeCursorAssetMapCache.get(recordingData); + if (cached) { + return cached; + } + + const assetMap = new Map(recordingData.assets.map((asset) => [asset.id, asset])); + nativeCursorAssetMapCache.set(recordingData, assetMap); + return assetMap; +} + +function getNativeCursorAsset(recordingData: CursorRecordingData, assetId: string) { + return getNativeCursorAssetMap(recordingData).get(assetId) ?? null; +} + +interface PrettyNativeCursorAsset { + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +const PRETTY_NATIVE_CURSOR_ASSETS: Partial> = { + arrow: { + imageDataUrl: arrowUrl, + width: 32, + height: 32, + hotspotX: 16.25, + hotspotY: 15.03, + }, + text: { + imageDataUrl: textUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + pointer: { + imageDataUrl: pointerUrl, + width: 32, + height: 33, + hotspotX: 16.65, + hotspotY: 14.24, + }, + crosshair: { + imageDataUrl: crosshairUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "open-hand": { + imageDataUrl: openHandUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 9, + }, + "closed-hand": { + imageDataUrl: closedHandUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 9, + }, + "resize-ew": { + imageDataUrl: resizeEwUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-ns": { + imageDataUrl: resizeNsUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-nesw": { + imageDataUrl: resizeNeswUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "resize-nwse": { + imageDataUrl: resizeNwseUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + move: { + imageDataUrl: moveUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "not-allowed": { + imageDataUrl: notAllowedUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + wait: { + imageDataUrl: waitUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 16, + }, + "app-starting": { + imageDataUrl: appStartingUrl, + width: 32, + height: 32, + hotspotX: 7.25, + hotspotY: 4.03, + }, + help: { + imageDataUrl: helpUrl, + width: 32, + height: 32, + hotspotX: 7.25, + hotspotY: 4.03, + }, + "up-arrow": { + imageDataUrl: upArrowUrl, + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 3, + }, +}; + +function resolveUntypedPrettyNativeCursorAsset(asset: NativeCursorAsset) { + if ( + asset.cursorType || + asset.width < 24 || + asset.width > 64 || + asset.height < 24 || + asset.height > 64 + ) { + return null; + } + + const hotspotXNorm = asset.hotspotX / asset.width; + const hotspotYNorm = asset.hotspotY / asset.height; + const looksLikeChromiumGrabCursor = + hotspotXNorm >= 0.22 && hotspotXNorm <= 0.55 && hotspotYNorm >= 0.2 && hotspotYNorm <= 0.45; + + return looksLikeChromiumGrabCursor ? (PRETTY_NATIVE_CURSOR_ASSETS["open-hand"] ?? null) : null; +} + +export function hasNativeCursorRecordingData( + recordingData: CursorRecordingData | null | undefined, +): recordingData is CursorRecordingData { + return Boolean( + recordingData && + recordingData.provider === "native" && + recordingData.samples.length > 0 && + recordingData.assets.length > 0, + ); +} + +export function createNativeCursorSmoothingState(): NativeCursorSmoothingState { + return { + cx: 0, + cy: 0, + lastTimeMs: null, + initialized: false, + }; +} + +export function resetNativeCursorSmoothingState(state: NativeCursorSmoothingState) { + state.cx = 0; + state.cy = 0; + state.lastTimeMs = null; + state.initialized = false; +} + +export function createNativeCursorMotionBlurState(): NativeCursorMotionBlurState { + return { + x: 0, + y: 0, + lastTimeMs: null, + initialized: false, + }; +} + +export function resetNativeCursorMotionBlurState(state: NativeCursorMotionBlurState) { + state.x = 0; + state.y = 0; + state.lastTimeMs = null; + state.initialized = false; +} + +export function smoothNativeCursorSample({ + forceSnap = false, + sample, + smoothing, + state, + timeMs, +}: { + forceSnap?: boolean; + sample: CursorRecordingSample; + smoothing: number; + state: NativeCursorSmoothingState; + timeMs: number; +}): CursorRecordingSample { + const clampedSmoothing = clamp(Number.isFinite(smoothing) ? smoothing : 0, 0, 0.98); + const previousTimeMs = state.lastTimeMs; + const shouldSnap = + forceSnap || + clampedSmoothing <= 0 || + !state.initialized || + previousTimeMs === null || + timeMs <= previousTimeMs; + + if (shouldSnap) { + state.cx = sample.cx; + state.cy = sample.cy; + state.lastTimeMs = timeMs; + state.initialized = true; + return sample; + } + + const frameCount = Math.max(1, (timeMs - previousTimeMs) / (1000 / 60)); + const alpha = 1 - Math.pow(clampedSmoothing, frameCount); + state.cx += (sample.cx - state.cx) * alpha; + state.cy += (sample.cy - state.cy) * alpha; + state.lastTimeMs = timeMs; + + return { + ...sample, + cx: state.cx, + cy: state.cy, + }; +} + +export function getNativeCursorClickBounceProgress( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +) { + if (!recordingData || recordingData.provider !== "native" || recordingData.samples.length === 0) { + return 0; + } + + for ( + let index = findNativeCursorSampleIndexAtOrBefore(recordingData.samples, timeMs); + index >= 0; + index -= 1 + ) { + const sample = recordingData.samples[index]; + const ageMs = timeMs - sample.timeMs; + if (ageMs > NATIVE_CURSOR_CLICK_ANIMATION_MS) { + return 0; + } + + if (sample.interactionType === "click") { + return 1 - ageMs / NATIVE_CURSOR_CLICK_ANIMATION_MS; + } + } + + return 0; +} + +export function getNativeCursorClickBounceScale(clickBounce: number, progress: number) { + if (progress <= 0 || clickBounce <= 0) { + return 1; + } + + const intensity = clamp(clickBounce, 0, 5) / 5; + const elapsed = 1 - clamp(progress, 0, 1); + if (elapsed < 0.38) { + const pressProgress = Math.sin((elapsed / 0.38) * Math.PI); + return 1 - pressProgress * intensity * 0.24; + } + + const reboundProgress = Math.sin(((elapsed - 0.38) / 0.62) * Math.PI); + return 1 + reboundProgress * intensity * 0.16; +} + +export function getNativeCursorMotionBlurPx({ + motionBlur, + point, + state, + timeMs, +}: { + motionBlur: number; + point: { x: number; y: number }; + state: NativeCursorMotionBlurState; + timeMs: number; +}) { + const clampedMotionBlur = clamp(Number.isFinite(motionBlur) ? motionBlur : 0, 0, 1); + const previousTimeMs = state.lastTimeMs; + const shouldSnap = + clampedMotionBlur <= 0 || + !state.initialized || + previousTimeMs === null || + timeMs <= previousTimeMs; + + if (shouldSnap) { + state.x = point.x; + state.y = point.y; + state.lastTimeMs = timeMs; + state.initialized = true; + return 0; + } + + const deltaMs = Math.max(1, timeMs - previousTimeMs); + const distance = Math.hypot(point.x - state.x, point.y - state.y); + const speedPxPerSecond = (distance / deltaMs) * 1000; + state.x = point.x; + state.y = point.y; + state.lastTimeMs = timeMs; + + return clamp(speedPxPerSecond * clampedMotionBlur * 0.004, 0, NATIVE_CURSOR_MOTION_BLUR_MAX_PX); +} + +function getCroppedCursorPosition(sample: CursorRecordingSample, cropRegion: CropRegion) { + if (cropRegion.width <= 0 || cropRegion.height <= 0) { + return null; + } + + const croppedCx = (sample.cx - cropRegion.x) / cropRegion.width; + const croppedCy = (sample.cy - cropRegion.y) / cropRegion.height; + + if (croppedCx < 0 || croppedCx > 1 || croppedCy < 0 || croppedCy > 1) { + return null; + } + + return { + cx: clamp(croppedCx, 0, 1), + cy: clamp(croppedCy, 0, 1), + }; +} + +function getNativeCursorMaskPoint(sample: CursorRecordingSample, cropRegion: CropRegion) { + const croppedPosition = getCroppedCursorPosition(sample, cropRegion); + if (!croppedPosition) { + return null; + } + + return new Point(croppedPosition.cx, croppedPosition.cy); +} + +export function resolveActiveNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!hasNativeCursorRecordingData(recordingData)) { + return null; + } + + const index = findNativeCursorSampleIndexAtOrBefore(recordingData.samples, timeMs); + if (index >= 0) { + const sample = recordingData.samples[index]; + + if (sample.visible === false || !sample.assetId) { + return null; + } + + const asset = getNativeCursorAsset(recordingData, sample.assetId); + if (!asset) { + return null; + } + + return { sample, asset }; + } + + return null; +} + +export function resolveInterpolatedNativeCursorFrame( + recordingData: CursorRecordingData | null | undefined, + timeMs: number, +): ActiveNativeCursorFrame | null { + if (!hasNativeCursorRecordingData(recordingData)) { + return null; + } + + const samples = recordingData.samples; + const activeIndex = findNativeCursorSampleIndexAtOrBefore(samples, timeMs); + + if (activeIndex < 0) { + return null; + } + + const activeSample = samples[activeIndex]; + if (activeSample.visible === false || !activeSample.assetId) { + return null; + } + + const asset = getNativeCursorAsset(recordingData, activeSample.assetId); + if (!asset) { + return null; + } + + const nextSample = samples[activeIndex + 1]; + if ( + !nextSample || + nextSample.timeMs <= activeSample.timeMs || + nextSample.visible === false || + nextSample.assetId !== activeSample.assetId || + timeMs <= activeSample.timeMs + ) { + return { asset, sample: activeSample }; + } + + const interpolation = clamp( + (timeMs - activeSample.timeMs) / (nextSample.timeMs - activeSample.timeMs), + 0, + 1, + ); + + return { + asset, + sample: { + ...activeSample, + cx: activeSample.cx + (nextSample.cx - activeSample.cx) * interpolation, + cy: activeSample.cy + (nextSample.cy - activeSample.cy) * interpolation, + }, + }; +} + +export function projectNativeCursorToLocal({ + cropRegion, + maskRect, + sample, +}: ProjectNativeCursorOptions) { + const maskPoint = getNativeCursorMaskPoint(sample, cropRegion); + if (!maskPoint) { + return null; + } + + return new Point( + maskRect.x + maskPoint.x * maskRect.width, + maskRect.y + maskPoint.y * maskRect.height, + ); +} + +export function projectNativeCursorToStage({ + cameraContainer, + videoContainerPosition, + ...options +}: ProjectNativeCursorToStageOptions) { + const localPoint = projectNativeCursorToLocal(options); + if (!localPoint) { + return null; + } + + return cameraContainer.toGlobal( + new Point(localPoint.x + videoContainerPosition.x, localPoint.y + videoContainerPosition.y), + ); +} + +export function getNativeCursorDisplayMetrics(asset: NativeCursorAsset, deviceScaleFactor: number) { + const scaleFactor = asset.scaleFactor ?? deviceScaleFactor ?? 1; + return { + width: asset.width / scaleFactor, + height: asset.height / scaleFactor, + hotspotX: asset.hotspotX / scaleFactor, + hotspotY: asset.hotspotY / scaleFactor, + }; +} + +export function resolvePrettyNativeCursorAsset( + asset: NativeCursorAsset, + sample?: CursorRecordingSample, +) { + const cursorType = sample?.cursorType ?? asset.cursorType ?? null; + return cursorType + ? (PRETTY_NATIVE_CURSOR_ASSETS[cursorType] ?? null) + : resolveUntypedPrettyNativeCursorAsset(asset); +} + +export function resolveNativeCursorRenderAsset( + asset: NativeCursorAsset, + deviceScaleFactor: number, + sample?: CursorRecordingSample, +) { + const prettyAsset = resolvePrettyNativeCursorAsset(asset, sample); + if (prettyAsset) { + return { + id: `pretty:${sample?.cursorType ?? asset.cursorType}`, + imageDataUrl: prettyAsset.imageDataUrl, + width: prettyAsset.width, + height: prettyAsset.height, + hotspotX: prettyAsset.hotspotX, + hotspotY: prettyAsset.hotspotY, + }; + } + + const metrics = getNativeCursorDisplayMetrics(asset, deviceScaleFactor); + return { + id: asset.id, + imageDataUrl: asset.imageDataUrl, + width: metrics.width, + height: metrics.height, + hotspotX: metrics.hotspotX, + hotspotY: metrics.hotspotY, + }; +} diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts new file mode 100644 index 000000000..0fb3708c5 --- /dev/null +++ b/src/lib/exporter/audioEncoder.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { AudioProcessor, downmixPlanarChannelsForExport } from "./audioEncoder"; + +describe("AudioProcessor.selectSupportedExportCodec", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("falls back to stereo when the source channel count cannot be encoded", async () => { + const isConfigSupported = vi.fn(async (config: AudioEncoderConfig) => ({ + config, + supported: + config.codec === "mp4a.40.2" && + config.sampleRate === 44100 && + config.numberOfChannels === 2, + })); + vi.stubGlobal("AudioEncoder", { isConfigSupported }); + + const codec = await AudioProcessor.selectSupportedExportCodec(44100, 8); + + expect(codec).toMatchObject({ + encoderCodec: "mp4a.40.2", + muxerCodec: "aac", + sampleRate: 44100, + numberOfChannels: 2, + }); + expect(isConfigSupported).toHaveBeenCalledWith({ + codec: "mp4a.40.2", + sampleRate: 44100, + numberOfChannels: 8, + bitrate: 128000, + }); + expect(isConfigSupported).toHaveBeenCalledWith({ + codec: "mp4a.40.2", + sampleRate: 44100, + numberOfChannels: 2, + bitrate: 128000, + }); + }); +}); + +describe("downmixPlanarChannelsForExport", () => { + it("preserves non-front Windows system audio channels when exporting stereo", () => { + const sourcePlanes = Array.from({ length: 8 }, (_, channel) => { + const plane = new Float32Array(2); + if (channel === 2) { + plane[0] = 0.8; + plane[1] = 0.4; + } + if (channel === 6) { + plane[0] = 0.2; + plane[1] = 0.1; + } + return plane; + }); + + const stereo = downmixPlanarChannelsForExport(sourcePlanes, 2); + + expect(stereo[0]).toBeGreaterThan(0); + expect(stereo[1]).toBeGreaterThan(0); + expect(stereo[2]).toBeGreaterThan(0); + expect(stereo[3]).toBeGreaterThan(0); + }); + + it("duplicates mono microphone audio when exporting stereo", () => { + const mono = new Float32Array([0.25, -0.5]); + + const stereo = downmixPlanarChannelsForExport([mono], 2); + + expect(Array.from(stereo)).toEqual([0.25, -0.5, 0.25, -0.5]); + }); +}); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 2391d083f..95227844d 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,15 +1,214 @@ import { WebDemuxer } from "web-demuxer"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; -import type { VideoMuxer } from "./muxer"; +import type { ExportAudioMuxerCodec, VideoMuxer } from "./muxer"; const AUDIO_BITRATE = 128_000; const DECODE_BACKPRESSURE_LIMIT = 20; const MIN_SPEED_REGION_DELTA_MS = 0.0001; const SEEK_TIMEOUT_MS = 5_000; +export interface ExportAudioCodec { + encoderCodec: string; + muxerCodec: ExportAudioMuxerCodec; + label: string; + sampleRate: number; + numberOfChannels: number; +} + +type ExportAudioCodecCandidate = Omit; + +const EXPORT_AUDIO_CODECS: ExportAudioCodecCandidate[] = [ + { encoderCodec: "mp4a.40.2", muxerCodec: "aac", label: "AAC" }, + { encoderCodec: "opus", muxerCodec: "opus", label: "Opus" }, +]; + +function averageChannels(sourcePlanes: Float32Array[], frame: number) { + let mixed = 0; + for (const plane of sourcePlanes) { + mixed += plane[frame] ?? 0; + } + return mixed / Math.max(1, sourcePlanes.length); +} + +function weightedSample( + sourcePlanes: Float32Array[], + frame: number, + weights: Array<[channel: number, weight: number]>, +) { + let mixed = 0; + let weightSum = 0; + for (const [channel, weight] of weights) { + const sample = sourcePlanes[channel]?.[frame]; + if (typeof sample !== "number") { + continue; + } + mixed += sample * weight; + weightSum += weight; + } + return weightSum > 0 ? mixed / weightSum : averageChannels(sourcePlanes, frame); +} + +function getStereoDownmixWeights(sourceChannels: number) { + const centerWeight = Math.SQRT1_2; + const surroundWeight = Math.SQRT1_2; + const lfeWeight = 0.5; + + if (sourceChannels >= 8) { + // Windows 7.1 order: FL, FR, FC, LFE, BL, BR, SL, SR. + return { + left: [ + [0, 1], + [2, centerWeight], + [3, lfeWeight], + [4, surroundWeight], + [6, surroundWeight], + ] satisfies Array<[number, number]>, + right: [ + [1, 1], + [2, centerWeight], + [3, lfeWeight], + [5, surroundWeight], + [7, surroundWeight], + ] satisfies Array<[number, number]>, + }; + } + + if (sourceChannels >= 6) { + // Windows 5.1 order: FL, FR, FC, LFE, BL, BR. + return { + left: [ + [0, 1], + [2, centerWeight], + [3, lfeWeight], + [4, surroundWeight], + ] satisfies Array<[number, number]>, + right: [ + [1, 1], + [2, centerWeight], + [3, lfeWeight], + [5, surroundWeight], + ] satisfies Array<[number, number]>, + }; + } + + if (sourceChannels >= 4) { + return { + left: [ + [0, 1], + [2, surroundWeight], + ] satisfies Array<[number, number]>, + right: [ + [1, 1], + [3, surroundWeight], + ] satisfies Array<[number, number]>, + }; + } + + return { + left: [ + [0, 1], + [2, centerWeight], + ] satisfies Array<[number, number]>, + right: [ + [1, 1], + [2, centerWeight], + ] satisfies Array<[number, number]>, + }; +} + +export function downmixPlanarChannelsForExport( + sourcePlanes: Float32Array[], + targetChannels: number, +): Float32Array { + const frameCount = sourcePlanes[0]?.length ?? 0; + const output = new Float32Array(frameCount * targetChannels); + + if (targetChannels === 1) { + for (let frame = 0; frame < frameCount; frame++) { + output[frame] = averageChannels(sourcePlanes, frame); + } + return output; + } + + if (targetChannels !== 2) { + throw new Error(`Unsupported target channel count: ${targetChannels}`); + } + + if (sourcePlanes.length === 1) { + output.set(sourcePlanes[0], 0); + output.set(sourcePlanes[0], frameCount); + return output; + } + + if (sourcePlanes.length === 2) { + output.set(sourcePlanes[0], 0); + output.set(sourcePlanes[1], frameCount); + return output; + } + + const weights = getStereoDownmixWeights(sourcePlanes.length); + for (let frame = 0; frame < frameCount; frame++) { + output[frame] = weightedSample(sourcePlanes, frame, weights.left); + output[frameCount + frame] = weightedSample(sourcePlanes, frame, weights.right); + } + return output; +} + export class AudioProcessor { private cancelled = false; + static async selectSupportedExportCodec( + sampleRate: number, + numberOfChannels: number, + ): Promise { + const channelOptions = [numberOfChannels]; + if (numberOfChannels > 2) { + channelOptions.push(2); + } + + if (!channelOptions.includes(1)) { + channelOptions.push(1); + } + + for (const codec of EXPORT_AUDIO_CODECS) { + for (const channels of channelOptions) { + const support = await AudioEncoder.isConfigSupported({ + codec: codec.encoderCodec, + sampleRate, + numberOfChannels: channels, + bitrate: AUDIO_BITRATE, + }); + if (support.supported) { + return { ...codec, sampleRate, numberOfChannels: channels }; + } + } + } + + return null; + } + + static async selectSupportedExportCodecForSource( + demuxer: WebDemuxer, + ): Promise { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = await demuxer.getDecoderConfig("audio"); + } catch { + return null; + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); + return null; + } + + return AudioProcessor.selectSupportedExportCodec( + audioConfig.sampleRate || 48000, + audioConfig.numberOfChannels || 2, + ); + } + /** * Audio export has two modes: * 1) no speed regions -> fast WebCodecs trim-only pipeline @@ -22,6 +221,7 @@ export class AudioProcessor { trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, validatedDurationSec: number, + exportCodec: ExportAudioCodec, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSpeedRegions = speedRegions @@ -39,7 +239,7 @@ export class AudioProcessor { validatedDurationSec, ); if (!this.cancelled && renderedAudioBlob.size > 0) { - await this.muxRenderedAudioBlob(renderedAudioBlob, muxer); + await this.muxRenderedAudioBlob(renderedAudioBlob, muxer, exportCodec); return; } return; @@ -49,7 +249,7 @@ export class AudioProcessor { // The +0.5s buffer mirrors streamingDecoder.decodeAll's read window so the trim-only // and speed-aware paths agree on how far to read past the validated duration boundary. const readEndSec = validatedDurationSec + 0.5; - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec, exportCodec); } // Legacy trim-only path. This is still used for projects without speed regions. @@ -58,6 +258,7 @@ export class AudioProcessor { muxer: VideoMuxer, sortedTrims: TrimRegion[], readEndSec?: number, + exportCodec?: ExportAudioCodec, ): Promise { let audioConfig: AudioDecoderConfig; try { @@ -136,17 +337,28 @@ export class AudioProcessor { const sampleRate = audioConfig.sampleRate || 48000; const channels = audioConfig.numberOfChannels || 2; + const selectedCodec = + exportCodec ?? (await AudioProcessor.selectSupportedExportCodec(sampleRate, channels)); + if (!selectedCodec) { + console.warn("[AudioProcessor] No supported audio export codec, skipping audio"); + for (const frame of decodedFrames) frame.close(); + return; + } + const outputSampleRate = selectedCodec.sampleRate || sampleRate; + const outputChannels = selectedCodec.numberOfChannels || channels; const encodeConfig: AudioEncoderConfig = { - codec: "opus", - sampleRate, - numberOfChannels: channels, + codec: selectedCodec.encoderCodec, + sampleRate: outputSampleRate, + numberOfChannels: outputChannels, bitrate: AUDIO_BITRATE, }; const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); if (!encodeSupport.supported) { - console.warn("[AudioProcessor] Opus encoding not supported, skipping audio"); + console.warn( + `[AudioProcessor] ${selectedCodec.label} encoding not supported, skipping audio`, + ); for (const frame of decodedFrames) frame.close(); return; } @@ -163,7 +375,11 @@ export class AudioProcessor { const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); const adjustedTimestampUs = audioData.timestamp - trimOffsetMs * 1000; - const adjusted = this.cloneWithTimestamp(audioData, Math.max(0, adjustedTimestampUs)); + const adjusted = this.cloneForEncoding( + audioData, + Math.max(0, adjustedTimestampUs), + outputChannels, + ); audioData.close(); encoder.encode(adjusted); @@ -388,7 +604,11 @@ export class AudioProcessor { } // Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer. - private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { + private async muxRenderedAudioBlob( + blob: Blob, + muxer: VideoMuxer, + exportCodec: ExportAudioCodec, + ): Promise { if (this.cancelled) return; const file = new File([blob], "speed-audio.webm", { type: blob.type || "audio/webm" }); @@ -397,28 +617,7 @@ export class AudioProcessor { try { await demuxer.load(file); - const audioConfig = await demuxer.getDecoderConfig("audio"); - const reader = demuxer.read("audio").getReader(); - let isFirstChunk = true; - - try { - while (!this.cancelled) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - if (isFirstChunk) { - await muxer.addAudioChunk(chunk, { decoderConfig: audioConfig }); - isFirstChunk = false; - } else { - await muxer.addAudioChunk(chunk); - } - } - } finally { - try { - await reader.cancel(); - } catch { - /* reader already closed */ - } - } + await this.processTrimOnlyAudio(demuxer, muxer, [], undefined, exportCodec); } finally { try { demuxer.destroy(); @@ -541,7 +740,15 @@ export class AudioProcessor { ); } - private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { + private cloneForEncoding( + src: AudioData, + newTimestamp: number, + targetChannels: number, + ): AudioData { + if (targetChannels !== src.numberOfChannels) { + return this.downmixWithTimestamp(src, newTimestamp, targetChannels); + } + if (!src.format) { throw new Error("AudioData format is required for cloning"); } @@ -571,6 +778,37 @@ export class AudioProcessor { }); } + private downmixWithTimestamp( + src: AudioData, + newTimestamp: number, + targetChannels: number, + ): AudioData { + const sourceChannels = src.numberOfChannels; + const frameCount = src.numberOfFrames; + if (targetChannels < 1 || targetChannels > 2) { + throw new Error(`Unsupported target channel count: ${targetChannels}`); + } + + const sourcePlanes = Array.from({ length: sourceChannels }, () => new Float32Array(frameCount)); + for (let channel = 0; channel < sourceChannels; channel++) { + src.copyTo(sourcePlanes[channel], { + format: "f32-planar", + planeIndex: channel, + }); + } + + const output = downmixPlanarChannelsForExport(sourcePlanes, targetChannels); + + return new AudioData({ + format: "f32-planar", + sampleRate: src.sampleRate, + numberOfFrames: frameCount, + numberOfChannels: targetChannels, + timestamp: newTimestamp, + data: output.buffer instanceof ArrayBuffer ? output.buffer : output.slice().buffer, + }); + } + private isInTrimRegion(timestampMs: number, trims: TrimRegion[]): boolean { return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs); } diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 5906c703c..d57155d00 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -57,8 +57,22 @@ import { type Size, type StyledRenderRect, } from "@/lib/compositeLayout"; +import { + createNativeCursorMotionBlurState, + createNativeCursorSmoothingState, + getNativeCursorClickBounceProgress, + getNativeCursorClickBounceScale, + getNativeCursorMotionBlurPx, + projectNativeCursorToLocal, + resetNativeCursorMotionBlurState, + resetNativeCursorSmoothingState, + resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, + smoothNativeCursorSample, +} from "@/lib/cursor/nativeCursor"; import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; +import type { CursorRecordingData } from "@/native/contracts"; import { renderAnnotations } from "./annotationRenderer"; import { getLinearGradientPoints, @@ -80,6 +94,11 @@ interface FrameRenderConfig { borderRadius?: number; padding?: number; cropRegion: CropRegion; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; videoWidth: number; videoHeight: number; webcamSize?: Size | null; @@ -137,11 +156,15 @@ export class FrameRenderer { private rasterCtx: CanvasRenderingContext2D | null = null; private threeDPass: ThreeDPass | null = null; private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D }; + private cursorImageCache = new Map(); + private warnedKeys = new Set(); private config: FrameRenderConfig; private animationState: AnimationState; private layoutCache: LayoutCache | null = null; private currentVideoTime = 0; private motionBlurState: MotionBlurState = createMotionBlurState(); + private nativeCursorSmoothingState = createNativeCursorSmoothingState(); + private nativeCursorMotionBlurState = createNativeCursorMotionBlurState(); private smoothedAutoFocus: { cx: number; cy: number } | null = null; private prevAnimationTimeMs: number | null = null; private prevTargetProgress = 0; @@ -469,6 +492,8 @@ export class FrameRenderer { } } + await this.drawNativeCursor(timeMs); + // Render annotations on top of foreground (so they rotate with recording). if ( this.config.annotationRegions && @@ -544,6 +569,106 @@ export class FrameRenderer { } } + private async drawNativeCursor(timeMs: number) { + if (!this.foregroundCtx || !this.layoutCache) { + return; + } + + if ((this.config.cursorScale ?? 1) <= 0) { + resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); + resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); + return; + } + + const activeNativeCursor = resolveInterpolatedNativeCursorFrame( + this.config.cursorRecordingData, + timeMs, + ); + if (!activeNativeCursor) { + resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); + resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); + return; + } + const displaySample = smoothNativeCursorSample({ + sample: activeNativeCursor.sample, + smoothing: this.config.cursorSmoothing ?? 0, + state: this.nativeCursorSmoothingState, + timeMs, + }); + + const projectedPoint = projectNativeCursorToLocal({ + cropRegion: this.config.cropRegion, + maskRect: this.layoutCache.maskRect, + sample: displaySample, + }); + if (!projectedPoint) { + resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); + resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); + return; + } + + const renderAsset = resolveNativeCursorRenderAsset(activeNativeCursor.asset, 1, displaySample); + let image: HTMLImageElement; + try { + image = await this.getCursorImage(renderAsset); + } catch (error) { + this.warnOnce("native-cursor-image-load", "Failed to load native cursor asset", error); + return; + } + const scale = + Math.max(0, this.config.cursorScale ?? 1) * + getNativeCursorClickBounceScale( + this.config.cursorClickBounce ?? 0, + getNativeCursorClickBounceProgress(this.config.cursorRecordingData, timeMs), + ); + const appliedScale = this.animationState.appliedScale; + const canvasX = projectedPoint.x * appliedScale + this.animationState.x; + const canvasY = projectedPoint.y * appliedScale + this.animationState.y; + const blurPx = getNativeCursorMotionBlurPx({ + motionBlur: this.config.cursorMotionBlur ?? 0, + point: { x: canvasX, y: canvasY }, + state: this.nativeCursorMotionBlurState, + timeMs, + }); + const previousFilter = this.foregroundCtx.filter; + if (blurPx > 0) { + this.foregroundCtx.filter = `blur(${blurPx.toFixed(2)}px)`; + } + this.foregroundCtx.drawImage( + image, + canvasX - renderAsset.hotspotX * scale * appliedScale, + canvasY - renderAsset.hotspotY * scale * appliedScale, + renderAsset.width * scale * appliedScale, + renderAsset.height * scale * appliedScale, + ); + this.foregroundCtx.filter = previousFilter; + } + + private async getCursorImage(asset: { id: string; imageDataUrl: string }) { + const cachedImage = this.cursorImageCache.get(asset.id); + if (cachedImage) { + return cachedImage; + } + + const image = new Image(); + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`)); + image.src = asset.imageDataUrl; + }); + + this.cursorImageCache.set(asset.id, image); + return image; + } + + private warnOnce(key: string, message: string, error: unknown) { + if (this.warnedKeys.has(key)) { + return; + } + this.warnedKeys.add(key); + console.warn(`[FrameRenderer] ${message}:`, error); + } + private updateLayout(webcamFrame?: VideoFrame | null): void { if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return; @@ -1001,5 +1126,6 @@ export class FrameRenderer { this.threeDPass.destroy(); this.threeDPass = null; } + this.cursorImageCache.clear(); } } diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 0d7a432ba..6ff3f8793 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -9,6 +9,7 @@ import type { ZoomRegion, } from "@/components/video-editor/types"; import { BackgroundLoadError } from "@/lib/wallpaper"; +import type { CursorRecordingData } from "@/native/contracts"; import { getPlatform } from "@/utils/platformUtils"; import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; import { FrameRenderer } from "./frameRenderer"; @@ -47,6 +48,11 @@ interface GifExporterConfig { webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -151,6 +157,11 @@ export class GifExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, + cursorSmoothing: this.config.cursorSmoothing, + cursorMotionBlur: this.config.cursorMotionBlur, + cursorClickBounce: this.config.cursorClickBounce, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index a51123877..95d41ad2e 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -8,6 +8,8 @@ import { } from "mediabunny"; import type { ExportConfig } from "./types"; +export type ExportAudioMuxerCodec = "aac" | "opus"; + export class VideoMuxer { private output: Output | null = null; private videoSource: EncodedVideoPacketSource | null = null; @@ -15,10 +17,12 @@ export class VideoMuxer { private hasAudio: boolean; private target: BufferTarget | null = null; private config: ExportConfig; + private audioCodec: ExportAudioMuxerCodec; - constructor(config: ExportConfig, hasAudio = false) { + constructor(config: ExportConfig, hasAudio = false, audioCodec: ExportAudioMuxerCodec = "aac") { this.config = config; this.hasAudio = hasAudio; + this.audioCodec = audioCodec; } async initialize(): Promise { @@ -40,7 +44,7 @@ export class VideoMuxer { // Create audio source if needed if (this.hasAudio) { - this.audioSource = new EncodedAudioPacketSource("opus"); + this.audioSource = new EncodedAudioPacketSource(this.audioCodec); this.output.addAudioTrack(this.audioSource); } diff --git a/src/lib/exporter/videoExporter.test.ts b/src/lib/exporter/videoExporter.test.ts new file mode 100644 index 000000000..d168b62b7 --- /dev/null +++ b/src/lib/exporter/videoExporter.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { + getSourceCopyFastPathBlockers, + isSourceCopyFastPathEligible, + type VideoExporterConfig, +} from "./videoExporter"; + +function createConfig(overrides: Partial = {}): VideoExporterConfig { + return { + videoUrl: "recording.mp4", + width: 1920, + height: 1080, + frameRate: 60, + bitrate: 30_000_000, + wallpaper: "#000000", + zoomRegions: [], + trimRegions: [], + speedRegions: [], + showShadow: false, + shadowIntensity: 0, + showBlur: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + ...overrides, + }; +} + +describe("isSourceCopyFastPathEligible", () => { + it("allows a no-op MP4 export at source dimensions", () => { + expect( + isSourceCopyFastPathEligible(createConfig(), { + width: 1920, + height: 1080, + }), + ).toBe(true); + }); + + it("rejects timeline edits and frame-level effects", () => { + const videoInfo = { width: 1920, height: 1080 }; + + expect( + isSourceCopyFastPathEligible( + createConfig({ trimRegions: [{ id: "trim", startMs: 100, endMs: 200 }] }), + videoInfo, + ), + ).toBe(false); + expect( + isSourceCopyFastPathEligible( + createConfig({ + speedRegions: [{ id: "speed", startMs: 100, endMs: 200, speed: 1.5 }], + }), + videoInfo, + ), + ).toBe(false); + expect( + isSourceCopyFastPathEligible( + createConfig({ + zoomRegions: [ + { + id: "zoom", + startMs: 100, + endMs: 200, + depth: 2, + focus: { cx: 0.5, cy: 0.5 }, + }, + ], + }), + videoInfo, + ), + ).toBe(false); + expect(isSourceCopyFastPathEligible(createConfig({ showBlur: true }), videoInfo)).toBe(false); + }); + + it("rejects resizing and overlays", () => { + const videoInfo = { width: 1920, height: 1080 }; + + expect(isSourceCopyFastPathEligible(createConfig({ width: 1280 }), videoInfo)).toBe(false); + expect( + isSourceCopyFastPathEligible( + createConfig({ + cursorScale: 2, + }), + videoInfo, + ), + ).toBe(false); + expect( + isSourceCopyFastPathEligible( + createConfig({ + cursorScale: 2, + cursorRecordingData: { + version: 2, + provider: "native", + assets: [ + { + id: "cursor", + platform: "win32", + imageDataUrl: "data:image/png;base64,AA==", + width: 32, + height: 32, + hotspotX: 0, + hotspotY: 0, + }, + ], + samples: [{ timeMs: 0, cx: 0.5, cy: 0.5, visible: true, assetId: "cursor" }], + }, + }), + videoInfo, + ), + ).toBe(false); + expect( + isSourceCopyFastPathEligible( + createConfig({ + cursorHighlight: { + enabled: true, + style: "ring", + sizePx: 24, + color: "#ffffff", + opacity: 1, + onlyOnClicks: false, + clickEmphasisDurationMs: 350, + offsetXNorm: 0, + offsetYNorm: 0, + }, + cursorTelemetry: [{ timeMs: 0, cx: 0.5, cy: 0.5 }], + }), + videoInfo, + ), + ).toBe(false); + }); +}); + +describe("getSourceCopyFastPathBlockers", () => { + it("reports the source-size mismatch that blocks copy-only export", () => { + expect( + getSourceCopyFastPathBlockers(createConfig({ height: 1080 }), { + width: 1920, + height: 1032, + }), + ).toContain("output-size 1920x1080 differs from source 1920x1032"); + }); +}); diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index e064ba7c9..10525a1e1 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -8,6 +8,7 @@ import type { ZoomRegion, } from "@/components/video-editor/types"; import { BackgroundLoadError } from "@/lib/wallpaper"; +import type { CursorRecordingData } from "@/native/contracts"; import { getPlatform } from "@/utils/platformUtils"; import { AsyncVideoFrameQueue } from "./asyncVideoFrameQueue"; import { AudioProcessor } from "./audioEncoder"; @@ -19,7 +20,7 @@ import type { ExportConfig, ExportProgress, ExportResult } from "./types"; const ENCODER_STALL_TIMEOUT_MS = 15_000; const ENCODER_FLUSH_TIMEOUT_MS = 20_000; -interface VideoExporterConfig extends ExportConfig { +export interface VideoExporterConfig extends ExportConfig { videoUrl: string; webcamVideoUrl?: string; wallpaper: string; @@ -38,6 +39,11 @@ interface VideoExporterConfig extends ExportConfig { webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; + cursorRecordingData?: CursorRecordingData | null; + cursorScale?: number; + cursorSmoothing?: number; + cursorMotionBlur?: number; + cursorClickBounce?: number; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -47,6 +53,93 @@ interface VideoExporterConfig extends ExportConfig { onProgress?: (progress: ExportProgress) => void; } +const SOURCE_COPY_EPSILON = 0.0001; + +function hasActiveTimeRegions(regions?: Array<{ startMs: number; endMs: number }>) { + return Boolean(regions?.some((region) => region.endMs - region.startMs > SOURCE_COPY_EPSILON)); +} + +function hasActiveSpeedRegions(regions?: SpeedRegion[]) { + return Boolean( + regions?.some( + (region) => + region.endMs - region.startMs > SOURCE_COPY_EPSILON && + Math.abs(region.speed - 1) > SOURCE_COPY_EPSILON, + ), + ); +} + +function hasNativeCursorOverlay(config: VideoExporterConfig) { + return (config.cursorScale ?? 0) > 0; +} + +function hasCursorHighlightOverlay(config: VideoExporterConfig) { + return Boolean( + config.cursorHighlight?.enabled && config.cursorTelemetry && config.cursorTelemetry.length > 0, + ); +} + +function isDefaultCrop(cropRegion: CropRegion) { + return ( + Math.abs(cropRegion.x) <= SOURCE_COPY_EPSILON && + Math.abs(cropRegion.y) <= SOURCE_COPY_EPSILON && + Math.abs(cropRegion.width - 1) <= SOURCE_COPY_EPSILON && + Math.abs(cropRegion.height - 1) <= SOURCE_COPY_EPSILON + ); +} + +export function isSourceCopyFastPathEligible( + config: VideoExporterConfig, + videoInfo: { width: number; height: number }, +) { + return getSourceCopyFastPathBlockers(config, videoInfo).length === 0; +} + +export function getSourceCopyFastPathBlockers( + config: VideoExporterConfig, + videoInfo: { width: number; height: number }, +) { + const blockers: string[] = []; + + if (config.width !== videoInfo.width || config.height !== videoInfo.height) { + blockers.push( + `output-size ${config.width}x${config.height} differs from source ${videoInfo.width}x${videoInfo.height}`, + ); + } + if (config.webcamVideoUrl) blockers.push("webcam overlay is enabled"); + if (hasActiveTimeRegions(config.trimRegions)) blockers.push("trim regions are present"); + if (hasActiveSpeedRegions(config.speedRegions)) blockers.push("speed regions are present"); + if (hasActiveTimeRegions(config.zoomRegions)) blockers.push("zoom regions are present"); + if (hasActiveTimeRegions(config.annotationRegions)) + blockers.push("annotation regions are present"); + if (hasNativeCursorOverlay(config)) blockers.push("editable cursor overlay is enabled"); + if (hasCursorHighlightOverlay(config)) blockers.push("cursor highlight overlay is enabled"); + if (!isDefaultCrop(config.cropRegion)) blockers.push("crop is not default"); + if ((config.padding ?? 0) > SOURCE_COPY_EPSILON) blockers.push("padding is not zero"); + if ((config.videoPadding ?? 0) > SOURCE_COPY_EPSILON) blockers.push("video padding is not zero"); + if ((config.borderRadius ?? 0) > SOURCE_COPY_EPSILON) blockers.push("roundness is not zero"); + if (config.showShadow || config.shadowIntensity > SOURCE_COPY_EPSILON) { + blockers.push("shadow is enabled"); + } + if (config.showBlur) blockers.push("background blur is enabled"); + if ((config.motionBlurAmount ?? 0) > SOURCE_COPY_EPSILON) blockers.push("motion blur is enabled"); + + return blockers; +} + +function isMp4Source(videoUrl: string, blob: Blob) { + if (blob.type.toLowerCase().includes("mp4")) { + return true; + } + + try { + const path = new URL(videoUrl, window.location.href).pathname; + return path.toLowerCase().endsWith(".mp4"); + } catch { + return videoUrl.toLowerCase().split(/[?#]/, 1)[0].endsWith(".mp4"); + } +} + export class VideoExporter { private config: VideoExporterConfig; private streamingDecoder: StreamingVideoDecoder | null = null; @@ -127,6 +220,11 @@ export class VideoExporter { const streamingDecoder = new StreamingVideoDecoder(); this.streamingDecoder = streamingDecoder; const videoInfo = await streamingDecoder.loadMetadata(this.config.videoUrl); + const sourceCopyResult = await this.trySourceCopyFastPath(videoInfo); + if (sourceCopyResult) { + return sourceCopyResult; + } + let webcamInfo: Awaited> | null = null; if (this.config.webcamVideoUrl) { webcamDecoder = new StreamingVideoDecoder(); @@ -146,6 +244,11 @@ export class VideoExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, + cursorRecordingData: this.config.cursorRecordingData, + cursorScale: this.config.cursorScale, + cursorSmoothing: this.config.cursorSmoothing, + cursorMotionBlur: this.config.cursorMotionBlur, + cursorClickBounce: this.config.cursorClickBounce, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, @@ -167,8 +270,17 @@ export class VideoExporter { await this.initializeEncoder(encoderPreference); - const hasAudio = videoInfo.hasAudio; - const muxer = new VideoMuxer(this.config, hasAudio); + const sourceDemuxer = streamingDecoder.getDemuxer(); + const audioExportCodec = + videoInfo.hasAudio && sourceDemuxer + ? await AudioProcessor.selectSupportedExportCodecForSource(sourceDemuxer) + : null; + if (videoInfo.hasAudio && !audioExportCodec) { + console.warn("[VideoExporter] No supported audio export codec, exporting video-only."); + } + + const hasAudio = Boolean(audioExportCodec); + const muxer = new VideoMuxer(this.config, hasAudio, audioExportCodec?.muxerCodec); this.muxer = muxer; await muxer.initialize(); @@ -350,7 +462,7 @@ export class VideoExporter { phase: "finalizing", }); - if (hasAudio && !this.cancelled) { + if (hasAudio && audioExportCodec && !this.cancelled) { const demuxer = streamingDecoder.getDemuxer(); if (demuxer) { console.log("[VideoExporter] Processing audio track..."); @@ -362,6 +474,7 @@ export class VideoExporter { this.config.trimRegions, this.config.speedRegions, videoInfo.duration, + audioExportCodec, ); } } @@ -545,6 +658,70 @@ export class VideoExporter { return ["prefer-hardware", "prefer-software"]; } + private async trySourceCopyFastPath(videoInfo: { width: number; height: number }) { + const blockers = getSourceCopyFastPathBlockers(this.config, videoInfo); + if (blockers.length > 0) { + console.info("[VideoExporter] source-copy fast path disabled", { + blockers, + output: { width: this.config.width, height: this.config.height }, + source: videoInfo, + }); + return null; + } + + const sourceBlob = await this.loadSourceBlob(); + if (!sourceBlob || !isMp4Source(this.config.videoUrl, sourceBlob)) { + console.info("[VideoExporter] source-copy fast path disabled", { + blockers: ["source is not a readable MP4"], + source: videoInfo, + }); + return null; + } + + if (this.cancelled) { + return { success: false, error: "Export cancelled" }; + } + + this.reportProgress({ + currentFrame: 1, + totalFrames: 1, + percentage: 100, + estimatedTimeRemaining: 0, + phase: "finalizing", + }); + console.info("[VideoExporter] using source-copy fast path", { + source: videoInfo, + bytes: sourceBlob.size, + }); + + return { + success: true, + blob: sourceBlob.type ? sourceBlob : new Blob([sourceBlob], { type: "video/mp4" }), + } satisfies ExportResult; + } + + private async loadSourceBlob() { + const videoUrl = this.config.videoUrl; + const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); + + if (!isRemoteUrl && window.electronAPI?.readBinaryFile) { + const result = await window.electronAPI.readBinaryFile(videoUrl); + if (!result.success || !result.data) { + return null; + } + + const type = videoUrl.toLowerCase().split(/[?#]/, 1)[0].endsWith(".mp4") ? "video/mp4" : ""; + return new Blob([result.data], type ? { type } : undefined); + } + + const response = await fetch(videoUrl); + if (!response.ok) { + return null; + } + + return response.blob(); + } + private reportProgress(progress: ExportProgress): void { this.config.onProgress?.(progress); } diff --git a/src/lib/nativeWindowsRecording.ts b/src/lib/nativeWindowsRecording.ts new file mode 100644 index 000000000..5e0685171 --- /dev/null +++ b/src/lib/nativeWindowsRecording.ts @@ -0,0 +1,60 @@ +export type NativeWindowsSourceType = "display" | "window"; + +export type NativeWindowsRecordingRequest = { + recordingId?: number; + source: { + type: NativeWindowsSourceType; + sourceId: string; + displayId?: number; + windowHandle?: string; + }; + video: { + fps: number; + width: number; + height: number; + }; + audio: { + system: { + enabled: boolean; + }; + microphone: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + gain: number; + }; + }; + webcam: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + directShowClsid?: string; + width: number; + height: number; + fps: number; + }; + cursor: { + mode: import("./recordingSession").CursorCaptureMode; + }; +}; + +export type NativeWindowsRecordingStartResult = { + success: boolean; + recordingId?: number; + path?: string; + helperPath?: string; + error?: string; +}; + +export function parseWindowHandleFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("window:")) { + return null; + } + + const handlePart = sourceId.split(":")[1]; + if (!handlePart || !/^\d+$/.test(handlePart)) { + return null; + } + + return handlePart; +} diff --git a/src/lib/recordingSession.ts b/src/lib/recordingSession.ts index 17cf7c1fa..f5ebf9ca2 100644 --- a/src/lib/recordingSession.ts +++ b/src/lib/recordingSession.ts @@ -1,8 +1,11 @@ export interface ProjectMedia { screenVideoPath: string; webcamVideoPath?: string; + cursorCaptureMode?: CursorCaptureMode; } +export type CursorCaptureMode = "editable-overlay" | "system"; + export interface RecordingSession extends ProjectMedia { createdAt: number; } @@ -16,6 +19,11 @@ export interface StoreRecordedSessionInput { screen: RecordedVideoAssetInput; webcam?: RecordedVideoAssetInput; createdAt?: number; + cursorCaptureMode?: CursorCaptureMode; +} + +export function normalizeCursorCaptureMode(value: unknown): CursorCaptureMode | undefined { + return value === "editable-overlay" || value === "system" ? value : undefined; } function normalizePath(value: unknown): string | undefined { @@ -40,12 +48,13 @@ export function normalizeProjectMedia(candidate: unknown): ProjectMedia | null { } const webcamVideoPath = normalizePath(raw.webcamVideoPath); + const cursorCaptureMode = normalizeCursorCaptureMode(raw.cursorCaptureMode); - return webcamVideoPath - ? { screenVideoPath, webcamVideoPath } - : { - screenVideoPath, - }; + return { + screenVideoPath, + ...(webcamVideoPath ? { webcamVideoPath } : {}), + ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + }; } export function normalizeRecordingSession(candidate: unknown): RecordingSession | null { diff --git a/src/native/client.ts b/src/native/client.ts new file mode 100644 index 000000000..3f53ce483 --- /dev/null +++ b/src/native/client.ts @@ -0,0 +1,133 @@ +import { + type CursorCapabilities, + type CursorRecordingData, + type CursorTelemetryPoint, + NATIVE_BRIDGE_CHANNEL, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectContext, + type ProjectFileResult, + type ProjectPathResult, + type SystemCapabilities, +} from "./contracts"; + +function createRequestId() { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + + return `req-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +function getElectronBridge() { + if (!window.electronAPI?.invokeNativeBridge) { + throw new Error( + `Native bridge unavailable. Expected ${NATIVE_BRIDGE_CHANNEL} transport in preload.`, + ); + } + + return window.electronAPI.invokeNativeBridge; +} + +export async function invokeNativeBridge( + request: NativeBridgeRequest, +): Promise> { + const invoke = getElectronBridge(); + return invoke({ + ...request, + requestId: request.requestId ?? createRequestId(), + }); +} + +export async function requireNativeBridgeData(request: NativeBridgeRequest): Promise { + const response = await invokeNativeBridge(request); + if (!response.ok) { + throw new Error(response.error.message); + } + + return response.data; +} + +export const nativeBridgeClient = { + rawInvoke: invokeNativeBridge, + system: { + getPlatform: () => + requireNativeBridgeData({ + domain: "system", + action: "getPlatform", + }), + getAssetBasePath: () => + requireNativeBridgeData({ + domain: "system", + action: "getAssetBasePath", + }), + getCapabilities: () => + requireNativeBridgeData({ + domain: "system", + action: "getCapabilities", + }), + }, + project: { + getCurrentContext: () => + requireNativeBridgeData({ + domain: "project", + action: "getCurrentContext", + }), + saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) => + requireNativeBridgeData({ + domain: "project", + action: "saveProjectFile", + payload: { + projectData, + suggestedName, + existingProjectPath, + }, + }), + loadProjectFile: () => + requireNativeBridgeData({ + domain: "project", + action: "loadProjectFile", + }), + loadCurrentProjectFile: () => + requireNativeBridgeData({ + domain: "project", + action: "loadCurrentProjectFile", + }), + setCurrentVideoPath: (path: string) => + requireNativeBridgeData({ + domain: "project", + action: "setCurrentVideoPath", + payload: { path }, + }), + getCurrentVideoPath: () => + requireNativeBridgeData({ + domain: "project", + action: "getCurrentVideoPath", + }), + clearCurrentVideoPath: () => + requireNativeBridgeData({ + domain: "project", + action: "clearCurrentVideoPath", + }), + }, + cursor: { + getCapabilities: () => + requireNativeBridgeData({ + domain: "cursor", + action: "getCapabilities", + }), + getRecordingData: (videoPath?: string) => + requireNativeBridgeData({ + domain: "cursor", + action: "getRecordingData", + payload: videoPath ? { videoPath } : {}, + }), + getTelemetry: (videoPath?: string) => + requireNativeBridgeData({ + domain: "cursor", + action: "getTelemetry", + payload: videoPath ? { videoPath } : {}, + }), + }, +}; diff --git a/src/native/contracts.ts b/src/native/contracts.ts new file mode 100644 index 000000000..6836095ac --- /dev/null +++ b/src/native/contracts.ts @@ -0,0 +1,229 @@ +export const NATIVE_BRIDGE_CHANNEL = "native-bridge:invoke"; +export const NATIVE_BRIDGE_VERSION = 1; + +export type NativePlatform = "darwin" | "win32" | "linux"; +export type CursorProviderKind = "native" | "none"; +export type NativeCursorType = + | "arrow" + | "text" + | "pointer" + | "crosshair" + | "open-hand" + | "closed-hand" + | "resize-ew" + | "resize-ns" + | "resize-nesw" + | "resize-nwse" + | "move" + | "not-allowed" + | "wait" + | "app-starting" + | "help" + | "up-arrow"; + +export interface CursorTelemetryPoint { + timeMs: number; + cx: number; + cy: number; +} + +export interface CursorRecordingSample extends CursorTelemetryPoint { + assetId?: string | null; + visible?: boolean; + cursorType?: NativeCursorType | null; + interactionType?: "move" | "click" | "mouseup"; +} + +export interface NativeCursorAsset { + id: string; + platform: NativePlatform; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; + scaleFactor?: number; + cursorType?: NativeCursorType | null; +} + +export interface CursorRecordingData { + version: number; + provider: CursorProviderKind; + samples: CursorRecordingSample[]; + assets: NativeCursorAsset[]; +} + +export interface CursorCapabilities { + telemetry: boolean; + systemAssets: boolean; + provider: CursorProviderKind; +} + +export interface SystemCapabilities { + bridgeVersion: typeof NATIVE_BRIDGE_VERSION; + platform: NativePlatform; + cursor: CursorCapabilities; + project: { + currentContext: boolean; + }; +} + +export interface ProjectContext { + currentProjectPath: string | null; + currentVideoPath: string | null; +} + +export interface ProjectPathResult { + success: boolean; + path?: string; + message?: string; + canceled?: boolean; + error?: string; +} + +export interface ProjectFileResult { + success: boolean; + path?: string; + project?: unknown; + message?: string; + canceled?: boolean; + error?: string; +} + +export type NativeBridgeErrorCode = + | "INVALID_REQUEST" + | "UNSUPPORTED_ACTION" + | "NOT_FOUND" + | "UNAVAILABLE" + | "INTERNAL_ERROR"; + +export interface NativeBridgeError { + code: NativeBridgeErrorCode; + message: string; + retryable: boolean; +} + +export interface NativeBridgeMeta { + version: typeof NATIVE_BRIDGE_VERSION; + requestId: string; + timestampMs: number; +} + +export interface NativeBridgeSuccess { + ok: true; + data: TData; + meta: NativeBridgeMeta; +} + +export interface NativeBridgeFailure { + ok: false; + error: NativeBridgeError; + meta: NativeBridgeMeta; +} + +export type NativeBridgeResponse = + | NativeBridgeSuccess + | NativeBridgeFailure; + +type EmptyPayload = Record; + +export type NativeBridgeRequest = + | { + domain: "system"; + action: "getPlatform"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "system"; + action: "getAssetBasePath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "system"; + action: "getCapabilities"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "getCurrentContext"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "saveProjectFile"; + payload: { + projectData: unknown; + suggestedName?: string; + existingProjectPath?: string; + }; + requestId?: string; + } + | { + domain: "project"; + action: "loadProjectFile"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "loadCurrentProjectFile"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "setCurrentVideoPath"; + payload: { + path: string; + }; + requestId?: string; + } + | { + domain: "project"; + action: "getCurrentVideoPath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "project"; + action: "clearCurrentVideoPath"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "cursor"; + action: "getCapabilities"; + payload?: EmptyPayload; + requestId?: string; + } + | { + domain: "cursor"; + action: "getTelemetry"; + payload?: { + videoPath?: string; + }; + requestId?: string; + } + | { + domain: "cursor"; + action: "getRecordingData"; + payload?: { + videoPath?: string; + }; + requestId?: string; + }; + +export type NativeBridgeEventName = + | "project.contextChanged" + | "cursor.providerChanged" + | "cursor.telemetryLoaded"; + +export interface NativeBridgeEvent { + name: NativeBridgeEventName; + payload: TPayload; + meta: NativeBridgeMeta; +} diff --git a/src/native/hooks/useCursorRecordingData.ts b/src/native/hooks/useCursorRecordingData.ts new file mode 100644 index 000000000..6b3451a88 --- /dev/null +++ b/src/native/hooks/useCursorRecordingData.ts @@ -0,0 +1,61 @@ +import { useEffect, useState } from "react"; +import type { CursorRecordingData } from "@/native/contracts"; +import { nativeBridgeClient } from "../client"; + +interface UseCursorRecordingDataResult { + data: CursorRecordingData | null; + loading: boolean; + error: string | null; +} + +export function useCursorRecordingData(videoPath: string | null): UseCursorRecordingDataResult { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + + async function loadCursorRecordingData() { + if (!videoPath) { + setData(null); + setLoading(false); + setError(null); + return; + } + + setLoading(true); + setError(null); + + try { + const nextData = await nativeBridgeClient.cursor.getRecordingData(videoPath); + if (!cancelled) { + setData(nextData); + } + } catch (nextError) { + if (!cancelled) { + setData(null); + setError( + nextError instanceof Error ? nextError.message : "Failed to load cursor recording data", + ); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + } + + loadCursorRecordingData(); + + return () => { + cancelled = true; + }; + }, [videoPath]); + + return { + data, + loading, + error, + }; +} diff --git a/src/native/hooks/useCursorTelemetry.ts b/src/native/hooks/useCursorTelemetry.ts new file mode 100644 index 000000000..161768054 --- /dev/null +++ b/src/native/hooks/useCursorTelemetry.ts @@ -0,0 +1,61 @@ +import { useEffect, useState } from "react"; +import type { CursorTelemetryPoint } from "@/components/video-editor/types"; +import { nativeBridgeClient } from "../client"; + +interface UseCursorTelemetryResult { + samples: CursorTelemetryPoint[]; + loading: boolean; + error: string | null; +} + +export function useCursorTelemetry(videoPath: string | null): UseCursorTelemetryResult { + const [samples, setSamples] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + + async function loadCursorTelemetry() { + if (!videoPath) { + setSamples([]); + setLoading(false); + setError(null); + return; + } + + setLoading(true); + setError(null); + + try { + const nextSamples = await nativeBridgeClient.cursor.getTelemetry(videoPath); + if (!cancelled) { + setSamples(nextSamples); + } + } catch (nextError) { + if (!cancelled) { + setSamples([]); + setError( + nextError instanceof Error ? nextError.message : "Failed to load cursor telemetry", + ); + } + } finally { + if (!cancelled) { + setLoading(false); + } + } + } + + loadCursorTelemetry(); + + return () => { + cancelled = true; + }; + }, [videoPath]); + + return { + samples, + loading, + error, + }; +} diff --git a/src/native/index.ts b/src/native/index.ts new file mode 100644 index 000000000..817d1cf14 --- /dev/null +++ b/src/native/index.ts @@ -0,0 +1,4 @@ +export * from "./client"; +export * from "./contracts"; +export * from "./hooks/useCursorRecordingData"; +export * from "./hooks/useCursorTelemetry"; diff --git a/tests/e2e/windows-native-checklist.spec.ts b/tests/e2e/windows-native-checklist.spec.ts new file mode 100644 index 000000000..d19a1fd64 --- /dev/null +++ b/tests/e2e/windows-native-checklist.spec.ts @@ -0,0 +1,322 @@ +import { once } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Page } from "@playwright/test"; +import { _electron as electron, expect, test } from "@playwright/test"; +import { NATIVE_BRIDGE_CHANNEL, NATIVE_BRIDGE_VERSION } from "../../src/native/contracts"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(__dirname, "../.."); +const MAIN_JS = path.join(ROOT, "dist-electron/main.js"); +const TEST_VIDEO = path.join(__dirname, "../fixtures/sample.webm"); + +async function launchApp() { + const testUserDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-e2e-")); + const app = await electron.launch({ + args: [ + MAIN_JS, + "--no-sandbox", + "--enable-unsafe-swiftshader", + "--lang=en-US", + `--user-data-dir=${testUserDataDir}`, + ], + env: { + ...process.env, + ELECTRON_USER_DATA_DIR: testUserDataDir, + HEADLESS: process.env["HEADLESS"] ?? "true", + LANG: "en_US.UTF-8", + LC_ALL: "en_US.UTF-8", + LANGUAGE: "en_US", + }, + }); + + const childProcess = app.process(); + childProcess.stdout?.on("data", (d) => process.stdout.write(`[electron] ${d}`)); + childProcess.stderr?.on("data", (d) => process.stderr.write(`[electron] ${d}`)); + ( + app as ElectronApplication & { + __testUserDataDir?: string; + __childProcess?: ReturnType; + } + ).__testUserDataDir = testUserDataDir; + ( + app as ElectronApplication & { + __testUserDataDir?: string; + __childProcess?: ReturnType; + } + ).__childProcess = childProcess; + + return app; +} + +async function closeApp(app: ElectronApplication) { + const childProcess = ( + app as ElectronApplication & { + __childProcess?: ReturnType; + } + ).__childProcess; + await Promise.race([app.close(), new Promise((resolve) => setTimeout(resolve, 5_000))]); + if (childProcess && childProcess.exitCode === null && childProcess.signalCode === null) { + if (!childProcess.killed) { + childProcess.kill(); + } + await Promise.race([ + once(childProcess, "close"), + new Promise((resolve) => setTimeout(resolve, 5_000)), + ]); + } + const testUserDataDir = (app as ElectronApplication & { __testUserDataDir?: string }) + .__testUserDataDir; + if (testUserDataDir && fs.existsSync(testUserDataDir)) { + fs.rmSync(testUserDataDir, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + } +} + +async function copyFixtureToRecordings(app: ElectronApplication, fileName: string) { + const userDataDir = await app.evaluate(({ app: electronApp }) => { + return electronApp.getPath("userData"); + }); + const recordingsDir = path.join(userDataDir, "recordings"); + const targetPath = path.join(recordingsDir, fileName); + fs.mkdirSync(recordingsDir, { recursive: true }); + fs.copyFileSync(TEST_VIDEO, targetPath); + return targetPath; +} + +async function dismissLanguagePrompt(page: Page) { + const keepCurrentLanguage = page + .getByRole("button") + .filter({ hasText: /Keep current language|Conserver la langue actuelle/ }); + if ((await keepCurrentLanguage.count()) > 0) { + await keepCurrentLanguage.click(); + } +} + +type ElectronApplication = Awaited>; + +test.describe("Windows native checklist smoke tests", () => { + test.skip(process.platform !== "win32", "Windows native capture is Windows-only."); + + test("source selector opens, lists thumbnails, and selects a screen/window source", async () => { + const app = await launchApp(); + + try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await dismissLanguagePrompt(hudWindow); + + await expect(hudWindow.getByTestId("launch-record-button")).toBeDisabled(); + await expect(hudWindow.getByTestId("launch-source-selector-button")).toBeVisible(); + await expect(hudWindow.getByTestId("launch-system-audio-button")).toBeEnabled(); + await expect(hudWindow.getByTestId("launch-microphone-button")).toBeEnabled(); + + await hudWindow.getByTestId("launch-source-selector-button").click(); + const sourceWindow = await app.waitForEvent("window", { + predicate: (w) => w.url().includes("windowType=source-selector"), + timeout: 15_000, + }); + await sourceWindow.waitForLoadState("domcontentloaded"); + + const cards = sourceWindow.getByTestId("source-selector-card"); + await expect.poll(() => cards.count(), { timeout: 15_000 }).toBeGreaterThan(0); + + const thumbnails = await cards.locator("img").evaluateAll((imgs) => + imgs.map((img) => ({ + alt: img.getAttribute("alt"), + src: img.getAttribute("src"), + })), + ); + expect(thumbnails.some((item) => item.alt && item.src?.startsWith("data:image"))).toBe(true); + + const hasScreen = await sourceWindow + .locator('[data-testid="source-selector-card"][data-source-kind="screen"]') + .count() + .then((count) => count > 0); + const hasWindow = await sourceWindow + .locator('[data-testid="source-selector-card"][data-source-kind="window"]') + .count() + .then((count) => count > 0); + expect(hasScreen || hasWindow).toBe(true); + + await expect(sourceWindow.getByTestId("source-selector-share-button")).toBeDisabled(); + await cards.first().click(); + await expect(sourceWindow.getByTestId("source-selector-share-button")).toBeEnabled(); + await sourceWindow.getByTestId("source-selector-share-button").click(); + + await expect + .poll( + () => + hudWindow.evaluate(async () => { + return await window.electronAPI.getSelectedSource(); + }), + { timeout: 10_000 }, + ) + .not.toBeNull(); + await expect(hudWindow.getByTestId("launch-record-button")).toBeEnabled(); + } finally { + await closeApp(app); + } + }); + + test("launch window opens an existing video into the editor and playback controls respond", async () => { + const app = await launchApp(); + let testVideoInRecordings = ""; + + try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await dismissLanguagePrompt(hudWindow); + testVideoInRecordings = await copyFixtureToRecordings(app, "checklist-sample.webm"); + + await app.evaluate(({ ipcMain }, videoPath) => { + ipcMain.removeHandler("open-video-file-picker"); + ipcMain.handle("open-video-file-picker", () => ({ + success: true, + path: videoPath, + })); + }, testVideoInRecordings); + + await hudWindow.getByTestId("launch-open-video-button").click(); + const editorWindow = await app.waitForEvent("window", { + predicate: (w) => w.url().includes("windowType=editor"), + timeout: 15_000, + }); + await editorWindow.waitForLoadState("domcontentloaded"); + await expect(editorWindow.getByText("Loading video...")).not.toBeVisible({ timeout: 20_000 }); + + const playButton = editorWindow.locator( + 'button[aria-label="Play"], button[aria-label="Lire"]', + ); + await expect(playButton).toBeVisible({ timeout: 10_000 }); + await playButton.click(); + + const seekInput = editorWindow.locator('input[type="range"]').first(); + await expect(seekInput).toBeVisible(); + await seekInput.evaluate((input) => { + const range = input as HTMLInputElement; + range.value = "0.25"; + range.dispatchEvent(new Event("input", { bubbles: true })); + range.dispatchEvent(new Event("change", { bubbles: true })); + }); + await expect.poll(() => seekInput.inputValue(), { timeout: 10_000 }).not.toBe("0"); + + await expect( + editorWindow.getByText("Background").or(editorWindow.getByText("Arrière-plan")), + ).toBeVisible(); + await expect(editorWindow.getByTestId("testId-export-button")).toBeVisible(); + } finally { + await closeApp(app); + if (testVideoInRecordings && fs.existsSync(testVideoInRecordings)) { + fs.unlinkSync(testVideoInRecordings); + } + } + }); + + test("launch window opens an existing project into the editor", async () => { + const app = await launchApp(); + let testVideoInRecordings = ""; + let projectPath = ""; + + try { + const hudWindow = await app.firstWindow({ timeout: 60_000 }); + await hudWindow.waitForLoadState("domcontentloaded"); + await dismissLanguagePrompt(hudWindow); + testVideoInRecordings = await copyFixtureToRecordings(app, "checklist-project-sample.webm"); + projectPath = path.join(os.tmpdir(), `openscreen-checklist-${Date.now()}.openscreen`); + const project = { + version: 2, + videoPath: testVideoInRecordings, + editor: {}, + }; + fs.writeFileSync(projectPath, JSON.stringify(project), "utf-8"); + + await app.evaluate( + ({ ipcMain }, payload) => { + ipcMain.removeHandler(payload.nativeBridgeChannel); + ipcMain.handle(payload.nativeBridgeChannel, (_event, request) => { + const success = (data: unknown) => ({ + ok: true, + data, + meta: { + version: payload.nativeBridgeVersion, + requestId: request.requestId ?? "checklist-project-load", + timestampMs: Date.now(), + }, + }); + + if (request.domain === "project" && request.action === "loadProjectFile") { + return success({ + success: true, + path: payload.projectPath, + project: payload.project, + }); + } + if (request.domain === "project" && request.action === "loadCurrentProjectFile") { + return success({ success: false, canceled: true }); + } + if (request.domain === "project" && request.action === "getCurrentVideoPath") { + return success({ success: true, path: payload.videoPath }); + } + if (request.domain === "system" && request.action === "getPlatform") { + return success("win32"); + } + if (request.domain === "system" && request.action === "getAssetBasePath") { + return success(null); + } + if (request.domain === "cursor" && request.action === "getRecordingData") { + return success({ version: 2, provider: "none", samples: [], assets: [] }); + } + if (request.domain === "cursor" && request.action === "getTelemetry") { + return success([]); + } + + return { + ok: false, + error: { + code: "UNSUPPORTED_ACTION", + message: `Unexpected native bridge request in test: ${request.domain}.${request.action}`, + retryable: false, + }, + meta: { + version: payload.nativeBridgeVersion, + requestId: request.requestId ?? "checklist-project-load", + timestampMs: Date.now(), + }, + }; + }); + }, + { + projectPath, + project, + videoPath: testVideoInRecordings, + nativeBridgeChannel: NATIVE_BRIDGE_CHANNEL, + nativeBridgeVersion: NATIVE_BRIDGE_VERSION, + }, + ); + + await hudWindow.getByTestId("launch-open-project-button").click(); + const editorWindow = await app.waitForEvent("window", { + predicate: (w) => w.url().includes("windowType=editor"), + timeout: 15_000, + }); + await editorWindow.waitForLoadState("domcontentloaded"); + await expect(editorWindow.getByText("Loading video...")).not.toBeVisible({ timeout: 20_000 }); + await expect(editorWindow.getByTestId("testId-export-button")).toBeVisible(); + } finally { + await closeApp(app); + if (testVideoInRecordings && fs.existsSync(testVideoInRecordings)) { + fs.unlinkSync(testVideoInRecordings); + } + if (projectPath && fs.existsSync(projectPath)) { + fs.unlinkSync(projectPath); + } + } + }); +});