From 2bec0d9e407d7e721b97d97394d8c15d1bcd72f9 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 28 Jul 2026 22:59:56 +0200 Subject: [PATCH 1/3] refactor(export): drive GIF from the video exporter's own frame walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GIF had its own loop over the LIVE PREVIEW Player, stepping one SOURCE frame per OUTPUT frame. Consequences, both proven by the new test: - a 4 s / 60 fps source at 12 fps produced 48 frames taken from the first 0.8 s — right frame count, 5x slow motion, 80% of the recording missing; - it could not decode files run_composited_multi handles (Player::open -> 'frame sans texture D3D11'), so it was not merely mistimed. Extract the format-agnostic half of run_multi_inner as walk_composited_timeline (clip iteration, decoder reuse, availability clamping, scene windowing, keyframe seeks, cursor binding, speed segments, output-time advancement) and drive both exporters through it. MP4 keeps its encoder/mux/audio; GIF supplies readback + quantize. 'Which source frame belongs at output frame N' now has exactly one definition. Also fixes the dither, which was a measurable no-op: it diffused error against round() of an already-integer channel, so err stayed 0 for every pixel by induction. Fused into the index mapping, where the error against the CHOSEN palette entry actually exists. tests/export_timing.rs pins both: MP4 frame count vs output fps, and a GIF whose per-frame palettes must span a source that changes colour every second — the assertion frame count alone cannot make. MP4 verified unchanged (120 frames) before and after the extraction. --- crates/compositor-view-napi/src/lib.rs | 75 ++++-- crates/compositor/src/gif_export.rs | 299 ++++++++++++++--------- crates/compositor/src/pipeline.rs | 250 +++++++++++-------- crates/compositor/tests/export_timing.rs | 210 ++++++++++++++++ 4 files changed, 607 insertions(+), 227 deletions(-) create mode 100644 crates/compositor/tests/export_timing.rs diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index b3a4b951d..9b2e42881 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -499,11 +499,13 @@ pub struct GifParamsInput { /// `Compositor` (équivalent de `cfg.cursor = false` dans /// `run_composited_multi`). pub struct ExportGifTask { - screen_path: String, - webcam_path: String, - /// Option (pas Option) parce que napi-rs n'expose - /// pas PathBuf en type d'entrée pratique. - cursor_path: Option, + /// Same clip list the MP4 export takes — GIF is now a multiclip export + /// driven by the same walk, not a single-file special case. + clips: Vec, + /// Scene JSON from the app, exactly as `exportMulti` receives it: it + /// carries background, layout, webcam and cursor. Absent/invalid → no + /// scene, same as a preview that was never configured. + scene_json: Option, out_path: PathBuf, params: GifExportParams, on_progress: Option>, @@ -520,12 +522,45 @@ impl Task for ExportGifTask { // RT avec la preview (sa propre `Compositor::new_sized`) mais le // 3D engine de la preview pollue quand même, d'où la pause. let _previews = PreviewPause::begin(); + + // Same construction as ExportMultiTask — GIF and MP4 differ only in the + // encoder, so everything up to it is built identically. + let gpu = Gpu::create(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; + let mut cfg = config::all().pop().expect("au moins une config"); // C8 + cfg.zoom = false; + cfg.layout_anim = false; + cfg.mblur_n = 1; + + let scene = self.scene_json.as_deref().and_then(|j| Scene::from_json(j).ok()); + if let Some(scene) = &scene { + cfg.bg_blur = scene.effects.blur; + cfg.cursor = scene.cursor.show; + } else { + cfg.cursor = false; + } + + let width = self + .params + .width + .unwrap_or(openscreen_compositor::gif_export::DEFAULT_GIF_WIDTH); + let height = self + .params + .height + .unwrap_or(openscreen_compositor::gif_export::DEFAULT_GIF_HEIGHT); + let comp = Compositor::new_sized(&gpu, width, height) + .map_err(|e| Error::from_reason(format!("{e:#}")))?; + if let Some(scene) = &scene { + comp.set_live_params(live_params_from_scene(scene)); + } + comp.set_scene(scene); + let mut progress = throttled_progress(self.on_progress.take()); openscreen_compositor::gif_export::export_gif( - &self.screen_path, - &self.webcam_path, - self.cursor_path.as_deref(), + &self.clips, &self.out_path, + &gpu, + &comp, + &cfg, &self.params, &mut progress, ) @@ -554,13 +589,26 @@ impl Task for ExportGifTask { /// throttled à ~10/s comme l'export MP4 (voir `throttled_progress`). #[napi] pub fn export_gif( - screen_path: String, - webcam_path: String, - cursor_path: Option, + clips: Vec, out_path: String, + scene_json: Option, params: Option, on_progress: Option, ) -> Result> { + // Deliberately the same argument shape as `export_multi`: the caller builds + // one clip list and one scene, and picks the container. Cursor comes from + // the scene like every other effect — there is no GIF-specific input left. + let clips = clips + .into_iter() + .map(|c| pipeline::ClipSource { + screen: c.screen_path, + webcam: c.webcam_path, + source_start_sec: c.source_start_sec, + source_end_sec: c.source_end_sec, + webcam_offset_sec: c.webcam_offset_sec, + has_audio: c.has_audio, + }) + .collect(); let gif_params = params .map(|p| GifExportParams { width: p.width, @@ -571,9 +619,8 @@ pub fn export_gif( }) .unwrap_or_default(); Ok(AsyncTask::new(ExportGifTask { - screen_path, - webcam_path, - cursor_path, + clips, + scene_json, out_path: PathBuf::from(out_path), params: gif_params, on_progress: make_progress_tsfn(on_progress)?, diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index 2fe600ede..56bb59431 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -25,13 +25,18 @@ //! //! ## What's in this file //! -//! - `export_gif` — the orchestrator. Opens the same `Player` the live -//! preview drives, constructs a `Compositor` sized to the requested -//! output dims, and per output frame: `Player::step` → -//! `Compositor::compose_frame` → `Compositor::readback_direct` → -//! optional Floyd-Steinberg dither → nearest-color index → -//! `GifWriter::write_frame`. Reports the same `GifStats` shape the -//! MP4 `pipeline::Stats` returns. +//! - `export_gif` — the orchestrator. Drives `pipeline::walk_composited_timeline`, +//! the SAME clip walk the MP4 exporter uses, so clip iteration, speed +//! segments and output-time decoder advancement have exactly one +//! definition. Per output frame the walk composes, then this module does +//! `Compositor::readback_direct` → palette → optional fused +//! Floyd-Steinberg → `GifWriter::write_frame`. Reports the same `GifStats` +//! shape the MP4 `pipeline::Stats` returns. +//! +//! It previously ran its own loop over the live-preview `Player`, stepping +//! one SOURCE frame per OUTPUT frame — which made a 30 s/60 fps recording +//! export as 6 s of content stretched over 30 s, and could not decode files +//! the MP4 path handled. `tests/export_timing.rs` pins the behaviour. //! - `GifWriter` — GIF89a format writer: header, optional Netscape 2.0 //! loop extension, per-frame Graphics Control Extension + Image //! Descriptor + LZW image data, trailer. Pure std `Write`. @@ -65,9 +70,8 @@ use crate::compositor::Compositor; use crate::config::Cfg; -use crate::cursor::CursorTrack; use crate::d3d::Gpu; -use crate::live::Player; +use crate::pipeline::{walk_composited_timeline, ClipSource, Decoder}; use anyhow::{anyhow, bail, Context, Result}; use std::collections::HashMap; use std::fs::File; @@ -143,21 +147,28 @@ impl Default for GifExportParams { } } -/// Drive a single-clip GIF export end-to-end. Mirrors the shape of -/// `pipeline::run_composited` so the bench can compare apples to -/// apples once the readback cost has been measured. +/// Drive a multiclip GIF export end-to-end. +/// +/// Deliberately the same shape as `pipeline::run_composited_multi`, and +/// deliberately driven by the same `walk_composited_timeline`: the clip walk, +/// the speed segments and the output-time decoder advancement are the video +/// exporter's, not a second implementation. Only the per-frame sink differs — +/// MP4 hands the composed texture to a hardware NV12 encoder, GIF reads it back +/// to the CPU and quantizes it to 256 colours. +/// /// A failed run leaves a truncated GIF under exactly the name the user thinks /// they exported. Remove it rather than leave it lying around — same contract /// as `discard_partial_output` on the MP4 path. pub fn export_gif( - screen: &str, - webcam: &str, - cursor_json: Option<&str>, + clips: &[ClipSource], out_path: &Path, + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, params: &GifExportParams, progress: &mut dyn FnMut(u64), ) -> Result { - let result = export_gif_inner(screen, webcam, cursor_json, out_path, params, progress); + let result = export_gif_inner(clips, out_path, gpu, comp, cfg, params, progress); if result.is_err() { let _ = std::fs::remove_file(out_path); } @@ -165,53 +176,25 @@ pub fn export_gif( } fn export_gif_inner( - screen: &str, - webcam: &str, - cursor_json: Option<&str>, + clips: &[ClipSource], out_path: &Path, + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, params: &GifExportParams, progress: &mut dyn FnMut(u64), ) -> Result { + if clips.is_empty() { + bail!("export_gif: aucun clip à exporter"); + } + // The caller builds the compositor at the output size (same contract as + // `run_composited_multi` / `ExportParams`), so these must agree with what + // `readback_direct` hands back — asserted per frame below. let width = params.width.unwrap_or(DEFAULT_GIF_WIDTH); let height = params.height.unwrap_or(DEFAULT_GIF_HEIGHT); let fps = params.fps.unwrap_or(DEFAULT_GIF_FPS).max(1); let dither = params.dither; - // Native compositor: D3D11 device + offscreen RT sized to the - // output. Same idiom as `run_composited` (one Cfg, full C8 - // effects), but zoom / layout anim / motion blur are off — GIF is - // a still-timeline artefact, the moving-camera cost that makes C8 - // relevant for MP4 doesn't move the needle on a 256-colour - // palette. - let gpu = Gpu::create(false).map_err(|e| anyhow!("export_gif: gpu init: {e:#}"))?; - let comp = Compositor::new_sized(&gpu, width, height) - .map_err(|e| anyhow!("export_gif: compositor: {e:#}"))?; - if let Some(cursor_path) = cursor_json { - match CursorTrack::load(cursor_path, 0.0, 24.0 * 3600.0) { - Ok(track) => comp.set_cursor(track), - Err(e) => bail!("export_gif: cursor.json load: {e:#}"), - } - } - let cfg = Cfg { - name: "gif", - composite: true, - cursor: cursor_json.is_some(), - ..Cfg::c8() - }; - - // Open the screen + webcam. Use the existing `Player` so cursor - // framing and webcam lockstep are identical to the live preview. - let mut player = unsafe { Player::open(screen, webcam, &gpu) } - .map_err(|e| anyhow!("export_gif: player open: {e:#}"))?; - - // Frame plan: the source decodes at 60 fps (fixture) and the - // output is `fps` — so we drive the player one step per output - // frame and stop after `target_frames` (i.e. the source time is - // `target_frames / fps`). We don't know the source duration up - // front; we let the player EOF when it has no more frames. - let target_frames = estimate_target_frames(&player, fps) - .ok_or_else(|| anyhow!("export_gif: source has zero decodable frames"))?; - // Set up the GIF writer up front: file + global header. We use a // per-frame local palette (the standard "high-quality" form: a // palette tuned to each frame's colours), so the global palette in @@ -229,6 +212,12 @@ fn export_gif_inner( // `100 / fps` cs per frame, rounded to the nearest unit the // GIF spec supports. u16 caps at 65535 — 10.9 minutes per // frame, plenty. + // + // ponytail: integer centiseconds can't express every fps exactly + // (12 → 8 cs → 12.5 fps). `video_duration_s` below is computed + // from the delays actually written, so the reported duration never + // disagrees with the file. Fractional accumulation if a viewer ever + // cares about the ~4% drift. let delay_cs: u16 = (100_u32 / fps).max(1) as u16; // Pre-allocate the per-frame index buffer. Reused across @@ -245,8 +234,8 @@ fn export_gif_inner( let mut palette_rgb: Vec = vec![0u8; PALETTE_COLORS * 3]; let t0 = Instant::now(); - let mut frames: u64 = 0; - { + let scene = comp.scene_snapshot(); + let frames = { let mut gw = GifWriter::new(&mut writer, width as u16, height as u16)?; gw.write_header()?; // Netscape 2.0 application extension drives the loop count. @@ -258,81 +247,83 @@ fn export_gif_inner( }; gw.write_netscape_loop(loops)?; - for n in 0..target_frames { - // Drive the player one output frame. `Player::step` is - // the same path the live preview's render thread uses. - let more = unsafe { player.step(&comp, &cfg) } - .map_err(|e| anyhow!("export_gif: player.step @ frame {n}: {e:#}"))?; - if !more { - break; - } - // CPU readback of the staged RT (RGBA8 - // tightly-packed, `width * height * 4` bytes). The - // dominant per-frame cost. - let (rw, rh, rgba) = unsafe { comp.readback_direct() } - .map_err(|e| anyhow!("export_gif: readback @ frame {n}: {e:#}"))?; - debug_assert_eq!(rw, width); - debug_assert_eq!(rh, height); - debug_assert_eq!(rgba.len(), (width as usize) * (height as usize) * 4); - - // Refresh the palette on a schedule. Building the - // histogram and running median-cut is O(unique colors) - // — fast enough at 480p and our 30-frame cadence. - if n % PALETTE_REQUANTIZE_EVERY == 0 { - build_palette_median_cut(&rgba, PALETTE_COLORS, &mut palette_rgb); - } - - // Quantize (with optional dithering). The dither pass - // mutates a working copy of the readback pixels - // (in-place error propagation); the quantize pass - // then walks the (possibly dithered) pixels once and - // writes indices. - let mut rgba_buf = rgba; - if dither { - floyd_steinberg(&mut rgba_buf, width, height, &mut err_cur, &mut err_next); - } - map_to_indices(&palette_rgb, &rgba_buf, &mut indices); + let mut screen_decs: HashMap = HashMap::new(); + let mut webcam_decs: HashMap = HashMap::new(); + screen_decs.insert(clips[0].screen.clone(), unsafe { + Decoder::open(&clips[0].screen, gpu)? + }); + + let frames = unsafe { + walk_composited_timeline( + clips, + gpu, + comp, + cfg, + fps as i32, + &scene, + &mut screen_decs, + &mut webcam_decs, + &mut |frame_index| { + // CPU readback of the staged RT (RGBA8 tightly-packed, + // `width * height * 4` bytes). The dominant per-frame cost, + // and the reason GIF can't use the MP4 zero-copy sink. + let (rw, rh, rgba) = comp + .readback_direct() + .map_err(|e| anyhow!("export_gif: readback @ frame {frame_index}: {e:#}"))?; + debug_assert_eq!(rw, width); + debug_assert_eq!(rh, height); + + // Refresh the palette on a schedule. Building the histogram + // and running median-cut is O(unique colors) — fast enough at + // 480p on our 30-frame cadence. + if frame_index % PALETTE_REQUANTIZE_EVERY == 0 { + build_palette_median_cut(&rgba, PALETTE_COLORS, &mut palette_rgb); + } - // Per-frame palette (GIF local palette, written by - // `write_frame`). - gw.write_frame(&indices, &palette_rgb, delay_cs, fps)?; + // Quantize (with optional dithering). The dither pass diffuses + // the error against the CHOSEN PALETTE ENTRY, so it has to run + // fused with the index mapping — see `map_to_indices_dithered`. + if dither { + map_to_indices_dithered( + &palette_rgb, + &rgba, + width, + height, + &mut err_cur, + &mut err_next, + &mut indices, + ); + } else { + map_to_indices(&palette_rgb, &rgba, &mut indices); + } - frames += 1; - progress(frames); - } + // Per-frame palette (GIF local palette, written by `write_frame`). + gw.write_frame(&indices, &palette_rgb, delay_cs, fps)?; + progress(frame_index + 1); + Ok(()) + }, + // GIF has no audio track, so clip boundaries need no work. + &mut |_, _, _, _| Ok(()), + )? + }; - // Trailer is written on drop (the writer's `Drop` flushes - // any pending bytes and appends `0x3B`). gw.finish()?; - } + frames + }; // Drop the writer before stat-ing the file so the trailer is // flushed. drop(writer); let wall_s = t0.elapsed().as_secs_f64(); let fps_actual = if wall_s > 0.0 { frames as f64 / wall_s } else { 0.0 }; - let video_duration_s = frames as f64 / fps as f64; + // From the delays actually written, not from the requested fps — see the + // `delay_cs` note above. + let video_duration_s = frames as f64 * (delay_cs as f64 / 100.0); let file_bytes = std::fs::metadata(out_path).map(|m| m.len()).unwrap_or(0); Ok(GifStats { frames, wall_s, fps: fps_actual, video_duration_s, file_bytes }) } -/// Estimate how many output frames a single-clip export will produce. -/// -/// The Player doesn't expose its source duration up front without -/// seeking to the EOF. We open the screen decoder, ask for the -/// duration, and floor to the nearest output frame. Returns `None` -/// when the source is undecodable or the duration is zero — the caller -/// should fail the export with a clear message in that case. -fn estimate_target_frames(player: &Player, fps: u32) -> Option { - // `screen_duration_sec` is `unsafe` because it touches the raw - // decoder. We just opened the player above and never moved it, so - // the contract is satisfied — but the `unsafe` keyword on the - // Player method has to be wrapped. - let dur = unsafe { player.screen_duration_sec() }?; - let out = (dur * fps as f64).floor() as u64; - if out == 0 { None } else { Some(out) } -} // ===================================================================== // GIF89a format writer (pure std::io::Write). @@ -876,6 +867,82 @@ fn map_to_indices(palette_rgb: &[u8], rgba: &[u8], indices: &mut [u8]) { } } +/// Nearest-palette mapping with Floyd-Steinberg error diffusion, fused. +/// +/// Fused on purpose. A separate dither pass has nothing to diffuse against: +/// quantizing to `round()` of an already-integer channel gives an error of +/// exactly zero for every pixel, so the whole pass is a no-op that still costs +/// a full float traversal. The error that matters is `pixel - palette[chosen]`, +/// which only exists once the palette entry has been picked — hence one loop. +/// +/// Kernel is the standard 7/16 right, 3/16 below-left, 5/16 below, +/// 1/16 below-right. Two row buffers keep the working set at O(width). +#[allow(clippy::too_many_arguments)] +fn map_to_indices_dithered( + palette_rgb: &[u8], + rgba: &[u8], + width: u32, + height: u32, + err_cur: &mut [f32], + err_next: &mut [f32], + indices: &mut [u8], +) { + let w = width as usize; + let h = height as usize; + debug_assert_eq!(indices.len(), w * h); + debug_assert_eq!(palette_rgb.len(), PALETTE_COLORS * 3); + + err_cur.fill(0.0); + err_next.fill(0.0); + + for y in 0..h { + for x in 0..w { + let base = (y * w + x) * 4; + let e = x * 3; + // Channel value carrying the error diffused into this pixel. + let cr = (rgba[base] as f32 + err_cur[e]).clamp(0.0, 255.0); + let cg = (rgba[base + 1] as f32 + err_cur[e + 1]).clamp(0.0, 255.0); + let cb = (rgba[base + 2] as f32 + err_cur[e + 2]).clamp(0.0, 255.0); + + let mut best_idx = 0usize; + let mut best_dist = f32::MAX; + for k in 0..PALETTE_COLORS { + let dr = cr - palette_rgb[k * 3] as f32; + let dg = cg - palette_rgb[k * 3 + 1] as f32; + let db = cb - palette_rgb[k * 3 + 2] as f32; + let dist = dr * dr + dg * dg + db * db; + if dist < best_dist { + best_dist = dist; + best_idx = k; + } + } + indices[y * w + x] = best_idx as u8; + + // THE error: distance to the colour actually written. + let er = cr - palette_rgb[best_idx * 3] as f32; + let eg = cg - palette_rgb[best_idx * 3 + 1] as f32; + let eb = cb - palette_rgb[best_idx * 3 + 2] as f32; + + let mut spread = |slot: &mut [f32], idx: usize, f: f32| { + slot[idx] += er * f; + slot[idx + 1] += eg * f; + slot[idx + 2] += eb * f; + }; + if x + 1 < w { + spread(err_cur, e + 3, 7.0 / 16.0); + spread(err_next, e + 3, 1.0 / 16.0); + } + if x > 0 { + spread(err_next, e - 3, 3.0 / 16.0); + } + spread(err_next, e, 5.0 / 16.0); + } + // Next row becomes current; clear the far row for reuse. + err_cur.copy_from_slice(err_next); + err_next.fill(0.0); + } +} + // ===================================================================== // Floyd-Steinberg dither. // ===================================================================== diff --git a/crates/compositor/src/pipeline.rs b/crates/compositor/src/pipeline.rs index 2bcf278f1..f2e03c302 100644 --- a/crates/compositor/src/pipeline.rs +++ b/crates/compositor/src/pipeline.rs @@ -11,7 +11,8 @@ use crate::config::Cfg; use crate::cursor::CursorTrack; use crate::d3d::Gpu; use crate::ffi::*; -use crate::regions::speed_segments_for_window; +use crate::regions::{speed_segments_for_window, SpeedSegment}; +use crate::scene::Scene; use anyhow::{anyhow, bail, Result}; use std::collections::HashMap; use std::ffi::{c_void, CString}; @@ -1038,82 +1039,39 @@ impl Default for ExportParams { } } -unsafe fn run_multi_inner( + +/// The format-agnostic half of a multiclip export: clip iteration, decoder +/// reuse, availability clamping, per-clip scene windowing, keyframe seeks, +/// cursor binding, speed segments, and — the part that matters — advancing the +/// decoders by OUTPUT time rather than by source frames. +/// +/// MP4 and GIF differ only in what they do with a composed frame (hardware NV12 +/// encode vs CPU readback + palette quantize), so that is all they supply here. +/// Sharing this walk is what keeps "which source frame belongs at output frame +/// N" defined exactly once: a GIF driven by its own loop is how the slow-motion +/// truncation bug happened. +/// +/// `on_frame` runs after `compose_frame` with the running output index; +/// `on_clip_end` runs once per clip with its clamped source window, the frames +/// it produced, and the speed segments used (MP4 needs those for audio). +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn walk_composited_timeline( clips: &[ClipSource], - out: &str, gpu: &Gpu, comp: &Compositor, cfg: &Cfg, - params: &ExportParams, - progress: &mut dyn FnMut(u64), -) -> Result { - if clips.is_empty() { - bail!("aucun clip à exporter"); - } - let (out_w, out_h) = (params.width, params.height); - // décodeurs ouverts une fois par chemin, réutilisés entre clips (screen ≠ webcam → 2 maps - // pour deux &mut indépendants). - let mut screen_decs: HashMap = HashMap::new(); - let mut webcam_decs: HashMap = HashMap::new(); - - // fps de sortie : choix explicite de l'app si fourni, sinon dérivé du 1er clip (recordings - // uniformes) — comportement historique. - screen_decs.insert(clips[0].screen.clone(), Decoder::open(&clips[0].screen, gpu)?); - let out_fps = params - .fps - .unwrap_or_else(|| screen_decs[&clips[0].screen].fps().round().max(1.0) as u32) - as i32; - - // Curseur : la scène (déjà posée par l'appelant via comp.set_scene) pilote tout — même - // parité que le live. Piste par chemin ÉCRAN distinct (convention sidecar `.cursor.json`, - // temps ABSOLU non re-basé : chaque décodeur avance dans le même référentiel que la piste). - let scene = comp.scene_snapshot(); + out_fps: i32, + scene: &Option, + screen_decs: &mut HashMap, + webcam_decs: &mut HashMap, + on_frame: &mut dyn FnMut(u64) -> Result<()>, + on_clip_end: &mut dyn FnMut(usize, f64, u64, &[SpeedSegment]) -> Result<()>, +) -> Result { let cursor_enabled = scene.as_ref().map(|s| s.cursor.show).unwrap_or(false); let cursor_smoothing = scene.as_ref().map(|s| s.cursor.smoothing).unwrap_or(0.0); let mut cursor_tracks: HashMap = HashMap::new(); let mut cursor_active_path: Option = None; - - // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- - let (mut enc_hwdev, mut enc_frames) = make_enc_frames(gpu, out_w as i32, out_h as i32)?; - // débit proportionnel à la surface de sortie (référence : 8Mbps @ 1920x1080), plancher - // 2Mbps pour rester regardable sur les petites tailles. - let bit_rate = ((out_w as i64 * out_h as i64 * 8_000_000) / (1920 * 1080)).max(2_000_000); - let mut enc = VideoEncoder::open( - ¶ms.codec, - out_w as i32, - out_h as i32, - out_fps, - bit_rate, - enc_frames, - )?; - let ectx = enc.ctx; - - let mut octx: *mut AVFormatContext = ptr::null_mut(); - let outc = CString::new(out)?; - averr( - avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), - "alloc_output_context2", - )?; - let ostream = avformat_new_stream(octx, ptr::null()); - if ostream.is_null() { - bail!("video avformat_new_stream"); - } - averr(avcodec_parameters_from_context((*ostream).codecpar, ectx), "params_from_ctx")?; - (*ostream).time_base = (*ectx).time_base; - // Les deux streams doivent exister avant le header MP4 ; l'AAC reste ouvert pendant le - // rendu puis reçoit le PCM assemblé à partir des comptes de frames réellement produits. - let mut audio_encoder = AacEncoder::open(octx)?; - let mut pb: *mut AVIOContext = ptr::null_mut(); - averr(avio_open(&mut pb, outc.as_ptr(), AVIO_FLAG_WRITE as i32), "avio_open")?; - sn_fmt_set_pb(octx, pb); - averr(avformat_write_header(octx, ptr::null_mut()), "write_header")?; - - let opkt = av_packet_alloc(); let mut frames: u64 = 0; - let mut clip_frame_counts = vec![0u64; clips.len()]; - let mut clip_pcm: Vec> = - std::iter::repeat_with(|| None).take(clips.len()).collect(); - let t0 = Instant::now(); for (clip_index, clip) in clips.iter().enumerate() { if !screen_decs.contains_key(&clip.screen) { @@ -1242,40 +1200,138 @@ unsafe fn run_multi_inner( } comp.compose_frame(sf, wf, frames as f32, cfg)?; - let outf = av_frame_alloc(); - averr(av_hwframe_get_buffer(enc_frames, outf, 0), "hwframe_get_buffer")?; - let out_tex = (*outf).data[0] as *mut c_void; - let out_slice = (*outf).data[1] as u32; - comp.rgb_to_nv12_scaled(out_w, out_h, out_tex, out_slice)?; - (*outf).pts = frames as i64; - enc.send(outf)?; - drain_encoder(ectx, octx, ostream, opkt)?; - av_frame_free(&mut (outf as *mut _)); + on_frame(frames)?; frames += 1; - progress(frames); } } - clip_frame_counts[clip_index] = frames - frames_before_clip; - if clip.has_audio && clip_frame_counts[clip_index] > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( - &pcm, - &speed_segments, - out_fps as f64, - )); + on_clip_end(clip_index, source_end_sec, frames - frames_before_clip, &speed_segments)?; + } + + comp.set_cursor_time(None); + comp.set_timeline_time(None); + Ok(frames) +} + +unsafe fn run_multi_inner( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), +) -> Result { + if clips.is_empty() { + bail!("aucun clip à exporter"); + } + let (out_w, out_h) = (params.width, params.height); + // décodeurs ouverts une fois par chemin, réutilisés entre clips (screen ≠ webcam → 2 maps + // pour deux &mut indépendants). + let mut screen_decs: HashMap = HashMap::new(); + let mut webcam_decs: HashMap = HashMap::new(); + + // fps de sortie : choix explicite de l'app si fourni, sinon dérivé du 1er clip (recordings + // uniformes) — comportement historique. + screen_decs.insert(clips[0].screen.clone(), Decoder::open(&clips[0].screen, gpu)?); + let out_fps = params + .fps + .unwrap_or_else(|| screen_decs[&clips[0].screen].fps().round().max(1.0) as u32) + as i32; + + // La scène (déjà posée par l'appelant via comp.set_scene) pilote le curseur et le + // fenêtrage par clip ; `walk_composited_timeline` s'en charge. + let scene = comp.scene_snapshot(); + + // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- + let (mut enc_hwdev, mut enc_frames) = make_enc_frames(gpu, out_w as i32, out_h as i32)?; + // débit proportionnel à la surface de sortie (référence : 8Mbps @ 1920x1080), plancher + // 2Mbps pour rester regardable sur les petites tailles. + let bit_rate = ((out_w as i64 * out_h as i64 * 8_000_000) / (1920 * 1080)).max(2_000_000); + let mut enc = VideoEncoder::open( + ¶ms.codec, + out_w as i32, + out_h as i32, + out_fps, + bit_rate, + enc_frames, + )?; + let ectx = enc.ctx; + + let mut octx: *mut AVFormatContext = ptr::null_mut(); + let outc = CString::new(out)?; + averr( + avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), + "alloc_output_context2", + )?; + let ostream = avformat_new_stream(octx, ptr::null()); + if ostream.is_null() { + bail!("video avformat_new_stream"); + } + averr(avcodec_parameters_from_context((*ostream).codecpar, ectx), "params_from_ctx")?; + (*ostream).time_base = (*ectx).time_base; + // Les deux streams doivent exister avant le header MP4 ; l'AAC reste ouvert pendant le + // rendu puis reçoit le PCM assemblé à partir des comptes de frames réellement produits. + let mut audio_encoder = AacEncoder::open(octx)?; + let mut pb: *mut AVIOContext = ptr::null_mut(); + averr(avio_open(&mut pb, outc.as_ptr(), AVIO_FLAG_WRITE as i32), "avio_open")?; + sn_fmt_set_pb(octx, pb); + averr(avformat_write_header(octx, ptr::null_mut()), "write_header")?; + + let opkt = av_packet_alloc(); + let mut clip_frame_counts = vec![0u64; clips.len()]; + let mut clip_pcm: Vec> = + std::iter::repeat_with(|| None).take(clips.len()).collect(); + let t0 = Instant::now(); + + let frames = walk_composited_timeline( + clips, + gpu, + comp, + cfg, + out_fps, + &scene, + &mut screen_decs, + &mut webcam_decs, + &mut |frame_index| { + // Hardware path: the composed texture goes straight into an NV12 + // encoder frame, so nothing ever descends to system memory. + let outf = av_frame_alloc(); + averr(av_hwframe_get_buffer(enc_frames, outf, 0), "hwframe_get_buffer")?; + let out_tex = (*outf).data[0] as *mut c_void; + let out_slice = (*outf).data[1] as u32; + comp.rgb_to_nv12_scaled(out_w, out_h, out_tex, out_slice)?; + (*outf).pts = frame_index as i64; + enc.send(outf)?; + drain_encoder(ectx, octx, ostream, opkt)?; + av_frame_free(&mut (outf as *mut _)); + progress(frame_index + 1); + Ok(()) + }, + &mut |clip_index, source_end_sec, frames_in_clip, speed_segments| { + clip_frame_counts[clip_index] = frames_in_clip; + let clip = &clips[clip_index]; + if clip.has_audio && frames_in_clip > 0 { + match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { + Ok(Some(pcm)) => { + clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( + &pcm, + speed_segments, + out_fps as f64, + )); + } + Ok(None) => eprintln!( + "[pipeline] warning: clip #{} déclaré audio mais sans flux décodable; silence conservé", + clip_index, + ), + Err(error) => eprintln!( + "[pipeline] warning: décodage audio du clip #{} échoué ({error:#}); silence conservé", + clip_index, + ), } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{} déclaré audio mais sans flux décodable; silence conservé", - clip_index, - ), - Err(error) => eprintln!( - "[pipeline] warning: décodage audio du clip #{} échoué ({error:#}); silence conservé", - clip_index, - ), } - } - } + Ok(()) + }, + )?; comp.set_cursor_time(None); comp.set_timeline_time(None); diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs new file mode 100644 index 000000000..ac67d07bb --- /dev/null +++ b/crates/compositor/tests/export_timing.rs @@ -0,0 +1,210 @@ +//! Frame-timing regression net for the export paths. +//! +//! The bug class this exists to catch is silent: an exporter that advances its +//! decoder by one SOURCE frame per OUTPUT frame emits the right number of +//! frames while covering only `out_fps / source_fps` of the recording. Frame +//! count alone therefore proves nothing — the content has to be checked. +//! +//! So the fixture is a 4 s / 60 fps clip whose colour changes every second +//! (red → green → blue → white). A correct export spans all four colours; a +//! mis-advanced one stays red for its whole length. +//! +//! Needs a D3D11 GPU and the generated media, so it is opt-in: set +//! OPENSCREEN_TEST_MEDIA to a directory holding `screen_colors.mp4` and +//! `webcam_gray.mp4`. Without it every test here skips (no CI builds this +//! crate today — see the Rust-CI gap noted in the PR). +//! +//! Regenerate the media with the vendored ffmpeg: +//! for c in red green blue white; do ffmpeg -f lavfi \ +//! -i "color=c=$c:size=640x360:duration=1:rate=60" -c:v libopenh264 \ +//! -g 60 -pix_fmt yuv420p seg_$c.mp4; done +//! ffmpeg -f concat -safe 0 -i concat.txt -c copy screen_colors.mp4 +//! ffmpeg -f lavfi -i "color=c=gray:size=320x240:duration=4:rate=60" \ +//! -c:v libopenh264 -g 60 -pix_fmt yuv420p webcam_gray.mp4 + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config::Cfg; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::gif_export::{self, GifExportParams}; +use openscreen_compositor::pipeline::{self, ClipSource, ExportCodec, ExportParams}; +use std::path::PathBuf; + +const SOURCE_SEC: f64 = 4.0; + +/// Per-frame local colour tables, in order, from a GIF89a file. +/// +/// Enough of the format to walk block-to-block: extensions are skipped by +/// their sub-block chain, image descriptors yield their local table and then +/// their LZW data is skipped the same way. No LZW decode — the palette alone +/// says which colours a frame is made of, which is all the timing assertions +/// need. +fn gif_frame_palettes(bytes: &[u8]) -> Vec> { + fn table_len(packed: u8) -> usize { + if packed & 0x80 == 0 { + 0 + } else { + 3 * (1usize << ((packed & 0x07) + 1)) + } + } + /// Skips a `len,data…,0` sub-block chain, returning the position after it. + fn skip_sub_blocks(bytes: &[u8], mut p: usize) -> usize { + while p < bytes.len() && bytes[p] != 0 { + p += 1 + bytes[p] as usize; + } + p + 1 + } + + let mut palettes = Vec::new(); + let mut p = 6; // "GIF89a" + let packed = bytes[p + 4]; + p += 7 + table_len(packed); // logical screen descriptor + global table + + while p < bytes.len() { + match bytes[p] { + 0x21 => p = skip_sub_blocks(bytes, p + 2), // extension: 0x21, label, chain + 0x2C => { + let packed = bytes[p + 9]; + let start = p + 10; + let len = table_len(packed); + palettes.push( + bytes[start..start + len] + .chunks_exact(3) + .map(|c| [c[0], c[1], c[2]]) + .collect(), + ); + p = skip_sub_blocks(bytes, start + len + 1); // +1 = LZW min code size + } + _ => break, // 0x3B trailer, or done + } + } + palettes +} + +/// Mean `R - B` across a palette. The fixture's first second is pure red +/// (large positive) and its last is white (≈ 0), so this single number +/// separates "covered the timeline" from "stuck on frame 0". +fn redness(palette: &[[u8; 3]]) -> f64 { + if palette.is_empty() { + return 0.0; + } + palette + .iter() + .map(|c| c[0] as f64 - c[2] as f64) + .sum::() + / palette.len() as f64 +} + +fn media_dir() -> Option { + let dir = PathBuf::from(std::env::var("OPENSCREEN_TEST_MEDIA").ok()?); + dir.join("screen_colors.mp4").exists().then_some(dir) +} + +/// One clip covering the whole fixture. +fn whole_clip(dir: &PathBuf) -> ClipSource { + ClipSource { + screen: dir.join("screen_colors.mp4").to_string_lossy().into_owned(), + webcam: dir.join("webcam_gray.mp4").to_string_lossy().into_owned(), + source_start_sec: 0.0, + source_end_sec: SOURCE_SEC, + webcam_offset_sec: 0.0, + has_audio: false, + } +} + +/// MP4 at 30 fps over a 4 s source must emit 120 frames — i.e. the walk is +/// driven by OUTPUT time, not by "one source frame per output frame" (which +/// would still emit 120 frames but cover only 2 s of the recording; the GIF +/// test below is the one that catches the coverage half). +#[test] +fn mp4_export_frame_count_follows_output_fps() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let gpu = Gpu::create(false).expect("gpu"); + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + }; + let comp = Compositor::new_sized(&gpu, params.width, params.height).expect("compositor"); + let out = dir.join("out_timing.mp4"); + let stats = pipeline::run_composited_multi( + &[whole_clip(&dir)], + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("mp4 export"); + + assert_eq!( + stats.frames, 120, + "4 s of source at 30 fps out must be 120 frames, got {}", + stats.frames + ); + let probed = pipeline::probe_frame_count(&out.to_string_lossy()).expect("probe"); + assert_eq!(probed, 120, "muxed file disagrees with the reported count"); +} + +/// The GIF must cover the WHOLE timeline, not just its first +/// `out_fps / source_fps` slice. +/// +/// This is the assertion frame count cannot make: an exporter that advances one +/// source frame per output frame still writes 48 frames for a 4 s / 12 fps +/// request — it just takes them all from the first 0.8 s, so every frame is red +/// and the GIF plays 5x slow. Comparing the first and last frame palettes +/// catches exactly that. +#[test] +fn gif_export_spans_the_whole_timeline() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let out = dir.join("out_timing.gif"); + let params = GifExportParams { + width: Some(320), + height: Some(180), + fps: Some(12), + loop_count: None, + dither: false, + }; + let gpu = Gpu::create(false).expect("gpu"); + // Same contract as the MP4 path: the caller sizes the compositor to the output. + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositor"); + let stats = gif_export::export_gif( + &[whole_clip(&dir)], + &out, + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("gif export"); + + assert_eq!( + stats.frames, 48, + "4 s at 12 fps must be 48 frames, got {}", + stats.frames + ); + + let bytes = std::fs::read(&out).expect("read gif"); + let palettes = gif_frame_palettes(&bytes); + assert_eq!(palettes.len(), 48, "GIF carries {} frames", palettes.len()); + + let first = redness(&palettes[0]); + let last = redness(&palettes[palettes.len() - 1]); + assert!( + first > 40.0, + "first frame should be red-dominated (redness {first:.1})" + ); + assert!( + last < first - 40.0, + "last frame still looks like the first — the export never advanced past \ + the opening red second (first {first:.1}, last {last:.1})" + ); +} From d8c313de411fc8f79cc7aee0aad89f44870e9075 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 28 Jul 2026 23:23:29 +0200 Subject: [PATCH 2/3] feat(export): route GIF through the native exporter, delete the pixi path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes the wiring #200 left open (the IPC route did not exist, so nothing could reach the native path even with the flag on) and removes the renderer-side GIF renderer it replaces. Wiring: exportGif takes the same clips + scene as exportMulti, gains the missing 'exportGif' dispatcher case, a client method, and resolveSceneAssetPaths (the GIF path never had it). ExportDialog now has one native branch for both formats — GIF differs only in output size preset, frame rate and loop count. Deleted, all consumer-less once GIF stopped using them: gifExporter, its documentExporter adapter, frameRenderer, the WGSL shaders it owned (composite/shadowCascade/evaluate), threeDPass, cropSchedule, frameExtract, timestampedVideoFrameQueue, and the NATIVE_GIF_EXPORT_ENABLED flag. gif.js and @types/gif.js drop out of package.json. This ends the second graphical SSOT: preview and export were both native already, but GIF still rasterised through a parallel WGSL implementation that had to be hand-synced with shaders.hlsl — and silently degraded when its 3D pass failed to init ('rotation fields will be ignored'), producing different pixels from the preview with no error. pixi.js stays: cursor rendering (pixiCursorRenderer, nativeCursor, CursorPreviewLayer) and zoomTransform still use it. That is the remaining half. The browser test suite went with it — its only test was the pixi GIF exporter, and the config existed to give Pixi software WebGL in headless CI. Test-file typecheck errors drop 80 -> 74; baseline lowered to match. --- .github/workflows/ci.yml | 4 +- electron/ipc/nativeBridge.ts | 22 + .../services/compositorViewService.ts | 26 +- electron/native/compositor-view/addon.d.ts | 21 +- package-lock.json | 49 +- package.json | 6 - src/components/ai-edition/ExportDialog.tsx | 131 +- .../exporter/documentExporter.test.ts | 107 -- .../ai-edition/exporter/documentExporter.ts | 286 --- src/lib/exporter/cropSchedule.ts | 26 - src/lib/exporter/featureFlags.ts | 11 - src/lib/exporter/frameExtract.ts | 87 - src/lib/exporter/frameRenderer.test.ts | 97 -- src/lib/exporter/frameRenderer.ts | 1545 ----------------- src/lib/exporter/gifExporter.browser.test.ts | 88 - src/lib/exporter/gifExporter.test.ts | 19 - src/lib/exporter/gifExporter.ts | 420 ----- src/lib/exporter/index.ts | 4 +- src/lib/exporter/threeDPass.ts | 341 ---- .../timestampedVideoFrameQueue.test.ts | 93 - .../exporter/timestampedVideoFrameQueue.ts | 110 -- src/lib/exporter/types.ts | 6 + src/lib/exporter/wgsl/composite.wgsl.ts | 192 -- src/lib/exporter/wgsl/evaluate.ts | 367 ---- src/lib/exporter/wgsl/shadowCascade.wgsl.ts | 195 --- src/native/compositorViewClient.ts | 17 + src/native/contracts.ts | 16 +- vitest.browser.config.ts | 28 - 28 files changed, 151 insertions(+), 4163 deletions(-) delete mode 100644 src/lib/ai-edition/exporter/documentExporter.test.ts delete mode 100644 src/lib/ai-edition/exporter/documentExporter.ts delete mode 100644 src/lib/exporter/cropSchedule.ts delete mode 100644 src/lib/exporter/featureFlags.ts delete mode 100644 src/lib/exporter/frameExtract.ts delete mode 100644 src/lib/exporter/frameRenderer.test.ts delete mode 100644 src/lib/exporter/frameRenderer.ts delete mode 100644 src/lib/exporter/gifExporter.browser.test.ts delete mode 100644 src/lib/exporter/gifExporter.test.ts delete mode 100644 src/lib/exporter/gifExporter.ts delete mode 100644 src/lib/exporter/threeDPass.ts delete mode 100644 src/lib/exporter/timestampedVideoFrameQueue.test.ts delete mode 100644 src/lib/exporter/timestampedVideoFrameQueue.ts delete mode 100644 src/lib/exporter/wgsl/composite.wgsl.ts delete mode 100644 src/lib/exporter/wgsl/evaluate.ts delete mode 100644 src/lib/exporter/wgsl/shadowCascade.wgsl.ts delete mode 100644 vitest.browser.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adbb039db..382618b35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Typecheck tests against a baseline shell: bash env: - BASELINE: 80 + BASELINE: 74 run: | set -uo pipefail COUNT=$(npx tsc -p tsconfig.test.json --noEmit 2>&1 | grep -c 'error TS' || true) @@ -88,8 +88,6 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - run: npm run test - - run: npm run test:browser:install - - run: npm run test:browser build: name: Build diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 56009ba1f..b411ae2e9 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -413,6 +413,28 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { } return createSuccessResponse(requestId, stats); } + case "exportGif": { + const sender = event.sender; + const stats = await compositorViewService.exportGif( + request.payload.clips, + request.payload.outPath, + request.payload.sceneJson, + request.payload.params, + (frames) => { + if (!sender.isDestroyed()) { + sender.send("export:native-progress", frames); + } + }, + ); + if (!stats) { + return createErrorResponse( + requestId, + "UNAVAILABLE", + "Native compositor addon not present.", + ); + } + return createSuccessResponse(requestId, stats); + } default: return createErrorResponse( requestId, diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index c2932a89a..2e835d4b1 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -494,17 +494,17 @@ export class CompositorViewService { ); } - /** Native single-clip GIF export (slice 1, behind `NATIVE_GIF_EXPORT_ENABLED`). - * Mirrors `exportMulti`'s shape, but the slice-1 surface is deliberately small: - * one screen + one webcam file, optional cursor sidecar (`.cursor.json`), - * no multiclip, no app `SceneDescription` (the Player drives the compositing, - * same as the live preview). Returns null when the addon is absent — the renderer - * treats that as "fall back to the legacy `gif.js` path" without raising. */ + /** Native GIF export. Same inputs as `exportMulti` — one clip list, one scene — + * because it is the same render: both drive `walk_composited_timeline` in the + * compositor crate and differ only in the encoder. The scene carries background, + * layout, webcam and cursor, so there is no GIF-specific input. + * + * Returns null when the addon is absent, which the renderer surfaces as a failed + * export — there is no longer a renderer-side GIF path to fall back to. */ async exportGif( - screenPath: string, - webcamPath: string, - cursorPath?: string | null, + clips: ClipInput[], outPath?: string, + sceneJson?: string, params?: GifParamsInput, onProgress?: (frames: number) => void, ): Promise { @@ -513,6 +513,12 @@ export class CompositorViewService { return null; } const target = outPath ?? path.join(app.getPath("temp"), "openscreen-native-export.gif"); - return addon.exportGif(screenPath, webcamPath, cursorPath ?? null, target, params, onProgress); + return addon.exportGif( + clips, + target, + sceneJson ? resolveSceneAssetPaths(sceneJson) : undefined, + params, + onProgress, + ); } } diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 401a4b6a2..67e617515 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -148,20 +148,17 @@ export interface CompositorViewAddon { params?: ExportParamsInput, onProgress?: (frames: number) => void, ): Promise; - /** Native single-clip GIF export (slice 1, behind `NATIVE_GIF_EXPORT_ENABLED`). - * Mirrors `exportMulti`'s shape, but the slice-1 PR keeps the surface small: - * one screen + one webcam file (no multiclip), no app `SceneDescription` (the - * Player drives the compositing, same as the live preview), and no - * encoder-config codec pick — GIF is one codec. `cursorPath` follows the - * sidecar convention (`.cursor.json`); `null`/missing → render - * without cursor. `params` defaults to 854×480, 12 fps, infinite loop, no - * dithering (`GifExportParams::default`). `onProgress(frames)` is throttled - * to ~10/s like the MP4 path. */ + /** Native GIF export. Identical inputs to `exportMulti` — same clips, same + * scene — because it is the same render: both drive `walk_composited_timeline` + * in the compositor crate and differ only in the encoder. Cursor, background, + * layout and webcam all come from the scene, so there is no GIF-specific + * input. No codec pick: GIF is one codec. `params` defaults to 854×480, + * 12 fps, infinite loop, no dithering (`GifExportParams::default`). + * `onProgress(frames)` is throttled to ~10/s like the MP4 path. */ exportGif( - screenPath: string, - webcamPath: string, - cursorPath: string | null, + clips: ClipInput[], outPath: string, + sceneJson?: string, params?: GifParamsInput, onProgress?: (frames: number) => void, ): Promise; diff --git a/package-lock.json b/package-lock.json index c73d08aa5..8b2875dbd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "openscreen", - "version": "1.8.0-rc.3", + "version": "1.8.0-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openscreen", - "version": "1.8.0-rc.3", + "version": "1.8.0-rc.4", "dependencies": { "@fix-webm-duration/fix": "^1.0.1", "@langchain/anthropic": "^1.3.26", @@ -28,14 +28,12 @@ "@tiptap/extension-text-style": "^3.27.1", "@tiptap/react": "^3.27.1", "@tiptap/starter-kit": "^3.27.1", - "@types/gif.js": "^0.2.5", "@uiw/color-convert": "^2.10.1", "@uiw/react-color-block": "^2.10.1", "@uiw/react-color-colorful": "^2.9.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "deepagents": "^1.10.5", - "gif.js": "^0.2.0", "i18next": "^23.16.0", "langchain": "^1.2.39", "lucide-react": "^0.545.0", @@ -69,8 +67,6 @@ "@types/react": "^18.3.28", "@types/react-dom": "^18.3.7", "@vitejs/plugin-react": "^5.2.0", - "@vitest/browser": "^4.1.4", - "@vitest/browser-playwright": "^4.1.4", "autoprefixer": "^10.5.0", "electron": "^41.2.1", "electron-builder": "^26.8.1", @@ -729,7 +725,9 @@ "resolved": "https://registry.npmjs.org/@blazediff/core/-/core-1.9.1.tgz", "integrity": "sha512-ehg3jIkYKulZh+8om/O25vkvSsXXwC+skXmyA87FFx6A/45eqOkZsBltMw/TVteb0mloiGT8oGRTcjRAz66zaA==", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/@bramus/specificity": { "version": "2.4.2", @@ -2383,7 +2381,9 @@ "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", "integrity": "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==", "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/@radix-ui/number": { "version": "1.1.1", @@ -4488,12 +4488,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/events": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz", - "integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==", - "license": "MIT" - }, "node_modules/@types/fs-extra": { "version": "9.0.13", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", @@ -4504,15 +4498,6 @@ "@types/node": "*" } }, - "node_modules/@types/gif.js": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/@types/gif.js/-/gif.js-0.2.5.tgz", - "integrity": "sha512-OdDQYh9v7td9ztjaooBSqjUBAyAuui2xwDDmQcyRLd6c9T0iWgkebAoCBEdEEBoZG3ekJE/6UnH63Dzq0S3bvw==", - "license": "MIT", - "dependencies": { - "@types/events": "*" - } - }, "node_modules/@types/gradient-parser": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/@types/gradient-parser/-/gradient-parser-0.1.5.tgz", @@ -4856,6 +4841,8 @@ "integrity": "sha512-iCDGI8c4yg+xmjUg2VsygdAUSIIB4x5Rht/P68OXy1hPELKXHDkzh87lkuTcdYmemRChDkEpB426MmDjzC0ziA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@blazediff/core": "1.9.1", "@vitest/mocker": "4.1.5", @@ -4879,6 +4866,8 @@ "integrity": "sha512-CWy0lBQJq97nionyJJdnaU4961IXTl43a7UCu5nHy51IoKxAt6PVIJLo+76rVl7KOOgcWHNkG4kbJu/pW7knvA==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@vitest/browser": "4.1.5", "@vitest/mocker": "4.1.5", @@ -7420,12 +7409,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/gif.js": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/gif.js/-/gif.js-0.2.0.tgz", - "integrity": "sha512-bYxCoT8OZKmbxY8RN4qDiYuj4nrQDTzgLRcFVovyona1PTWNePzI4nzOmotnlOFIzTk/ZxAHtv+TfVLiBWj/hw==", - "license": "MIT" - }, "node_modules/gifuct-js": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/gifuct-js/-/gifuct-js-2.1.2.tgz", @@ -8820,6 +8803,8 @@ "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -9444,6 +9429,8 @@ "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=14.19.0" } @@ -10570,6 +10557,8 @@ "integrity": "sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", @@ -11178,6 +11167,8 @@ "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=6" } diff --git a/package.json b/package.json index 46210df76..8602a807f 100644 --- a/package.json +++ b/package.json @@ -50,8 +50,6 @@ "diagnostic:run": "node scripts/diagnostic-tool/diagnostic.mjs", "diagnostic:smoke:win": "node scripts/diagnostic-tool/diagnostic.mjs --duration 3", "build-vite": "tsc && vite build", - "test:browser": "vitest --config vitest.browser.config.ts --run", - "test:browser:install": "playwright install --with-deps chromium-headless-shell", "test:e2e": "playwright test", "test:e2e:windows-native-checklist": "playwright test tests/e2e/windows-native-checklist.spec.ts", "prepare": "husky", @@ -78,14 +76,12 @@ "@tiptap/extension-text-style": "^3.27.1", "@tiptap/react": "^3.27.1", "@tiptap/starter-kit": "^3.27.1", - "@types/gif.js": "^0.2.5", "@uiw/color-convert": "^2.10.1", "@uiw/react-color-block": "^2.10.1", "@uiw/react-color-colorful": "^2.9.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "deepagents": "^1.10.5", - "gif.js": "^0.2.0", "i18next": "^23.16.0", "langchain": "^1.2.39", "lucide-react": "^0.545.0", @@ -119,8 +115,6 @@ "@types/react": "^18.3.28", "@types/react-dom": "^18.3.7", "@vitejs/plugin-react": "^5.2.0", - "@vitest/browser": "^4.1.4", - "@vitest/browser-playwright": "^4.1.4", "autoprefixer": "^10.5.0", "electron": "^41.2.1", "electron-builder": "^26.8.1", diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index eab949305..0d09dca37 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -1,6 +1,6 @@ // Export dialog for the new editor. Wires together: // 1. pickExportSavePath (native save dialog) -// 2. exportAxcutDocument (renders frames + muxes mp4/gif) +// 2. the native D3D exporter (exportMultiNative / exportGifNative) // 3. writeExportToPath (writes the resulting buffer to disk) // // Format/quality/GIF options live in the dialog's local state. The @@ -18,11 +18,6 @@ import { pickExtremeDims, resolveAspectRatioValue, } from "@/lib/ai-edition/document/outputFormat"; -import { - type DocumentExportOptions, - type ExportVideoCodec, - exportAxcutDocument, -} from "@/lib/ai-edition/exporter/documentExporter"; import type { AxcutDocument } from "@/lib/ai-edition/schema"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; @@ -30,14 +25,14 @@ import { type ExportFormat, type ExportProgress, type ExportQuality, + type ExportVideoCodec, GIF_FRAME_RATES, GIF_SIZE_PRESETS, type GifFrameRate, type GifSizePreset, } from "@/lib/exporter"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; -import { exportMultiNative } from "@/native"; -import { nativeBridgeClient } from "@/native/client"; +import { exportGifNative, exportMultiNative } from "@/native"; import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { ModalShell } from "./Modals"; @@ -156,13 +151,10 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { () => (document ? collectEffectiveClipDims(document) : []), [document], ); - // Largest clip's true (cropped) footprint — used only by the GIF export path below (its - // own, separate sizing option), which sizes to the best available footage the same way - // this dialog always has. - const referenceSource = useMemo( - () => pickExtremeDims(effectiveClipDims, "largest"), - [effectiveClipDims], - ); + // (The "largest clip" pick lived here for the old renderer-side GIF path, which + // sized to the best available footage independently of the quality tier. GIF now + // goes through the same native exporter as MP4 and shares its sizing, so only the + // smallest-clip pick below is still needed.) // Smallest clip's true (cropped) footprint on the timeline — a multiclip timeline can mix // crops/resolutions, so this is what "Source" quality actually targets: sizing to the @@ -193,6 +185,25 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { // quality actually uses these as its target size; 720p/1080p target a fixed short side // regardless (`calculateDimensionsForShortSide`), so this only changes what "Source" // resolves to. + // GIF is 8-bit indexed and grows fast with area, so the size preset caps the + // output height rather than following the quality tier. `original` keeps the + // tier's dims; the native side falls back to its own defaults when undefined. + const gifOutputDims = ( + preset: GifSizePreset, + tierDims: { width: number; height: number } | null, + ): { width?: number; height?: number } => { + if (!tierDims) return {}; + const maxHeight = GIF_SIZE_PRESETS[preset].maxHeight; + if (!Number.isFinite(maxHeight) || tierDims.height <= maxHeight) { + return { width: tierDims.width, height: tierDims.height }; + } + const scale = maxHeight / tierDims.height; + // Even dimensions: the compositor rasterises to this size and the readback + // assumes a tightly-packed RGBA buffer. + const even = (n: number) => Math.max(2, Math.round(n * scale) & ~1); + return { width: even(tierDims.width), height: even(tierDims.height) }; + }; + const tierOutputDims = (value: ExportQuality) => smallestSource ? calculateMp4ExportSettings({ @@ -252,14 +263,19 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { return; } - // MP4: the native D3D exporter is the only path (no CPU/web fallback — that - // path silently regressed to an ultra-slow CPU render once, which is exactly - // the failure mode a flag-gated fallback invites). Background/layout/webcam/ - // cursor/effects come from the same scene as the live preview. - if (format === "mp4") { + // Both formats go through the native D3D exporter — same clips, same scene, + // same frame walk in the compositor crate; only the container differs. There + // is no CPU/web fallback: that path silently regressed to an ultra-slow CPU + // render once, which is exactly the failure mode a flag-gated fallback + // invites. Background/layout/webcam/cursor/effects come from the same scene + // as the live preview, so an export can no longer disagree with what the + // user previewed. + { setPhase("rendering"); // Render the real timeline when there are clips; else fall back to the fixture. const clips = buildNativeClipList(document); + // GIF runs at its own frame rate, so the progress total has to use it. + const outFps = format === "gif" ? gifFrameRate : fps; // Total frames the encoder will produce, known upfront from the timeline (sum of // each clip's trimmed source duration) — the native side only reports frames // AFTER encoding one (onNativeExportProgress), it doesn't know/send a total, so @@ -268,7 +284,7 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { (sum, c) => sum + Math.max(0, c.sourceEndSec - c.sourceStartSec), 0, ); - const totalFrames = Math.max(1, Math.round(totalDurationSec * fps)); + const totalFrames = Math.max(1, Math.round(totalDurationSec * outFps)); const startedAt = Date.now(); const unsubscribeProgress = window.electronAPI?.onNativeExportProgress?.((frames) => { const elapsedS = (Date.now() - startedAt) / 1000; @@ -287,12 +303,22 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { if (clips.length === 0) { throw new Error(t("exportDialog.nothingToExport")); } - const stats = await exportMultiNative(clips, pickedPath, sceneJson, { - width: outDims?.width, - height: outDims?.height, - fps, - codec, - }); + const stats = + format === "gif" + ? await exportGifNative(clips, pickedPath, sceneJson, { + // GIF is 256-colour and grows fast; cap the long edge at the + // chosen preset rather than exporting at source size. + ...gifOutputDims(gifSize, outDims), + fps: gifFrameRate, + // 0 = infinite, the historical GIF default; 1 = play once. + loopCount: gifLoop ? 0 : 1, + }) + : await exportMultiNative(clips, pickedPath, sceneJson, { + width: outDims?.width, + height: outDims?.height, + fps, + codec, + }); setSavedPath(pickedPath); setPhase("done"); toast.success(t("exportDialog.exportedVideo"), { @@ -315,57 +341,6 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { } return; } - - // GIF: no native encoder for this format yet, so this is the only - // implementation — not a fallback for MP4, a distinct code path. - setPhase("rendering"); - const options: DocumentExportOptions = { - format, - quality, - frameRate: fps, - codec, - gifFrameRate, - gifLoop, - gifSizePreset: gifSize, - // Size the output to the largest clip on the timeline (see referenceSource), - // not just the primary asset, so "Source" matches the shown size. - sourceWidth: referenceSource?.width ?? asset.video?.width, - sourceHeight: referenceSource?.height ?? asset.video?.height, - onProgress: (p) => setProgress(p), - }; - - try { - const result = await exportAxcutDocument(document, options); - if (!result.success || !result.blob) { - throw new Error(result.error ?? t("exportDialog.exportFailed")); - } - setPhase("writing"); - const arrayBuffer = await result.blob.arrayBuffer(); - const writeResult = await window.electronAPI?.writeExportToPath?.(arrayBuffer, pickedPath); - if (!writeResult?.success) { - throw new Error(writeResult?.error ?? t("exportDialog.failedToWriteFile")); - } - setSavedPath(pickedPath); - setPhase("done"); - toast.success(t("exportDialog.exportedGif"), { - description: pickedPath, - action: { - label: t("exportDialog.showInFolder"), - onClick: () => { - void window.electronAPI?.revealInFolder?.(pickedPath); - }, - }, - }); - // Touch the bridge so it stays referenced even when export - // is invoked from a non-Electron shim. - void nativeBridgeClient.aiEdition.llmGetSnapshot; - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - setPhase("error"); - toast.error(t("exportDialog.exportFailed"), { - description: err instanceof Error ? err.message : String(err), - }); - } }; const isBusy = phase === "rendering" || phase === "writing" || phase === "configuring"; diff --git a/src/lib/ai-edition/exporter/documentExporter.test.ts b/src/lib/ai-edition/exporter/documentExporter.test.ts deleted file mode 100644 index e973f4bd8..000000000 --- a/src/lib/ai-edition/exporter/documentExporter.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { CursorRecordingData } from "@/native/contracts"; -import type { AxcutAsset, AxcutClip, AxcutDocument, AxcutTrimRange } from "../schema"; -import type { DocumentExportOptions } from "./documentExporter"; -import { computeCropSchedule, computeExportTrimRegions } from "./documentExporter"; - -function clip(p: Partial & Pick): AxcutClip { - return { - assetId: "a", - sourceStartSec: 0, - sourceEndSec: 10, - timelineStartSec: 0, - timelineEndSec: 10, - wordRefs: [], - origin: "user", - reason: "", - ...p, - }; -} -function trim(p: Partial & Pick): AxcutTrimRange { - return { assetId: "a", startSec: 0, endSec: 1, origin: "user", reason: "", ...p }; -} - -describe("computeExportTrimRegions", () => { - it("cuts everything outside the kept clip ranges (clip in/out)", () => { - // One clip keeps source 2..8 of a 10s asset → cut 0..2 and 8..10. - const clips = [clip({ id: "c1", sourceStartSec: 2, sourceEndSec: 8 })]; - expect(computeExportTrimRegions(10, clips, [], "a")).toEqual([ - { id: "trim_1", startMs: 0, endMs: 2000 }, - { id: "trim_2", startMs: 8000, endMs: 10000 }, - ]); - }); - - it("also cuts a mid-clip trim (previously dropped from export)", () => { - // Full clip 0..10, plus a trim removing source 4..6 → export must cut it. - const clips = [clip({ id: "c1", sourceStartSec: 0, sourceEndSec: 10 })]; - const trims = [trim({ id: "t1", startSec: 4, endSec: 6 })]; - expect(computeExportTrimRegions(10, clips, trims, "a")).toEqual([ - { id: "trim_1", startMs: 4000, endMs: 6000 }, - ]); - }); - - it("merges overlapping clip-gap and trim cuts", () => { - // Clip keeps 0..8 (cut 8..10); a trim 7..9 overlaps the tail cut → merge to 7..10. - const clips = [clip({ id: "c1", sourceStartSec: 0, sourceEndSec: 8 })]; - const trims = [trim({ id: "t1", startSec: 7, endSec: 9 })]; - expect(computeExportTrimRegions(10, clips, trims, "a")).toEqual([ - { id: "trim_1", startMs: 7000, endMs: 10000 }, - ]); - }); - - it("ignores trims that belong to a different asset", () => { - const clips = [clip({ id: "c1", sourceStartSec: 0, sourceEndSec: 10 })]; - const trims = [trim({ id: "t1", assetId: "other", startSec: 3, endSec: 5 })]; - expect(computeExportTrimRegions(10, clips, trims, "a")).toEqual([]); - }); -}); - -describe("computeCropSchedule", () => { - function assetA(durationSec?: number): AxcutAsset { - return { - kind: "video", - id: "a", - label: "Asset A", - originalPath: "/a.mp4", - cameraTrack: null, - ...(durationSec !== undefined ? { durationSec } : {}), - }; - } - - it("builds one schedule entry per clip, defaulting to the identity crop", () => { - const clips = [ - clip({ id: "c1", sourceStartSec: 0, sourceEndSec: 3 }), - clip({ - id: "c2", - sourceStartSec: 3, - sourceEndSec: 6, - cropRegion: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 }, - }), - ]; - expect(computeCropSchedule(clips, assetA(6))).toEqual([ - { startSec: 0, endSec: 3, cropRegion: { x: 0, y: 0, width: 1, height: 1 } }, - { - startSec: 3, - endSec: 6, - cropRegion: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 }, - }, - ]); - }); - - it("falls back to asset.durationSec when a clip's sourceEndSec is unset", () => { - const clips = [clip({ id: "c1", sourceStartSec: 0, sourceEndSec: undefined })]; - expect(computeCropSchedule(clips, assetA(12))).toEqual([ - { startSec: 0, endSec: 12, cropRegion: { x: 0, y: 0, width: 1, height: 1 } }, - ]); - }); - - it("excludes clips that belong to a different asset", () => { - const clips = [ - clip({ id: "c1", assetId: "a" }), - clip({ id: "c2", assetId: "other", cropRegion: { x: 0.1, y: 0.1, width: 0.5, height: 0.5 } }), - ]; - expect(computeCropSchedule(clips, assetA(10))).toEqual([ - { startSec: 0, endSec: 10, cropRegion: { x: 0, y: 0, width: 1, height: 1 } }, - ]); - }); -}); diff --git a/src/lib/ai-edition/exporter/documentExporter.ts b/src/lib/ai-edition/exporter/documentExporter.ts deleted file mode 100644 index 351ac0ed4..000000000 --- a/src/lib/ai-edition/exporter/documentExporter.ts +++ /dev/null @@ -1,286 +0,0 @@ -// Adapter: feeds an AxcutDocument into GifExporter. GIF is the only format -// rendered here — MP4 goes through the native D3D exporter, which ExportDialog -// calls directly. FrameRenderer + StreamingVideoDecoder do the rendering -// (annotations, zoom, blur, webcam, cursor). -// -// ponytail: the existing exporter accepts trimRegions (removed spans in source -// time). Two contributors are merged into them (see computeExportTrimRegions): -// the inverse of the kept clip ranges (clip in/out) AND the DSL `trimRanges` -// (mid-clip trims — previously dropped from the render). We also pull -// zoom/annotations from the document (ms units, same as legacy) and -// appearance/cursor/webcam from legacyEditor (passthrough blob). The webcam -// file path lives on the primary asset's cameraTrack (P4 — per-asset, since a -// project can hold multiple recordings; export only ever handles the primary -// asset today, so no per-clip camera stitching is needed here). - -import { toFileUrl } from "@/components/video-editor/projectPersistence"; -import { - type AnnotationRegion, - type CameraFullscreenRegion, - type CropRegion, - type SpeedRegion, - type TrimRegion, - type ZoomRegion, -} from "@/components/video-editor/types"; -import { - type CropScheduleEntry, - type ExportFormat, - type ExportQuality, - GifExporter, - type GifFrameRate, - type GifSizePreset, -} from "@/lib/exporter"; -import type { ExportProgress } from "@/lib/exporter/types"; -import type { CursorRecordingData, CursorTelemetryPoint } from "@/native/contracts"; -import { - captionCuesToTextRegions, - deriveCaptionCues, - getCaptionSettings, - getCaptionTranslations, -} from "../captions"; -import { createId } from "../document/ids"; -import { type Interval, normalizeIntervals, primaryAssetDuration } from "../document/timeline"; -import type { AxcutAsset, AxcutDocument } from "../schema"; -import { resolveClipSourceEndSec } from "../timeline/clipDuration"; -import { projectRegionsToSourceTime } from "../timeline/region-ventilation"; - -export type ExportVideoCodec = "h264" | "h265" | "vp9"; - -export interface DocumentExportOptions { - quality: ExportQuality; - format: ExportFormat; - frameRate?: number; - codec?: ExportVideoCodec; - gifFrameRate?: GifFrameRate; - gifLoop?: boolean; - gifSizePreset?: GifSizePreset; - sourceWidth?: number; - sourceHeight?: number; - webcamVideoUrl?: string; - cursorRecordingData?: CursorRecordingData | null; - cursorTelemetry?: CursorTelemetryPoint[]; - cursorClickTimestamps?: number[]; - cursorScale?: number; - cursorSmoothing?: number; - cursorMotionBlur?: number; - cursorClickBounce?: number; - cursorClipToBounds?: boolean; - previewWidth?: number; - previewHeight?: number; - onProgress?: (progress: ExportProgress) => void; -} - -// The source-time spans the exporter must CUT. Two independent contributors, -// merged so both reach the render (this is the fix for trims being silently -// dropped from the export — the preview + aggregated transcript already applied -// them): -// 1. Clip in/out: everything OUTSIDE the kept clip source ranges (head/tail -// trimming + gaps between clips). -// 2. Trims: the DSL `trimRanges` — removals INSIDE a clip that don't split it. -export function computeExportTrimRegions( - sourceDurationSec: number, - clips: AxcutDocument["timeline"]["clips"], - trimRanges: AxcutDocument["timeline"]["trimRanges"], - primaryAssetId: string, -): TrimRegion[] { - // Kept source ranges = clip in/out points, clamped to the real source - // duration and merged. (Computed directly, not via timelineIntervals + a - // fake empty-asset document, whose primaryAssetDuration would be 0 and - // clamp every kept interval away.) - const keptIntervals = normalizeIntervals( - sourceDurationSec, - clips.map((c) => ({ startSec: c.sourceStartSec, endSec: c.sourceEndSec ?? sourceDurationSec })), - ); - - // Cuts from clip in/out = the complement of the kept intervals over [0, dur]. - const cuts: Interval[] = []; - let cursor = 0; - for (const interval of keptIntervals) { - if (interval.startSec > cursor) cuts.push({ startSec: cursor, endSec: interval.startSec }); - cursor = Math.max(cursor, interval.endSec); - } - if (cursor < sourceDurationSec) cuts.push({ startSec: cursor, endSec: sourceDurationSec }); - - // Cuts from trims (primary asset, source time). - for (const trim of trimRanges) { - if (trim.assetId === primaryAssetId) - cuts.push({ startSec: trim.startSec, endSec: trim.endSec }); - } - - // Merge/normalize so overlapping clip-gap + trim cuts collapse cleanly. - return normalizeIntervals(sourceDurationSec, cuts).map((iv, i) => ({ - id: `trim_${i + 1}`, - startMs: Math.round(iv.startSec * 1000), - endMs: Math.round(iv.endSec * 1000), - })); -} - -const IDENTITY_CROP: CropRegion = { x: 0, y: 0, width: 1, height: 1 }; - -// Crop is per-clip (clipSchema.cropRegion), applied clip-by-clip — not -// per-frame interpolation. The export renderer switches to whichever entry's -// [startSec, endSec) covers the current frame's SOURCE time right before -// rendering it (see VideoExporter/GifExporter + FrameRenderer.setCropRegion). -// Only clips on `primaryAsset` matter here — export renders one continuous -// source video (the primary asset), so a clip pointing at a different asset -// has no meaningful source-time range against this one. -export function computeCropSchedule( - clips: AxcutDocument["timeline"]["clips"], - primaryAsset: AxcutAsset, -): CropScheduleEntry[] { - const primaryAssetId = primaryAsset.id; - return clips - .filter((c) => c.assetId === primaryAssetId) - .map((c) => ({ - startSec: c.sourceStartSec, - endSec: resolveClipSourceEndSec(c, primaryAsset), - cropRegion: c.cropRegion ?? IDENTITY_CROP, - })); -} - -function extractLegacyField( - legacy: Record | null, - key: string, - fallback: T, -): T { - if (legacy && typeof legacy[key] === typeof fallback) { - return legacy[key] as T; - } - return fallback; -} - -export async function exportAxcutDocument( - document: AxcutDocument, - options: DocumentExportOptions, -): Promise { - const asset = - document.assets.find((a) => a.id === document.project.primaryAssetId) ?? document.assets[0]; - if (!asset) { - return { success: false, error: "No asset to export." }; - } - - const videoUrl = toFileUrl(asset.originalPath); - const sourceDurationSec = asset.durationSec ?? primaryAssetDuration(document); - const trimRegions = computeExportTrimRegions( - sourceDurationSec, - document.timeline.clips, - document.timeline.trimRanges, - asset.id, - ); - // Effects are authored in virtual (edited-timeline) time, but the export - // frame loop matches them against each frame's *source* time — so project - // them onto source ranges through the clips they cover (clip in/out + order), - // splitting any region that straddles a clip boundary. This is what makes a - // multi-clip export's zooms/annotations/speed land on the same frames as the - // preview. Identity single-clip projects are unchanged (source == virtual). - const clips = document.timeline.clips; - const zoomRegions = projectRegionsToSourceTime( - document.zoomRanges as unknown as ZoomRegion[], - clips, - () => createId("zoom"), - ); - // Captions are derived from the transcript at export time, never stored — so a - // project exported after re-transcribing or after a style change always writes - // the captions the editor was showing, with no regeneration step in between. - // They ride the annotation text renderer (same wrapping/plate/alignment) and - // go through the same virtual→source projection, so preview and file agree. - const captionSettings = getCaptionSettings(document); - const captionRegions = captionCuesToTextRegions( - deriveCaptionCues(document, captionSettings, getCaptionTranslations(document)), - captionSettings, - ); - const annotationRegions = projectRegionsToSourceTime( - [...(document.annotations as unknown as AnnotationRegion[]), ...captionRegions], - clips, - () => createId("ann"), - ); - const legacy = document.legacyEditor as Record | null; - - const wallpaper = extractLegacyField(legacy, "wallpaper", ""); - const shadowIntensity = extractLegacyField(legacy, "shadowIntensity", 0); - const showBlur = extractLegacyField(legacy, "showBlur", false); - const motionBlurAmount = extractLegacyField(legacy, "motionBlurAmount", 0); - const borderRadius = extractLegacyField(legacy, "borderRadius", 0); - const padding = extractLegacyField(legacy, "padding", 50); - const cropSchedule = computeCropSchedule(clips, asset); - const cropRegion: CropRegion = IDENTITY_CROP; - const webcamLayoutPreset = extractLegacyField(legacy, "webcamLayoutPreset", "picture-in-picture"); - const webcamMaskShape = extractLegacyField(legacy, "webcamMaskShape", "rectangle"); - const webcamMirrored = extractLegacyField(legacy, "webcamMirrored", false); - const webcamReactiveZoom = extractLegacyField(legacy, "webcamReactiveZoom", true); - const webcamSizePreset = extractLegacyField(legacy, "webcamSizePreset", 25); - const webcamPosition = extractLegacyField(legacy, "webcamPosition", null); - const cursorTheme = extractLegacyField(legacy, "cursorTheme", ""); - const speedRegions: SpeedRegion[] = projectRegionsToSourceTime( - extractLegacyField(legacy, "speedRegions", []), - clips, - () => createId("speed"), - ); - const cameraFullscreenRegions: CameraFullscreenRegion[] = projectRegionsToSourceTime( - extractLegacyField(legacy, "cameraFullscreenRegions", []), - clips, - () => createId("camfull"), - ); - - const cameraTrack = asset.cameraTrack; - - const commonConfig = { - videoUrl, - // ponytail: the camera is a derived stream from cameraTrack; the legacy - // exporter accepts webcamVideoUrl as a visual-only second source. - webcamVideoUrl: - cameraTrack && cameraTrack.visible && cameraTrack.sourcePath - ? toFileUrl(cameraTrack.sourcePath) - : options.webcamVideoUrl, - wallpaper, - zoomRegions, - trimRegions, - speedRegions, - cameraFullscreenRegions, - showShadow: shadowIntensity > 0, - shadowIntensity, - showBlur, - motionBlurAmount, - borderRadius, - padding, - cropRegion, - cropSchedule, - annotationRegions, - webcamLayoutPreset: webcamLayoutPreset as - | "picture-in-picture" - | "no-webcam" - | "vertical-stack" - | "dual-frame", - webcamMaskShape: webcamMaskShape as "rectangle" | "circle" | "square" | "rounded", - webcamMirrored, - webcamReactiveZoom, - webcamSizePreset, - webcamPosition, - cursorRecordingData: options.cursorRecordingData ?? null, - cursorScale: options.cursorScale ?? 0, - cursorSmoothing: options.cursorSmoothing, - cursorMotionBlur: options.cursorMotionBlur, - cursorClickBounce: options.cursorClickBounce, - cursorClipToBounds: options.cursorClipToBounds, - cursorTheme, - cursorTelemetry: options.cursorTelemetry, - cursorClickTimestamps: options.cursorClickTimestamps, - previewWidth: options.previewWidth, - previewHeight: options.previewHeight, - onProgress: options.onProgress, - }; - - // GIF is the only format this adapter renders. MP4 goes through the native - // D3D exporter (`exportMultiNative`), which ExportDialog calls directly. - const exporter = new GifExporter({ - ...commonConfig, - width: 1280, - height: 720, - frameRate: options.gifFrameRate ?? 15, - loop: options.gifLoop ?? true, - sizePreset: options.gifSizePreset ?? "medium", - } as unknown as ConstructorParameters[0]); - return exporter.export(); -} - -import type { ExportResult } from "@/lib/exporter/types"; diff --git a/src/lib/exporter/cropSchedule.ts b/src/lib/exporter/cropSchedule.ts deleted file mode 100644 index b6749ad58..000000000 --- a/src/lib/exporter/cropSchedule.ts +++ /dev/null @@ -1,26 +0,0 @@ -// Per-clip crop lookup, shared by the GIF exporter and the document adapter. -// Lived in videoExporter.ts until the web MP4 path was removed; GIF is the only -// remaining renderer that resolves a crop off a timeline. - -import type { CropRegion } from "@/components/video-editor/types"; - -export interface CropScheduleEntry { - startSec: number; - endSec: number; - cropRegion: CropRegion; -} - -/** Finds which clip's crop applies at a given SOURCE-media timestamp — the - * first schedule entry whose [startSec, endSec) covers it, falling back to - * `fallback` when the schedule is absent or nothing covers it (e.g. a gap). */ -export function resolveCropAt( - schedule: CropScheduleEntry[] | undefined, - sourceSec: number, - fallback: CropRegion, -): CropRegion { - if (!schedule || schedule.length === 0) return fallback; - const covering = schedule.find( - (entry) => sourceSec >= entry.startSec && sourceSec < entry.endSec, - ); - return covering?.cropRegion ?? fallback; -} diff --git a/src/lib/exporter/featureFlags.ts b/src/lib/exporter/featureFlags.ts deleted file mode 100644 index 0968cdb0f..000000000 --- a/src/lib/exporter/featureFlags.ts +++ /dev/null @@ -1,11 +0,0 @@ -// ponytail: gates the native GIF export path added in slice 1 of the -// D3D ↔ Pixi cleanup roadmap. Stays `false` for this PR — the renderer -// still routes GIF through `gif.js` via `src/lib/exporter/gifExporter.ts`. -// The native path is wired end-to-end (Rust → napi addon → -// `compositorViewService.exportGif` → TS contract) but flipping this on -// is a follow-up: the bench in `crates/poc-d3d/src/bench.rs` is the -// honest signal that decides whether the readback is fast enough to be -// a win (or whether a 5× regression makes it not worth the swap — see -// the `Native GIF export — initial bench` section of -// `technical-documentation/engineering/rendering-performance.md`). -export const NATIVE_GIF_EXPORT_ENABLED = false; diff --git a/src/lib/exporter/frameExtract.ts b/src/lib/exporter/frameExtract.ts deleted file mode 100644 index bee21b493..000000000 --- a/src/lib/exporter/frameExtract.ts +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Pulls composited pixels off the canvas as raw bytes for the native encoder. - * - * `new VideoFrame(canvas)` is LAZY — it does not read anything back; the - * GPU->CPU descent happens inside copyTo(). Timing the constructor measures - * nothing (that mistake is why the v2 spec originally blamed readback for a - * cost it does not have). - * - * Chromium will only hand us BGRA here: copyTo({format: "NV12"}) throws - * NotSupportedError. Packing to NV12 on the GPU before extraction is the next - * step (3.0 MB/frame instead of 7.9, measured ~2x end to end) — see the v2 - * spec's Phase 4. - */ - -export interface FrameLayoutPlane { - offset: number; - stride: number; -} - -/** - * Raw video has no stride: ffmpeg reads width*4 bytes per row, forever. If - * Chromium ever pads rows, feeding the buffer through unchanged would skew - * every frame into diagonal garbage — visibly broken, but only at runtime and - * only on whatever machine pads. Fail loudly instead. - */ -export function assertTightBgraLayout( - layout: readonly FrameLayoutPlane[], - width: number, - height: number, - byteLength: number, -): void { - if (layout.length !== 1) { - throw new Error(`Expected 1 BGRA plane from the canvas, got ${layout.length}`); - } - const plane = layout[0]; - const tightStride = width * 4; - if (plane.stride !== tightStride) { - throw new Error( - `Canvas frame is padded (stride ${plane.stride}, expected ${tightStride}). ` + - "Raw video cannot carry stride; the frame would be skewed.", - ); - } - if (plane.offset !== 0) { - throw new Error(`Canvas frame plane starts at ${plane.offset}, expected 0`); - } - const expected = tightStride * height; - if (byteLength !== expected) { - throw new Error(`Canvas frame is ${byteLength} bytes, expected ${expected}`); - } -} - -/** - * Reusable extraction buffer. The frame is copied by IPC during send(), so the - * caller may refill this as soon as the sink's write() resolves — which saves - * allocating (and collecting) ~8 MB per frame. - */ -export class CanvasFrameExtractor { - private buffer: ArrayBuffer | null = null; - - constructor( - private readonly width: number, - private readonly height: number, - ) {} - - /** BGRA is what Chromium yields from a canvas; ffmpeg is told to expect it. */ - readonly pixelFormat = "bgra" as const; - - async extract(canvas: HTMLCanvasElement | OffscreenCanvas): Promise { - const frame = new VideoFrame(canvas as HTMLCanvasElement, { timestamp: 0 }); - try { - const size = frame.allocationSize(); - this.buffer ??= new ArrayBuffer(size); - if (this.buffer.byteLength !== size) { - // Output size is fixed for the whole export; a change here means the - // renderer was reconfigured mid-flight and ffmpeg's -s is now wrong. - throw new Error( - `Frame size changed mid-export (${this.buffer.byteLength} -> ${size} bytes)`, - ); - } - const layout = await frame.copyTo(this.buffer); - assertTightBgraLayout(layout, this.width, this.height, size); - return this.buffer; - } finally { - frame.close(); - } - } -} diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts deleted file mode 100644 index 16ba6bde0..000000000 --- a/src/lib/exporter/frameRenderer.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { drawWebcamFrameImage } from "./webcamFrameDrawing"; - -type DrawCall = - | ["drawImage", unknown, number, number, number, number, number, number, number, number] - | ["restore"] - | ["save"] - | ["scale", number, number] - | ["translate", number, number]; - -function createMockCanvasContext() { - const calls: DrawCall[] = []; - const ctx = { - drawImage: ( - image: CanvasImageSource, - sx: number, - sy: number, - sw: number, - sh: number, - dx: number, - dy: number, - dw: number, - dh: number, - ) => calls.push(["drawImage", image, sx, sy, sw, sh, dx, dy, dw, dh]), - restore: () => calls.push(["restore"]), - save: () => calls.push(["save"]), - scale: (x: number, y: number) => calls.push(["scale", x, y]), - translate: (x: number, y: number) => calls.push(["translate", x, y]), - }; - - return { calls, ctx }; -} - -describe("drawWebcamFrameImage", () => { - it("draws the webcam frame into the layout rect by default", () => { - const { calls, ctx } = createMockCanvasContext(); - const frame = {} as CanvasImageSource; - - drawWebcamFrameImage( - ctx, - frame, - { x: 12, y: 8, width: 640, height: 360 }, - { x: 100, y: 50, width: 320, height: 180 }, - ); - - expect(calls).toEqual([["drawImage", frame, 12, 8, 640, 360, 100, 50, 320, 180]]); - }); - - it("mirrors around the webcam rect without changing the crop", () => { - const { calls, ctx } = createMockCanvasContext(); - const frame = {} as CanvasImageSource; - - drawWebcamFrameImage( - ctx, - frame, - { x: 12, y: 8, width: 640, height: 360 }, - { x: 100, y: 50, width: 320, height: 180 }, - true, - ); - - expect(calls).toEqual([ - ["save"], - ["translate", 420, 50], - ["scale", -1, 1], - ["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180], - ["restore"], - ]); - }); - - it("restores the canvas context if mirrored drawing fails", () => { - const { calls, ctx } = createMockCanvasContext(); - const frame = {} as CanvasImageSource; - const error = new Error("draw failed"); - ctx.drawImage = () => { - calls.push(["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180]); - throw error; - }; - - expect(() => - drawWebcamFrameImage( - ctx, - frame, - { x: 12, y: 8, width: 640, height: 360 }, - { x: 100, y: 50, width: 320, height: 180 }, - true, - ), - ).toThrow(error); - - expect(calls).toEqual([ - ["save"], - ["translate", 420, 50], - ["scale", -1, 1], - ["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180], - ["restore"], - ]); - }); -}); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts deleted file mode 100644 index 6f42581ba..000000000 --- a/src/lib/exporter/frameRenderer.ts +++ /dev/null @@ -1,1545 +0,0 @@ -import { - Application, - BlurFilter, - Container, - Graphics, - Sprite, - Texture, - type TextureSourceLike, -} from "pixi.js"; -import { MotionBlurFilter } from "pixi-filters/motion-blur"; -import type { - AnnotationRegion, - CameraFullscreenRegion, - CropRegion, - Rotation3D, - SpeedRegion, - WebcamLayoutPreset, - WebcamSizePreset, - ZoomRegion, -} from "@/components/video-editor/types"; -import { - DEFAULT_ROTATION_3D, - getZoomScale, - isRotation3DIdentity, - lerpRotation3D, -} from "@/components/video-editor/types"; -import { - computeCameraFullscreenRect, - computeCompositeLayout, - getWebcamLayoutPresetDefinition, - reactiveWebcamScale, - resolveWebcamReactiveZoom, - type Size, - type StyledRenderRect, -} from "@/lib/compositeLayout"; -import { getSmoothedCursorPath } from "@/lib/cursor/cursorPathSmoothing"; -import { - createNativeCursorMotionBlurState, - getNativeCursorClickBounceProgress, - getNativeCursorClickBounceScale, - getNativeCursorMotionBlurPx, - projectNativeCursorToLocal, - resetNativeCursorMotionBlurState, - resolveInterpolatedNativeCursorFrame, - resolveNativeCursorRenderAsset, -} from "@/lib/cursor/nativeCursor"; -import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; -import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; -import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils"; -import { AUTO_FOLLOW_PARAMS, DEFAULT_FOCUS } from "@/lib/zoomMath/constants"; -import { advanceFollowFocus } from "@/lib/zoomMath/cursorFollowUtils"; -import { clampFocusToScale } from "@/lib/zoomMath/focusUtils"; -import { findDominantRegion } from "@/lib/zoomMath/zoomRegionUtils"; -import { createZoomSpringState, resetZoomSpring, stepZoomSpring } from "@/lib/zoomMath/zoomSpring"; -import { - applyZoomTransform, - computeFocusFromTransform, - computeZoomTransform, - createMotionBlurState, - type MotionBlurState, -} from "@/lib/zoomMath/zoomTransform"; -import type { CursorRecordingData } from "@/native/contracts"; -import { renderAnnotations } from "./annotationRenderer"; -import { - getLinearGradientPoints, - getRadialGradientShape, - parseCssGradient, - resolveLinearGradientAngle, -} from "./gradientParser"; -import { createThreeDPass, type ThreeDPass } from "./threeDPass"; -import { drawWebcamFrameImage } from "./webcamFrameDrawing"; - -/** - * Ask for CPU-backed 2D canvases (`willReadFrequently`) outside Linux. - * - * The default is GPU-backed, which makes compositing cheap and reading back - * expensive. That is the right trade for WebCodecs, which encodes straight from - * the GPU texture and never descends. It is the wrong trade for the native - * encoder, which needs the pixels on the CPU for EVERY frame — there, the - * readback measured 54-87ms/frame and became ~75% of the loop. - * - * Read at runtime so one app session can measure both settings: - * localStorage.setItem("openscreen.readFrequently", "1") - * - * Temporary scaffold: once the native path is the only path, this becomes a - * plain constant rather than a question. - */ -function cpuCanvasRequested(): boolean { - try { - return localStorage.getItem("openscreen.readFrequently") === "1"; - } catch { - return false; - } -} - -/** - * Restore the pre-2026-07-17 compositor: a fresh GPU texture per frame, the mask - * retessellated per frame, and clearRect before every full-frame draw. - * - * Exists so the three fixes can be ATTRIBUTED. Comparing across bench sessions - * would not do: this machine drifts more between sessions than the fixes are - * worth, which is the failure mode this whole investigation keeps tripping over. - * With the flag, one interleaved run measures old against new on one thermal - * state, and the repeat proves it. - * - * localStorage.setItem("openscreen.legacyCompositor", "1") - */ -function legacyCompositorRequested(): boolean { - try { - return localStorage.getItem("openscreen.legacyCompositor") === "1"; - } catch { - return false; - } -} - -/** - * Diagnostic: run the shadow's cache-miss path but SKIP the filter chain. - * - * A cache miss costs 16.7 ms/frame (Annex B), and that number is two different - * things stacked: three chained gaussians, and the full-frame Canvas2D plumbing - * that feeds them (a silhouette copy, a source-in fill, a filtered blit — 2 Mpx - * each). They do not have the same fix. A shader only helps if the gaussians are - * the cost; if it is the plumbing, the fix is to stop touching the whole frame. - * - * Timing the ops individually would answer nothing — Canvas2D is as lazy as the - * GPU, so a timer around a drawImage measures submission and bills the work to - * whatever syncs next (§7.4). Hence a flag and an arm PAIR instead: same path, - * one filter, one none, both fenced. - * - * The output is wrong on purpose (no shadow is drawn). Diagnostic only. - * - * localStorage.setItem("openscreen.shadowNoFilter", "1") - */ -function shadowFilterDisabled(): boolean { - try { - return localStorage.getItem("openscreen.shadowNoFilter") === "1"; - } catch { - return false; - } -} - -/** - * The recording's drop shadow, as a CSS filter chain. - * - * Three chained shadows, each blurring the alpha of the PREVIOUS stage's output - * (its own shadow included) — that cascade is what gives the falloff, and it is - * why this cannot be swapped for an SDF without changing the picture. - * - * Was duplicated verbatim on the flat and the 3D path; they must not drift. - */ -function shadowFilterChain(intensity: number): string { - const offset = 12 * intensity; - return ( - `drop-shadow(0 ${offset}px ${48 * intensity}px rgba(0,0,0,${0.7 * intensity})) ` + - `drop-shadow(0 ${offset / 3}px ${16 * intensity}px rgba(0,0,0,${0.5 * intensity})) ` + - `drop-shadow(0 ${offset / 6}px ${8 * intensity}px rgba(0,0,0,${0.3 * intensity}))` - ); -} - -interface FrameRenderConfig { - width: number; - height: number; - wallpaper: string; - zoomRegions: ZoomRegion[]; - cameraFullscreenRegions?: CameraFullscreenRegion[]; - showShadow: boolean; - shadowIntensity: number; - showBlur: boolean; - motionBlurAmount?: number; - borderRadius?: number; - padding?: number; - cropRegion: CropRegion; - cursorRecordingData?: CursorRecordingData | null; - cursorScale?: number; - cursorSmoothing?: number; - cursorMotionBlur?: number; - cursorClickBounce?: number; - cursorClipToBounds?: boolean; - cursorTheme?: string; - videoWidth: number; - videoHeight: number; - webcamSize?: Size | null; - webcamLayoutPreset?: WebcamLayoutPreset; - webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; - webcamMirrored?: boolean; - webcamReactiveZoom?: boolean; - webcamSizePreset?: WebcamSizePreset; - webcamPosition?: { cx: number; cy: number } | null; - annotationRegions?: AnnotationRegion[]; - speedRegions?: SpeedRegion[]; - previewWidth?: number; - previewHeight?: number; - cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[]; - cursorClickTimestamps?: number[]; - platform: string; -} - -interface AnimationState { - scale: number; - focusX: number; - focusY: number; - progress: number; - x: number; - y: number; - appliedScale: number; - cameraFullscreenProgress: number; -} - -interface LayoutCache { - stageSize: { width: number; height: number }; - videoSize: { width: number; height: number }; - baseScale: number; - baseOffset: { x: number; y: number }; - maskRect: { x: number; y: number; width: number; height: number }; - croppedRect: { x: number; y: number; width: number; height: number }; - maskBorderRadius: number; - webcamRect: StyledRenderRect | null; -} - -// Renders video frames with all effects (background, zoom, crop, blur, shadow) to an offscreen canvas for export. - -export class FrameRenderer { - private app: Application | null = null; - private cameraContainer: Container | null = null; - private videoContainer: Container | null = null; - private videoSprite: Sprite | null = null; - /** Source geometry behind videoSprite's texture — see renderFrame's reuse. */ - private videoFrameSize: { width: number; height: number } | null = null; - /** Shape currently tessellated into maskGraphics, so it is rebuilt only on change. */ - private maskShape: { width: number; height: number; radius: number } | null = null; - /** Bench-only: undo the compositor fixes so they can be measured. */ - private legacyCompositor = legacyCompositorRequested(); - /** Bench-only: price the filter chain apart from the plumbing feeding it. */ - private shadowNoFilter = shadowFilterDisabled(); - /** Shadow filter output, keyed by the geometry it was computed for. */ - private shadowCache: { key: string; canvas: HTMLCanvasElement } | null = null; - /** - * How often the geometry key held, and how often it paid for the filter chain. - * - * The miss rate IS the Step-3 decision input (see rendering-architecture.md - * §13): the cache captures the shadow's cost on still frames, and a moving - * camera must miss by design. Only a count says which case a real timeline is. - */ - private shadowCacheHits = 0; - private shadowCacheMisses = 0; - /** Scratch holding videoCanvas's alpha as an opaque black shape. */ - private shadowSilhouetteCanvas: HTMLCanvasElement | null = null; - /** The wallpaper, blurred once. It is a still image — see blurredBackgroundLayer. */ - private blurredBackground: HTMLCanvasElement | null = null; - /** Which backgroundSprite blurredBackground was built from, so a reload invalidates it. */ - private blurredBackgroundSource: HTMLCanvasElement | null = null; - private backgroundSprite: HTMLCanvasElement | null = null; - private maskGraphics: Graphics | null = null; - private blurFilter: BlurFilter | null = null; - private motionBlurFilter: MotionBlurFilter | null = null; - private shadowCanvas: HTMLCanvasElement | null = null; - private shadowCtx: CanvasRenderingContext2D | null = null; - private compositeCanvas: HTMLCanvasElement | null = null; - private compositeCtx: CanvasRenderingContext2D | null = null; - private foregroundCanvas: HTMLCanvasElement | null = null; - private foregroundCtx: CanvasRenderingContext2D | null = null; - private rasterCanvas: HTMLCanvasElement | null = null; - private rasterCtx: CanvasRenderingContext2D | null = null; - private threeDPass: ThreeDPass | null = null; - private currentRotation3D: Rotation3D = { ...DEFAULT_ROTATION_3D }; - private cursorImageCache = new Map(); - private warnedKeys = new Set(); - private config: FrameRenderConfig; - private animationState: AnimationState; - private layoutCache: LayoutCache | null = null; - private currentVideoTime = 0; - private motionBlurState: MotionBlurState = createMotionBlurState(); - private nativeCursorMotionBlurState = createNativeCursorMotionBlurState(); - private smoothedAutoFocus: { cx: number; cy: number } | null = null; - private prevAnimationTimeMs: number | null = null; - private zoomSpringState = createZoomSpringState(); - private prevTargetProgress = 0; - private isLinux = false; - /** CPU-backed 2D canvases — see cpuCanvasRequested(). Fixed per instance. */ - private cpuCanvas = false; - - constructor(config: FrameRenderConfig) { - this.config = config; - this.isLinux = config.platform === "linux"; - this.cpuCanvas = this.isLinux || cpuCanvasRequested(); - this.animationState = { - scale: 1, - focusX: DEFAULT_FOCUS.cx, - focusY: DEFAULT_FOCUS.cy, - progress: 0, - x: 0, - y: 0, - appliedScale: 1, - cameraFullscreenProgress: 0, - }; - } - - /** - * Swaps the active crop before rendering the next frame — crop is per-clip - * (see clipSchema.cropRegion), not a single value for the whole export. - * `updateLayout()` (called at the top of every `renderFrame`) reads - * `this.config.cropRegion` fresh each time, and the cursor overlay does - * the same, so mutating it here is picked up correctly by both without - * any extra cache invalidation. - */ - setCropRegion(cropRegion: CropRegion): void { - this.config.cropRegion = cropRegion; - } - - /** - * Reconfigure the renderer's INPUT for the next segment of a multi-asset - * export (v2 segment loop). Output dimensions stay fixed for the whole - * export; only the source-dependent fields change. `updateLayout()` reads - * these from `this.config` fresh every frame and `renderFrame` swaps the - * video texture per frame, so mutating config here is picked up with no - * teardown. Virtual-time zoom/annotation state is intentionally preserved - * (those effects span segments), but per-source cursor/auto-focus continuity - * is reset so a new asset doesn't inherit the previous one's smoothed motion. - */ - setSource(source: { - videoWidth: number; - videoHeight: number; - webcamSize?: Size | null; - cursorRecordingData?: CursorRecordingData | null; - cursorScale?: number; - // Timeline-authored effects PROJECTED to this segment's SOURCE time — the - // export loop passes each frame its source timestamp, so zoom/annotation - // (and cursor samples) all match in one coordinate system even under speed. - zoomRegions?: ZoomRegion[]; - annotationRegions?: AnnotationRegion[]; - speedRegions?: SpeedRegion[]; - }): void { - this.config.videoWidth = source.videoWidth; - this.config.videoHeight = source.videoHeight; - this.config.webcamSize = source.webcamSize ?? null; - this.config.cursorRecordingData = source.cursorRecordingData ?? null; - this.config.cursorScale = source.cursorScale ?? 0; - if (source.zoomRegions) this.config.zoomRegions = source.zoomRegions; - if (source.annotationRegions) this.config.annotationRegions = source.annotationRegions; - if (source.speedRegions) this.config.speedRegions = source.speedRegions; - this.layoutCache = null; - this.smoothedAutoFocus = null; - this.zoomSpringState = createZoomSpringState(); - this.prevAnimationTimeMs = null; - resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); - } - - async initialize(): Promise { - const canvas = document.createElement("canvas"); - canvas.width = this.config.width; - canvas.height = this.config.height; - - // colorSpace isn't available on all platforms - try { - if (canvas && "colorSpace" in canvas) { - canvas.colorSpace = "srgb"; - } - } catch (error) { - console.warn("[FrameRenderer] colorSpace not supported on this platform:", error); - } - - this.app = new Application(); - await this.app.init({ - canvas, - width: this.config.width, - height: this.config.height, - backgroundAlpha: 0, - antialias: true, - resolution: 1, - autoDensity: true, - }); - - this.cameraContainer = new Container(); - this.videoContainer = new Container(); - this.app.stage.addChild(this.cameraContainer); - this.cameraContainer.addChild(this.videoContainer); - - // Background renders separately, not in PixiJS - await this.setupBackground(); - - this.blurFilter = new BlurFilter(); - this.blurFilter.quality = 5; - this.blurFilter.resolution = this.app.renderer.resolution; - this.blurFilter.blur = 0; - this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); - this.videoContainer.filters = [this.blurFilter, this.motionBlurFilter]; - - // Composite canvas: final output with shadows - this.compositeCanvas = document.createElement("canvas"); - this.compositeCanvas.width = this.config.width; - this.compositeCanvas.height = this.config.height; - - // Hint frequent CPU readback: Linux exports via getImageData every frame, - // and so does the native encoder path (see cpuCanvasRequested). - this.compositeCtx = this.compositeCanvas.getContext("2d", { - willReadFrequently: this.cpuCanvas, - }); - - if (!this.compositeCtx) { - throw new Error("Failed to get 2D context for composite canvas"); - } - - // willReadFrequently is a HINT: report what Chromium actually granted, not - // what we asked for. An arm that silently no-ops would otherwise read as - // "the lever does not help" when it means "the lever was never pulled". - console.warn( - `[export perf] canvas requested willReadFrequently=${this.cpuCanvas} granted=${ - this.compositeCtx.getContextAttributes?.()?.willReadFrequently - }`, - ); - - this.rasterCanvas = document.createElement("canvas"); - this.rasterCanvas.width = this.config.width; - this.rasterCanvas.height = this.config.height; - this.rasterCtx = this.rasterCanvas.getContext("2d"); - if (!this.rasterCtx) { - throw new Error("Failed to get 2D context for raster canvas"); - } - - // Foreground (transparent): recording + shadow + webcam + cursor + annotations. - // The 3D pass operates only on this layer so the wallpaper stays flat behind it. - this.foregroundCanvas = document.createElement("canvas"); - this.foregroundCanvas.width = this.config.width; - this.foregroundCanvas.height = this.config.height; - // Flips WITH the composite canvas, never against it: a GPU-backed - // foreground drawn onto a CPU-backed composite would force its own - // readback on every drawImage — worse than either setting alone. - this.foregroundCtx = this.foregroundCanvas.getContext("2d", { - willReadFrequently: this.cpuCanvas, - }); - if (!this.foregroundCtx) { - throw new Error("Failed to get 2D context for foreground canvas"); - } - - if (this.config.showShadow) { - this.shadowCanvas = document.createElement("canvas"); - this.shadowCanvas.width = this.config.width; - this.shadowCanvas.height = this.config.height; - this.shadowCtx = this.shadowCanvas.getContext("2d", { - willReadFrequently: false, - }); - - if (!this.shadowCtx) { - throw new Error("Failed to get 2D context for shadow canvas"); - } - } - - this.maskGraphics = new Graphics(); - this.videoContainer.addChild(this.maskGraphics); - this.videoContainer.mask = this.maskGraphics; - - try { - this.threeDPass = createThreeDPass(this.config.width, this.config.height); - } catch (error) { - console.warn("[FrameRenderer] 3D pass unavailable, rotation fields will be ignored:", error); - this.threeDPass = null; - } - } - - private async setupBackground(): Promise { - const wallpaper = this.config.wallpaper; - - const bgCanvas = document.createElement("canvas"); - bgCanvas.width = this.config.width; - bgCanvas.height = this.config.height; - const bgCtx = bgCanvas.getContext("2d")!; - - const classified = classifyWallpaper(wallpaper); - - if (classified.kind === "color") { - bgCtx.fillStyle = classified.value; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } else if (classified.kind === "gradient") { - const parsedGradient = parseCssGradient(classified.value); - if (!parsedGradient) { - throw new BackgroundLoadError(classified.value); - } - const gradient = - parsedGradient.type === "linear" - ? (() => { - const points = getLinearGradientPoints( - resolveLinearGradientAngle(parsedGradient.descriptor), - this.config.width, - this.config.height, - ); - return bgCtx.createLinearGradient(points.x0, points.y0, points.x1, points.y1); - })() - : (() => { - const shape = getRadialGradientShape( - parsedGradient.descriptor, - this.config.width, - this.config.height, - ); - return bgCtx.createRadialGradient( - shape.cx, - shape.cy, - 0, - shape.cx, - shape.cy, - shape.radius, - ); - })(); - - parsedGradient.stops.forEach((stop) => { - gradient.addColorStop(stop.offset, stop.color); - }); - - bgCtx.fillStyle = gradient; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } else { - const imageUrl = resolveImageWallpaperUrl(classified.path); - const img = new Image(); - if (imageUrl.startsWith("http") && !imageUrl.startsWith(window.location.origin)) { - img.crossOrigin = "anonymous"; - } - - try { - await new Promise((resolve, reject) => { - img.onload = () => resolve(); - img.onerror = (err) => reject(err); - img.src = imageUrl; - }); - } catch (err) { - throw new BackgroundLoadError(imageUrl, err); - } - - const imgAspect = img.width / img.height; - const canvasAspect = this.config.width / this.config.height; - - let drawWidth: number; - let drawHeight: number; - let drawX: number; - let drawY: number; - - if (imgAspect > canvasAspect) { - drawHeight = this.config.height; - drawWidth = drawHeight * imgAspect; - drawX = (this.config.width - drawWidth) / 2; - drawY = 0; - } else { - drawWidth = this.config.width; - drawHeight = drawWidth / imgAspect; - drawX = 0; - drawY = (this.config.height - drawHeight) / 2; - } - - bgCtx.drawImage(img, drawX, drawY, drawWidth, drawHeight); - } - - this.backgroundSprite = bgCanvas; - } - - async renderFrame( - videoFrame: VideoFrame, - timestamp: number, - webcamFrame?: VideoFrame | null, - ): Promise { - if (!this.app || !this.videoContainer || !this.cameraContainer) { - throw new Error("Renderer not initialized"); - } - - this.currentVideoTime = timestamp / 1000000; - - // Reuse the GPU texture across frames. Texture.from() allocates a new - // TextureSource per call (every VideoFrame is a distinct object, so nothing - // caches), and destroy(true) frees the GL texture behind it — an allocate + - // upload + free of a 1080p texture on every single frame. Swapping the - // resource re-uploads the pixels into the texture we already have. - // - // Only valid while the geometry is unchanged: segments can carry different - // source sizes, so a size change still rebuilds (and frees) the source. - const frameWidth = videoFrame.displayWidth; - const frameHeight = videoFrame.displayHeight; - const sizeUnchanged = - !this.legacyCompositor && - this.videoFrameSize?.width === frameWidth && - this.videoFrameSize?.height === frameHeight; - - if (!this.videoSprite) { - const texture = Texture.from(videoFrame as unknown as TextureSourceLike); - this.videoSprite = new Sprite(texture); - this.videoContainer.addChild(this.videoSprite); - this.videoFrameSize = { width: frameWidth, height: frameHeight }; - } else if (sizeUnchanged) { - const source = this.videoSprite.texture.source; - source.resource = videoFrame; - source.update(); - } else { - // Destroy old texture before swapping to avoid a leak - const oldTexture = this.videoSprite.texture; - const newTexture = Texture.from(videoFrame as unknown as TextureSourceLike); - this.videoSprite.texture = newTexture; - oldTexture.destroy(true); - this.videoFrameSize = { width: frameWidth, height: frameHeight }; - } - - this.updateLayout(webcamFrame); - - const timeMs = this.currentVideoTime * 1000; - const TICKS_PER_FRAME = 1; - - let maxMotionIntensity = 0; - for (let i = 0; i < TICKS_PER_FRAME; i++) { - const motionIntensity = this.updateAnimationState(timeMs); - maxMotionIntensity = Math.max(maxMotionIntensity, motionIntensity); - } - - const layoutCache = this.layoutCache; - if (!layoutCache) { - throw new Error("Layout cache not initialized"); - } - - // Feed the spring-smoothed transform (appliedScale/x/y) via transformOverride, like the - // preview. Without it applyZoomTransform recomputes the camera from the raw eased target and - // the spring is discarded, so the export snaps to the target every frame while the preview - // glides (very visible for auto-focus, whose target pans with the cursor). It also keeps the - // camera, mask, and cursor (which already read appliedScale/x/y) consistent. - applyZoomTransform({ - cameraContainer: this.cameraContainer, - blurFilter: this.blurFilter, - motionBlurFilter: this.motionBlurFilter, - stageSize: layoutCache.stageSize, - baseMask: layoutCache.maskRect, - zoomScale: this.animationState.scale, - zoomProgress: this.animationState.progress, - focusX: this.animationState.focusX, - focusY: this.animationState.focusY, - motionIntensity: maxMotionIntensity, - isPlaying: true, - motionBlurAmount: this.config.motionBlurAmount ?? 0, - motionBlurState: this.motionBlurState, - frameTimeMs: timeMs, - transformOverride: { - scale: this.animationState.appliedScale, - x: this.animationState.x, - y: this.animationState.y, - }, - }); - - // Render the PixiJS stage (video only, transparent background) - this.app.renderer.render(this.app.stage); - - // Skip baking the shadow when the rotation pass will run; bilinear sampling would - // alias it to a hard edge. Re-applied fresh after rotation. - const willRotate = !isRotation3DIdentity(this.currentRotation3D); - this.compositeWithShadows(webcamFrame, !willRotate); - - await this.drawNativeCursor(timeMs); - - // Annotations go on top of foreground so they rotate with the recording - if ( - this.config.annotationRegions && - this.config.annotationRegions.length > 0 && - this.foregroundCtx - ) { - const previewWidth = this.config.previewWidth ?? this.config.width; - const previewHeight = this.config.previewHeight ?? this.config.height; - const scaleX = this.config.width / previewWidth; - const scaleY = this.config.height / previewHeight; - const scaleFactor = (scaleX + scaleY) / 2; - - await renderAnnotations( - this.foregroundCtx, - this.config.annotationRegions, - this.config.width, - this.config.height, - timeMs, - scaleFactor, - ); - } - - // Rotate foreground only; wallpaper (on compositeCanvas) stays untouched - if (willRotate && this.threeDPass && this.foregroundCanvas && this.foregroundCtx) { - const passCanvas = this.threeDPass.apply(this.foregroundCanvas, this.currentRotation3D); - const w = this.foregroundCanvas.width; - const h = this.foregroundCanvas.height; - this.foregroundCtx.clearRect(0, 0, w, h); - if (this.isLinux) { - // drawImage(webglCanvas) is unreliable on Linux/Wayland, so use readPixels - const pixels = this.threeDPass.readPixels(); - const imageData = this.foregroundCtx.createImageData(w, h); - imageData.data.set(pixels); - this.foregroundCtx.putImageData(imageData, 0, 0); - } else { - this.foregroundCtx.drawImage(passCanvas, 0, 0); - } - } - - // Apply shadow fresh on the rotated silhouette. Flat path already baked it in - // compositeWithShadows, so guard on willRotate to avoid doubling. Same 3-layer - // filter chain as the flat path to keep the soft Gaussian intact. - if ( - willRotate && - this.config.showShadow && - this.config.shadowIntensity > 0 && - this.shadowCanvas && - this.shadowCtx && - this.foregroundCanvas - ) { - const shadowCtx = this.shadowCtx; - const w = this.foregroundCanvas.width; - const h = this.foregroundCanvas.height; - shadowCtx.clearRect(0, 0, w, h); - shadowCtx.save(); - // NOT cacheable, unlike the flat path: this blurs foregroundCanvas, whose - // alpha carries the webcam, the cursor and the annotations — it changes - // every frame, so there is no geometry to key on. The 3D path stays at - // full price until it is restructured. - shadowCtx.filter = shadowFilterChain(this.config.shadowIntensity); - shadowCtx.drawImage(this.foregroundCanvas, 0, 0, w, h); - shadowCtx.restore(); - if (this.compositeCtx) { - this.compositeCtx.drawImage(this.shadowCanvas, 0, 0); - } - } else if (this.compositeCtx && this.foregroundCanvas) { - // Flat path or 3D-without-shadow: stamp foreground directly - this.compositeCtx.drawImage(this.foregroundCanvas, 0, 0); - } - } - - // Video's on-screen boundary including the zoom camera transform. The PIXI mask - // lives inside cameraContainer, so during zoom the visible video extends beyond - // the static maskRect and a static clip would crop it. Mirrors the preview. - private cameraAwareMaskRect() { - if (!this.layoutCache) return null; - const { x: maskX, y: maskY, width: maskW, height: maskH } = this.layoutCache.maskRect; - const camS = this.animationState.appliedScale; - const camX = this.animationState.x; - const camY = this.animationState.y; - // No stage clamping: the canvas clips to its own bounds, matching CSS inset(). - // Clamping x/y would pin rounded corners to the stage edge instead of the true - // mask boundary, mismatching preview/export when zoom/pan pushes the mask off-stage. - return { - x: camX + camS * maskX, - y: camY + camS * maskY, - width: camS * maskW, - height: camS * maskH, - br: this.layoutCache.maskBorderRadius * camS, - }; - } - - private async drawNativeCursor(timeMs: number) { - if (!this.foregroundCtx || !this.layoutCache) { - return; - } - - if ((this.config.cursorScale ?? 1) <= 0) { - resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); - return; - } - - const activeNativeCursor = resolveInterpolatedNativeCursorFrame( - this.config.cursorRecordingData, - timeMs, - ); - if (!activeNativeCursor) { - resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); - return; - } - // Position comes from the precomputed smoothed path (deterministic, matches preview); - // the frame still supplies the cursor image, type, and click timing. - const smoothedPos = getSmoothedCursorPath( - this.config.cursorRecordingData, - this.config.cursorSmoothing ?? 0, - )?.sampleAt(timeMs); - const displaySample = smoothedPos - ? { ...activeNativeCursor.sample, cx: smoothedPos.cx, cy: smoothedPos.cy } - : activeNativeCursor.sample; - - const projectedPoint = projectNativeCursorToLocal({ - cropRegion: this.config.cropRegion, - maskRect: this.layoutCache.croppedRect, - sample: displaySample, - }); - if (!projectedPoint) { - resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); - return; - } - - const renderAsset = resolveNativeCursorRenderAsset( - activeNativeCursor.asset, - 1, - displaySample, - this.config.cursorTheme, - ); - let image: HTMLImageElement; - try { - image = await this.getCursorImage(renderAsset); - } catch (error) { - this.warnOnce("native-cursor-image-load", "Failed to load native cursor asset", error); - return; - } - const scale = - Math.max(0, this.config.cursorScale ?? 1) * - getNativeCursorClickBounceScale( - this.config.cursorClickBounce ?? 0, - getNativeCursorClickBounceProgress(this.config.cursorRecordingData, timeMs), - ); - const appliedScale = this.animationState.appliedScale; - // Normalize cursor size to croppedRect.width (the painted video width). - // The preview path still uses screenRect.width; they agree in cover mode but - // differ in fit-to-height letterbox — known asymmetry pending the preview-path - // follow-up to project the cursor onto the cropped sub-rect as well. - const sizeNorm = - this.layoutCache.videoSize.width > 0 - ? this.layoutCache.maskRect.width / this.layoutCache.videoSize.width - : 1; - const canvasX = projectedPoint.x * appliedScale + this.animationState.x; - const canvasY = projectedPoint.y * appliedScale + this.animationState.y; - const blurPx = getNativeCursorMotionBlurPx({ - motionBlur: this.config.cursorMotionBlur ?? 0, - point: { x: canvasX, y: canvasY }, - state: this.nativeCursorMotionBlurState, - timeMs, - }); - // Clip only when explicitly enabled; by default the cursor may overflow the canvas - const cursorClip = this.config.cursorClipToBounds === true ? this.cameraAwareMaskRect() : null; - this.foregroundCtx.save(); - this.foregroundCtx.beginPath(); - if (cursorClip) { - this.foregroundCtx.roundRect( - cursorClip.x, - cursorClip.y, - cursorClip.width, - cursorClip.height, - cursorClip.br, - ); - this.foregroundCtx.clip(); - } - const previousFilter = this.foregroundCtx.filter; - if (blurPx > 0) { - this.foregroundCtx.filter = `blur(${blurPx.toFixed(2)}px)`; - } - this.foregroundCtx.drawImage( - image, - canvasX - renderAsset.hotspotX * scale * appliedScale * sizeNorm, - canvasY - renderAsset.hotspotY * scale * appliedScale * sizeNorm, - renderAsset.width * scale * appliedScale * sizeNorm, - renderAsset.height * scale * appliedScale * sizeNorm, - ); - this.foregroundCtx.filter = previousFilter; - this.foregroundCtx.restore(); - } - - private async getCursorImage(asset: { id: string; imageDataUrl: string }) { - const cachedImage = this.cursorImageCache.get(asset.id); - if (cachedImage) { - return cachedImage; - } - - const image = new Image(); - await new Promise((resolve, reject) => { - image.onload = () => resolve(); - image.onerror = () => reject(new Error(`Failed to load cursor asset ${asset.id}`)); - image.src = asset.imageDataUrl; - }); - - this.cursorImageCache.set(asset.id, image); - return image; - } - - private warnOnce(key: string, message: string, error: unknown) { - if (this.warnedKeys.has(key)) { - return; - } - this.warnedKeys.add(key); - console.warn(`[FrameRenderer] ${message}:`, error); - } - - private updateLayout(webcamFrame?: VideoFrame | null): void { - if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return; - - const { width, height } = this.config; - const { cropRegion, borderRadius = 0, padding = 0 } = this.config; - const videoWidth = this.config.videoWidth; - const videoHeight = this.config.videoHeight; - - const cropStartX = cropRegion.x; - const cropStartY = cropRegion.y; - const cropEndX = cropRegion.x + cropRegion.width; - const cropEndY = cropRegion.y + cropRegion.height; - - const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); - const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); - - // Padding is a percentage (0-100), where 50% ~ 0.8 scale. It applies to every - // preset — in the block layouts it insets the welded screen+camera block as - // one, which is the whole point of welding them (see `computeCompositeLayout`). - const paddingScale = 1.0 - (padding / 100) * 0.4; - const viewportWidth = width * paddingScale; - const viewportHeight = height * paddingScale; - const compositeLayout = computeCompositeLayout({ - canvasSize: { width, height }, - maxContentSize: { width: viewportWidth, height: viewportHeight }, - screenSize: { width: croppedVideoWidth, height: croppedVideoHeight }, - webcamSize: webcamFrame ? this.config.webcamSize : null, - layoutPreset: this.config.webcamLayoutPreset, - webcamSizePreset: this.config.webcamSizePreset, - webcamPosition: this.config.webcamPosition, - webcamMaskShape: this.config.webcamMaskShape, - }); - if (!compositeLayout) return; - - const screenRect = compositeLayout.screenRect; - - // Cover mode scales to fill the rect (may crop), otherwise fit-to-width - let scale: number; - if (compositeLayout.screenCover) { - scale = Math.max( - screenRect.width / croppedVideoWidth, - screenRect.height / croppedVideoHeight, - ); - } else { - scale = screenRect.width / croppedVideoWidth; - } - - this.videoSprite.width = videoWidth * scale; - this.videoSprite.height = videoHeight * scale; - - // Center the cropped region within the screenRect - const croppedDisplayWidth = croppedVideoWidth * scale; - const croppedDisplayHeight = croppedVideoHeight * scale; - const coverOffsetX = (screenRect.width - croppedDisplayWidth) / 2; - const coverOffsetY = (screenRect.height - croppedDisplayHeight) / 2; - - const cropPixelX = cropStartX * videoWidth * scale; - const cropPixelY = cropStartY * videoHeight * scale; - this.videoSprite.x = -cropPixelX + coverOffsetX; - this.videoSprite.y = -cropPixelY + coverOffsetY; - - this.videoContainer.x = screenRect.x; - this.videoContainer.y = screenRect.y; - - // Scale border radius by the export/preview canvas ratio - const previewWidth = this.config.previewWidth ?? this.config.width; - const previewHeight = this.config.previewHeight ?? this.config.height; - const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight); - const scaledBorderRadius = - compositeLayout.screenBorderRadius != null - ? compositeLayout.screenBorderRadius - : compositeLayout.screenCover - ? 0 - : borderRadius * canvasScaleFactor; - - // Retessellate only when the shape actually changes. The mask is a rounded - // rect: while nothing zooms or resizes it is byte-identical from frame to - // frame, and clear()/roundRect()/fill() rebuilds its geometry every time. - // A zoom easing does change it per frame — that is the case this cannot help. - if ( - this.legacyCompositor || - this.maskShape?.width !== screenRect.width || - this.maskShape?.height !== screenRect.height || - this.maskShape?.radius !== scaledBorderRadius - ) { - this.maskGraphics.clear(); - this.maskGraphics.roundRect(0, 0, screenRect.width, screenRect.height, scaledBorderRadius); - this.maskGraphics.fill({ color: 0xffffff }); - this.maskShape = { - width: screenRect.width, - height: screenRect.height, - radius: scaledBorderRadius, - }; - } - - // baseOffset is the stage position of the full (uncropped) sprite's top-left, matching - // preview semantics, so consumers (e.g. cursor highlight) can map normalized - // recording-space coords to stage coords uniformly: - // stagePos = baseOffset + (cx, cy) * (videoWidth, videoHeight) * baseScale - this.layoutCache = { - stageSize: { width, height }, - videoSize: { width: croppedVideoWidth, height: croppedVideoHeight }, - baseScale: scale, - baseOffset: { - x: compositeLayout.screenRect.x + coverOffsetX - cropPixelX, - y: compositeLayout.screenRect.y + coverOffsetY - cropPixelY, - }, - maskRect: compositeLayout.screenRect, - croppedRect: { - x: compositeLayout.screenRect.x + coverOffsetX, - y: compositeLayout.screenRect.y + coverOffsetY, - width: croppedDisplayWidth, - height: croppedDisplayHeight, - }, - maskBorderRadius: scaledBorderRadius, - webcamRect: compositeLayout.webcamRect, - }; - } - - private updateAnimationState(timeMs: number): number { - if (!this.cameraContainer || !this.layoutCache) return 0; - - this.animationState.cameraFullscreenProgress = computeCameraFullscreenProgress( - this.config.cameraFullscreenRegions ?? [], - timeMs, - ); - - const { region, strength, blendedScale, rotation3D, transition } = findDominantRegion( - this.config.zoomRegions, - timeMs, - { connectZooms: true, cursorTelemetry: this.config.cursorTelemetry }, - ); - - const defaultFocus = DEFAULT_FOCUS; - let targetScaleFactor = 1; - let targetFocus = { ...defaultFocus }; - let targetProgress = 0; - - this.currentRotation3D = - region && strength > 0 - ? lerpRotation3D(DEFAULT_ROTATION_3D, rotation3D, strength) - : { ...DEFAULT_ROTATION_3D }; - - if (region && strength > 0) { - const zoomScale = blendedScale ?? getZoomScale(region); - const regionFocus = clampFocusToScale(region.focus, zoomScale); - - targetScaleFactor = zoomScale; - targetFocus = regionFocus; - targetProgress = strength; - - // Adaptive smoothing for auto-follow mode - if (region.focusMode === "auto" && !transition) { - const raw = targetFocus; - const dtMs = this.prevAnimationTimeMs != null ? timeMs - this.prevAnimationTimeMs : 0; - const isZoomingIn = targetProgress < 0.999 && targetProgress >= this.prevTargetProgress; - if (targetProgress >= 0.999) { - // Full zoom: move faster when far, decelerate when close - const prev = this.smoothedAutoFocus ?? raw; - const smoothed = advanceFollowFocus(prev, raw, dtMs, AUTO_FOLLOW_PARAMS); - this.smoothedAutoFocus = smoothed; - targetFocus = smoothed; - } else if (isZoomingIn) { - // Track cursor directly while zooming in; keep ref in sync to avoid a snap - // when full-zoom begins - this.smoothedAutoFocus = raw; - } else { - // Zoom-out: keep smoothing to avoid a snap at the start - const prev = this.smoothedAutoFocus ?? raw; - const smoothed = advanceFollowFocus(prev, raw, dtMs, AUTO_FOLLOW_PARAMS); - this.smoothedAutoFocus = smoothed; - targetFocus = smoothed; - } - } else if (region.focusMode !== "auto") { - this.smoothedAutoFocus = null; - } - this.prevTargetProgress = targetProgress; - - if (transition) { - const startTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: transition.startScale, - zoomProgress: 1, - focusX: transition.startFocus.cx, - focusY: transition.startFocus.cy, - }); - const endTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: transition.endScale, - zoomProgress: 1, - focusX: transition.endFocus.cx, - focusY: transition.endFocus.cy, - }); - - const interpolatedTransform = { - scale: - startTransform.scale + - (endTransform.scale - startTransform.scale) * transition.progress, - x: startTransform.x + (endTransform.x - startTransform.x) * transition.progress, - y: startTransform.y + (endTransform.y - startTransform.y) * transition.progress, - }; - - targetScaleFactor = interpolatedTransform.scale; - targetFocus = computeFocusFromTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: interpolatedTransform.scale, - x: interpolatedTransform.x, - y: interpolatedTransform.y, - }); - targetProgress = 1; - } - } - - const state = this.animationState; - - const prevScale = state.appliedScale; - const prevX = state.x; - const prevY = state.y; - - state.scale = targetScaleFactor; - state.focusX = targetFocus.cx; - state.focusY = targetFocus.cy; - state.progress = targetProgress; - - const projectedTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: state.scale, - zoomProgress: state.progress, - focusX: state.focusX, - focusY: state.focusY, - }); - - // Spring-chase the eased target (same as preview) so exported motion glides past the jerk - // at the steep start of the ease. Stepped by content time; snapped on the first frame or - // any large time jump. - const dtMs = this.prevAnimationTimeMs != null ? timeMs - this.prevAnimationTimeMs : 0; - let appliedScale: number; - let appliedX: number; - let appliedY: number; - if (this.prevAnimationTimeMs == null || dtMs <= 0 || dtMs > 80) { - resetZoomSpring(this.zoomSpringState, projectedTransform); - appliedScale = projectedTransform.scale; - appliedX = projectedTransform.x; - appliedY = projectedTransform.y; - } else { - const sprung = stepZoomSpring(this.zoomSpringState, projectedTransform, dtMs); - appliedScale = sprung.scale; - appliedX = sprung.x; - appliedY = sprung.y; - } - - state.x = appliedX; - state.y = appliedY; - state.appliedScale = appliedScale; - - this.prevAnimationTimeMs = timeMs; - - return Math.max( - Math.abs(appliedScale - prevScale), - Math.abs(appliedX - prevX) / Math.max(1, this.layoutCache.stageSize.width), - Math.abs(appliedY - prevY) / Math.max(1, this.layoutCache.stageSize.height), - ); - } - - // On Linux/Wayland the implicit GPU-to-2D texture-sharing path behind - // drawImage(webglCanvas) can fail silently (EGL/Ozone), giving green/empty - // frames. gl.readPixels copies GPU to CPU directly, bypassing that path. - private readbackVideoCanvas(): HTMLCanvasElement { - const glCanvas = this.app!.canvas as HTMLCanvasElement; - const gl = - (glCanvas.getContext("webgl2") as WebGL2RenderingContext | null) ?? - (glCanvas.getContext("webgl") as WebGLRenderingContext | null); - - if (!gl || !this.rasterCanvas || !this.rasterCtx) { - return glCanvas; - } - - const w = glCanvas.width; - const h = glCanvas.height; - const buf = new Uint8Array(w * h * 4); - gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf); - - // readPixels returns rows bottom-to-top; flip vertically - const rowSize = w * 4; - const temp = new Uint8Array(rowSize); - for (let top = 0, bot = h - 1; top < bot; top++, bot--) { - const tOff = top * rowSize; - const bOff = bot * rowSize; - temp.set(buf.subarray(tOff, tOff + rowSize)); - buf.copyWithin(tOff, bOff, bOff + rowSize); - buf.set(temp, bOff); - } - - const imageData = new ImageData(new Uint8ClampedArray(buf.buffer), w, h); - this.rasterCtx.putImageData(imageData, 0, 0); - - return this.rasterCanvas; - } - - // applyShadowToRecording is false when the 3D pass will rotate this canvas next; - // the shadow is re-applied after rotation to avoid aliasing. - /** - * The wallpaper, blurred once instead of 1418 times. - * - * backgroundSprite is rasterised once at load and never touched again — it is - * a still image. Blurring it per frame recomputes an identical result every - * time; measured at ~5 ms/frame. - * - * Byte-identical by construction: the original blurs while scaling to w*h, so - * the blur lands in destination space. Doing that once into a w*h canvas and - * blitting it 1:1 is the same operation, hoisted. - * - * Returns null when there is nothing to hoist (no blur), so the caller keeps - * drawing the sprite directly rather than through a pointless copy. - */ - private blurredBackgroundLayer(w: number, h: number): HTMLCanvasElement | null { - if (!this.config.showBlur || !this.backgroundSprite || this.legacyCompositor) return null; - // Keyed on the sprite's identity, not just the size: loading a different - // wallpaper replaces the sprite, and a size-only check would keep serving - // the previous one's blur. - if ( - this.blurredBackgroundSource === this.backgroundSprite && - this.blurredBackground?.width === w && - this.blurredBackground.height === h - ) { - return this.blurredBackground; - } - const canvas = document.createElement("canvas"); - canvas.width = w; - canvas.height = h; - const ctx = canvas.getContext("2d"); - if (!ctx) return null; - ctx.filter = "blur(6px)"; // Canvas blur is weaker than CSS - ctx.drawImage(this.backgroundSprite, 0, 0, w, h); - ctx.filter = "none"; - this.blurredBackground = canvas; - this.blurredBackgroundSource = this.backgroundSprite; - return canvas; - } - - /** - * Everything the shadow's shape depends on. If two frames agree here, the - * filter output is identical and re-running it is pure waste. - * - * Taken from the world transforms rather than from the layout inputs, so a - * new way to move the recording cannot silently bypass the cache and serve a - * stale shadow. A zoom easing changes this every frame — by design: that case - * MUST miss. - */ - private shadowGeometryKey(): string { - const v = this.videoContainer?.worldTransform; - const s = this.app?.stage?.worldTransform; - const m = this.maskShape; - return [ - v ? `${v.a},${v.b},${v.c},${v.d},${v.tx},${v.ty}` : "-", - s ? `${s.a},${s.b},${s.c},${s.d},${s.tx},${s.ty}` : "-", - m ? `${m.width},${m.height},${m.radius}` : "-", - this.config.shadowIntensity, - ].join("|"); - } - - /** - * The shadow layer, rebuilt only when the geometry moves. - * - * drop-shadow reads SourceAlpha only, and videoCanvas's alpha is just the - * masked rounded rect — so the filter blurs 2M pixels of VIDEO to compute - * something that depends on nothing but the silhouette. Measured: ~30 ms per - * frame, twice the cost of the entire rest of the compositor. - * - * So it runs against the silhouette instead, and the silhouette is taken from - * videoCanvas's own alpha (source-in over black) rather than re-derived from - * the layout — same pixels, same anti-aliasing, nothing to keep in sync. - * - * Returns the filter output, which is `silhouette OVER shadow`: the caller - * draws videoCanvas on top, and it covers the silhouette exactly. - */ - private cachedShadowLayer( - videoCanvas: HTMLCanvasElement, - w: number, - h: number, - ): HTMLCanvasElement | null { - const key = this.shadowGeometryKey(); - if (this.shadowCache?.key === key) { - this.shadowCacheHits++; - return this.shadowCache.canvas; - } - this.shadowCacheMisses++; - - if (!this.shadowSilhouetteCanvas) { - this.shadowSilhouetteCanvas = document.createElement("canvas"); - this.shadowSilhouetteCanvas.width = w; - this.shadowSilhouetteCanvas.height = h; - } - if (!this.shadowCache) { - const canvas = document.createElement("canvas"); - canvas.width = w; - canvas.height = h; - this.shadowCache = { key: "", canvas }; - } - const silCtx = this.shadowSilhouetteCanvas.getContext("2d"); - const outCtx = this.shadowCache.canvas.getContext("2d"); - if (!silCtx || !outCtx) return null; - - // Silhouette: videoCanvas's exact alpha, filled black. - silCtx.globalCompositeOperation = "copy"; - silCtx.drawImage(videoCanvas, 0, 0, w, h); - silCtx.globalCompositeOperation = "source-in"; - silCtx.fillStyle = "#000"; - silCtx.fillRect(0, 0, w, h); - silCtx.globalCompositeOperation = "source-over"; - - outCtx.globalCompositeOperation = "copy"; - // Diagnostic arm: everything but the gaussians, so the pair prices them. - if (!this.shadowNoFilter) outCtx.filter = shadowFilterChain(this.config.shadowIntensity); - outCtx.drawImage(this.shadowSilhouetteCanvas, 0, 0, w, h); - outCtx.filter = "none"; - outCtx.globalCompositeOperation = "source-over"; - - this.shadowCache.key = key; - return this.shadowCache.canvas; - } - - private compositeWithShadows( - webcamFrame: VideoFrame | null | undefined, - applyShadowToRecording: boolean, - ): void { - if ( - !this.compositeCanvas || - !this.compositeCtx || - !this.foregroundCanvas || - !this.foregroundCtx || - !this.app - ) - return; - - const videoCanvas = this.isLinux - ? this.readbackVideoCanvas() - : (this.app.canvas as HTMLCanvasElement); - - const bgCtx = this.compositeCtx; - const fgCtx = this.foregroundCtx; - const w = this.compositeCanvas.width; - const h = this.compositeCanvas.height; - - // Background (compositeCanvas): wallpaper only. Stays flat, never touched by the - // 3D rotation pass, matching the preview. - // - // "copy" replaces the whole destination, so the clearRect that used to - // precede it is redundant: the draw below covers all w*h. Two full-frame - // passes become one. It is only safe BECAUSE the draw covers everything — - // hence the else branch below still clears explicitly. - if (this.legacyCompositor) bgCtx.clearRect(0, 0, w, h); - if (this.backgroundSprite) { - const bgCanvas = this.blurredBackgroundLayer(w, h) ?? this.backgroundSprite; - const stillNeedsBlur = this.config.showBlur && bgCanvas === this.backgroundSprite; - bgCtx.save(); - if (!this.legacyCompositor) bgCtx.globalCompositeOperation = "copy"; - if (stillNeedsBlur) { - bgCtx.filter = "blur(6px)"; // Canvas blur is weaker than CSS - } - bgCtx.drawImage(bgCanvas, 0, 0, w, h); - bgCtx.restore(); - } else { - // Nothing is drawn, so this is the only thing dropping the previous frame. - bgCtx.clearRect(0, 0, w, h); - console.warn("[FrameRenderer] No background sprite found during compositing!"); - } - - // Foreground (transparent): recording + webcam. Shadow baked here only on the - // flat path; the 3D path applies it after rotation (see renderFrame). - // - // Same trick as the background: both branches below draw a full w*h image, - // so "copy" subsumes the clear. restore() puts source-over back before the - // webcam is drawn on top — with "copy" still set, it would wipe the frame. - if (this.legacyCompositor) fgCtx.clearRect(0, 0, w, h); - fgCtx.save(); - if (!this.legacyCompositor) fgCtx.globalCompositeOperation = "copy"; - if ( - applyShadowToRecording && - this.config.showShadow && - this.config.shadowIntensity > 0 && - this.shadowCanvas && - this.shadowCtx - ) { - const cached = this.legacyCompositor ? null : this.cachedShadowLayer(videoCanvas, w, h); - if (cached) { - // The filter ran once for this geometry; per frame this is one blit. - fgCtx.drawImage(cached, 0, 0, w, h); - fgCtx.globalCompositeOperation = "source-over"; - fgCtx.drawImage(videoCanvas, 0, 0, w, h); - } else { - const shadowCtx = this.shadowCtx; - shadowCtx.clearRect(0, 0, w, h); - shadowCtx.save(); - shadowCtx.filter = shadowFilterChain(this.config.shadowIntensity); - shadowCtx.drawImage(videoCanvas, 0, 0, w, h); - shadowCtx.restore(); - fgCtx.drawImage(this.shadowCanvas, 0, 0, w, h); - } - } else { - fgCtx.drawImage(videoCanvas, 0, 0, w, h); - } - fgCtx.restore(); - - const webcamRect = this.layoutCache?.webcamRect ?? null; - if (webcamFrame && webcamRect) { - const preset = getWebcamLayoutPresetDefinition(this.config.webcamLayoutPreset); - const cameraFullProgress = this.animationState.cameraFullscreenProgress; - let drawRect: StyledRenderRect; - if (cameraFullProgress > 0) { - // Full Camera takes over the webcam's size/position entirely, growing it to BE - // the frame — no margin, no rounding, no mask left (see - // computeCameraFullscreenRect). Reactive zoom is ignored for this frame (see - // design notes 6.4): mixing "shrink for zoom" and "grow to full" in the same frame - // doesn't make sense. - drawRect = computeCameraFullscreenRect( - webcamRect, - { width: this.config.width, height: this.config.height }, - cameraFullProgress, - ); - } else { - // Scale the PiP webcam inversely with the eased zoom, anchoring the shrink to the - // docked corner (bottom-right by default) like the preview, so it stays flush to the - // edges instead of drifting toward center. - const reactiveFactor = resolveWebcamReactiveZoom( - this.config.webcamLayoutPreset, - this.config.webcamReactiveZoom, - ) - ? reactiveWebcamScale(this.animationState.appliedScale) - : 1; - const camPos = this.config.webcamPosition; - const biasX = (camPos ? camPos.cx >= 0.5 : true) ? 1 : 0; - const biasY = (camPos ? camPos.cy >= 0.5 : true) ? 1 : 0; - drawRect = - reactiveFactor < 1 - ? { - width: webcamRect.width * reactiveFactor, - height: webcamRect.height * reactiveFactor, - x: webcamRect.x + webcamRect.width * (1 - reactiveFactor) * biasX, - y: webcamRect.y + webcamRect.height * (1 - reactiveFactor) * biasY, - borderRadius: webcamRect.borderRadius * reactiveFactor, - } - : webcamRect; - } - const sourceWidth = - ("displayWidth" in webcamFrame && webcamFrame.displayWidth > 0 - ? webcamFrame.displayWidth - : webcamFrame.codedWidth) || webcamRect.width; - const sourceHeight = - ("displayHeight" in webcamFrame && webcamFrame.displayHeight > 0 - ? webcamFrame.displayHeight - : webcamFrame.codedHeight) || webcamRect.height; - const sourceAspect = sourceWidth / sourceHeight; - // The crop follows the box actually being DRAWN, not the layout box: Full - // Camera walks the box from the layout's aspect ratio to the frame's, and a - // crop pinned to the layout's aspect would stretch the face all the way there. - const targetAspect = drawRect.width / drawRect.height; - const sourceCropWidth = - sourceAspect > targetAspect ? Math.round(sourceHeight * targetAspect) : sourceWidth; - const sourceCropHeight = - sourceAspect > targetAspect ? sourceHeight : Math.round(sourceWidth / targetAspect); - const sourceCropX = Math.max(0, Math.round((sourceWidth - sourceCropWidth) / 2)); - const sourceCropY = Math.max(0, Math.round((sourceHeight - sourceCropHeight) / 2)); - fgCtx.save(); - drawCanvasClipPath( - fgCtx, - drawRect.x, - drawRect.y, - drawRect.width, - drawRect.height, - drawRect.maskShape ?? this.config.webcamMaskShape ?? "rectangle", - drawRect.borderRadius, - ); - // The drop shadow belongs to the floating bubble, so it recedes with it: at - // full screen the camera is the frame and nothing may frame it. Zero blur and - // zero offset leave the shadow exactly under the opaque camera, invisible. - const shadowFade = 1 - cameraFullProgress; - if (preset.shadow && shadowFade > 0) { - fgCtx.shadowColor = preset.shadow.color; - fgCtx.shadowBlur = preset.shadow.blur * shadowFade; - fgCtx.shadowOffsetX = preset.shadow.offsetX * shadowFade; - fgCtx.shadowOffsetY = preset.shadow.offsetY * shadowFade; - } - fgCtx.fillStyle = "#000000"; - fgCtx.fill(); - fgCtx.clip(); - drawWebcamFrameImage( - fgCtx, - webcamFrame as unknown as CanvasImageSource, - { - x: sourceCropX, - y: sourceCropY, - width: sourceCropWidth, - height: sourceCropHeight, - }, - { - x: drawRect.x, - y: drawRect.y, - width: drawRect.width, - height: drawRect.height, - }, - this.config.webcamMirrored, - ); - fgCtx.restore(); - } - } - - getCanvas(): HTMLCanvasElement { - if (!this.compositeCanvas) { - throw new Error("Renderer not initialized"); - } - return this.compositeCanvas; - } - - /** Shadow cache hits/misses for the frames rendered so far — see the fields. */ - shadowCacheStats(): { hits: number; misses: number } { - return { hits: this.shadowCacheHits, misses: this.shadowCacheMisses }; - } - - /** - * Bench-only sync point — gate G0, technical-documentation/engineering/rendering-performance.md - * - * Draw calls queue GPU work; the cost lands on whichever operation first needs - * the pixels, which today is the encoder — so the compositor's execution is - * billed to `encodeWait`. Forcing execution HERE moves that cost into the - * caller's `fence` timer and the stage table finally says where the time goes. - * - * gl.finish() drains the Pixi context. Canvas2D has no finish(): the 1×1 - * getImageData is the equivalent — it forces the pending raster work of the - * whole surface chain (pixi canvas → shadow/foreground → composite) to - * complete, while transferring a single pixel. - */ - finishGpuWork(): void { - const gl = (this.app?.renderer as unknown as { gl?: WebGLRenderingContext } | null)?.gl; - gl?.finish(); - this.compositeCtx?.getImageData(0, 0, 1, 1); - } - - destroy(): void { - if (this.videoSprite) { - this.videoSprite.destroy(); - this.videoSprite = null; - } - this.backgroundSprite = null; - if (this.app) { - this.app.destroy(true, { - children: true, - texture: true, - textureSource: true, - }); - this.app = null; - } - this.cameraContainer = null; - this.videoContainer = null; - this.maskGraphics = null; - this.blurFilter = null; - this.motionBlurFilter = null; - this.shadowCanvas = null; - this.shadowCtx = null; - this.compositeCanvas = null; - this.compositeCtx = null; - this.foregroundCanvas = null; - this.foregroundCtx = null; - this.rasterCanvas = null; - this.rasterCtx = null; - if (this.threeDPass) { - this.threeDPass.destroy(); - this.threeDPass = null; - } - this.cursorImageCache.clear(); - } -} diff --git a/src/lib/exporter/gifExporter.browser.test.ts b/src/lib/exporter/gifExporter.browser.test.ts deleted file mode 100644 index 1d9607608..000000000 --- a/src/lib/exporter/gifExporter.browser.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { describe, expect, it } from "vitest"; -import sampleVideoUrl from "../../../tests/fixtures/sample.webm?url"; -import { BackgroundLoadError } from "../wallpaper"; -import { GifExporter } from "./gifExporter"; -import type { ExportProgress } from "./types"; - -describe("GifExporter (real browser)", () => { - it("exports a valid GIF blob from a real video", async () => { - const progressEvents: ExportProgress[] = []; - - const exporter = new GifExporter({ - videoUrl: sampleVideoUrl, - width: 320, - height: 180, - frameRate: 15, - loop: true, - sizePreset: "medium", - wallpaper: "#1a1a2e", - zoomRegions: [], - showShadow: false, - shadowIntensity: 0, - showBlur: false, - cropRegion: { x: 0, y: 0, width: 1, height: 1 }, - onProgress: (p) => progressEvents.push(p), - }); - - const result = await exporter.export(); - - expect(result.success, result.error).toBe(true); - expect(result.blob).toBeInstanceOf(Blob); - - const buf = await result.blob!.arrayBuffer(); - const header = new TextDecoder().decode(new Uint8Array(buf, 0, 6)); - expect(header).toMatch(/^GIF8[79]a/); - - expect(result.blob!.size).toBeGreaterThan(1024); - - expect(progressEvents.length).toBeGreaterThan(0); - - const finalizing = progressEvents.filter((p) => p.phase === "finalizing"); - expect(finalizing.length).toBeGreaterThan(0); - expect(finalizing.at(-1)!.percentage).toBe(100); - }); - - it("exports successfully with an image wallpaper (served by Vite dev server)", async () => { - const exporter = new GifExporter({ - videoUrl: sampleVideoUrl, - width: 320, - height: 180, - frameRate: 15, - loop: true, - sizePreset: "medium", - wallpaper: "/wallpapers/wallpaper1.jpg", - zoomRegions: [], - showShadow: false, - shadowIntensity: 0, - showBlur: false, - cropRegion: { x: 0, y: 0, width: 1, height: 1 }, - }); - - const result = await exporter.export(); - expect(result.success, result.error).toBe(true); - expect(result.blob!.size).toBeGreaterThan(1024); - }); - - it("throws BackgroundLoadError when wallpaper fails to load (no silent black fallback)", async () => { - const exporter = new GifExporter({ - videoUrl: sampleVideoUrl, - width: 320, - height: 180, - frameRate: 15, - loop: true, - sizePreset: "medium", - wallpaper: "/wallpapers/does-not-exist.jpg", - zoomRegions: [], - showShadow: false, - shadowIntensity: 0, - showBlur: false, - cropRegion: { x: 0, y: 0, width: 1, height: 1 }, - }); - - const rejection = exporter.export(); - await expect(rejection).rejects.toBeInstanceOf(BackgroundLoadError); - await expect(rejection).rejects.toMatchObject({ - url: expect.stringContaining("does-not-exist"), - }); - }); -}); diff --git a/src/lib/exporter/gifExporter.test.ts b/src/lib/exporter/gifExporter.test.ts deleted file mode 100644 index 1ad16717a..000000000 --- a/src/lib/exporter/gifExporter.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { calculateOutputDimensions } from "./gifExporter"; -import { GIF_SIZE_PRESETS } from "./types"; - -describe("calculateOutputDimensions", () => { - it("uses the selected aspect ratio for scaled GIF exports", () => { - expect(calculateOutputDimensions(1080, 1920, "medium", GIF_SIZE_PRESETS, 16 / 9)).toEqual({ - width: 1280, - height: 720, - }); - }); - - it("fits original-size GIF exports within the source bounds at the selected aspect ratio", () => { - expect(calculateOutputDimensions(1080, 1920, "original", GIF_SIZE_PRESETS, 16 / 9)).toEqual({ - width: 1080, - height: 606, - }); - }); -}); diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts deleted file mode 100644 index 60f57bc50..000000000 --- a/src/lib/exporter/gifExporter.ts +++ /dev/null @@ -1,420 +0,0 @@ -import GIF from "gif.js"; -import type { - AnnotationRegion, - CameraFullscreenRegion, - CropRegion, - SpeedRegion, - TrimRegion, - WebcamLayoutPreset, - WebcamSizePreset, - ZoomRegion, -} from "@/components/video-editor/types"; -import { BackgroundLoadError } from "@/lib/wallpaper"; -import type { CursorRecordingData } from "@/native/contracts"; -import { getPlatform } from "@/utils/platformUtils"; -import { type CropScheduleEntry, resolveCropAt } from "./cropSchedule"; -import { FrameRenderer } from "./frameRenderer"; -import { StreamingVideoDecoder } from "./streamingDecoder"; -import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue"; -import type { - ExportProgress, - ExportResult, - GIF_SIZE_PRESETS, - GifFrameRate, - GifSizePreset, -} from "./types"; - -const GIF_WORKER_URL = new URL("gif.js/dist/gif.worker.js", import.meta.url).toString(); - -interface GifExporterConfig { - videoUrl: string; - webcamVideoUrl?: string; - width: number; - height: number; - frameRate: GifFrameRate; - loop: boolean; - sizePreset: GifSizePreset; - wallpaper: string; - zoomRegions: ZoomRegion[]; - cameraFullscreenRegions?: CameraFullscreenRegion[]; - trimRegions?: TrimRegion[]; - speedRegions?: SpeedRegion[]; - showShadow: boolean; - shadowIntensity: number; - showBlur: boolean; - motionBlurAmount?: number; - borderRadius?: number; - padding?: number; - videoPadding?: number; - cropRegion: CropRegion; - /** Per-clip crop, in source-media time — see VideoExporterConfig.cropSchedule. */ - cropSchedule?: CropScheduleEntry[]; - webcamLayoutPreset?: WebcamLayoutPreset; - webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; - webcamMirrored?: boolean; - webcamReactiveZoom?: boolean; - webcamSizePreset?: WebcamSizePreset; - webcamPosition?: { cx: number; cy: number } | null; - cursorRecordingData?: CursorRecordingData | null; - cursorScale?: number; - cursorSmoothing?: number; - cursorMotionBlur?: number; - cursorClickBounce?: number; - cursorClipToBounds?: boolean; - cursorTheme?: string; - annotationRegions?: AnnotationRegion[]; - previewWidth?: number; - previewHeight?: number; - cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[]; - cursorClickTimestamps?: number[]; - onProgress?: (progress: ExportProgress) => void; -} - -/** - * Calculate output dimensions based on size preset and source dimensions while preserving aspect ratio. - * @param sourceWidth - Original video width - * @param sourceHeight - Original video height - * @param sizePreset - The size preset to use - * @param sizePresets - The size presets configuration - * @returns The calculated output dimensions - */ -export function calculateOutputDimensions( - sourceWidth: number, - sourceHeight: number, - sizePreset: GifSizePreset, - sizePresets: typeof GIF_SIZE_PRESETS, - targetAspectRatio = sourceWidth / sourceHeight, -): { width: number; height: number } { - const preset = sizePresets[sizePreset]; - const maxHeight = preset.maxHeight; - const aspectRatio = - Number.isFinite(targetAspectRatio) && targetAspectRatio > 0 - ? targetAspectRatio - : sourceWidth / sourceHeight; - - const toEven = (value: number) => { - const evenValue = Math.max(2, Math.floor(value / 2) * 2); - return evenValue; - }; - - if (sizePreset === "original") { - const sourceAspect = sourceWidth / sourceHeight; - if (aspectRatio >= sourceAspect) { - const width = toEven(sourceWidth); - const height = toEven(width / aspectRatio); - return { width, height }; - } - - const height = toEven(sourceHeight); - const width = toEven(height * aspectRatio); - return { width, height }; - } - - const targetHeight = maxHeight; - const targetWidth = Math.round(targetHeight * aspectRatio); - - return { - width: toEven(targetWidth), - height: toEven(targetHeight), - }; -} - -export class GifExporter { - private config: GifExporterConfig; - private streamingDecoder: StreamingVideoDecoder | null = null; - private webcamDecoder: StreamingVideoDecoder | null = null; - private renderer: FrameRenderer | null = null; - private gif: GIF | null = null; - private cancelled = false; - - constructor(config: GifExporterConfig) { - this.config = config; - } - - async export(): Promise { - let webcamFrameQueue: TimestampedVideoFrameQueue | null = null; - - const warnings: string[] = []; - const onWarning = (message: string) => warnings.push(message); - - try { - const platform = getPlatform(); - - this.cleanup(); - this.cancelled = false; - - this.streamingDecoder = new StreamingVideoDecoder(); - const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl); - let webcamInfo: Awaited> | null = null; - if (this.config.webcamVideoUrl) { - this.webcamDecoder = new StreamingVideoDecoder(); - webcamInfo = await this.webcamDecoder.loadMetadata(this.config.webcamVideoUrl); - } - - this.renderer = new FrameRenderer({ - width: this.config.width, - height: this.config.height, - wallpaper: this.config.wallpaper, - zoomRegions: this.config.zoomRegions, - cameraFullscreenRegions: this.config.cameraFullscreenRegions, - showShadow: this.config.showShadow, - shadowIntensity: this.config.shadowIntensity, - showBlur: this.config.showBlur, - motionBlurAmount: this.config.motionBlurAmount, - borderRadius: this.config.borderRadius, - padding: this.config.padding, - cropRegion: this.config.cropRegion, - cursorRecordingData: this.config.cursorRecordingData, - cursorScale: this.config.cursorScale, - cursorSmoothing: this.config.cursorSmoothing, - cursorMotionBlur: this.config.cursorMotionBlur, - cursorClickBounce: this.config.cursorClickBounce, - cursorClipToBounds: this.config.cursorClipToBounds, - cursorTheme: this.config.cursorTheme, - videoWidth: videoInfo.width, - videoHeight: videoInfo.height, - webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, - webcamLayoutPreset: this.config.webcamLayoutPreset, - webcamMaskShape: this.config.webcamMaskShape, - webcamMirrored: this.config.webcamMirrored, - webcamReactiveZoom: this.config.webcamReactiveZoom, - webcamSizePreset: this.config.webcamSizePreset, - webcamPosition: this.config.webcamPosition, - annotationRegions: this.config.annotationRegions, - speedRegions: this.config.speedRegions, - previewWidth: this.config.previewWidth, - previewHeight: this.config.previewHeight, - cursorTelemetry: this.config.cursorTelemetry, - cursorClickTimestamps: this.config.cursorClickTimestamps, - platform, - }); - await this.renderer.initialize(); - - // gif.js repeat: 0 = infinite loop, 1 = play once - const repeat = this.config.loop ? 0 : 1; - const cores = navigator.hardwareConcurrency || 4; - const WORKER_COUNT = Math.max(1, Math.min(8, cores - 1)); - this.gif = new GIF({ - workers: WORKER_COUNT, - quality: 10, - width: this.config.width, - height: this.config.height, - workerScript: GIF_WORKER_URL, - repeat, - background: "#000000", - transparent: null, - dither: "FloydSteinberg", - }); - - // Effective duration and frame count, excluding trim regions - const { effectiveDuration, totalFrames } = this.streamingDecoder.getExportMetrics( - this.config.frameRate, - this.config.trimRegions, - this.config.speedRegions, - ); - - // gif.js wants frame delay in ms - const frameDelay = Math.round(1000 / this.config.frameRate); - - console.log("[GifExporter] Original duration:", videoInfo.duration, "s"); - console.log("[GifExporter] Effective duration:", effectiveDuration, "s"); - console.log("[GifExporter] Total frames to export:", totalFrames); - console.log("[GifExporter] Frame rate:", this.config.frameRate, "FPS"); - console.log("[GifExporter] Frame delay:", frameDelay, "ms"); - console.log("[GifExporter] Loop:", this.config.loop ? "infinite" : "once"); - console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)"); - - let frameIndex = 0; - webcamFrameQueue = this.config.webcamVideoUrl ? new TimestampedVideoFrameQueue() : null; - let stopWebcamDecode = false; - let webcamDecodeError: Error | null = null; - const webcamDecodePromise = - this.webcamDecoder && webcamFrameQueue - ? (() => { - const queue = webcamFrameQueue; - return this.webcamDecoder - .decodeAll( - this.config.frameRate, - this.config.trimRegions, - this.config.speedRegions, - async (webcamFrame, _exportTimestampUs, webcamSourceTimestampMs) => { - while (queue.length >= 12 && !this.cancelled && !stopWebcamDecode) { - await new Promise((resolve) => setTimeout(resolve, 2)); - } - if (this.cancelled || stopWebcamDecode) { - webcamFrame.close(); - return; - } - queue.enqueue(webcamFrame, webcamSourceTimestampMs); - }, - onWarning, - ) - .catch((error) => { - webcamDecodeError = error instanceof Error ? error : new Error(String(error)); - throw error; - }) - .finally(() => { - if (webcamDecodeError) { - queue.fail(webcamDecodeError); - } else { - queue.close(); - } - }); - })() - : null; - - // Stream decode and process frames, no seeking - await this.streamingDecoder.decodeAll( - this.config.frameRate, - this.config.trimRegions, - this.config.speedRegions, - async (videoFrame, _exportTimestampUs, sourceTimestampMs) => { - let webcamFrame: VideoFrame | null = null; - try { - if (this.cancelled) { - return; - } - - webcamFrame = webcamFrameQueue - ? await webcamFrameQueue.frameAt(sourceTimestampMs) - : null; - const renderer = this.renderer; - if (this.cancelled || !renderer) { - return; - } - - const sourceTimestampUs = sourceTimestampMs * 1000; // us - renderer.setCropRegion( - resolveCropAt( - this.config.cropSchedule, - sourceTimestampMs / 1000, - this.config.cropRegion, - ), - ); - await renderer.renderFrame(videoFrame, sourceTimestampUs, webcamFrame); - - const canvas = renderer.getCanvas(); - - this.gif!.addFrame(canvas, { delay: frameDelay, copy: true }); - - frameIndex++; - - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame: frameIndex, - totalFrames, - percentage: (frameIndex / totalFrames) * 100, - estimatedTimeRemaining: 0, - }); - } - } finally { - videoFrame.close(); - webcamFrame?.close(); - } - }, - onWarning, - ); - - if (this.cancelled) { - return { success: false, error: "Export cancelled" }; - } - - stopWebcamDecode = true; - webcamFrameQueue?.destroy(); - this.webcamDecoder?.cancel(); - await webcamDecodePromise; - - // Now in the finalizing phase - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame: totalFrames, - totalFrames, - percentage: 100, - estimatedTimeRemaining: 0, - phase: "finalizing", - }); - } - - const blob = await new Promise((resolve, _reject) => { - this.gif!.on("finished", (blob: Blob) => { - resolve(blob); - }); - - this.gif!.on("progress", (progress: number) => { - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame: totalFrames, - totalFrames, - percentage: 100, - estimatedTimeRemaining: 0, - phase: "finalizing", - renderProgress: Math.round(progress * 100), - }); - } - }); - - // gif.js has no typed 'error' event; the outer try/catch handles failures - this.gif!.render(); - }); - - return { success: true, blob, warnings: warnings.length > 0 ? warnings : undefined }; - } catch (error) { - if (error instanceof BackgroundLoadError) { - throw error; - } - console.error("GIF Export error:", error); - return { - success: false, - error: error instanceof Error ? error.message : String(error), - }; - } finally { - webcamFrameQueue?.destroy(); - this.cleanup(); - } - } - - cancel(): void { - this.cancelled = true; - if (this.streamingDecoder) { - this.streamingDecoder.cancel(); - } - if (this.webcamDecoder) { - this.webcamDecoder.cancel(); - } - if (this.gif) { - this.gif.abort(); - } - this.cleanup(); - } - - private cleanup(): void { - if (this.streamingDecoder) { - try { - this.streamingDecoder.destroy(); - } catch (e) { - console.warn("Error destroying streaming decoder:", e); - } - this.streamingDecoder = null; - } - - if (this.webcamDecoder) { - try { - this.webcamDecoder.destroy(); - } catch (e) { - console.warn("Error destroying webcam decoder:", e); - } - this.webcamDecoder = null; - } - - if (this.renderer) { - try { - this.renderer.destroy(); - } catch (e) { - console.warn("Error destroying renderer:", e); - } - this.renderer = null; - } - - this.gif = null; - } -} diff --git a/src/lib/exporter/index.ts b/src/lib/exporter/index.ts index 33cabd3a6..299e45c54 100644 --- a/src/lib/exporter/index.ts +++ b/src/lib/exporter/index.ts @@ -1,6 +1,3 @@ -export { type CropScheduleEntry, resolveCropAt } from "./cropSchedule"; -export { FrameRenderer } from "./frameRenderer"; -export { calculateOutputDimensions, GifExporter } from "./gifExporter"; export { calculateEffectiveSourceDimensions, calculateMp4ExportSettings, @@ -14,6 +11,7 @@ export type { ExportQuality, ExportResult, ExportSettings, + ExportVideoCodec, GifExportConfig, GifFrameRate, GifSizePreset, diff --git a/src/lib/exporter/threeDPass.ts b/src/lib/exporter/threeDPass.ts deleted file mode 100644 index 657c5ff2a..000000000 --- a/src/lib/exporter/threeDPass.ts +++ /dev/null @@ -1,341 +0,0 @@ -import type { Rotation3D } from "@/components/video-editor/types"; -import { - computeRotation3DContainScale, - isRotation3DIdentity, - rotation3DPerspective, -} from "@/components/video-editor/types"; - -// Rotation math is done in CSS convention (+y down) to match the preview, then -// gl_Position.y is flipped so WebGL clip space (+y up) lands the input's top edge -// at the top of the viewport. -const VERTEX_SHADER = `#version 300 es -in vec2 aPos; -in vec2 aUV; -out vec2 vUV; -uniform mat4 uMvp; -uniform vec2 uSize; -void main() { - vUV = aUV; - vec2 px = (aPos - 0.5) * uSize; - vec4 clip = uMvp * vec4(px, 0.0, 1.0); - clip.y = -clip.y; - gl_Position = clip; -} -`; - -const FRAGMENT_SHADER = `#version 300 es -precision highp float; -in vec2 vUV; -out vec4 fragColor; -uniform sampler2D uTex; -void main() { - fragColor = texture(uTex, vUV); -} -`; - -function deg2rad(deg: number): number { - return (deg * Math.PI) / 180; -} - -function multiplyMat4(a: Float32Array, b: Float32Array): Float32Array { - const out = new Float32Array(16); - for (let i = 0; i < 4; i += 1) { - for (let j = 0; j < 4; j += 1) { - let s = 0; - for (let k = 0; k < 4; k += 1) { - s += a[k * 4 + j] * b[i * 4 + k]; - } - out[i * 4 + j] = s; - } - } - return out; -} - -function rotationXMat(rad: number): Float32Array { - const c = Math.cos(rad); - const s = Math.sin(rad); - return new Float32Array([1, 0, 0, 0, 0, c, s, 0, 0, -s, c, 0, 0, 0, 0, 1]); -} - -function rotationYMat(rad: number): Float32Array { - const c = Math.cos(rad); - const s = Math.sin(rad); - return new Float32Array([c, 0, -s, 0, 0, 1, 0, 0, s, 0, c, 0, 0, 0, 0, 1]); -} - -function rotationZMat(rad: number): Float32Array { - const c = Math.cos(rad); - const s = Math.sin(rad); - return new Float32Array([c, s, 0, 0, -s, c, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]); -} - -function translationMat(x: number, y: number, z: number): Float32Array { - return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, x, y, z, 1]); -} - -function perspectiveMat(fovY: number, aspect: number, near: number, far: number): Float32Array { - const f = 1 / Math.tan(fovY / 2); - const nf = 1 / (near - far); - return new Float32Array([ - f / aspect, - 0, - 0, - 0, - 0, - f, - 0, - 0, - 0, - 0, - (far + near) * nf, - -1, - 0, - 0, - 2 * far * near * nf, - 0, - ]); -} - -function scaleMat(s: number): Float32Array { - return new Float32Array([s, 0, 0, 0, 0, s, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]); -} - -export function buildMvpMatrix(rot: Rotation3D, w: number, h: number): Float32Array { - const rx = rotationXMat(deg2rad(rot.rotationX)); - const ry = rotationYMat(deg2rad(rot.rotationY)); - const rz = rotationZMat(deg2rad(rot.rotationZ)); - const rotMat = multiplyMat4(rz, multiplyMat4(ry, rx)); - - const perspective = rotation3DPerspective(w, h); - const containScale = computeRotation3DContainScale(rot, w, h, perspective); - const rotScaled = multiplyMat4(rotMat, scaleMat(containScale)); - - const d = perspective; - const fovY = 2 * Math.atan2(h / 2, d); - const proj = perspectiveMat(fovY, w / h, 0.1, d * 4 + Math.max(w, h)); - const view = translationMat(0, 0, -d); - return multiplyMat4(proj, multiplyMat4(view, rotScaled)); -} - -function compileShader(gl: WebGL2RenderingContext, type: number, source: string): WebGLShader { - const shader = gl.createShader(type); - if (!shader) throw new Error("Failed to create shader"); - gl.shaderSource(shader, source); - gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - const info = gl.getShaderInfoLog(shader); - gl.deleteShader(shader); - throw new Error(`Shader compile failed: ${info}`); - } - return shader; -} - -function createProgram(gl: WebGL2RenderingContext): WebGLProgram { - const vs = compileShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER); - const fs = compileShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER); - const program = gl.createProgram(); - if (!program) throw new Error("Failed to create program"); - gl.attachShader(program, vs); - gl.attachShader(program, fs); - gl.linkProgram(program); - if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - const info = gl.getProgramInfoLog(program); - gl.deleteProgram(program); - throw new Error(`Program link failed: ${info}`); - } - gl.deleteShader(vs); - gl.deleteShader(fs); - return program; -} - -export interface ThreeDPass { - apply(srcCanvas: HTMLCanvasElement | OffscreenCanvas, rot: Rotation3D): HTMLCanvasElement; - /** Read the last apply() result as ImageData-ready pixels, for platforms where drawImage(webglCanvas) is unreliable. */ - readPixels(): Uint8ClampedArray; - resize(width: number, height: number): void; - destroy(): void; -} - -export function createThreeDPass(width: number, height: number): ThreeDPass { - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - const gl = canvas.getContext("webgl2", { premultipliedAlpha: true, alpha: true }); - if (!gl) throw new Error("WebGL2 not available for 3D pass"); - - const program = createProgram(gl); - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API, not a React hook - gl.useProgram(program); - - const aPos = gl.getAttribLocation(program, "aPos"); - const aUV = gl.getAttribLocation(program, "aUV"); - const uMvp = gl.getUniformLocation(program, "uMvp"); - const uSize = gl.getUniformLocation(program, "uSize"); - const uTex = gl.getUniformLocation(program, "uTex"); - - const vao = gl.createVertexArray(); - gl.bindVertexArray(vao); - - // Quad as two triangles. pos.y is 0 (top) to 1 (bottom) per CSS convention; UV.y - // is inverted so that with UNPACK_FLIP_Y_WEBGL the top of the input lands at the - // top of the rendered quad. - // TL: pos(0,0) uv(0,1) TR: pos(1,0) uv(1,1) - // BL: pos(0,1) uv(0,0) BR: pos(1,1) uv(1,0) - const verts = new Float32Array([ - // aPos.x, aPos.y, aUV.x, aUV.y - 0, - 0, - 0, - 1, // TL - 1, - 0, - 1, - 1, // TR - 0, - 1, - 0, - 0, // BL - 0, - 1, - 0, - 0, // BL - 1, - 0, - 1, - 1, // TR (was 1,0,1,0, broken) - 1, - 1, - 1, - 0, // BR - ]); - const vbo = gl.createBuffer(); - gl.bindBuffer(gl.ARRAY_BUFFER, vbo); - gl.bufferData(gl.ARRAY_BUFFER, verts, gl.STATIC_DRAW); - gl.enableVertexAttribArray(aPos); - gl.vertexAttribPointer(aPos, 2, gl.FLOAT, false, 16, 0); - gl.enableVertexAttribArray(aUV); - gl.vertexAttribPointer(aUV, 2, gl.FLOAT, false, 16, 8); - - const texture = gl.createTexture(); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - // Plain bilinear, no mipmaps. Even at our moderate angles (<=22deg) the receding - // edge picks a smaller mip level, softening the rounded-corner AA ramp and shadow - // falloff (corners look hard, shadows grimy). Sampling level 0 keeps source crispness. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - - // Anisotropic filtering still helps without mipmaps: at oblique angles it samples - // multiple texels along the gradient at level 0, recovering detail bilinear loses. - // Cap to the device max (16x typical). - const anisoExt = - gl.getExtension("EXT_texture_filter_anisotropic") || - gl.getExtension("MOZ_EXT_texture_filter_anisotropic") || - gl.getExtension("WEBKIT_EXT_texture_filter_anisotropic"); - if (anisoExt) { - const maxAniso = gl.getParameter(anisoExt.MAX_TEXTURE_MAX_ANISOTROPY_EXT) as number; - gl.texParameterf(gl.TEXTURE_2D, anisoExt.TEXTURE_MAX_ANISOTROPY_EXT, Math.min(16, maxAniso)); - } - gl.uniform1i(uTex, 0); - - let currentSize = { width, height }; - - const apply = ( - srcCanvas: HTMLCanvasElement | OffscreenCanvas, - rot: Rotation3D, - ): HTMLCanvasElement => { - gl.viewport(0, 0, currentSize.width, currentSize.height); - gl.clearColor(0, 0, 0, 0); - gl.clear(gl.COLOR_BUFFER_BIT); - gl.useProgram(program); - gl.bindVertexArray(vao); - - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); - // Premultiply on upload. The source 2D canvas is non-premultiplied (alpha=0 areas - // have RGB=0), so bilinear filtering across a shape edge in that space gives - // half-strength color, showing as a dark halo on rounded corners and grimy shadows. - // Premultiplying makes the filter math match compositing, so edges stay crisp. - gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.RGBA, - gl.RGBA, - gl.UNSIGNED_BYTE, - srcCanvas as TexImageSource, - ); - - const mvp = isRotation3DIdentity(rot) - ? buildMvpMatrix( - { rotationX: 0, rotationY: 0, rotationZ: 0 }, - currentSize.width, - currentSize.height, - ) - : buildMvpMatrix(rot, currentSize.width, currentSize.height); - gl.uniformMatrix4fv(uMvp, false, mvp); - gl.uniform2f(uSize, currentSize.width, currentSize.height); - - gl.drawArrays(gl.TRIANGLES, 0, 6); - return canvas; - }; - - const resize = (w: number, h: number) => { - if (w === currentSize.width && h === currentSize.height) return; - canvas.width = w; - canvas.height = h; - currentSize = { width: w, height: h }; - }; - - const readPixels = (): Uint8ClampedArray => { - const w = currentSize.width; - const h = currentSize.height; - const buf = new Uint8Array(w * h * 4); - gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf); - // readPixels is bottom-up, so flip to top-down. Also un-premultiply: the - // framebuffer is premultiplied (UNPACK_PREMULTIPLY_ALPHA_WEBGL on upload) but - // ImageData expects non-premultiplied, else semi-transparent pixels read too dark. - const rowSize = w * 4; - const out = new Uint8ClampedArray(buf.length); - for (let row = 0; row < h; row += 1) { - const src = (h - 1 - row) * rowSize; - const dst = row * rowSize; - for (let col = 0; col < rowSize; col += 4) { - const r = buf[src + col]; - const g = buf[src + col + 1]; - const b = buf[src + col + 2]; - const a = buf[src + col + 3]; - if (a === 0) { - out[dst + col] = 0; - out[dst + col + 1] = 0; - out[dst + col + 2] = 0; - out[dst + col + 3] = 0; - } else if (a === 255) { - out[dst + col] = r; - out[dst + col + 1] = g; - out[dst + col + 2] = b; - out[dst + col + 3] = 255; - } else { - const inv = 255 / a; - out[dst + col] = Math.min(255, Math.round(r * inv)); - out[dst + col + 1] = Math.min(255, Math.round(g * inv)); - out[dst + col + 2] = Math.min(255, Math.round(b * inv)); - out[dst + col + 3] = a; - } - } - } - return out; - }; - - const destroy = () => { - gl.deleteProgram(program); - gl.deleteBuffer(vbo); - gl.deleteVertexArray(vao); - gl.deleteTexture(texture); - }; - - return { apply, readPixels, resize, destroy }; -} diff --git a/src/lib/exporter/timestampedVideoFrameQueue.test.ts b/src/lib/exporter/timestampedVideoFrameQueue.test.ts deleted file mode 100644 index 9645c9859..000000000 --- a/src/lib/exporter/timestampedVideoFrameQueue.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { TimestampedVideoFrameQueue } from "./timestampedVideoFrameQueue"; - -class MockVideoFrame { - timestamp: number; - closed = false; - - constructor(source: MockVideoFrame | number) { - this.timestamp = typeof source === "number" ? source : source.timestamp; - } - - close() { - this.closed = true; - } -} - -function restoreVideoFrame(originalVideoFrame: typeof globalThis.VideoFrame | undefined) { - if (originalVideoFrame === undefined) { - delete (globalThis as { VideoFrame?: typeof globalThis.VideoFrame }).VideoFrame; - return; - } - - vi.stubGlobal("VideoFrame", originalVideoFrame); -} - -describe("TimestampedVideoFrameQueue", () => { - it("samples the latest webcam frame at or before the requested source timestamp", async () => { - const originalVideoFrame = globalThis.VideoFrame; - vi.stubGlobal("VideoFrame", MockVideoFrame); - try { - const queue = new TimestampedVideoFrameQueue(); - const frame0 = new MockVideoFrame(0) as unknown as VideoFrame; - const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame; - const frame66 = new MockVideoFrame(66_000) as unknown as VideoFrame; - - queue.enqueue(frame0, 0); - queue.enqueue(frame33, 33); - queue.enqueue(frame66, 66); - queue.close(); - - const sampled0 = await queue.frameAt(0); - const sampled20 = await queue.frameAt(20); - const sampled40 = await queue.frameAt(40); - const sampled80 = await queue.frameAt(80); - - expect(sampled0?.timestamp).toBe(0); - expect(sampled20?.timestamp).toBe(0); - expect(sampled40?.timestamp).toBe(33_000); - expect(sampled80?.timestamp).toBe(66_000); - - sampled0?.close(); - sampled20?.close(); - sampled40?.close(); - sampled80?.close(); - queue.destroy(); - } finally { - restoreVideoFrame(originalVideoFrame); - } - }); - - it("waits for a newer frame before falling back to the held frame while open", async () => { - const originalVideoFrame = globalThis.VideoFrame; - vi.stubGlobal("VideoFrame", MockVideoFrame); - try { - const queue = new TimestampedVideoFrameQueue(); - const frame0 = new MockVideoFrame(0) as unknown as VideoFrame; - const frame33 = new MockVideoFrame(33_000) as unknown as VideoFrame; - - queue.enqueue(frame0, 0); - const sampled0 = await queue.frameAt(0); - let resolved = false; - const pending = queue.frameAt(33).then((frame) => { - resolved = true; - return frame; - }); - - await Promise.resolve(); - expect(resolved).toBe(false); - - queue.enqueue(frame33, 33); - const sampled33 = await pending; - - expect(sampled0?.timestamp).toBe(0); - expect(sampled33?.timestamp).toBe(33_000); - - sampled0?.close(); - sampled33?.close(); - queue.destroy(); - } finally { - restoreVideoFrame(originalVideoFrame); - } - }); -}); diff --git a/src/lib/exporter/timestampedVideoFrameQueue.ts b/src/lib/exporter/timestampedVideoFrameQueue.ts deleted file mode 100644 index 86c0fe91d..000000000 --- a/src/lib/exporter/timestampedVideoFrameQueue.ts +++ /dev/null @@ -1,110 +0,0 @@ -type TimestampedVideoFrame = { - frame: VideoFrame; - sourceTimestampMs: number; -}; - -type PendingConsumer = { - resolve: () => void; - reject: (error: Error) => void; -}; - -const TIMESTAMP_EPSILON_MS = 0.5; - -export class TimestampedVideoFrameQueue { - private frames: TimestampedVideoFrame[] = []; - private consumers: PendingConsumer[] = []; - private error: Error | null = null; - private closed = false; - private heldFrame: TimestampedVideoFrame | null = null; - - get length() { - return this.frames.length; - } - - enqueue(frame: VideoFrame, sourceTimestampMs: number) { - if (this.closed) { - frame.close(); - return; - } - - this.frames.push({ frame, sourceTimestampMs }); - const consumers = this.consumers.splice(0); - for (const consumer of consumers) { - consumer.resolve(); - } - } - - fail(error: Error) { - this.error = error; - this.closed = true; - const consumers = this.consumers.splice(0); - for (const consumer of consumers) { - consumer.reject(error); - } - this.closeOwnedFrames(); - } - - close() { - this.closed = true; - const consumers = this.consumers.splice(0); - for (const consumer of consumers) { - consumer.resolve(); - } - } - - async frameAt(sourceTimestampMs: number): Promise { - for (;;) { - if (this.error) { - throw this.error; - } - - const next = this.frames[0] ?? null; - if (next && next.sourceTimestampMs <= sourceTimestampMs + TIMESTAMP_EPSILON_MS) { - this.replaceHeldFrame(this.frames.shift() ?? null); - continue; - } - - if ( - this.heldFrame && - (next || - this.closed || - this.heldFrame.sourceTimestampMs >= sourceTimestampMs - TIMESTAMP_EPSILON_MS) - ) { - return new VideoFrame(this.heldFrame.frame, { - timestamp: this.heldFrame.frame.timestamp, - }); - } - - if (next || this.closed) { - return null; - } - - await new Promise((resolve, reject) => { - this.consumers.push({ resolve, reject }); - }); - } - } - - destroy() { - this.close(); - this.closeOwnedFrames(); - } - - private replaceHeldFrame(frame: TimestampedVideoFrame | null) { - if (this.heldFrame) { - this.heldFrame.frame.close(); - } - this.heldFrame = frame; - } - - private closeOwnedFrames() { - if (this.heldFrame) { - this.heldFrame.frame.close(); - this.heldFrame = null; - } - for (const item of this.frames) { - item.frame.close(); - } - this.frames = []; - } -} diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index e779a27b6..78a5731de 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -33,6 +33,12 @@ export type ExportQuality = "medium" | "good" | "source"; // GIF Export Types export type ExportFormat = "mp4" | "gif"; +/** Video codec for the MP4 container. GIF is a single codec, so this doesn't + * apply to it. (`vp9` is offered by the type but the native exporter rejects + * it with a clear message — there is no hardware AMF equivalent and the + * software path measured too slow to ship.) */ +export type ExportVideoCodec = "h264" | "h265" | "vp9"; + export type GifFrameRate = 15 | 20 | 25 | 30; export type GifSizePreset = "medium" | "large" | "original"; diff --git a/src/lib/exporter/wgsl/composite.wgsl.ts b/src/lib/exporter/wgsl/composite.wgsl.ts deleted file mode 100644 index d13f7693b..000000000 --- a/src/lib/exporter/wgsl/composite.wgsl.ts +++ /dev/null @@ -1,192 +0,0 @@ -// The compositor: one WGSL program, one draw call, every pixel of the frame. -// -// WGSL and not GLSL/Pixi on purpose (rendering-architecture.md §12): this text -// runs unmodified in the browser (WebGPU) and natively (wgpu), so the shader and -// the evaluator — the product's actual substance — stay portable, and the host -// question stays a bindings swap rather than an architectural bet. -// -// Everything here is a pure function of the uniforms. There is no cache, and that -// is the point: a shader recomputes 2 Mpx per frame at a cost the Canvas2D path -// paid to AVOID recomputing. The 2D cache existed because a CSS filter cost 14 ms; -// the paradigm does not carry over. -// -// The shadow is the exception worth reading (see shadowCascade.wgsl.ts): it needs -// a real blur, so it is prepared in its own passes and sampled here. - -export const COMPOSITE_WGSL = /* wgsl */ ` - -/** - * Every member is a vec4f, and that is deliberate. - * - * WGSL aligns a vec4f to 16 bytes and a vec2f to 8, so a struct of mixed scalars - * and vectors has padding holes the CPU side must reproduce EXACTLY — get one - * offset wrong and the shader reads a radius where a rectangle should be, draws - * nothing, and reports no error. (It happened: the first version of this file - * packed 24 sequential floats against a struct whose real layout was 128 bytes - * with holes at 8, 36 and 92.) All-vec4 has no holes to get wrong: field n is at - * byte 16n, always. Flags travel as floats for the same reason — no u32/f32 - * interleaving to mis-pack. - */ -struct Uniforms { - // stage.xy | videoRadius | shadowIntensity - a : vec4f, - // The recording's box AFTER the camera: x, y, w, h - videoRect : vec4f, - // Source crop, normalised: x, y, w, h - crop : vec4f, - // Webcam destination: x, y, w, h - webcamRect : vec4f, - // webcamRadius | motionBlur.x | motionBlur.y | unused - b : vec4f, - // webcamShape | webcamMirrored | hasWebcam | hasShadow (0/1 as floats) - flags : vec4f, - // Webcam source sub-rect — the cover crop, normalised: x, y, w, h - webcamSrc : vec4f, -}; - -@group(0) @binding(0) var u : Uniforms; -@group(0) @binding(1) var samp : sampler; -@group(0) @binding(2) var videoTex : texture_external; -@group(0) @binding(3) var webcamTex : texture_external; -@group(0) @binding(4) var bgTex : texture_2d; -@group(0) @binding(5) var shadowTex : texture_2d; - -struct VsOut { - @builtin(position) pos : vec4f, - @location(0) uv : vec2f, -}; - -// Full-screen triangle. No vertex buffer: three points, clipped to the viewport. -@vertex -fn vs(@builtin(vertex_index) i : u32) -> VsOut { - var out : VsOut; - let x = f32((i << 1u) & 2u); - let y = f32(i & 2u); - out.uv = vec2f(x, y); - out.pos = vec4f(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); - return out; -} - -/** - * Signed distance to a rounded box, negative inside. - * - * This is geometry, not a shadow approximation: it decides which pixels the - * recording covers, exactly as the Canvas2D mask's rounded rect did. §13's ban is - * on approximating the shadow's FALLOFF with an SDF, and this never touches it. - */ -fn sdRoundBox(p : vec2f, halfSize : vec2f, r : f32) -> f32 { - let rr = min(r, min(halfSize.x, halfSize.y)); - let q = abs(p) - halfSize + vec2f(rr); - return length(max(q, vec2f(0.0))) + min(max(q.x, q.y), 0.0) - rr; -} - -/** Coverage of a shape at a pixel, antialiased over one pixel of distance. */ -fn coverage(dist : f32) -> f32 { - return clamp(0.5 - dist, 0.0, 1.0); -} - -fn shapeDistance(px : vec2f, rect : vec4f, radius : f32, shape : f32) -> f32 { - let centre = rect.xy + rect.zw * 0.5; - let half = rect.zw * 0.5; - let p = px - centre; - if (shape > 1.5 && shape < 2.5) { - // Circle: an ellipse inscribed in the rect, so a non-square webcam still - // masks to the shape the layout asked for. - let n = p / max(half, vec2f(1.0)); - return (length(n) - 1.0) * min(half.x, half.y); - } - if (shape > 2.5) { - let s = min(half.x, half.y); - return sdRoundBox(p, vec2f(s), 0.0); - } - if (shape > 0.5) { - return sdRoundBox(p, half, radius); - } - return sdRoundBox(p, half, 0.0); -} - -/** Map a stage pixel to the recording's source UV, through the crop. */ -fn videoUv(px : vec2f) -> vec2f { - let local = (px - u.videoRect.xy) / max(u.videoRect.zw, vec2f(1.0)); - return u.crop.xy + local * u.crop.zw; -} - -fn sampleVideo(px : vec2f) -> vec4f { - return textureSampleBaseClampToEdge(videoTex, samp, videoUv(px)); -} - -/** - * The recording, with directional motion blur when the camera is moving. - * - * Taps are skipped entirely at rest — the common case must not pay for the rare - * one, and a branch is free next to nine texture fetches. - */ -fn sampleVideoBlurred(px : vec2f) -> vec4f { - let motion = u.b.yz; - if (length(motion) < 0.5) { - return sampleVideo(px); - } - var acc = vec4f(0.0); - let taps = 9; - for (var i = 0; i < taps; i = i + 1) { - let t = (f32(i) / f32(taps - 1)) - 0.5; - acc = acc + sampleVideo(px + motion * t); - } - return acc / f32(taps); -} - -fn overComposite(dst : vec4f, src : vec4f) -> vec4f { - let a = src.a + dst.a * (1.0 - src.a); - let rgb = src.rgb * src.a + dst.rgb * dst.a * (1.0 - src.a); - return vec4f(select(rgb / a, vec3f(0.0), a <= 0.0), a); -} - -@fragment -fn fs(in : VsOut) -> @location(0) vec4f { - let px = in.uv * u.a.xy; - - // 1. Background. Pre-blurred at init if the document asks for blur: it is a - // still image, so blurring it per frame would be work with no output. - var colour = textureSample(bgTex, samp, in.uv); - colour = vec4f(colour.rgb, 1.0); - - // 2. Shadow, prepared by the cascade passes. Under everything, cut to nothing - // where the recording will cover it anyway. - if (u.flags.w > 0.5) { - let s = textureSample(shadowTex, samp, in.uv).a; - colour = overComposite(colour, vec4f(0.0, 0.0, 0.0, s)); - } - - // 3. The recording: rounded, masked, motion-blurred. - let vd = sdRoundBox( - px - (u.videoRect.xy + u.videoRect.zw * 0.5), - u.videoRect.zw * 0.5, - u.a.z - ); - let vc = coverage(vd); - if (vc > 0.0) { - let video = sampleVideoBlurred(px); - colour = overComposite(colour, vec4f(video.rgb, vc)); - } - - // 4. The webcam, on top, in its own shape. - if (u.flags.z > 0.5) { - let wd = shapeDistance(px, u.webcamRect, u.b.x, u.flags.x); - let wc = coverage(wd); - if (wc > 0.0) { - var wuv = (px - u.webcamRect.xy) / max(u.webcamRect.zw, vec2f(1.0)); - if (u.flags.y > 0.5) { - wuv.x = 1.0 - wuv.x; - } - // Through the cover crop: the box's aspect ratio is not the camera's (a block - // layout hands it a column slot, Full Camera walks it out to the whole frame), - // so the box selects a sub-rect of the source rather than stretching all of it. - let wsrc = u.webcamSrc.xy + clamp(wuv, vec2f(0.0), vec2f(1.0)) * u.webcamSrc.zw; - let cam = textureSampleBaseClampToEdge(webcamTex, samp, wsrc); - colour = overComposite(colour, vec4f(cam.rgb, wc)); - } - } - - return vec4f(colour.rgb, 1.0); -} -`; diff --git a/src/lib/exporter/wgsl/evaluate.ts b/src/lib/exporter/wgsl/evaluate.ts deleted file mode 100644 index 968da2de3..000000000 --- a/src/lib/exporter/wgsl/evaluate.ts +++ /dev/null @@ -1,367 +0,0 @@ -// evaluate — (document appearance, t) → FrameState. Pure CPU, no GPU, no canvas. -// -// The architecture's load-bearing idea (rendering-architecture.md §8a): every -// per-frame appearance decision is a deterministic function of the document and -// a time. Extracting it means preview and export cannot drift on layout, easing -// or timing, because there is only one place that decides them. -// -// This is an EXTRACTION, not a rewrite: the geometry already lives in pure -// modules (computeCompositeLayout, zoomSpring, zoomTransform, cameraFullscreen). -// evaluate() calls exactly what FrameRenderer.updateLayout/updateAnimationState -// call, in the same order, so the WGSL compositor inherits their behaviour rather -// than reimplementing it — including the spring, which is the part nobody would -// reproduce identically by eye. -// -// The one thing this file owns that FrameRenderer does not: `velocity`, and the -// SHAPE of the output. The state is a plain struct, sized to become ~200 bytes of -// uniforms. - -import type { - CameraFullscreenRegion, - CropRegion, - Rotation3D, - WebcamMaskShape, - ZoomRegion, -} from "@/components/video-editor/types"; -import { DEFAULT_ROTATION_3D, getZoomScale, lerpRotation3D } from "@/components/video-editor/types"; -import { - computeCameraFullscreenRect, - computeCompositeLayout, - reactiveWebcamScale, - resolveWebcamReactiveZoom, - type Size, - type StyledRenderRect, - type WebcamLayoutPreset, - type WebcamSizePreset, -} from "@/lib/compositeLayout"; -import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils"; -import { AUTO_FOLLOW_PARAMS, DEFAULT_FOCUS } from "@/lib/zoomMath/constants"; -import { advanceFollowFocus } from "@/lib/zoomMath/cursorFollowUtils"; -import { clampFocusToScale } from "@/lib/zoomMath/focusUtils"; -import { findDominantRegion } from "@/lib/zoomMath/zoomRegionUtils"; -import { - createZoomSpringState, - resetZoomSpring, - stepZoomSpring, - type ZoomSpringState, -} from "@/lib/zoomMath/zoomSpring"; -import { computeFocusFromTransform, computeZoomTransform } from "@/lib/zoomMath/zoomTransform"; - -/** Everything about the document that does not depend on `t`. */ -export interface EvaluateScene { - outputSize: Size; - videoSize: Size; - webcamSize: Size | null; - cropRegion: CropRegion; - padding: number; - borderRadius: number; - shadowIntensity: number; - motionBlurAmount: number; - zoomRegions: ZoomRegion[]; - cameraFullscreenRegions: CameraFullscreenRegion[]; - webcamLayoutPreset: WebcamLayoutPreset; - webcamSizePreset: WebcamSizePreset; - webcamMaskShape: WebcamMaskShape; - webcamPosition: { cx: number; cy: number } | null; - webcamMirrored: boolean; - webcamReactiveZoom: boolean; - cursorTelemetry?: import("@/components/video-editor/types").CursorTelemetryPoint[]; -} - -export interface Rect { - x: number; - y: number; - width: number; - height: number; -} - -export interface FrameState { - /** Where the (cropped) recording lands on the stage, before the camera moves. */ - screenRect: Rect; - /** Corner radius of the recording, in output pixels, BEFORE the camera scales it. */ - borderRadius: number; - /** The camera: applied (spring-smoothed) scale + translation, in output pixels. */ - camera: { scale: number; x: number; y: number }; - /** The recording's on-screen box WITH the camera applied — what the shadow follows. */ - cameraRect: Rect; - cameraBorderRadius: number; - /** Source-space crop, normalised 0..1. */ - crop: CropRegion; - /** Webcam destination, already carrying reactive-zoom shrink and Full Camera growth. */ - webcamRect: (StyledRenderRect & { shape: WebcamMaskShape }) | null; - webcamMirrored: boolean; - /** Per-frame camera movement, normalised. Drives motion blur; 0 on a still frame. */ - velocity: number; - motionBlurAmount: number; - shadowIntensity: number; - rotation3D: Rotation3D; - cameraFullscreenProgress: number; -} - -/** - * The parts of the animation that depend on the PREVIOUS frame. - * - * evaluate() is a pure function of (scene, t, prev) — not of (scene, t) alone, - * because the zoom spring and the auto-focus smoother are integrators: they chase - * their target over time. Threading their state through the signature keeps the - * function pure and, more to the point, keeps it honest — a renderer that seeks - * must reset this, and the type makes that impossible to forget. - */ -export interface EvaluateMemory { - spring: ZoomSpringState; - prevTimeMs: number | null; - prevTargetProgress: number; - smoothedAutoFocus: { cx: number; cy: number } | null; - prevCamera: { scale: number; x: number; y: number } | null; -} - -export function createEvaluateMemory(): EvaluateMemory { - return { - spring: createZoomSpringState(), - prevTimeMs: null, - prevTargetProgress: 0, - smoothedAutoFocus: null, - prevCamera: null, - }; -} - -/** Layout is independent of `t`, so it is computed once per (scene, webcam presence). */ -export function evaluateLayout(scene: EvaluateScene, hasWebcam: boolean) { - const { width, height } = scene.outputSize; - const crop = scene.cropRegion; - const croppedVideo = { - width: scene.videoSize.width * crop.width, - height: scene.videoSize.height * crop.height, - }; - - // Padding is a percentage (0-100) where 50% ~ 0.8 scale, applied to every preset - // (in the block layouts it insets the welded screen+camera block as one). Same - // constants as the Canvas2D path, on purpose. - const paddingScale = 1.0 - (scene.padding / 100) * 0.4; - - const layout = computeCompositeLayout({ - canvasSize: { width, height }, - maxContentSize: { width: width * paddingScale, height: height * paddingScale }, - screenSize: croppedVideo, - webcamSize: hasWebcam ? scene.webcamSize : null, - layoutPreset: scene.webcamLayoutPreset, - webcamSizePreset: scene.webcamSizePreset, - webcamPosition: scene.webcamPosition, - webcamMaskShape: scene.webcamMaskShape, - }); - if (!layout) return null; - - const screenRect = layout.screenRect; - const scale = layout.screenCover - ? Math.max(screenRect.width / croppedVideo.width, screenRect.height / croppedVideo.height) - : screenRect.width / croppedVideo.width; - - // The mask is the visible box of the recording: the cropped source at `scale`, - // centred in screenRect, clipped to it. Cover mode overflows and is cut. - const displayed = { width: croppedVideo.width * scale, height: croppedVideo.height * scale }; - const maskRect = { - x: screenRect.x + Math.max(0, (screenRect.width - displayed.width) / 2), - y: screenRect.y + Math.max(0, (screenRect.height - displayed.height) / 2), - width: Math.min(screenRect.width, displayed.width), - height: Math.min(screenRect.height, displayed.height), - }; - - return { - stageSize: { width, height }, - maskRect, - borderRadius: scene.borderRadius, - webcamRect: layout.webcamRect ?? null, - }; -} - -export type EvaluateLayout = NonNullable>; - -export function evaluate( - scene: EvaluateScene, - layout: EvaluateLayout, - timeMs: number, - memory: EvaluateMemory, -): FrameState { - const cameraFullscreenProgress = computeCameraFullscreenProgress( - scene.cameraFullscreenRegions, - timeMs, - ); - - const { region, strength, blendedScale, rotation3D, transition } = findDominantRegion( - scene.zoomRegions, - timeMs, - { connectZooms: true, cursorTelemetry: scene.cursorTelemetry }, - ); - - let targetScale = 1; - let targetFocus = { ...DEFAULT_FOCUS }; - let targetProgress = 0; - const currentRotation3D = - region && strength > 0 - ? lerpRotation3D(DEFAULT_ROTATION_3D, rotation3D, strength) - : { ...DEFAULT_ROTATION_3D }; - - const dtMs = memory.prevTimeMs != null ? timeMs - memory.prevTimeMs : 0; - - if (region && strength > 0) { - const zoomScale = blendedScale ?? getZoomScale(region); - targetScale = zoomScale; - targetFocus = clampFocusToScale(region.focus, zoomScale); - targetProgress = strength; - - if (region.focusMode === "auto" && !transition) { - const raw = targetFocus; - const isZoomingIn = targetProgress < 0.999 && targetProgress >= memory.prevTargetProgress; - if (targetProgress >= 0.999 || !isZoomingIn) { - const prev = memory.smoothedAutoFocus ?? raw; - const smoothed = advanceFollowFocus(prev, raw, dtMs, AUTO_FOLLOW_PARAMS); - memory.smoothedAutoFocus = smoothed; - targetFocus = smoothed; - } else { - memory.smoothedAutoFocus = raw; - } - } else if (region.focusMode !== "auto") { - memory.smoothedAutoFocus = null; - } - memory.prevTargetProgress = targetProgress; - - if (transition) { - const start = computeZoomTransform({ - stageSize: layout.stageSize, - baseMask: layout.maskRect, - zoomScale: transition.startScale, - zoomProgress: 1, - focusX: transition.startFocus.cx, - focusY: transition.startFocus.cy, - }); - const end = computeZoomTransform({ - stageSize: layout.stageSize, - baseMask: layout.maskRect, - zoomScale: transition.endScale, - zoomProgress: 1, - focusX: transition.endFocus.cx, - focusY: transition.endFocus.cy, - }); - const t = transition.progress; - const interpolated = { - scale: start.scale + (end.scale - start.scale) * t, - x: start.x + (end.x - start.x) * t, - y: start.y + (end.y - start.y) * t, - }; - targetScale = interpolated.scale; - targetFocus = computeFocusFromTransform({ - stageSize: layout.stageSize, - baseMask: layout.maskRect, - zoomScale: interpolated.scale, - x: interpolated.x, - y: interpolated.y, - }); - targetProgress = 1; - } - } - - const projected = computeZoomTransform({ - stageSize: layout.stageSize, - baseMask: layout.maskRect, - zoomScale: targetScale, - zoomProgress: targetProgress, - focusX: targetFocus.cx, - focusY: targetFocus.cy, - }); - - // Spring-chase the eased target, exactly as the preview does, so the export - // glides past the jerk at the steep start of the ease instead of snapping to - // the target every frame. Snapped on the first frame or any large time jump - // (a seek): integrating across a gap would fling the camera. - let camera: { scale: number; x: number; y: number }; - if (memory.prevTimeMs == null || dtMs <= 0 || dtMs > 80) { - resetZoomSpring(memory.spring, projected); - camera = { scale: projected.scale, x: projected.x, y: projected.y }; - } else { - camera = stepZoomSpring(memory.spring, projected, dtMs); - } - - const prev = memory.prevCamera; - const velocity = prev - ? Math.max( - Math.abs(camera.scale - prev.scale), - Math.abs(camera.x - prev.x) / Math.max(1, layout.stageSize.width), - Math.abs(camera.y - prev.y) / Math.max(1, layout.stageSize.height), - ) - : 0; - memory.prevCamera = { ...camera }; - memory.prevTimeMs = timeMs; - - // The recording's box with the camera applied. The shader needs this directly: - // it is what the rounded corners, the mask and the shadow are all cut from. - const m = layout.maskRect; - const cameraRect = { - x: camera.x + camera.scale * m.x, - y: camera.y + camera.scale * m.y, - width: camera.scale * m.width, - height: camera.scale * m.height, - }; - - return { - screenRect: m, - borderRadius: layout.borderRadius, - camera, - cameraRect, - cameraBorderRadius: layout.borderRadius * camera.scale, - crop: scene.cropRegion, - webcamRect: evaluateWebcamRect(scene, layout, camera.scale, cameraFullscreenProgress), - webcamMirrored: scene.webcamMirrored, - velocity, - motionBlurAmount: scene.motionBlurAmount, - shadowIntensity: scene.shadowIntensity, - rotation3D: currentRotation3D, - cameraFullscreenProgress, - }; -} - -/** - * Where the webcam lands: docked PiP, shrunk by the zoom, or grown to fill the - * stage during a Full Camera region. - * - * Both movements are the reason the shadow cache could never hold on this - * product's real timelines: they change the webcam's geometry every frame of an - * animation, and the user's answer to §13 is that they are the norm. - */ -function evaluateWebcamRect( - scene: EvaluateScene, - layout: EvaluateLayout, - appliedScale: number, - fullscreenProgress: number, -): (StyledRenderRect & { shape: WebcamMaskShape }) | null { - const base = layout.webcamRect; - if (!base) return null; - const shape = base.maskShape ?? scene.webcamMaskShape ?? "rectangle"; - - if (fullscreenProgress > 0) { - // Full Camera owns size, position AND shape outright; reactive zoom is ignored - // for the frame (mixing "shrink for zoom" and "grow to full" means nothing). - const r = computeCameraFullscreenRect(base, scene.outputSize, fullscreenProgress); - return { ...r, shape: r.maskShape ?? "rectangle" }; - } - - const reactive = resolveWebcamReactiveZoom(scene.webcamLayoutPreset, scene.webcamReactiveZoom) - ? reactiveWebcamScale(appliedScale) - : 1; - if (!(reactive < 1)) return { ...base, shape }; - - // Anchor the shrink to the docked corner (bottom-right by default), like the - // preview, so the bubble stays flush with the edges instead of drifting toward - // the centre. Same bias rule as the Canvas2D path — deliberately identical. - const pos = scene.webcamPosition; - const biasX = (pos ? pos.cx >= 0.5 : true) ? 1 : 0; - const biasY = (pos ? pos.cy >= 0.5 : true) ? 1 : 0; - return { - ...base, - x: base.x + base.width * (1 - reactive) * biasX, - y: base.y + base.height * (1 - reactive) * biasY, - width: base.width * reactive, - height: base.height * reactive, - borderRadius: base.borderRadius * reactive, - shape, - }; -} diff --git a/src/lib/exporter/wgsl/shadowCascade.wgsl.ts b/src/lib/exporter/wgsl/shadowCascade.wgsl.ts deleted file mode 100644 index a01c3c084..000000000 --- a/src/lib/exporter/wgsl/shadowCascade.wgsl.ts +++ /dev/null @@ -1,195 +0,0 @@ -// The drop shadow, computed — not cached. -// -// The Canvas2D path chains three CSS drop-shadows over the video, at 14.2 ms per -// moving frame (Annex B.1), and caches the result by geometry to avoid paying it. -// That cache is a 2D artefact: it exists because the filter is expensive, and a -// moving camera — which the product says is the norm — misses it by construction. -// On the GPU there is nothing to cache: the shadow is redrawn every frame. -// -// EXACTNESS is the constraint (§13). The ban is on approximating the falloff with -// an SDF/smoothstep — a DIFFERENT shape. It is not a ban on another implementation -// of the same shape. CSS `drop-shadow(0 dy blur rgba)` is feGaussianBlur on -// SourceAlpha with stdDeviation = blur/2, and the SVG filter spec defines that, -// for our radii, as three successive BOX blurs of a stated width: -// -// d = floor(s * 3 * sqrt(2*PI) / 4 + 0.5) -// d odd : three box-blurs of width d, centred. -// d even : two box-blurs of width d (centred left, then right), then one of d+1. -// -// That is what runs below — the spec's own algorithm, on the device that suits it. -// Box blurs are separable and cheap here; nothing is approximated. -// -// Whether Chromium's Skia follows the spec's letter is NOT assumed: the pixel-diff -// against the Canvas2D output is the gate, and it is what makes this file -// falsifiable rather than merely plausible. - -/** One CSS drop-shadow stage: blur radius (px), y offset (px), alpha. */ -export interface ShadowStage { - blur: number; - offsetY: number; - alpha: number; -} - -/** - * The three stages, matching shadowFilterChain() in frameRenderer.ts exactly. - * - * Duplicated deliberately rather than imported: the Canvas2D chain is the thing - * under test, and a shared constant would make the two agree by construction and - * prove nothing. If they drift, the pixel-diff must fail. - */ -export function shadowStages(intensity: number): ShadowStage[] { - const offset = 12 * intensity; - return [ - { blur: 48 * intensity, offsetY: offset, alpha: 0.7 * intensity }, - { blur: 16 * intensity, offsetY: offset / 3, alpha: 0.5 * intensity }, - { blur: 8 * intensity, offsetY: offset / 6, alpha: 0.3 * intensity }, - ]; -} - -/** - * The SVG filter spec's box widths for a gaussian of stdDeviation `s`. - * - * Returns the three box widths and their centring offsets. The even case is the - * subtle one: the spec compensates a box of even width by centring it left, then - * right, then widening the third — which is why this is a table and not a loop. - */ -export function boxesForStdDeviation(s: number): { width: number; offset: number }[] { - if (s <= 0) return []; - const d = Math.floor((s * 3 * Math.sqrt(2 * Math.PI)) / 4 + 0.5); - if (d < 1) return []; - if (d % 2 === 1) { - const half = (d - 1) / 2; - return [ - { width: d, offset: -half }, - { width: d, offset: -half }, - { width: d, offset: -half }, - ]; - } - return [ - { width: d, offset: -d / 2 }, - { width: d, offset: -d / 2 + 1 }, - { width: d + 1, offset: -d / 2 }, - ]; -} - -/** stdDeviation of a CSS drop-shadow's blur radius. */ -export const stdDeviationForBlur = (blur: number) => blur / 2; - -export const SHADOW_WGSL = /* wgsl */ ` - -// All-vec4, for the same reason as composite.wgsl.ts: no alignment holes for the -// CPU packing to get wrong. The first version declared (vec2f, vec4f, f32, f32) -// and WGSL laid it out at 48 bytes with a hole at 8 — against a 32-byte buffer. -// That is a validation error, a black frame, and no exception anywhere. -struct SilhouetteU { - // stage.xy | radius | unused - a : vec4f, - // rect: x, y, w, h - rect : vec4f, -}; - -struct BlurU { - // Texel step: (1/width, 0) horizontal, (0, 1/height) vertical. - step : vec2f, - // Box width in texels, and where the box starts relative to the pixel. - width : i32, - offset : i32, -}; - -struct StageU { - // offsetY (in texels, along v) | alpha | unused | unused - a : vec4f, -}; - -struct VsOut { - @builtin(position) pos : vec4f, - @location(0) uv : vec2f, -}; - -@vertex -fn vs(@builtin(vertex_index) i : u32) -> VsOut { - var out : VsOut; - let x = f32((i << 1u) & 2u); - let y = f32(i & 2u); - out.uv = vec2f(x, y); - out.pos = vec4f(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); - return out; -} - -// ---- pass 1: the silhouette ------------------------------------------------- -// The alpha the CSS chain actually blurs. The video is opaque and masked by a -// rounded rect, so its alpha IS that rounded rect — no video pixel takes part. - -@group(0) @binding(0) var su : SilhouetteU; - -fn sdRoundBox(p : vec2f, halfSize : vec2f, r : f32) -> f32 { - let rr = min(r, min(halfSize.x, halfSize.y)); - let q = abs(p) - halfSize + vec2f(rr); - return length(max(q, vec2f(0.0))) + min(max(q.x, q.y), 0.0) - rr; -} - -@fragment -fn fsSilhouette(in : VsOut) -> @location(0) vec4f { - let px = in.uv * su.a.xy; - let d = sdRoundBox(px - (su.rect.xy + su.rect.zw * 0.5), su.rect.zw * 0.5, su.a.z); - let a = clamp(0.5 - d, 0.0, 1.0); - return vec4f(0.0, 0.0, 0.0, a); -} - -// ---- pass 2: one box blur, one axis ---------------------------------------- -// Separable, so a d x d box costs 2d taps rather than d*d. Three of these per -// gaussian, per the SVG spec. - -@group(0) @binding(0) var bu : BlurU; -@group(0) @binding(1) var blurSamp : sampler; -@group(0) @binding(2) var blurSrc : texture_2d; - -@fragment -fn fsBox(in : VsOut) -> @location(0) vec4f { - var acc = vec4f(0.0); - for (var i = 0; i < bu.width; i = i + 1) { - let o = f32(bu.offset + i); - acc = acc + textureSample(blurSrc, blurSamp, in.uv + bu.step * o); - } - return acc / f32(bu.width); -} - -// ---- pass 3: one cascade stage --------------------------------------------- -// stage_out = source OVER shadow(source), where shadow is the blurred alpha, -// offset down and tinted. Each stage shadows the PREVIOUS stage's output — its -// own shadow included. That cascade is what gives the falloff, and it is why -// this cannot collapse into a single blur. - -@group(0) @binding(0) var stu : StageU; -@group(0) @binding(1) var stageSamp : sampler; -@group(0) @binding(2) var stageSrc : texture_2d; // the un-blurred source -@group(0) @binding(3) var stageBlur : texture_2d; // its blurred alpha - -@fragment -fn fsStage(in : VsOut) -> @location(0) vec4f { - let src = textureSample(stageSrc, stageSamp, in.uv); - let shadowAlpha = textureSample(stageBlur, stageSamp, in.uv - vec2f(0.0, stu.a.x)).a * stu.a.y; - // src is black-tinted throughout, so only alpha needs compositing. - let a = src.a + shadowAlpha * (1.0 - src.a); - return vec4f(0.0, 0.0, 0.0, a); -} - -// ---- pass 4: strip the silhouette ------------------------------------------ -// The cascade's output is "silhouette OVER shadows". The compositor draws the -// recording on top, which covers the silhouette exactly — but only where the -// recording is opaque. Subtracting it here keeps the shadow honest under the -// antialiased corners instead of double-darkening them. - -@group(0) @binding(0) var fu : SilhouetteU; -@group(0) @binding(1) var finalSamp : sampler; -@group(0) @binding(2) var finalSrc : texture_2d; - -@fragment -fn fsStrip(in : VsOut) -> @location(0) vec4f { - let px = in.uv * fu.a.xy; - let d = sdRoundBox(px - (fu.rect.xy + fu.rect.zw * 0.5), fu.rect.zw * 0.5, fu.a.z); - let sil = clamp(0.5 - d, 0.0, 1.0); - let a = textureSample(finalSrc, finalSamp, in.uv).a; - return vec4f(0.0, 0.0, 0.0, max(a - sil, 0.0)); -} -`; diff --git a/src/native/compositorViewClient.ts b/src/native/compositorViewClient.ts index 5edca550a..859dcb19f 100644 --- a/src/native/compositorViewClient.ts +++ b/src/native/compositorViewClient.ts @@ -10,6 +10,8 @@ import { requireNativeBridgeData } from "./client"; import type { CompositorClipInput, + CompositorExportGifParams, + CompositorExportGifResult, CompositorExportParams, CompositorExportResult, CompositorFramePacket, @@ -125,3 +127,18 @@ export function exportMultiNative( payload: { clips, outPath, sceneJson, params }, }); } + +/** Same clips and scene as `exportMultiNative` — the render is identical, only + * the container differs. */ +export function exportGifNative( + clips: CompositorClipInput[], + outPath?: string, + sceneJson?: string, + params?: CompositorExportGifParams, +): Promise { + return requireNativeBridgeData({ + domain: "compositor", + action: "exportGif", + payload: { clips, outPath, sceneJson, params }, + }); +} diff --git a/src/native/contracts.ts b/src/native/contracts.ts index a6965f2e9..e2ba5b18c 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -732,18 +732,14 @@ export type NativeBridgeRequest = | { domain: "compositor"; action: "exportGif"; - /** Slice 1 du chemin natif GIF (derrière `NATIVE_GIF_EXPORT_ENABLED`, - * flag off pour cette PR). Surface réduite à un seul clip (screen + - * webcam + cursor sidecar optionnel) pour matcher la fonction Rust - * `gif_export::export_gif` ; le multiclip / scene JSON viendront - * dans une slice ultérieure si la bench passe. */ + /** Même payload que `exportMulti` : GIF et MP4 sont le même rendu + * (`walk_composited_timeline` côté Rust) et ne diffèrent que par + * l'encodeur. La scène porte fond / layout / webcam / curseur, donc + * il n'y a aucune entrée spécifique au GIF. */ payload: { - screenPath: string; - webcamPath: string; - /** Sidecar `.cursor.json` selon la convention `ExportDialog`. - * Optionnel : null / absent → rend sans curseur. */ - cursorPath?: string | null; + clips: CompositorClipInput[]; outPath?: string; + sceneJson?: string; params?: CompositorExportGifParams; }; requestId?: string; diff --git a/vitest.browser.config.ts b/vitest.browser.config.ts deleted file mode 100644 index ba5cc426b..000000000 --- a/vitest.browser.config.ts +++ /dev/null @@ -1,28 +0,0 @@ -import path from "node:path"; -import { playwright } from "@vitest/browser-playwright"; -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - include: ["src/**/*.browser.test.{ts,tsx}"], - browser: { - enabled: true, - provider: playwright({ - launch: { - // Software WebGL so Pixi.js works in headless CI without a GPU. - args: ["--enable-unsafe-swiftshader", "--use-gl=swiftshader"], - }, - }), - headless: true, - instances: [{ browser: "chromium" }], - }, - testTimeout: 120_000, - hookTimeout: 30_000, - }, - resolve: { - alias: { - "@": path.resolve(__dirname, "src"), - }, - }, - assetsInclude: ["**/*.webm"], -}); From cc5af1d230291e2a79fcb01d360bcc8c408a6bcb Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Wed, 29 Jul 2026 08:11:25 +0200 Subject: [PATCH 3/3] =?UTF-8?q?chore(pixi):=20remove=20pixi.js=20=E2=80=94?= =?UTF-8?q?=20the=20last=20consumers=20were=20already=20dead=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correcting my own earlier claim that cursor and zoom still needed Pixi. They do not: the native compositor owns both. NativeCompositorOverlay hands the native view a cursorPath, buildSceneDescription sends cursor {show,size,smoothing} into the scene, crates/compositor/src/cursor.rs renders it, and regions.rs owns zoom scale/focus/rotation. D3D compositing is the pixel SSOT, exactly as designed. What was left was the same situation as the Pixi screen compositor before it was removed — code that outlived its mount: - CursorPreviewLayer was rendered by nothing but its own test, and it was the only path to pixiCursorRenderer and nativeCursor (all 15 exports of which had zero consumers outside that cluster); - zoomTransform's Pixi half (applyZoomTransform, createMotionBlurState, computeFocusFromTransform) had zero consumers. Only computeZoomTransform is live, and it is arithmetic — it just happened to share a module with the Pixi code, which is what kept pixi.js in the bundle at all. 249 lines -> 56. pixi.js and pixi-filters drop out of package.json, the lockfile and the vite manual-chunk config; the 484 kB pixi chunk is gone from the build. Test-file typecheck errors 74 -> 71, baseline lowered to match. --- .github/workflows/ci.yml | 2 +- package-lock.json | 113 +-- package.json | 2 - .../ai-edition/CursorPreviewLayer.module.css | 34 - .../ai-edition/CursorPreviewLayer.test.tsx | 212 ----- .../ai-edition/CursorPreviewLayer.tsx | 440 ---------- src/lib/cursor/nativeCursor.test.ts | 285 ------- src/lib/cursor/nativeCursor.ts | 626 -------------- src/lib/cursor/pixiCursorRenderer.ts | 762 ------------------ src/lib/zoomMath/zoomTransform.ts | 205 +---- vite.config.ts | 2 - 11 files changed, 7 insertions(+), 2676 deletions(-) delete mode 100644 src/components/ai-edition/CursorPreviewLayer.module.css delete mode 100644 src/components/ai-edition/CursorPreviewLayer.test.tsx delete mode 100644 src/components/ai-edition/CursorPreviewLayer.tsx delete mode 100644 src/lib/cursor/nativeCursor.test.ts delete mode 100644 src/lib/cursor/nativeCursor.ts delete mode 100644 src/lib/cursor/pixiCursorRenderer.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 382618b35..9a719360f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Typecheck tests against a baseline shell: bash env: - BASELINE: 74 + BASELINE: 71 run: | set -uo pipefail COUNT=$(npx tsc -p tsconfig.test.json --noEmit 2>&1 | grep -c 'error TS' || true) diff --git a/package-lock.json b/package-lock.json index 8b2875dbd..e7c3c517b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,8 +40,6 @@ "mediabunny": "^1.40.1", "motion": "^12.38.0", "mp4box": "^2.3.0", - "pixi-filters": "^6.1.5", - "pixi.js": "^8.18.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-icons": "^5.6.0", @@ -2354,12 +2352,6 @@ "@octokit/openapi-types": "^27.0.0" } }, - "node_modules/@pixi/colord": { - "version": "2.9.6", - "resolved": "https://registry.npmjs.org/@pixi/colord/-/colord-2.9.6.tgz", - "integrity": "sha512-nezytU2pw587fQstUu1AsJZDVEynjskwOL+kibwcdxsMBFqPsFFNA7xl0ii/gXuDi6M0xj3mfRJj8pBSc2jCfA==", - "license": "MIT" - }, "node_modules/@playwright/test": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", @@ -4498,12 +4490,6 @@ "@types/node": "*" } }, - "node_modules/@types/gradient-parser": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@types/gradient-parser/-/gradient-parser-0.1.5.tgz", - "integrity": "sha512-r7K3NkJz3A95WkVVmjs0NcchhHstC2C/VIYNX4JC6tieviUNo774FFeOHjThr3Vw/WCeMP9kAT77MKbIRlO/4w==", - "license": "MIT" - }, "node_modules/@types/http-cache-semantics": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", @@ -4999,16 +4985,11 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@webgpu/types": { - "version": "0.1.69", - "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.69.tgz", - "integrity": "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==", - "license": "BSD-3-Clause" - }, "node_modules/@xmldom/xmldom": { "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", "integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==", + "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -7409,15 +7390,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/gifuct-js": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/gifuct-js/-/gifuct-js-2.1.2.tgz", - "integrity": "sha512-rI2asw77u0mGgwhV3qA+OEgYqaDn5UNqgs+Bx0FGwSpuqfYn+Ir6RQY5ENNQ8SbIiG/m5gVa7CD5RriO4f4Lsg==", - "license": "MIT", - "dependencies": { - "js-binary-schema-parser": "^2.0.3" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -7970,12 +7942,6 @@ "node": ">=18" } }, - "node_modules/ismobilejs": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ismobilejs/-/ismobilejs-1.1.1.tgz", - "integrity": "sha512-VaFW53yt8QO61k2WJui0dHf4SlL8lxBofUuUmwBo0ljPk0Drz2TiuDW4jo3wDcv41qy/SxrJ+VAzJ/qYqsmzRw==", - "license": "MIT" - }, "node_modules/jake": { "version": "10.9.4", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", @@ -8004,12 +7970,6 @@ "jiti": "lib/jiti-cli.mjs" } }, - "node_modules/js-binary-schema-parser": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/js-binary-schema-parser/-/js-binary-schema-parser-2.0.3.tgz", - "integrity": "sha512-xezGJmOb4lk/M1ZZLTR/jaBHQ4gG/lqQnJqdIv4721DMggsa1bDVlHXNeHYogaIEHD9vCRv0fcL4hMA+Coarkg==", - "license": "MIT" - }, "node_modules/js-tiktoken": { "version": "1.0.21", "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", @@ -9210,12 +9170,6 @@ "node": ">=8" } }, - "node_modules/parse-svg-path": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/parse-svg-path/-/parse-svg-path-0.1.2.tgz", - "integrity": "sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ==", - "license": "MIT" - }, "node_modules/parse5": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", @@ -9320,62 +9274,6 @@ "node": ">= 6" } }, - "node_modules/pixi-filters": { - "version": "6.1.5", - "resolved": "https://registry.npmjs.org/pixi-filters/-/pixi-filters-6.1.5.tgz", - "integrity": "sha512-Ewb/J+kxAbaNN+0/ATJbglAJG+skGJfh7BIDP3ILIDdD6wWk1p0pGa25pVf1T8hGBOQSUNVAmwwJBwkj+cyLLA==", - "license": "MIT", - "dependencies": { - "@types/gradient-parser": "^0.1.2" - }, - "peerDependencies": { - "pixi.js": ">=8.0.0-0" - } - }, - "node_modules/pixi.js": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/pixi.js/-/pixi.js-8.18.1.tgz", - "integrity": "sha512-6LUPWYgulZhp/w4kam2XHXB0QedISZIqrJbRdHLLQ3csn5a38uzKxAp6B5j6s89QFYaIJbg95kvgTRcbgpO1ow==", - "license": "MIT", - "workspaces": [ - "examples", - "playground" - ], - "dependencies": { - "@pixi/colord": "^2.9.6", - "@types/earcut": "^3.0.0", - "@webgpu/types": "^0.1.69", - "@xmldom/xmldom": "^0.8.12", - "earcut": "^3.0.2", - "eventemitter3": "^5.0.1", - "gifuct-js": "^2.1.2", - "ismobilejs": "^1.1.1", - "parse-svg-path": "^0.1.2", - "tiny-lru": "^11.4.7" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/pixijs" - } - }, - "node_modules/pixi.js/node_modules/@types/earcut": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/earcut/-/earcut-3.0.0.tgz", - "integrity": "sha512-k/9fOUGO39yd2sCjrbAJvGDEQvRwRnQIZlBz43roGwUZo5SHAmyVvSFyaVVZkicRVCaDXPKlbxrUcBuJoSWunQ==", - "license": "MIT" - }, - "node_modules/pixi.js/node_modules/earcut": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", - "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==", - "license": "ISC" - }, - "node_modules/pixi.js/node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "license": "MIT" - }, "node_modules/playwright": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", @@ -11057,15 +10955,6 @@ "semver": "bin/semver" } }, - "node_modules/tiny-lru": { - "version": "11.4.7", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.4.7.tgz", - "integrity": "sha512-w/Te7uMUVeH0CR8vZIjr+XiN41V+30lkDdK+NRIDCUYKKuL9VcmaUEmaPISuwGhLlrTGh5yu18lENtR9axSxYw==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=12" - } - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", diff --git a/package.json b/package.json index 8602a807f..0c8c0194a 100644 --- a/package.json +++ b/package.json @@ -88,8 +88,6 @@ "mediabunny": "^1.40.1", "motion": "^12.38.0", "mp4box": "^2.3.0", - "pixi-filters": "^6.1.5", - "pixi.js": "^8.18.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-icons": "^5.6.0", diff --git a/src/components/ai-edition/CursorPreviewLayer.module.css b/src/components/ai-edition/CursorPreviewLayer.module.css deleted file mode 100644 index 8237d3aef..000000000 --- a/src/components/ai-edition/CursorPreviewLayer.module.css +++ /dev/null @@ -1,34 +0,0 @@ -/* Cursor preview layer. Transparent overlay that hosts a Pixi canvas + - a native-cursor . The Pixi canvas is appended by the component on - mount; the is positioned via translate3d in the same coordinate - space as the