From 5bc406c07ec2e9b84cd825bc8e3f1c23b5aec432 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Fri, 24 Jul 2026 23:35:18 -0600 Subject: [PATCH] fix(wayland): implement content fit geometry --- subwave_wayland/README.md | 8 +- subwave_wayland/src/geometry.rs | 156 ++++++++++++++++++++++ subwave_wayland/src/lib.rs | 2 + subwave_wayland/src/subsurface_manager.rs | 26 +++- subwave_wayland/src/subtitle_runtime.rs | 112 ++++++++++++---- subwave_wayland/src/video.rs | 30 +++-- subwave_wayland/src/video_player.rs | 111 +++++++++------ 7 files changed, 363 insertions(+), 82 deletions(-) create mode 100644 subwave_wayland/src/geometry.rs diff --git a/subwave_wayland/README.md b/subwave_wayland/README.md index 12ad296..f70189f 100644 --- a/subwave_wayland/README.md +++ b/subwave_wayland/README.md @@ -13,6 +13,7 @@ Wayland subsurface-based video output for Iced with HDR passthrough support. - [x] GStreamer pipeline with waylandsink - [x] Wayland display context sharing - [x] Pre-commit hook synchronization for position updates +- [x] `ContentFit` geometry for Contain, Cover, Fill, None, and ScaleDown - [x] Zero-copy video rendering with hardware acceleration ## Architecture @@ -32,6 +33,7 @@ This crate implements video playback using Wayland subsurfaces, which allows: - Iced widget that reserves space in layout - Accesses WaylandIntegration via thread-local storage - Updates subsurface position based on widget bounds + - Maps Iced `ContentFit` modes to GStreamer's render rectangle 3. **Pipeline** (`pipeline.rs`) - Dynamic pipeline creation @@ -86,8 +88,7 @@ This approach ensures proper video display and prevents transparency issues, but ### Next Steps 1. Subtitle management - Requires upstream gstreamer changes -2. Add content fit support through aspect ratio pipeline element -3. Playback speed integration +2. Playback speed integration ## Usage @@ -104,7 +105,8 @@ let video = SubsurfaceVideo::new(&url).expect("Failed to create video"); // Build an Iced widget to reserve layout space and drive updates let player = VideoPlayer::new(&video) .width(iced::Length::Fill) - .height(iced::Length::Fill); + .height(iced::Length::Fill) + .content_fit(iced::ContentFit::Contain); ``` Note: Keep the `SubsurfaceVideo` alive for the duration of playback. diff --git a/subwave_wayland/src/geometry.rs b/subwave_wayland/src/geometry.rs new file mode 100644 index 0000000..7310673 --- /dev/null +++ b/subwave_wayland/src/geometry.rs @@ -0,0 +1,156 @@ +use iced::{ContentFit, Size}; + +/// Video destination relative to the widget's Wayland surface. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct VideoRectangle { + pub(crate) x: i32, + pub(crate) y: i32, + pub(crate) width: i32, + pub(crate) height: i32, +} + +impl VideoRectangle { + pub(crate) fn fill(width: i32, height: i32) -> Self { + Self { + x: 0, + y: 0, + width, + height, + } + } +} + +/// Resolve an Iced content-fit mode into a Wayland render rectangle. +/// +/// The rectangle is relative to the video widget. Modes are centered except +/// for Iced's native-size `None` mode, which remains anchored at the top-left. +/// `Cover` can intentionally produce negative offsets and dimensions larger +/// than the widget so the compositor clips the excess at the top-level surface +/// boundary. +pub(crate) fn fit_video_rectangle( + content_fit: ContentFit, + source_width: i32, + source_height: i32, + target_width: i32, + target_height: i32, +) -> Option { + if source_width <= 0 || source_height <= 0 || target_width <= 0 || target_height <= 0 { + return None; + } + + let source = Size::new(source_width as f32, source_height as f32); + let target = Size::new(target_width as f32, target_height as f32); + let fitted = content_fit.fit(source, target); + + if !fitted.width.is_finite() + || !fitted.height.is_finite() + || fitted.width <= 0.0 + || fitted.height <= 0.0 + || fitted.width > i32::MAX as f32 + || fitted.height > i32::MAX as f32 + { + return None; + } + + let width = (fitted.width.round() as i32).max(1); + let height = (fitted.height.round() as i32).max(1); + let (x, y) = if content_fit == ContentFit::None { + (0, 0) + } else { + ( + ((target_width - width) as f64 / 2.0).round() as i32, + ((target_height - height) as f64 / 2.0).round() as i32, + ) + }; + + Some(VideoRectangle { + x, + y, + width, + height, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn contain_upscales_1080p_to_4k() { + assert_eq!( + fit_video_rectangle(ContentFit::Contain, 1920, 1080, 3840, 2160), + Some(VideoRectangle::fill(3840, 2160)) + ); + } + + #[test] + fn contain_letterboxes_and_centers() { + assert_eq!( + fit_video_rectangle(ContentFit::Contain, 1440, 1080, 3840, 2160), + Some(VideoRectangle { + x: 480, + y: 0, + width: 2880, + height: 2160, + }) + ); + } + + #[test] + fn cover_crops_and_centers() { + assert_eq!( + fit_video_rectangle(ContentFit::Cover, 1440, 1080, 3840, 2160), + Some(VideoRectangle { + x: 0, + y: -360, + width: 3840, + height: 2880, + }) + ); + } + + #[test] + fn fill_stretches_to_target() { + assert_eq!( + fit_video_rectangle(ContentFit::Fill, 1440, 1080, 3840, 2160), + Some(VideoRectangle::fill(3840, 2160)) + ); + } + + #[test] + fn none_keeps_native_size_at_top_left() { + assert_eq!( + fit_video_rectangle(ContentFit::None, 1920, 1080, 3840, 2160), + Some(VideoRectangle::fill(1920, 1080)) + ); + } + + #[test] + fn scale_down_never_upscales() { + assert_eq!( + fit_video_rectangle(ContentFit::ScaleDown, 1920, 1080, 3840, 2160), + Some(VideoRectangle { + x: 960, + y: 540, + width: 1920, + height: 1080, + }) + ); + assert_eq!( + fit_video_rectangle(ContentFit::ScaleDown, 3840, 2160, 1920, 1080), + Some(VideoRectangle::fill(1920, 1080)) + ); + } + + #[test] + fn rejects_non_positive_dimensions() { + assert_eq!( + fit_video_rectangle(ContentFit::Contain, 0, 1080, 3840, 2160), + None + ); + assert_eq!( + fit_video_rectangle(ContentFit::Contain, 1920, 1080, 3840, 0), + None + ); + } +} diff --git a/subwave_wayland/src/lib.rs b/subwave_wayland/src/lib.rs index cb8046d..64cf8fd 100644 --- a/subwave_wayland/src/lib.rs +++ b/subwave_wayland/src/lib.rs @@ -1,6 +1,8 @@ #[cfg(target_os = "linux")] pub mod color_management; #[cfg(target_os = "linux")] +mod geometry; +#[cfg(target_os = "linux")] pub mod gstplayflags; #[cfg(target_os = "linux")] pub mod internal; diff --git a/subwave_wayland/src/subsurface_manager.rs b/subwave_wayland/src/subsurface_manager.rs index ecd8751..e8d7228 100644 --- a/subwave_wayland/src/subsurface_manager.rs +++ b/subwave_wayland/src/subsurface_manager.rs @@ -1,4 +1,4 @@ -use crate::{Error, Result, WaylandIntegration}; +use crate::{geometry::VideoRectangle, Error, Result, WaylandIntegration}; use parking_lot::Mutex; use std::io::Write; use std::os::fd::AsFd; @@ -63,9 +63,12 @@ pub struct WaylandSubsurfaceManager { /// Current position relative to parent position: Arc>, - /// Current size + /// Current widget/canvas size. size: Arc>, + /// Current GStreamer destination relative to the widget canvas. + video_rectangle: Mutex>, + /// Flag indicating we need to update on next parent commit needs_update: Arc, @@ -100,6 +103,7 @@ impl std::fmt::Debug for WaylandSubsurfaceManager { f.debug_struct("WaylandVideoSubsurface") .field("position", &self.position.lock()) .field("size", &self.size.lock()) + .field("video_rectangle", &self.video_rectangle.lock()) .field( "needs_update", &self.needs_update.load(std::sync::atomic::Ordering::Relaxed), @@ -400,6 +404,7 @@ impl WaylandSubsurfaceManager { subtitle_viewport, position: Arc::new(Mutex::new((0, 0))), size: Arc::new(Mutex::new((0, 0))), + video_rectangle: Mutex::new(None), needs_update: Arc::new(AtomicBool::new(false)), shm: Some(shm), video_anchor_buffer, @@ -696,11 +701,19 @@ impl WaylandSubsurfaceManager { *self.position.lock() } - /// Get the current size + /// Get the current widget/canvas size. pub fn get_size(&self) -> (i32, i32) { *self.size.lock() } + pub(crate) fn set_video_rectangle(&self, rectangle: VideoRectangle) { + *self.video_rectangle.lock() = Some(rectangle); + } + + pub(crate) fn get_video_rectangle(&self) -> Option { + *self.video_rectangle.lock() + } + // Do we have use for this function? pub fn set_buffer_offset(&self, x: i32, y: i32) { self.video_surface.offset(x, y); @@ -798,6 +811,13 @@ impl WaylandSubsurfaceManager { Ok(()) } + /// Commit child-subsurface geometry requested by GStreamer. + /// + /// This does not damage, resize, or retag the transparent mapping anchor. + pub(crate) fn commit_video_host_state(&self) { + self.video_surface.commit(); + } + /// Force the mutable overlay surfaces to redraw. /// /// The video host is intentionally omitted because its transparent 1x1 diff --git a/subwave_wayland/src/subtitle_runtime.rs b/subwave_wayland/src/subtitle_runtime.rs index 4c17466..0c97bf9 100644 --- a/subwave_wayland/src/subtitle_runtime.rs +++ b/subwave_wayland/src/subtitle_runtime.rs @@ -1,6 +1,7 @@ use std::time::Duration; use crate::{ + geometry::VideoRectangle, pgs_decoder::PgsFrame, subtitle_scheduler::{DecodedSubtitleEvent, SubtitleAction, SubtitleScheduler}, }; @@ -67,6 +68,7 @@ pub(crate) fn compose_pgs_bitmap( pgs_height: u16, surface_width: i32, surface_height: i32, + video_rectangle: VideoRectangle, ) -> Option { if frames.is_empty() { return None; @@ -76,8 +78,8 @@ pub(crate) fn compose_pgs_bitmap( let surf_h = surface_height.max(1) as usize; let pgs_w = pgs_width.max(1) as f64; let pgs_h = pgs_height.max(1) as f64; - let scale_x = surf_w as f64 / pgs_w; - let scale_y = surf_h as f64 / pgs_h; + let scale_x = video_rectangle.width.max(1) as f64 / pgs_w; + let scale_y = video_rectangle.height.max(1) as f64 / pgs_h; let stride = surf_w * 4; let mut canvas = vec![0u8; stride * surf_h]; @@ -88,36 +90,32 @@ pub(crate) fn compose_pgs_bitmap( continue; } - let fx = (frame.x as f64 * scale_x) as usize; - let fy = (frame.y as f64 * scale_y) as usize; - let scaled_fw = ((frame.width as f64) * scale_x).ceil().max(1.0) as usize; - let scaled_fh = ((frame.height as f64) * scale_y).ceil().max(1.0) as usize; + let frame_left = video_rectangle.x as f64 + frame.x as f64 * scale_x; + let frame_top = video_rectangle.y as f64 + frame.y as f64 * scale_y; + let frame_right = frame_left + frame.width as f64 * scale_x; + let frame_bottom = frame_top + frame.height as f64 * scale_y; + let dest_x_start = (frame_left.floor() as i32).max(0) as usize; + let dest_y_start = (frame_top.floor() as i32).max(0) as usize; + let dest_x_end = (frame_right.ceil() as i32).max(0) as usize; + let dest_y_end = (frame_bottom.ceil() as i32).max(0) as usize; let src_stride = frame_w * 4; - for dy in 0..scaled_fh { - let canvas_y = fy + dy; - if canvas_y >= surf_h { - break; - } - let src_row = ((dy as f64) / scale_y).floor() as usize; - if src_row >= frame_h { + for canvas_y in dest_y_start..dest_y_end.min(surf_h) { + let source_y = ((canvas_y as f64 - frame_top) / scale_y).floor(); + if source_y < 0.0 || source_y >= frame_h as f64 { continue; } - let src_row_offset = src_row * src_stride; + let src_row_offset = source_y as usize * src_stride; - for dx in 0..scaled_fw { - let canvas_x = fx + dx; - if canvas_x >= surf_w { - break; - } - let src_col = ((dx as f64) / scale_x).floor() as usize; - if src_col >= frame_w { + for canvas_x in dest_x_start..dest_x_end.min(surf_w) { + let source_x = ((canvas_x as f64 - frame_left) / scale_x).floor(); + if source_x < 0.0 || source_x >= frame_w as f64 { continue; } - let src_offset = src_row_offset + src_col * 4; + let src_offset = src_row_offset + source_x as usize * 4; let dst_offset = canvas_y * stride + canvas_x * 4; - if src_offset + 4 <= frame.argb.len() && dst_offset + 4 <= canvas.len() { + if src_offset + 4 <= frame.argb.len() { canvas[dst_offset..dst_offset + 4] .copy_from_slice(&frame.argb[src_offset..src_offset + 4]); } @@ -136,3 +134,71 @@ pub(crate) fn compose_pgs_bitmap( pub(crate) fn duration_from_clock_time(clock_time: gstreamer::ClockTime) -> Duration { Duration::from_nanos(clock_time.nseconds()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn pixel(bitmap: &SubtitleBitmap, x: usize, y: usize) -> &[u8] { + let offset = y * bitmap.stride as usize + x * 4; + &bitmap.data[offset..offset + 4] + } + + #[test] + fn pgs_composition_uses_centered_video_rectangle() { + let frame = PgsFrame { + argb: vec![1, 2, 3, 4], + width: 1, + height: 1, + x: 0, + y: 0, + }; + let bitmap = compose_pgs_bitmap( + &[frame], + 2, + 2, + 6, + 4, + VideoRectangle { + x: 1, + y: 1, + width: 4, + height: 2, + }, + ) + .expect("subtitle bitmap"); + + assert_eq!(pixel(&bitmap, 0, 0), &[0, 0, 0, 0]); + assert_eq!(pixel(&bitmap, 1, 1), &[1, 2, 3, 4]); + assert_eq!(pixel(&bitmap, 2, 1), &[1, 2, 3, 4]); + assert_eq!(pixel(&bitmap, 3, 1), &[0, 0, 0, 0]); + } + + #[test] + fn pgs_composition_clips_cover_rectangle() { + let frame = PgsFrame { + argb: vec![1, 0, 0, 255, 2, 0, 0, 255], + width: 2, + height: 1, + x: 0, + y: 0, + }; + let bitmap = compose_pgs_bitmap( + &[frame], + 2, + 1, + 2, + 1, + VideoRectangle { + x: -1, + y: 0, + width: 4, + height: 1, + }, + ) + .expect("subtitle bitmap"); + + assert_eq!(pixel(&bitmap, 0, 0), &[1, 0, 0, 255]); + assert_eq!(pixel(&bitmap, 1, 0), &[2, 0, 0, 255]); + } +} diff --git a/subwave_wayland/src/video.rs b/subwave_wayland/src/video.rs index 9e1cf6a..ed21fd5 100644 --- a/subwave_wayland/src/video.rs +++ b/subwave_wayland/src/video.rs @@ -1,5 +1,6 @@ use crate::internal::Internal; use crate::{ + geometry::VideoRectangle, pipeline::SubsurfacePipeline, subsurface_manager::WaylandSubsurfaceManager, subtitle_runtime::{ @@ -972,12 +973,16 @@ impl SubsurfaceVideo { video_height, } => { let (surface_width, surface_height) = subsurface.get_size(); + let video_rectangle = subsurface + .get_video_rectangle() + .unwrap_or_else(|| VideoRectangle::fill(surface_width, surface_height)); if let Some(bitmap) = compose_pgs_bitmap( &frames, video_width, video_height, surface_width, surface_height, + video_rectangle, ) { let _ = subsurface.attach_subtitle_frame( &bitmap.data, @@ -1186,18 +1191,25 @@ impl SubsurfaceVideo { } } - pub fn set_video_size_position(&self, x_offset: i32, y_offset: i32, width: i32, height: i32) { - let (pipeline, subsurface) = { - let guard = self.0.read(); - (guard.pipeline.clone(), guard.subsurface.clone()) - }; - - if let Some(p) = pipeline { + /// Set GStreamer's video destination relative to the widget canvas without + /// changing the canvas used by backgrounds and subtitles. + pub(crate) fn set_video_render_rectangle( + &self, + x_offset: i32, + y_offset: i32, + width: i32, + height: i32, + ) { + if let Some(p) = self.0.read().pipeline.clone() { p.set_render_rectangle(x_offset, y_offset, width, height); } + } - if let Some(s) = subsurface { - s.set_size(width, height); + pub fn set_video_size_position(&self, x_offset: i32, y_offset: i32, width: i32, height: i32) { + self.set_video_render_rectangle(x_offset, y_offset, width, height); + + if let Some(subsurface) = self.0.read().subsurface.clone() { + subsurface.set_size(width, height); } } diff --git a/subwave_wayland/src/video_player.rs b/subwave_wayland/src/video_player.rs index 434af06..4c3dda9 100644 --- a/subwave_wayland/src/video_player.rs +++ b/subwave_wayland/src/video_player.rs @@ -1,4 +1,4 @@ -use crate::SubsurfaceVideo; +use crate::{geometry::fit_video_rectangle, SubsurfaceVideo}; use gstreamer::glib; type OnError<'a, Message> = Box Message + 'a>; @@ -18,7 +18,7 @@ pub type VideoHandle = Rc>>>; /// Note: This widget requires the wgpu renderer and Wayland platform pub struct VideoPlayer<'a, Message, Theme = iced::Theme> { video: &'a VideoHandle, - _content_fit: ContentFit, + content_fit: ContentFit, width: Length, height: Length, _on_end_of_stream: Option, @@ -32,7 +32,7 @@ impl<'a, Message, Theme> VideoPlayer<'a, Message, Theme> { pub fn new(video: &'a VideoHandle) -> Self { Self { video, - _content_fit: ContentFit::Contain, + content_fit: ContentFit::Contain, width: Length::Fill, height: Length::Fill, _on_end_of_stream: None, @@ -66,7 +66,7 @@ impl<'a, Message, Theme> VideoPlayer<'a, Message, Theme> { /// Set the content fit mode pub fn content_fit(self, content_fit: ContentFit) -> Self { VideoPlayer { - _content_fit: content_fit, + content_fit, ..self } } @@ -211,53 +211,76 @@ where } } - // TODO: Calculate and pass the correct aspect ratio to the video player pipeline seemlessly - // We should probably add the element to the pipeline on demand if the user changes the default fit mode if let Ok(guard) = self.video.try_borrow() { if let Some(video) = guard.as_ref() { - if let Some(resolution) = video.resolution() { - // Validate video dimensions - must be reasonable - if resolution.0 < 2 || resolution.1 < 2 { - log::debug!( - "WARNING: Invalid video dimensions detected: {}x{}, skipping render", - resolution.0, - resolution.1 + if let Some((video_width, video_height)) = video.resolution() { + let canvas_width = window_bounds.width.round() as i32; + let canvas_height = window_bounds.height.round() as i32; + + if let (Some(video_rectangle), Some(subsurface)) = ( + fit_video_rectangle( + self.content_fit, + video_width, + video_height, + canvas_width, + canvas_height, + ), + video.get_subsurface(), + ) { + let canvas_position = ( + window_bounds.x.round() as i32, + window_bounds.y.round() as i32, ); - return; // Skip this draw call until we have valid dimensions - } + let position_changed = subsurface.get_position() != canvas_position; + let canvas_changed = subsurface.get_size() != (canvas_width, canvas_height); + let rectangle_changed = + subsurface.get_video_rectangle() != Some(video_rectangle); + + if position_changed || canvas_changed || rectangle_changed { + log::info!( + "Updating video geometry: fit={:?}, source={}x{}, canvas={}x{}, rectangle=({}, {}, {}x{})", + self.content_fit, + video_width, + video_height, + canvas_width, + canvas_height, + video_rectangle.x, + video_rectangle.y, + video_rectangle.width, + video_rectangle.height, + ); + + if position_changed { + subsurface.set_position(canvas_position.0, canvas_position.1); + } + + if canvas_changed { + subsurface.update_background(canvas_width, canvas_height); + subsurface.set_size(canvas_width, canvas_height); + } + + if rectangle_changed { + video.set_video_render_rectangle( + video_rectangle.x, + video_rectangle.y, + video_rectangle.width, + video_rectangle.height, + ); + subsurface.set_video_rectangle(video_rectangle); + subsurface.commit_video_host_state(); + } + + if position_changed || canvas_changed { + subsurface.integration.trigger_pre_commit_hooks(); + subsurface.force_damage_and_commit(); + } - let _video_width = resolution.0; - let _video_height = resolution.1; - //let video_aspect = video_width / video_height; - - let widget_width = window_bounds.width; - let widget_height = window_bounds.height; - //let widget_aspect = widget_width / widget_height; - - // Apply the calculated viewport - if let Some(subsurface) = video.get_subsurface() { - let current_size = subsurface.get_size(); - let new_width = widget_width.round() as i32; - let new_height = widget_height.round() as i32; - - if current_size != (new_width, new_height) - && new_width > 0 - && new_height > 0 - { - log::info!("Setting new size to {}, {}", new_width, new_height); - subsurface.update_background(new_width, new_height); - subsurface.set_size(new_width, new_height); - video.set_video_size_position(0, 0, new_width, new_height); - subsurface.integration.trigger_pre_commit_hooks(); - subsurface.force_damage_and_commit(); - match subsurface.flush() { - Ok(_) => (), - Err(e) => log::debug!("Error: {:#?}", e), + if let Err(error) = subsurface.flush() { + log::debug!("Failed to flush video geometry: {error:#?}"); } } - // Pump updates (bus commands + subtitles) from the UI thread each draw - // We need a mutable reference to call tick() + // Pump updates (bus commands + subtitles) from the UI thread each draw. drop(guard); if let Ok(mut guard2) = self.video.try_borrow_mut() { if let Some(video_mut) = guard2.as_deref_mut() {