mse: Initial MSE traits w/ gstreamer backing

.github/workflows/rust.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-latest]
         rust: [stable]
         target: ["default"]
     steps:
@@ -36,7 +36,7 @@ jobs:
           gstreamer1.0-tools \
           libasound2-plugins \
           libfaad2 \
-          libffi7 \
+          libffi8 \
           libfftw3-single3 \
           libges-1.0-dev \
           libgstreamer-plugins-bad1.0-dev\
@@ -49,7 +49,6 @@ jobs:
           libopus0 \
           liborc-0.4-0 \
           liborc-0.4-dev \
-          libpulsedsp \
           libsamplerate0 \
           libspeexdsp1 \
           libtdb1 \

Cargo.toml

@@ -9,6 +9,7 @@ members = [
   "backends/gstreamer/render-unix",
   "examples",
   "examples/android/lib",
+  "mse",
   "player",
   "servo-media",
   "servo-media-derive",
@@ -19,11 +20,12 @@ members = [
 [workspace.dependencies]
 glib = "0.21"
 glib-sys = "0.21"
-gst = { package = "gstreamer", version = "0.24" }
+gst = { package = "gstreamer", version = "0.24", features = ["v1_24"] }
 gst-app = { package = "gstreamer-app", version = "0.24" }
 gst-audio = { package = "gstreamer-audio", version = "0.24" }
 gst-base = { package = "gstreamer-base", version = "0.24" }
 gst-gl = { package = "gstreamer-gl", version = "0.24" }
+gst-mse = { package = "gstreamer-mse", version = "0.24" }
 gst-play = { package = "gstreamer-play", version = "0.24" }
 gst-sdp = { package = "gstreamer-sdp", version = "0.24" }
 gst-video = { package = "gstreamer-video", version = "0.24" }

backends/gstreamer/Cargo.toml

@@ -18,6 +18,7 @@ gst-app = { workspace = true }
 gst-audio = { workspace = true }
 gst-video = { workspace = true }
 gst-base = { workspace = true }
+gst-mse = { workspace = true }
 gst-play = { workspace = true }
 gst-webrtc = { workspace = true }
 gst-sdp = { workspace = true }
@@ -29,6 +30,7 @@ once_cell = "1.18.0"
 servo-media = { path = "../../servo-media" }
 servo-media-audio = { path = "../../audio" }
 servo-media-gstreamer-render = { path = "render" }
+servo-media-mse = { path = "../../mse" }
 servo-media-player = { path = "../../player" }
 servo-media-streams = { path = "../../streams" }
 servo-media-traits = { path = "../../traits" }

backends/gstreamer/lib.rs

@@ -4,14 +4,18 @@ pub mod audio_stream_reader;
 mod datachannel;
 mod device_monitor;
 pub mod media_capture;
+pub mod media_source;
 pub mod media_stream;
 mod media_stream_source;
 pub mod player;
 mod registry_scanner;
 mod render;
 mod source;
+mod source_buffer;
+mod source_buffer_list;
 pub mod webrtc;
 
+use crate::media_source::GStreamerMediaSource;
 use device_monitor::GStreamerDeviceMonitor;
 use gst::prelude::*;
 use ipc_channel::ipc::IpcSender;
@@ -25,6 +29,7 @@ use servo_media_audio::context::{AudioContext, AudioContextOptions};
 use servo_media_audio::decoder::AudioDecoder;
 use servo_media_audio::sink::AudioSinkError;
 use servo_media_audio::{AudioBackend, AudioStreamReader};
+use servo_media_mse::MediaSource;
 use servo_media_player::audio::AudioRenderer;
 use servo_media_player::context::PlayerGLContext;
 use servo_media_player::video::VideoFrameRenderer;
@@ -301,6 +306,10 @@ impl Backend for GStreamerBackend {
     fn get_device_monitor(&self) -> Box<dyn MediaDeviceMonitor> {
         Box::new(GStreamerDeviceMonitor::new())
     }
+
+    fn create_mse_source(&self) -> Option<Box<dyn MediaSource>> {
+        Some(Box::new(GStreamerMediaSource::new()))
+    }
 }
 
 impl AudioBackend for GStreamerBackend {

backends/gstreamer/media_source.rs

@@ -0,0 +1,118 @@
+use crate::source_buffer::GStreamerSourceBuffer;
+use crate::source_buffer_list::GStreamerSourceBufferList;
+use gst::ClockTime;
+use gst_mse::{MediaSource as GstMediaSource, MediaSourceEOSError};
+use servo_media_mse::{EosError, MediaSource, ReadyState, SourceBuffer, SourceBufferList};
+
+#[repr(transparent)]
+pub struct GStreamerMediaSource {
+    inner: GstMediaSource,
+}
+
+impl GStreamerMediaSource {
+    pub fn new() -> Self {
+        GStreamerMediaSource {
+            inner: GstMediaSource::new(),
+        }
+    }
+
+    pub(crate) fn inner(&self) -> &GstMediaSource {
+        &self.inner
+    }
+
+    pub(crate) fn from_ref(inner: &GstMediaSource) -> &Self {
+        unsafe { &*(inner as *const GstMediaSource as *const GStreamerMediaSource) }
+    }
+}
+
+impl MediaSource for GStreamerMediaSource {
+    fn add_source_buffer(&self, ty: &str) -> Box<dyn SourceBuffer> {
+        let buffer = self.inner.add_source_buffer(ty).unwrap();
+        let buffer = GStreamerSourceBuffer::from(buffer);
+        Box::new(buffer)
+    }
+
+    fn remove_source_buffer(&self, source_buffer: &dyn SourceBuffer) {
+        let gst_source_buffer = source_buffer
+            .as_any()
+            .downcast_ref::<GStreamerSourceBuffer>()
+            .expect("Expected a GStreamer SourceBuffer");
+        self.inner.remove_source_buffer(gst_source_buffer.inner());
+    }
+
+    fn live_seekable_range(&self) -> (Option<f64>, Option<f64>) {
+        let range = self.inner.live_seekable_range();
+        (
+            Some(range.start().seconds_f64()),
+            Some(range.end().seconds_f64()),
+        )
+    }
+
+    fn set_live_seekable_range(&self, start: Option<f64>, end: Option<f64>) {
+        let start = start.map(ClockTime::from_seconds_f64);
+        let end = end.map(ClockTime::from_seconds_f64);
+        self.inner.set_live_seekable_range(start, end);
+    }
+
+    fn clear_live_seekable_range(&self) {
+        self.inner.clear_live_seekable_range();
+    }
+
+    fn active_source_buffers(&self) -> Box<dyn SourceBufferList> {
+        let list: GStreamerSourceBufferList = self.inner.active_source_buffers().into();
+        Box::new(list)
+    }
+
+    fn source_buffers(&self) -> Box<dyn SourceBufferList> {
+        let list: GStreamerSourceBufferList = self.inner.source_buffers().into();
+        Box::new(list)
+    }
+
+    fn duration(&self) -> Option<f64> {
+        self.inner.duration().map(|d| d.seconds_f64())
+    }
+
+    fn set_duration(&self, duration: f64) {
+        self.inner
+            .set_duration(ClockTime::from_seconds_f64(duration));
+    }
+
+    fn end_of_stream(&self, error: Option<EosError>) {
+        let _ = match error {
+            Some(EosError::Network) => self.inner.end_of_stream(MediaSourceEOSError::Network),
+            Some(EosError::Decode) => self.inner.end_of_stream(MediaSourceEOSError::Decode),
+            None => self.inner.end_of_stream(MediaSourceEOSError::None),
+        };
+    }
+
+    fn ready_state(&self) -> ReadyState {
+        match self.inner.ready_state() {
+            gst_mse::MediaSourceReadyState::Closed => ReadyState::Closed,
+            gst_mse::MediaSourceReadyState::Open => ReadyState::Open,
+            gst_mse::MediaSourceReadyState::Ended => ReadyState::Ended,
+            _ => unreachable!(),
+        }
+    }
+
+    fn on_source_open(&self, f: Box<dyn Fn(&dyn MediaSource) + Send + Sync>) {
+        self.inner.connect_on_source_open(move |s| {
+            f(Self::from_ref(s));
+        });
+    }
+
+    fn on_source_ended(&self, f: Box<dyn Fn(&dyn MediaSource) + Send + Sync>) {
+        self.inner.connect_on_source_ended(move |s| {
+            f(Self::from_ref(s));
+        });
+    }
+
+    fn on_source_close(&self, f: Box<dyn Fn(&dyn MediaSource) + Send + Sync>) {
+        self.inner.connect_on_source_close(move |s| {
+            f(Self::from_ref(s));
+        });
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+}

backends/gstreamer/player.rs

@@ -1,4 +1,5 @@
 use super::BACKEND_BASE_TIME;
+use crate::media_source::GStreamerMediaSource;
 use crate::media_stream::GStreamerMediaStream;
 use crate::media_stream_source::{register_servo_media_stream_src, ServoMediaStreamSrc};
 use crate::render::GStreamerRender;
@@ -9,9 +10,11 @@ use glib::prelude::*;
 use gst;
 use gst::prelude::*;
 use gst_app;
+use gst_mse::MseSrc;
 use gst_play;
 use gst_play::prelude::*;
 use ipc_channel::ipc::{channel, IpcReceiver, IpcSender};
+use servo_media_mse::MediaSource;
 use servo_media_player::audio::AudioRenderer;
 use servo_media_player::context::PlayerGLContext;
 use servo_media_player::metadata::Metadata;
@@ -108,6 +111,7 @@ impl AsRef<[f32]> for GStreamerAudioChunk {
 enum PlayerSource {
     Seekable(ServoSrc),
     Stream(ServoMediaStreamSrc),
+    Mse(MseSrc),
 }
 
 struct PlayerInner {
@@ -313,6 +317,19 @@ impl PlayerInner {
         self.player.set_video_track_enabled(enabled);
         Ok(())
     }
+
+    fn connect_media_source(&self, source: &dyn MediaSource) -> Result<(), PlayerError> {
+        debug_assert!(self.stream_type == StreamType::MSE);
+        if let Some(ref src) = self.source {
+            if let PlayerSource::Mse(src) = src {
+                if let Some(gst_source) = source.as_any().downcast_ref::<GStreamerMediaSource>() {
+                    gst_source.inner().attach(src);
+                    return Ok(());
+                }
+            }
+        }
+        Err(PlayerError::SetStreamFailed)
+    }
 }
 
 macro_rules! notify(
@@ -528,6 +545,7 @@ impl GStreamerPlayer {
                 })?;
                 "servosrc://".to_value()
             },
+            StreamType::MSE => "mse://".to_value(),
         };
         player.set_property("uri", &uri);
 
@@ -774,6 +792,16 @@ impl GStreamerPlayer {
                         });
                         PlayerSource::Stream(media_stream_src)
                     },
+                    StreamType::MSE => {
+                        let mse_src = source
+                            .dynamic_cast::<MseSrc>()
+                            .expect("Source element is expected to be a MseSrc!");
+                        let sender_clone = sender.clone();
+                        is_ready_clone.call_once(|| {
+                            let _ = notify!(sender_clone, Ok(()));
+                        });
+                        PlayerSource::Mse(mse_src)
+                    },
                 };
 
                 inner.set_src(source);
@@ -874,6 +902,13 @@ impl Player for GStreamerPlayer {
         let mut inner = inner.as_ref().unwrap().lock().unwrap();
         inner.set_video_track(stream_index, enabled)
     }
+
+    fn connect_media_source(&self, source: &dyn MediaSource) -> Result<(), PlayerError> {
+        self.setup()?;
+        let inner = self.inner.borrow();
+        let inner = inner.as_ref().unwrap().lock().unwrap();
+        inner.connect_media_source(source)
+    }
 }
 
 impl MediaInstance for GStreamerPlayer {