Videotoolbox macos

CHANGELOG.md

@@ -8,8 +8,8 @@
 - **video_rca**: Bump vtk-streaming to 0.3.4 for vpx encoder fix
   ([`f1ab553`](https://github.com/Kitware/trame-rca/commit/f1ab553d28049c728694cbaf58efce6fd5c39f38))
 
-- just call set_size if window size is different from last encode isntead of recreating encoder and
-  frame from sratch
+- just call set_size if window size is different from last encode instead of recreating encoder and
+  frame from scratch
 
 
 ## v2.6.2 (2026-07-02)

examples/01_vtk/vtk_cone_simple_video.py

@@ -2,7 +2,7 @@
 from trame.app import TrameApp
 from trame.decorators import change
 from trame.widgets import rca
-from trame.widgets import vuetify3 as v3
+from trame.widgets import html, vuetify3 as v3
 from trame.ui.vuetify3 import SinglePageLayout
 
 from vtkmodules.vtkFiltersSources import vtkConeSource
@@ -25,6 +25,8 @@ def __init__(self, server=None):
         self.render_window = self.setup_vtk()
 
         self._build_ui()
+        self.state.video_codec = "unavailable"
+        self._update_video_codec_label()
 
     def setup_vtk(self):
         renderer = vtkRenderer()
@@ -54,6 +56,17 @@ def update_cone(self, resolution, **kwargs):
         self.cone_source.SetResolution(resolution)
         self.view_handler.update()
 
+    def _update_video_codec_label(self):
+        # Reflect the encoder the video scheduler actually selected,
+        try:
+            from trame_rca.encoders.video_encoder import describe_encoder
+        except ImportError:
+            self.state.video_codec = "unavailable"
+            return
+        scheduler = getattr(self.view_handler, "_scheduler", None)
+        rca_encoder = getattr(scheduler, "_rca_encoder", None)
+        self.state.video_codec = describe_encoder(getattr(rca_encoder, "encoder", None))
+
     def update_reset_resolution(self):
         self.state.resolution = DEFAULT_RESOLUTION
 
@@ -73,6 +86,11 @@ def _build_ui(self):
                 )
 
                 v3.VBtn(icon="mdi-undo-variant", click=self.update_reset_resolution)
+            with layout.footer.clear() as footer:
+                footer.classes = "text-caption"
+                v3.VSpacer()
+                html.Span("Codec: {{video_codec}}")
+                v3.VSpacer()
 
             with layout.content:
                 view = rca.RemoteControlledArea(display="video-decoder")

examples/03_fps/fps_explore.py

@@ -57,40 +57,42 @@ def time_now_ms() -> int:
     return int(time.time_ns() / 1000000)
 
 
-def video_codec_label() -> str:
-    """Codec selection is automatic (see RcaVideoEncoder.__init__)."""
-    try:
-        from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL
-
-        if vtkNvEncoderGL.CheckAvailability():
-            return "h264 (nvenc)"
-        return "vp9 (libvpx)"
-    except ImportError:
-        return "unavailable"
-
-
 class ConeApp(TrameApp):
     def __init__(self, server=None):
         super().__init__(server)
 
         self.server.cli.add_argument("--encoder", default="turbo-jpeg")  # jpeg
         args, _ = self.server.cli.parse_known_args()
         self.encoder = args.encoder
-        self.state.video_codec = video_codec_label()
+        self.state.video_codec = "unavailable"
         self.state.display_mode = IMAGE
         self.state.stats = None
         self.state.stats_display = ""
         self.max_dt = 0
 
         self.render_window, self.cone_source = self.setup_vtk()
         self.build_ui()
+        # Label the codec from the encoder actually selected by the video scheduler
+        # (codec selection is automatic; see RcaVideoEncoder / vtkEncoderFactory).
+        self._update_video_codec_label()
 
     @property
     def view_handler(self):
         if self.state.display_mode == VIDEO:
             return self.video_view_handler
         return self.image_view_handler
 
+    def _update_video_codec_label(self):
+        # Reflect the encoder the video scheduler actually selected,
+        try:
+            from trame_rca.encoders.video_encoder import describe_encoder
+        except ImportError:
+            self.state.video_codec = "unavailable"
+            return
+        scheduler = getattr(self.video_view_handler, "_scheduler", None)
+        rca_encoder = getattr(scheduler, "_rca_encoder", None)
+        self.state.video_codec = describe_encoder(getattr(rca_encoder, "encoder", None))
+
     def setup_vtk(self):
         renderer = vtkRenderer()
         renderWindow = vtkRenderWindow()

pyproject.toml

@@ -48,7 +48,7 @@ turbo = [
 ]
 vtkstreaming = [
     "vtk",
-    "vtk-streaming>=0.3.5",
+    "vtk-streaming>=0.4.0",
 ]
 
 

src/trame_rca/encoders/video_encoder.py

@@ -1,39 +1,63 @@
 import logging
 from time import time_ns
-from typing import Callable
+from typing import Callable, Optional
 
 from vtk_streaming.vtkStreamingCore import (
     VTKPF_IYUV,
-    VTKVC_H264,
-    VTKVC_VP9,
     vtkCompressedVideoPacket,
+    vtkVideoCodecTypeUtilities,
 )
-from vtk_streaming.vtkStreamingEncode import vtkVideoEncoder
-from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL
+from vtk_streaming.vtkStreamingEncode import vtkEncoderFactory, vtkVideoEncoder
 from vtk_streaming.vtkStreamingOpenGL2 import vtkOpenGLVideoFrame
-from vtk_streaming.vtkStreamingVpxEncode import vtkVpxEncoder
 from vtkmodules.util.misc import calldata_type
 from vtkmodules.util.vtkConstants import VTK_OBJECT
 from vtkmodules.vtkCommonCore import vtkUnsignedCharArray
 from vtkmodules.vtkRenderingCore import vtkRenderWindow
 
 logger = logging.getLogger(__name__)
 
+# Backend selection is delegated to vtkEncoderFactory via a single ranked preference string
+# (VTK 9.7 override format): ';' separates keys strongest-to-weakest, ',' separates a key's
+# values strongest-to-weakest. Here: always take hardware when available. Among hardware,
+# prefer H265 > H264 > VP9
+_ENCODER_PREFERENCES = "Hardware=true;Codec=H265,H264,VP9"
 
-def create_vpx_encoder() -> vtkVpxEncoder:
-    encoder = vtkVpxEncoder()
-    encoder.SetCodec(VTKVC_VP9)
-    encoder.SetTargetCPUUsage(9)
-    encoder.SetRowBasedMultiThreading(False)  # crashes if True
-    return encoder
+
+def _tune_encoder(encoder: vtkVideoEncoder) -> None:
+    # Backend-specific knobs the base vtkVideoEncoder API does not expose. vtkEncoderFactory
+    # returns the concrete (downcast) encoder, so these setters are available when matched.
+    if encoder.GetClassName() == "vtkVpxEncoder":
+        encoder.SetTargetCPUUsage(9)
+        encoder.SetRowBasedMultiThreading(False)  # crashes if True
 
 
-def create_nvenc_encoder() -> vtkNvEncoderGL:
-    encoder = vtkNvEncoderGL()
-    encoder.SetCodec(VTKVC_H264)
+def create_encoder() -> Optional[vtkVideoEncoder]:
+    vtkEncoderFactory.SetPreferences(_ENCODER_PREFERENCES)
+    encoder = vtkEncoderFactory.CreateEncoder()
+    if encoder is None:
+        return None
+    _tune_encoder(encoder)
+    logger.info("Using %s", encoder.GetClassName())
     return encoder
 
 
+# Short backend names for a human-readable label; keyed by concrete encoder class.
+_BACKEND_LABELS = {
+    "vtkVideoToolboxEncoder": "VideoToolbox",
+    "vtkNvEncoderGL": "nvenc",
+    "vtkVpxEncoder": "libvpx",
+}
+
+
+def describe_encoder(encoder: Optional[vtkVideoEncoder]) -> str:
+    """Human-readable 'codec (backend)' label, e.g. 'h.265 (VideoToolbox)'."""
+    if encoder is None:
+        return "unavailable"
+    codec = vtkVideoCodecTypeUtilities.ToString(encoder.GetCodec())
+    backend = _BACKEND_LABELS.get(encoder.GetClassName(), encoder.GetClassName())
+    return f"{codec} ({backend})"
+
+
 def encode(
     video_packet: vtkCompressedVideoPacket, now_ms: int
 ) -> tuple[bytes, dict, int]:
@@ -57,19 +81,15 @@ def __init__(
         push_callback: Callable[[bytes, dict, int], None],
     ) -> None:
         self._render_window = render_window
-        self.encoder = None
         self.frame = None
         self._push_callback = push_callback
         self._window_size = None
 
-        if vtkNvEncoderGL.CheckAvailability():
-            logger.info("Using H264 through NVENC")
-            self._create_encoder = create_nvenc_encoder
-        else:
-            logger.info("Using VP9 through libvpx")
-            self._create_encoder = create_vpx_encoder
-
-        self.encoder = self._create_encoder()
+        self.encoder = create_encoder()
+        if self.encoder is None:
+            raise RuntimeError(
+                "No suitable video encoder is available on this machine."
+            )
         self._initialize(render_window)
 
     def _set_size(self, render_window_size: tuple[int]):
@@ -102,7 +122,7 @@ def _reset(self, render_window: vtkRenderWindow) -> None:
     @calldata_type(VTK_OBJECT)
     def _on_encoded_chunk(
         self,
-        _encoder: vtkNvEncoderGL | vtkVpxEncoder,
+        _encoder: vtkVideoEncoder,
         _event: str,
         video_packet: vtkCompressedVideoPacket,
     ) -> None:

vue-components/src/utils/FPSMonitor.js

@@ -2,16 +2,20 @@ export class FPSMonitor {
   constructor(
     windowSize = 255,
     windowStatSize = 10,
-    newInteractionThreshold = 1000
+    newInteractionThreshold = 1000,
+    fpsWindowMs = 1000
   ) {
     this.windowSize = windowSize;
     this.windowStatSize = windowStatSize;
     this.newInteractionThreshold = newInteractionThreshold;
+    // Sustained-fps window: count packets received in the last fpsWindowMs.
+    this.fpsWindowMs = fpsWindowMs;
     this.lastTS = 0;
     this.serverTime = [];
     this.clientTimes = [];
     this.packetSizes = [];
     this.statWindow = [];
+    this.fpsWindow = [];
   }
 
   trim() {
@@ -32,7 +36,8 @@ export class FPSMonitor {
     const start = this.statWindow[0];
     const end = this.statWindow[this.statWindow.length - 1];
     const dt = (end - start) / (this.statWindow.length - 1);
-    const avgFps = 1000 / dt;
+    // Sustained fps: number of packets received in the trailing fpsWindowMs.
+    const avgFps = (this.fpsWindow.length * 1000) / this.fpsWindowMs;
     const client = [];
     const server = [];
     const minMax = [0, 0];
@@ -75,6 +80,11 @@ export class FPSMonitor {
     this.packetSizes.push(size);
     this.serverTime.push(timeInMs);
     this.clientTimes.push(ts);
+    // Maintain a trailing window of arrival times for sustained-fps counting.
+    this.fpsWindow.push(ts);
+    while (this.fpsWindow.length && ts - this.fpsWindow[0] > this.fpsWindowMs) {
+      this.fpsWindow.shift();
+    }
     if (ts - this.lastTS < this.newInteractionThreshold) {
       this.statWindow.push(ts);
     } else {