From 505fc2798db35b51e44402bf4f203521766f9e37 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Thu, 9 Jul 2026 16:50:28 -0400 Subject: [PATCH 1/5] chore(vtk-streaming): bump version to 0.4.0 for apple encoders --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f414330..0ad03f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ turbo = [ ] vtkstreaming = [ "vtk", - "vtk-streaming>=0.3.5", + "vtk-streaming>=0.4.0", ] From 5002c84faa97ac78aa8a075f4595333579375eae Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Thu, 9 Jul 2026 16:29:45 -0400 Subject: [PATCH 2/5] feat(encoder): add VideoToolbox H265 encoder for macOS Prefer Apple's hardware VideoToolbox H265 encoder on darwin, falling back to NVENC (H264) and then libvpx (VP9) as before. --- src/trame_rca/encoders/video_encoder.py | 70 ++++++++++++++++--------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/src/trame_rca/encoders/video_encoder.py b/src/trame_rca/encoders/video_encoder.py index f2c8091..b3515a1 100644 --- a/src/trame_rca/encoders/video_encoder.py +++ b/src/trame_rca/encoders/video_encoder.py @@ -1,17 +1,14 @@ import logging from time import time_ns -from typing import Callable +from typing import Callable, Optional from vtk_streaming.vtkStreamingCore import ( VTKPF_IYUV, - VTKVC_H264, - VTKVC_VP9, vtkCompressedVideoPacket, + vtkVideoCodecTypeUtilities, ) -from vtk_streaming.vtkStreamingEncode import vtkVideoEncoder -from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL +from vtk_streaming.vtkStreamingEncode import vtkEncoderFactory, vtkVideoEncoder from vtk_streaming.vtkStreamingOpenGL2 import vtkOpenGLVideoFrame -from vtk_streaming.vtkStreamingVpxEncode import vtkVpxEncoder from vtkmodules.util.misc import calldata_type from vtkmodules.util.vtkConstants import VTK_OBJECT from vtkmodules.vtkCommonCore import vtkUnsignedCharArray @@ -19,21 +16,48 @@ logger = logging.getLogger(__name__) +# Backend selection is delegated to vtkEncoderFactory via a single ranked preference string +# (VTK 9.7 override format): ';' separates keys strongest-to-weakest, ',' separates a key's +# values strongest-to-weakest. Here: always take hardware when available. Among hardware, +# prefer H265 > H264 > VP9 +_ENCODER_PREFERENCES = "Hardware=true;Codec=H265,H264,VP9" -def create_vpx_encoder() -> vtkVpxEncoder: - encoder = vtkVpxEncoder() - encoder.SetCodec(VTKVC_VP9) - encoder.SetTargetCPUUsage(9) - encoder.SetRowBasedMultiThreading(False) # crashes if True - return encoder + +def _tune_encoder(encoder: vtkVideoEncoder) -> None: + # Backend-specific knobs the base vtkVideoEncoder API does not expose. vtkEncoderFactory + # returns the concrete (downcast) encoder, so these setters are available when matched. + if encoder.GetClassName() == "vtkVpxEncoder": + encoder.SetTargetCPUUsage(9) + encoder.SetRowBasedMultiThreading(False) # crashes if True -def create_nvenc_encoder() -> vtkNvEncoderGL: - encoder = vtkNvEncoderGL() - encoder.SetCodec(VTKVC_H264) +def create_encoder() -> Optional[vtkVideoEncoder]: + vtkEncoderFactory.SetPreferences(_ENCODER_PREFERENCES) + encoder = vtkEncoderFactory.CreateEncoder() + if encoder is None: + return None + _tune_encoder(encoder) + logger.info("Using %s", encoder.GetClassName()) return encoder +# Short backend names for a human-readable label; keyed by concrete encoder class. +_BACKEND_LABELS = { + "vtkVideoToolboxEncoder": "VideoToolbox", + "vtkNvEncoderGL": "nvenc", + "vtkVpxEncoder": "libvpx", +} + + +def describe_encoder(encoder: Optional[vtkVideoEncoder]) -> str: + """Human-readable 'codec (backend)' label, e.g. 'h.265 (VideoToolbox)'.""" + if encoder is None: + return "unavailable" + codec = vtkVideoCodecTypeUtilities.ToString(encoder.GetCodec()) + backend = _BACKEND_LABELS.get(encoder.GetClassName(), encoder.GetClassName()) + return f"{codec} ({backend})" + + def encode( video_packet: vtkCompressedVideoPacket, now_ms: int ) -> tuple[bytes, dict, int]: @@ -57,19 +81,15 @@ def __init__( push_callback: Callable[[bytes, dict, int], None], ) -> None: self._render_window = render_window - self.encoder = None self.frame = None self._push_callback = push_callback self._window_size = None - if vtkNvEncoderGL.CheckAvailability(): - logger.info("Using H264 through NVENC") - self._create_encoder = create_nvenc_encoder - else: - logger.info("Using VP9 through libvpx") - self._create_encoder = create_vpx_encoder - - self.encoder = self._create_encoder() + self.encoder = create_encoder() + if self.encoder is None: + raise RuntimeError( + "No suitable video encoder is available on this machine." + ) self._initialize(render_window) def _set_size(self, render_window_size: tuple[int]): @@ -102,7 +122,7 @@ def _reset(self, render_window: vtkRenderWindow) -> None: @calldata_type(VTK_OBJECT) def _on_encoded_chunk( self, - _encoder: vtkNvEncoderGL | vtkVpxEncoder, + _encoder: vtkVideoEncoder, _event: str, video_packet: vtkCompressedVideoPacket, ) -> None: From 105477436c091681698f67707b6354f53812ac3b Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Thu, 9 Jul 2026 16:29:50 -0400 Subject: [PATCH 3/5] chore(example): show VideoToolbox codec in fps_explore label Mirror RcaVideoEncoder selection so the footer reports "h265 (VideoToolbox)" on macOS when available. --- examples/01_vtk/vtk_cone_simple_video.py | 20 ++++++++++++++++- examples/03_fps/fps_explore.py | 28 +++++++++++++----------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/examples/01_vtk/vtk_cone_simple_video.py b/examples/01_vtk/vtk_cone_simple_video.py index 190f075..d38b2f1 100644 --- a/examples/01_vtk/vtk_cone_simple_video.py +++ b/examples/01_vtk/vtk_cone_simple_video.py @@ -2,7 +2,7 @@ from trame.app import TrameApp from trame.decorators import change from trame.widgets import rca -from trame.widgets import vuetify3 as v3 +from trame.widgets import html, vuetify3 as v3 from trame.ui.vuetify3 import SinglePageLayout from vtkmodules.vtkFiltersSources import vtkConeSource @@ -25,6 +25,8 @@ def __init__(self, server=None): self.render_window = self.setup_vtk() self._build_ui() + self.state.video_codec = "unavailable" + self._update_video_codec_label() def setup_vtk(self): renderer = vtkRenderer() @@ -54,6 +56,17 @@ def update_cone(self, resolution, **kwargs): self.cone_source.SetResolution(resolution) self.view_handler.update() + def _update_video_codec_label(self): + # Reflect the encoder the video scheduler actually selected, + try: + from trame_rca.encoders.video_encoder import describe_encoder + except ImportError: + self.state.video_codec = "unavailable" + return + scheduler = getattr(self.view_handler, "_scheduler", None) + rca_encoder = getattr(scheduler, "_rca_encoder", None) + self.state.video_codec = describe_encoder(getattr(rca_encoder, "encoder", None)) + def update_reset_resolution(self): self.state.resolution = DEFAULT_RESOLUTION @@ -73,6 +86,11 @@ def _build_ui(self): ) v3.VBtn(icon="mdi-undo-variant", click=self.update_reset_resolution) + with layout.footer.clear() as footer: + footer.classes = "text-caption" + v3.VSpacer() + html.Span("Codec: {{video_codec}}") + v3.VSpacer() with layout.content: view = rca.RemoteControlledArea(display="video-decoder") diff --git a/examples/03_fps/fps_explore.py b/examples/03_fps/fps_explore.py index 34b0a69..afe4fa1 100644 --- a/examples/03_fps/fps_explore.py +++ b/examples/03_fps/fps_explore.py @@ -57,18 +57,6 @@ def time_now_ms() -> int: return int(time.time_ns() / 1000000) -def video_codec_label() -> str: - """Codec selection is automatic (see RcaVideoEncoder.__init__).""" - try: - from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL - - if vtkNvEncoderGL.CheckAvailability(): - return "h264 (nvenc)" - return "vp9 (libvpx)" - except ImportError: - return "unavailable" - - class ConeApp(TrameApp): def __init__(self, server=None): super().__init__(server) @@ -76,7 +64,7 @@ def __init__(self, server=None): self.server.cli.add_argument("--encoder", default="turbo-jpeg") # jpeg args, _ = self.server.cli.parse_known_args() self.encoder = args.encoder - self.state.video_codec = video_codec_label() + self.state.video_codec = "unavailable" self.state.display_mode = IMAGE self.state.stats = None self.state.stats_display = "" @@ -84,6 +72,9 @@ def __init__(self, server=None): self.render_window, self.cone_source = self.setup_vtk() self.build_ui() + # Label the codec from the encoder actually selected by the video scheduler + # (codec selection is automatic; see RcaVideoEncoder / vtkEncoderFactory). + self._update_video_codec_label() @property def view_handler(self): @@ -91,6 +82,17 @@ def view_handler(self): return self.video_view_handler return self.image_view_handler + def _update_video_codec_label(self): + # Reflect the encoder the video scheduler actually selected, + try: + from trame_rca.encoders.video_encoder import describe_encoder + except ImportError: + self.state.video_codec = "unavailable" + return + scheduler = getattr(self.video_view_handler, "_scheduler", None) + rca_encoder = getattr(scheduler, "_rca_encoder", None) + self.state.video_codec = describe_encoder(getattr(rca_encoder, "encoder", None)) + def setup_vtk(self): renderer = vtkRenderer() renderWindow = vtkRenderWindow() From 4139033de4601aaae1b151ec0c43694fbf047f2e Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Thu, 9 Jul 2026 16:30:00 -0400 Subject: [PATCH 4/5] fix(stats): measure sustained fps over a trailing 1s window The gauge computed fps from the mean inter-arrival of the last 10 packets, which is biased toward delivery bursts and reads far above the target rate. Count packets received in the trailing 1000ms instead, independent of the burst/reset window used by the graph. --- vue-components/src/utils/FPSMonitor.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vue-components/src/utils/FPSMonitor.js b/vue-components/src/utils/FPSMonitor.js index 056085e..54672de 100644 --- a/vue-components/src/utils/FPSMonitor.js +++ b/vue-components/src/utils/FPSMonitor.js @@ -2,16 +2,20 @@ export class FPSMonitor { constructor( windowSize = 255, windowStatSize = 10, - newInteractionThreshold = 1000 + newInteractionThreshold = 1000, + fpsWindowMs = 1000 ) { this.windowSize = windowSize; this.windowStatSize = windowStatSize; this.newInteractionThreshold = newInteractionThreshold; + // Sustained-fps window: count packets received in the last fpsWindowMs. + this.fpsWindowMs = fpsWindowMs; this.lastTS = 0; this.serverTime = []; this.clientTimes = []; this.packetSizes = []; this.statWindow = []; + this.fpsWindow = []; } trim() { @@ -32,7 +36,8 @@ export class FPSMonitor { const start = this.statWindow[0]; const end = this.statWindow[this.statWindow.length - 1]; const dt = (end - start) / (this.statWindow.length - 1); - const avgFps = 1000 / dt; + // Sustained fps: number of packets received in the trailing fpsWindowMs. + const avgFps = (this.fpsWindow.length * 1000) / this.fpsWindowMs; const client = []; const server = []; const minMax = [0, 0]; @@ -75,6 +80,11 @@ export class FPSMonitor { this.packetSizes.push(size); this.serverTime.push(timeInMs); this.clientTimes.push(ts); + // Maintain a trailing window of arrival times for sustained-fps counting. + this.fpsWindow.push(ts); + while (this.fpsWindow.length && ts - this.fpsWindow[0] > this.fpsWindowMs) { + this.fpsWindow.shift(); + } if (ts - this.lastTS < this.newInteractionThreshold) { this.statWindow.push(ts); } else { From 6d8fbd3f3c74736bf07151b5305f0408e225fb16 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Sat, 11 Jul 2026 19:03:45 -0400 Subject: [PATCH 5/5] chore(changelog): fix spelling --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ca2ade..efc4e43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,8 @@ - **video_rca**: Bump vtk-streaming to 0.3.4 for vpx encoder fix ([`f1ab553`](https://github.com/Kitware/trame-rca/commit/f1ab553d28049c728694cbaf58efce6fd5c39f38)) -- just call set_size if window size is different from last encode isntead of recreating encoder and - frame from sratch +- just call set_size if window size is different from last encode instead of recreating encoder and + frame from scratch ## v2.6.2 (2026-07-02)