From 059cfca03e7e210398a510414c44939e359943a2 Mon Sep 17 00:00:00 2001 From: Jonathan Fontanez Date: Sat, 12 Sep 2026 23:22:17 -0400 Subject: [PATCH 1/5] feat(examples): ShaderEffect draws the fragment shader its config carries over every frame One host processor in the virtual-camera showcase takes fragment GLSL as config, lands each frame device-to-device in a texture it owns, and draws the shader over it as one fullscreen pass. Grayscale, vignette and pixelate ship beside it, each proven against a CPU reference over a buffer-backed frame, and a shader that does not compile is refused at setup naming the compiler's diagnostic while the rest of the graph keeps running. Refs #2216 Co-Authored-By: Claude Opus 5 --- .../processors/shader_effect.py | 150 ++++++++++ .../shaders/fullscreen_triangle.vert | 19 ++ .../processors/shaders/grayscale.frag | 18 ++ .../processors/shaders/pixelate.frag | 27 ++ .../processors/shaders/vignette.frag | 25 ++ examples/camera-virtual-camera/pyproject.toml | 22 +- .../tests/shader_effect_test_app.py | 97 +++++++ .../tests/shader_effect_test_processors.py | 112 ++++++++ .../tests/test_shader_effect.py | 264 ++++++++++++++++++ 9 files changed, 728 insertions(+), 6 deletions(-) create mode 100644 examples/camera-virtual-camera/processors/shader_effect.py create mode 100644 examples/camera-virtual-camera/processors/shaders/fullscreen_triangle.vert create mode 100644 examples/camera-virtual-camera/processors/shaders/grayscale.frag create mode 100644 examples/camera-virtual-camera/processors/shaders/pixelate.frag create mode 100644 examples/camera-virtual-camera/processors/shaders/vignette.frag create mode 100644 examples/camera-virtual-camera/tests/shader_effect_test_app.py create mode 100644 examples/camera-virtual-camera/tests/shader_effect_test_processors.py create mode 100644 examples/camera-virtual-camera/tests/test_shader_effect.py diff --git a/examples/camera-virtual-camera/processors/shader_effect.py b/examples/camera-virtual-camera/processors/shader_effect.py new file mode 100644 index 000000000..7040dac59 --- /dev/null +++ b/examples/camera-virtual-camera/processors/shader_effect.py @@ -0,0 +1,150 @@ +# Copyright (c) 2025 Jonathan Fontanez +# SPDX-License-Identifier: BUSL-1.1 +"""One effect host, any number of looks: a fragment shader taken from config. + +Importable as `processors.shader_effect:ShaderEffect`, which is the name the +engine spawns this processor's child interpreter with — and the name an agent +passes to `add_processor` to put a look into the running graph. + +The host is written once. A look is the GLSL a `config` carries: the engine +compiles it at `setup()`, draws it over every frame as one fullscreen pass, +and the pixels never leave the GPU. `shaders/` holds three to start from. + +What a fragment shader gets: the frame as a `sampler2D` under the name +`sampled_input_binding_name` says (`upstream_frame` unless the config names +another), `screen_uv` at location 0 running 0..1 from the top left, and one +colour output. It declares no other binding and no push constants. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cupy +from streamlib import ( + ProcessorOutputTextureRing, + RuntimeContextFullAccess, + RuntimeContextLimitedAccess, + VideoFrame, + input, + output, + processor, +) + +SHIPPED_SHADERS_DIRECTORY = Path(__file__).parent / "shaders" + +FULLSCREEN_TRIANGLE_VERTEX_GLSL = ( + SHIPPED_SHADERS_DIRECTORY / "fullscreen_triangle.vert" +).read_text(encoding="utf-8") + +# One format end to end: the camera publishes RGBA8 and a `VirtualCameraSink` +# samples RGBA8 on its way to the device's buffers. +TEXTURE_FORMAT = "rgba8_unorm" + +# The texture each incoming frame lands in and the shader samples. +SAMPLED_LANDING_TEXTURE_USAGE = ["texture_binding"] + +# The texture the pass renders into and the next processor samples. +RENDERED_OUTPUT_TEXTURE_USAGE = ["render_attachment", "texture_binding"] + + +@dataclass +class ShaderEffectConfig: + """The look: fragment GLSL, and the name it gives the frame it samples.""" + + fragment_glsl: str + sampled_input_binding_name: str = "upstream_frame" + + +class VideoFrameWithTheBagItArrivedIn: + """A typed read's target that keeps the bag beside the frame cast from it. + + The frame is constructed while the read is offering its claim, so it holds + the camera's pixels still for the landing copy; the bag is what gets + forwarded, because every key on it still describes the picture. + """ + + def __init__(self, **bag: Any) -> None: + self.bag = bag + self.video_frame = VideoFrame(**bag) + + +@processor(description="Draws the fragment shader its config carries over every frame") +class ShaderEffect: + """Frame in, the same frame through a fragment shader out.""" + + @input(delivery_profile="newest") + def video_from_upstream(self) -> None: ... + + @output() + def video_to_downstream(self) -> None: ... + + def __init__(self, config: ShaderEffectConfig) -> None: + self.fragment_glsl = config.fragment_glsl + self.sampled_input_binding_name = config.sampled_input_binding_name + + def setup(self, ctx: RuntimeContextFullAccess) -> None: + try: + self.graphics_kernel = ctx.gpu_full_access.create_graphics_kernel( + color_attachment_formats=[TEXTURE_FORMAT], + vertex_source=FULLSCREEN_TRIANGLE_VERTEX_GLSL, + fragment_source=self.fragment_glsl, + bindings={ + self.sampled_input_binding_name: ("sampled_texture", ["fragment"]) + }, + label="ShaderEffect", + ) + except Exception as refusal: + raise ValueError( + f"ShaderEffect could not build its pass from `fragment_glsl` sampling " + f"`{self.sampled_input_binding_name}` " + f"(`sampled_input_binding_name`): {refusal}" + ) from refusal + # Depth 1: the draw returns with the GPU work retired, and nothing + # outside this processor ever names a landing texture. + self.landing_texture_ring = ProcessorOutputTextureRing( + TEXTURE_FORMAT, SAMPLED_LANDING_TEXTURE_USAGE, depth=1 + ) + self.rendered_output_texture_ring = ProcessorOutputTextureRing( + TEXTURE_FORMAT, RENDERED_OUTPUT_TEXTURE_USAGE + ) + + def process(self, ctx: RuntimeContextLimitedAccess) -> None: + arrival = ctx.inputs.read( + "video_from_upstream", into=VideoFrameWithTheBagItArrivedIn + ) + if arrival is None: + return + frame = arrival.video_frame + + # A camera publishes buffer-backed frames and a draw binds + # texture-backed surfaces only, so each frame is copied device-to-device + # into a texture this processor owns. cupy does nothing here but that + # copy; the frame is a DLPack producer in its own right. + landing_texture = self.landing_texture_ring.next_texture_for_this_frame( + ctx.gpu_limited_access, frame.width, frame.height + ) + with landing_texture.as_device_tensor() as writable_landing_texture: + cupy.from_dlpack(writable_landing_texture)[...] = cupy.from_dlpack(frame) + + rendered_output_texture = ( + self.rendered_output_texture_ring.next_texture_for_this_frame( + ctx.gpu_limited_access, frame.width, frame.height + ) + ) + self.graphics_kernel.draw( + bindings={self.sampled_input_binding_name: landing_texture}, + color_targets=[rendered_output_texture], + extent=(frame.width, frame.height), + vertex_count=3, + ) + + # The upstream bag forwarded whole, with only the surface swapped: the + # capture stamp and the colour metadata still describe this picture, + # and a `VirtualCameraSink` sets the device's colorimetry from them. + rendered_bag = dict(arrival.bag) + rendered_bag["surface_id"] = rendered_output_texture.surface_id + # A per-frame layout override describes the surface it was published + # with, and this is a different surface. + rendered_bag.pop("texture_layout", None) + ctx.outputs.write("video_to_downstream", rendered_bag) diff --git a/examples/camera-virtual-camera/processors/shaders/fullscreen_triangle.vert b/examples/camera-virtual-camera/processors/shaders/fullscreen_triangle.vert new file mode 100644 index 000000000..3f564c9d4 --- /dev/null +++ b/examples/camera-virtual-camera/processors/shaders/fullscreen_triangle.vert @@ -0,0 +1,19 @@ +// Copyright (c) 2025 Jonathan Fontanez +// SPDX-License-Identifier: BUSL-1.1 + +// The vertex stage every look shares. No vertex buffer is reachable from a +// Python processor, so three vertices are fabricated from `gl_VertexIndex` and +// cover the whole viewport: +// vertex 0: pos(-1,-1), uv(0,0) vertex 1: pos(3,-1), uv(2,0) +// vertex 2: pos(-1,3), uv(0,2) +// `screen_uv` reaches a fragment as 0..1 across the frame, (0,0) at the top +// left — the same corner the frame's first texel is in. + +#version 450 + +layout(location = 0) out vec2 screen_uv; + +void main() { + screen_uv = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2); + gl_Position = vec4(screen_uv * 2.0 - 1.0, 0.0, 1.0); +} diff --git a/examples/camera-virtual-camera/processors/shaders/grayscale.frag b/examples/camera-virtual-camera/processors/shaders/grayscale.frag new file mode 100644 index 000000000..386fe018b --- /dev/null +++ b/examples/camera-virtual-camera/processors/shaders/grayscale.frag @@ -0,0 +1,18 @@ +// Copyright (c) 2025 Jonathan Fontanez +// SPDX-License-Identifier: BUSL-1.1 + +// Black and white: every pixel becomes its own brightness. + +#version 450 + +layout(location = 0) in vec2 screen_uv; +layout(location = 0) out vec4 painted_colour; + +layout(set = 0, binding = 0) uniform sampler2D upstream_frame; + +void main() { + vec4 source = texture(upstream_frame, screen_uv); + // BT.709 luma, the weights the HD standard published for it. + float luma = dot(source.rgb, vec3(0.2126, 0.7152, 0.0722)); + painted_colour = vec4(vec3(luma), source.a); +} diff --git a/examples/camera-virtual-camera/processors/shaders/pixelate.frag b/examples/camera-virtual-camera/processors/shaders/pixelate.frag new file mode 100644 index 000000000..64c2a2d88 --- /dev/null +++ b/examples/camera-virtual-camera/processors/shaders/pixelate.frag @@ -0,0 +1,27 @@ +// Copyright (c) 2025 Jonathan Fontanez +// SPDX-License-Identifier: BUSL-1.1 + +// Big square pixels: the picture is diced into cells and each cell takes the +// colour at its centre. + +#version 450 + +layout(location = 0) out vec4 painted_colour; + +layout(set = 0, binding = 0) uniform sampler2D upstream_frame; + +const int CELL_SIZE_IN_PIXELS = 16; + +void main() { + ivec2 at = ivec2(gl_FragCoord.xy); + ivec2 cell_origin = (at / CELL_SIZE_IN_PIXELS) * CELL_SIZE_IN_PIXELS; + // Clamped because a cell hanging off the right or bottom edge has its + // centre outside the frame. + ivec2 cell_centre = min( + cell_origin + CELL_SIZE_IN_PIXELS / 2, + textureSize(upstream_frame, 0) - 1 + ); + // texelFetch rather than texture(): the centre is an exact texel index, so + // there is nothing to filter. + painted_colour = texelFetch(upstream_frame, cell_centre, 0); +} diff --git a/examples/camera-virtual-camera/processors/shaders/vignette.frag b/examples/camera-virtual-camera/processors/shaders/vignette.frag new file mode 100644 index 000000000..79d6a9c4a --- /dev/null +++ b/examples/camera-virtual-camera/processors/shaders/vignette.frag @@ -0,0 +1,25 @@ +// Copyright (c) 2025 Jonathan Fontanez +// SPDX-License-Identifier: BUSL-1.1 + +// A lens vignette: the middle of the picture untouched, the corners fading +// towards black. + +#version 450 + +layout(location = 0) in vec2 screen_uv; +layout(location = 0) out vec4 painted_colour; + +layout(set = 0, binding = 0) uniform sampler2D upstream_frame; + +// Distances from the centre, in the frame's own 0..1 coordinates: nothing +// darkens inside the first, everything past the second is black. A corner sits +// at about 0.707, so it keeps a little of its picture. +const float FADE_STARTS_AT = 0.35; +const float FADE_ENDS_AT = 0.8; + +void main() { + vec4 source = texture(upstream_frame, screen_uv); + float distance_from_centre = distance(screen_uv, vec2(0.5)); + float light_kept = 1.0 - smoothstep(FADE_STARTS_AT, FADE_ENDS_AT, distance_from_centre); + painted_colour = vec4(source.rgb * light_kept, source.a); +} diff --git a/examples/camera-virtual-camera/pyproject.toml b/examples/camera-virtual-camera/pyproject.toml index 4f2a0d1aa..34aeffbc7 100644 --- a/examples/camera-virtual-camera/pyproject.toml +++ b/examples/camera-virtual-camera/pyproject.toml @@ -2,12 +2,16 @@ name = "camera-virtual-camera" version = "0.1.0" requires-python = ">=3.12" -# Both doors ship from 0.18.59, but a sink only takes a Python processor's -# frames from 0.18.60 — which is this example's second camera. 0.19.1 is the -# floor the example can actually run on: below it, a refused surface-store -# lookup leaked a plane fd per frame (#2207) and a desktop run died at about -# 30 s against the shell's 1024 descriptor limit. -dependencies = ["streamlib>=0.19.1", "numpy>=2.1"] +# `ShaderEffect` takes its look as a config class, which the wheel constructs +# from 0.21.0; a node serves the MCP recipe that inserts an effect between two +# running processors from 0.22.1. +dependencies = [ + "streamlib>=0.22.1", + "numpy>=2.1", + # `ShaderEffect`'s landing copy, device-to-device. Any DLPack-speaking GPU + # array package would serve; cupy is the smallest one that does nothing else. + "cupy-cuda13x>=14.2", +] # streamlib is served from its own simple index until the PyPI publication that # follows the project rename; everything else resolves from PyPI as usual. @@ -18,3 +22,9 @@ explicit = true [tool.uv.sources] streamlib = { index = "streamlib" } + +[dependency-groups] +dev = ["pytest>=8"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/examples/camera-virtual-camera/tests/shader_effect_test_app.py b/examples/camera-virtual-camera/tests/shader_effect_test_app.py new file mode 100644 index 000000000..0e75cf8cf --- /dev/null +++ b/examples/camera-virtual-camera/tests/shader_effect_test_app.py @@ -0,0 +1,97 @@ +# Copyright (c) 2025 Jonathan Fontanez +# SPDX-License-Identifier: BUSL-1.1 +"""Graphs a `ShaderEffect` test runs as a real `python` app. + +Launched with this example's directory on `PYTHONPATH`, so the app process and +every helper import `processors.shader_effect` under the same name the showcase +does. +""" + +import json +import sys + +from shader_effect_test_processors import ( + KnownPatternPixelBufferSource, + RenderedPixelReportingSink, +) +from streamlib import Runtime + +from processors.shader_effect import SHIPPED_SHADERS_DIRECTORY, ShaderEffect + +FRAGMENT_GLSL_THAT_DOES_NOT_COMPILE = """\ +#version 450 +layout(set = 0, binding = 0) uniform sampler2D upstream_frame; +layout(location = 0) out vec4 painted_colour; +void main() { + painted_colour = no_such_function(upstream_frame); +} +""" + + +def _shipped_fragment_glsl(shader_file_name: str) -> str: + return (SHIPPED_SHADERS_DIRECTORY / shader_file_name).read_text(encoding="utf-8") + + +def scenario_one_shipped_look( + shader_file_name: str, pixel_coordinates_to_report: "list[list[int]]" +) -> None: + """Known pattern → the shipped look → the reporting sink.""" + runtime = Runtime() + source = runtime.add(KnownPatternPixelBufferSource) + effect = runtime.add( + ShaderEffect, + config={"fragment_glsl": _shipped_fragment_glsl(shader_file_name)}, + ) + sink = runtime.add( + RenderedPixelReportingSink, + config={"pixel_coordinates_to_report": pixel_coordinates_to_report}, + ) + runtime.connect( + source.output("video_to_downstream"), effect.input("video_from_upstream") + ) + runtime.connect( + effect.output("video_to_downstream"), sink.input("video_from_upstream") + ) + runtime.run() + print("MARKER:CLEAN_EXIT", flush=True) + + +def scenario_a_look_that_does_not_compile_beside_one_that_does() -> None: + """The pattern fanned to a grayscale chain that reports and to an effect + whose shader cannot compile.""" + runtime = Runtime() + source = runtime.add(KnownPatternPixelBufferSource) + working_effect = runtime.add( + ShaderEffect, + config={"fragment_glsl": _shipped_fragment_glsl("grayscale.frag")}, + ) + sink = runtime.add( + RenderedPixelReportingSink, + config={"pixel_coordinates_to_report": [[0, 0]]}, + ) + effect_that_does_not_compile = runtime.add( + ShaderEffect, + config={"fragment_glsl": FRAGMENT_GLSL_THAT_DOES_NOT_COMPILE}, + ) + runtime.connect( + source.output("video_to_downstream"), + working_effect.input("video_from_upstream"), + ) + runtime.connect( + working_effect.output("video_to_downstream"), sink.input("video_from_upstream") + ) + runtime.connect( + source.output("video_to_downstream"), + effect_that_does_not_compile.input("video_from_upstream"), + ) + runtime.run() + print("MARKER:CLEAN_EXIT", flush=True) + + +if __name__ == "__main__": + if sys.argv[1] == "one_shipped_look": + scenario_one_shipped_look(sys.argv[2], json.loads(sys.argv[3])) + elif sys.argv[1] == "a_look_that_does_not_compile_beside_one_that_does": + scenario_a_look_that_does_not_compile_beside_one_that_does() + else: + raise SystemExit(f"no scenario named {sys.argv[1]!r}") diff --git a/examples/camera-virtual-camera/tests/shader_effect_test_processors.py b/examples/camera-virtual-camera/tests/shader_effect_test_processors.py new file mode 100644 index 000000000..bee0b33b2 --- /dev/null +++ b/examples/camera-virtual-camera/tests/shader_effect_test_processors.py @@ -0,0 +1,112 @@ +# Copyright (c) 2025 Jonathan Fontanez +# SPDX-License-Identifier: BUSL-1.1 +"""The two ends a `ShaderEffect` is tested between. + +The source publishes a known pattern the way `CameraSource` publishes a frame — +a buffer-backed surface — so the effect's landing copy runs exactly as it does +behind a camera. The sink reports the rendered pixels at the coordinates it was +configured with, one line per frame, for the test to hold against a reference +computed on the CPU. +""" + +import json +from dataclasses import dataclass, field + +from streamlib import ( + RuntimeContextFullAccess, + RuntimeContextLimitedAccess, + VideoFrame, + clock, + input, + log, + output, + processor, +) + +KNOWN_PATTERN_WIDTH = 64 +# Not square, so a shader that swaps its axes cannot pass. +KNOWN_PATTERN_HEIGHT = 48 + +RENDERED_PIXELS_MARKER = "MARKER:RENDERED_PIXELS " + + +def known_pattern_pixel_at(x: int, y: int) -> "tuple[int, int, int, int]": + """The RGBA8 value the source publishes at column `x`, row `y`.""" + return (4 * x, 5 * y, 96, 255) + + +@processor( + execution="continuous", + interval_ms=20, + description="Publishes a known RGBA pattern as a buffer-backed frame", +) +class KnownPatternPixelBufferSource: + """The pattern, filled once, published every tick.""" + + @output() + def video_to_downstream(self) -> None: ... + + def setup(self, ctx: RuntimeContextFullAccess) -> None: + # Held for the processor's life, so every id this publishes stays live. + self.known_pattern_pixel_buffer = ctx.gpu_full_access.acquire_pixel_buffer( + KNOWN_PATTERN_WIDTH, KNOWN_PATTERN_HEIGHT, "rgba" + ) + self.known_pattern_pixel_buffer.lock(read_only=False) + try: + pixels = self.known_pattern_pixel_buffer.as_numpy() + for y in range(KNOWN_PATTERN_HEIGHT): + for x in range(KNOWN_PATTERN_WIDTH): + pixels[y, x] = known_pattern_pixel_at(x, y) + finally: + self.known_pattern_pixel_buffer.unlock() + + def process(self, ctx: RuntimeContextLimitedAccess) -> None: + ctx.outputs.write( + "video_to_downstream", + { + "surface_id": self.known_pattern_pixel_buffer.surface_id, + "width": KNOWN_PATTERN_WIDTH, + "height": KNOWN_PATTERN_HEIGHT, + "timestamp_ns": clock.monotonic_now_ns(), + }, + ) + + +@dataclass +class RenderedPixelReportingSinkConfig: + """`[x, y]` pairs whose rendered value each frame's report carries.""" + + pixel_coordinates_to_report: "list[list[int]]" = field(default_factory=list) + + +@processor(description="Reports rendered pixels at configured coordinates") +class RenderedPixelReportingSink: + """One marker line per frame: the frame count and the requested pixels.""" + + @input(delivery_profile="newest") + def video_from_upstream(self) -> None: ... + + def __init__(self, config: RenderedPixelReportingSinkConfig) -> None: + self.pixel_coordinates_to_report = config.pixel_coordinates_to_report + self.frames_reported = 0 + + def process(self, ctx: RuntimeContextLimitedAccess) -> None: + frame = ctx.inputs.read("video_from_upstream", into=VideoFrame) + if frame is None: + return + with frame.cpu() as rendered_pixels: + reported_pixels = [ + [int(channel) for channel in rendered_pixels[y, x]] + for x, y in self.pixel_coordinates_to_report + ] + self.frames_reported += 1 + log.info( + RENDERED_PIXELS_MARKER + + json.dumps( + { + "frame": self.frames_reported, + "extent": [frame.width, frame.height], + "pixels": reported_pixels, + } + ) + ) diff --git a/examples/camera-virtual-camera/tests/test_shader_effect.py b/examples/camera-virtual-camera/tests/test_shader_effect.py new file mode 100644 index 000000000..ec523da26 --- /dev/null +++ b/examples/camera-virtual-camera/tests/test_shader_effect.py @@ -0,0 +1,264 @@ +# Copyright (c) 2025 Jonathan Fontanez +# SPDX-License-Identifier: BUSL-1.1 +"""`ShaderEffect`, driven as a real app: each shipped look, and a broken one. + +Every test here boots an engine and builds a kernel, so every test needs the +GPU: the shader compiler runs at kernel construction, behind the GPU context. +They run on a machine with one, from this example's venv — `uv run pytest`. +""" + +import json +import math +import os +import queue +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path + +import pytest +from shader_effect_test_processors import ( + KNOWN_PATTERN_HEIGHT, + KNOWN_PATTERN_WIDTH, + RENDERED_PIXELS_MARKER, + known_pattern_pixel_at, +) + +EXAMPLE_DIRECTORY = Path(__file__).resolve().parent.parent +TEST_APP = Path(__file__).resolve().parent / "shader_effect_test_app.py" + +# A cold engine boot stands up a GPU context and a helper interpreter per +# processor, and cupy's first import is slow; a real hang blows through it. +FIRST_RENDERED_FRAME_TIMEOUT_SECONDS = 90.0 +CLEAN_EXIT_TIMEOUT_SECONDS = 60.0 + +# An 8-bit target rounds, and a sampler may land a hair off a texel centre. +PIXEL_CHANNEL_TOLERANCE = 2 + + +class RunningTestApp: + """A `python shader_effect_test_app.py …` with its output pumped off the pipe.""" + + def __init__(self, *arguments: str) -> None: + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + path + for path in (str(EXAMPLE_DIRECTORY), environment.get("PYTHONPATH", "")) + if path + ) + self.process = subprocess.Popen( + [sys.executable, str(TEST_APP), *arguments], + cwd=TEST_APP.parent, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + self.output_lines: list[str] = [] + self._incoming: queue.Queue[str | None] = queue.Queue() + threading.Thread(target=self._pump_output, daemon=True).start() + + def _pump_output(self) -> None: + assert self.process.stdout is not None + for line in self.process.stdout: + self._incoming.put(line) + self._incoming.put(None) + + @property + def output(self) -> str: + return "".join(self.output_lines) + + def await_line( + self, matches: "Callable[[str], bool]", what: str, timeout: float + ) -> str: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError(f"no {what} within {timeout} s:\n{self.output}") + try: + line = self._incoming.get(timeout=min(0.5, remaining)) + except queue.Empty: + continue + if line is None: + raise AssertionError(f"the app exited before {what}:\n{self.output}") + self.output_lines.append(line) + if matches(line): + return line + + def interrupt_and_await_clean_exit(self) -> None: + self.process.send_signal(signal.SIGINT) + self.await_line( + lambda line: "MARKER:CLEAN_EXIT" in line, + "clean exit", + CLEAN_EXIT_TIMEOUT_SECONDS, + ) + assert self.process.wait(timeout=CLEAN_EXIT_TIMEOUT_SECONDS) == 0, self.output + + def kill_process_group(self) -> None: + if self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait() + + +@pytest.fixture +def start_test_app(): + """Hands out apps and kills their process groups however a test ends, so a + failed assertion never strands an engine holding the GPU.""" + started: list[RunningTestApp] = [] + + def start(*arguments: str) -> RunningTestApp: + app = RunningTestApp(*arguments) + started.append(app) + return app + + try: + yield start + finally: + for app in started: + app.kill_process_group() + + +def rendered_pixels_report_in(line: str) -> dict: + # Decoded up to the object's end: a forwarded helper record carries the + # processor's id after the message. + report, _ = json.JSONDecoder().raw_decode(line.split(RENDERED_PIXELS_MARKER, 1)[1]) + return report + + +def assert_rendered_matches_the_reference( + rendered: "list[int]", expected: "list[int]", what: str +) -> None: + assert all( + abs(rendered_channel - expected_channel) <= PIXEL_CHANNEL_TOLERANCE + for rendered_channel, expected_channel in zip(rendered, expected) + ), f"{what}: rendered {rendered}, the CPU reference is {expected}" + + +def unorm8(value: float) -> int: + return round(min(max(value, 0.0), 1.0) * 255) + + +def grayscale_reference_at(x: int, y: int) -> "list[int]": + red, green, blue, alpha = known_pattern_pixel_at(x, y) + luma = (0.2126 * red + 0.7152 * green + 0.0722 * blue) / 255 + return [unorm8(luma)] * 3 + [alpha] + + +def vignette_reference_at(x: int, y: int) -> "list[int]": + red, green, blue, alpha = known_pattern_pixel_at(x, y) + screen_u = (x + 0.5) / KNOWN_PATTERN_WIDTH + screen_v = (y + 0.5) / KNOWN_PATTERN_HEIGHT + distance_from_centre = math.hypot(screen_u - 0.5, screen_v - 0.5) + fade = min(max((distance_from_centre - 0.35) / (0.8 - 0.35), 0.0), 1.0) + light_kept = 1.0 - fade * fade * (3.0 - 2.0 * fade) + return [unorm8(channel / 255 * light_kept) for channel in (red, green, blue)] + [ + alpha + ] + + +def pixelate_reference_at(x: int, y: int) -> "list[int]": + cell_size = 16 + cell_centre_x = min( + (x // cell_size) * cell_size + cell_size // 2, KNOWN_PATTERN_WIDTH - 1 + ) + cell_centre_y = min( + (y // cell_size) * cell_size + cell_size // 2, KNOWN_PATTERN_HEIGHT - 1 + ) + return list(known_pattern_pixel_at(cell_centre_x, cell_centre_y)) + + +# One pixel per look, each chosen where the look changes the pattern: a corner +# the vignette darkens without blacking out, and a pixelate cell whose centre +# is not the pixel itself. +SHIPPED_LOOKS = [ + pytest.param("grayscale.frag", (37, 21), grayscale_reference_at, id="grayscale"), + pytest.param("vignette.frag", (5, 4), vignette_reference_at, id="vignette"), + pytest.param("pixelate.frag", (20, 37), pixelate_reference_at, id="pixelate"), +] + + +@pytest.mark.parametrize(("shader_file_name", "pixel", "reference_at"), SHIPPED_LOOKS) +def test_a_shipped_look_renders_the_cpu_reference_over_a_buffer_backed_frame( + start_test_app, shader_file_name, pixel, reference_at +): + x, y = pixel + expected = reference_at(x, y) + assert expected != list(known_pattern_pixel_at(x, y)), ( + "the chosen pixel must be one the look changes, or a pass that forwarded " + "the frame untouched would pass" + ) + + app = start_test_app("one_shipped_look", shader_file_name, json.dumps([[x, y]])) + report = rendered_pixels_report_in( + app.await_line( + lambda line: RENDERED_PIXELS_MARKER in line, + f"rendered frame from {shader_file_name}", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + ) + app.interrupt_and_await_clean_exit() + + assert report["extent"] == [KNOWN_PATTERN_WIDTH, KNOWN_PATTERN_HEIGHT] + assert_rendered_matches_the_reference( + report["pixels"][0], expected, f"{shader_file_name} at {pixel}" + ) + + +def test_a_look_that_does_not_compile_is_refused_at_setup_and_the_graph_keeps_running( + start_test_app, +): + app = start_test_app("a_look_that_does_not_compile_beside_one_that_does") + + # The engine's record of the failed setup names the processor, then carries + # the raised message and the compiler's diagnostic on the lines after it. + app.await_line( + lambda line: "Setup failed" in line and "[ShaderEffect 2]" in line, + "the engine refusing the broken look by its display name", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + refusal_message = app.await_line( + lambda line: "ShaderEffect could not build its pass" in line, + "the broken look's refusal message", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + assert "fragment_glsl" in refusal_message, ( + f"the refusal must name the config key: {refusal_message}" + ) + app.await_line( + lambda line: "error" in line and "no_such_function" in line, + "the compiler's own diagnostic in the refusal", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + + # Frames reported *after* the refusal: the working chain kept running + # through it rather than having finished before it. + first_report_after = rendered_pixels_report_in( + app.await_line( + lambda line: RENDERED_PIXELS_MARKER in line, + "a rendered frame after the refusal", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + ) + later_report = rendered_pixels_report_in( + app.await_line( + lambda line: ( + RENDERED_PIXELS_MARKER in line + and rendered_pixels_report_in(line)["frame"] + >= first_report_after["frame"] + 10 + ), + "ten more rendered frames", + FIRST_RENDERED_FRAME_TIMEOUT_SECONDS, + ) + ) + app.interrupt_and_await_clean_exit() + + assert_rendered_matches_the_reference( + later_report["pixels"][0], + grayscale_reference_at(0, 0), + "the working grayscale chain at (0, 0)", + ) From 28fc26acaa93c5bbdff73d5466a585bc1b23d529 Mon Sep 17 00:00:00 2001 From: Jonathan Fontanez Date: Sat, 12 Sep 2026 23:44:31 -0400 Subject: [PATCH 2/5] docs(examples): the virtual-camera README shows one ShaderEffect host carrying many looks Refs #2216 Co-Authored-By: Claude Opus 5 --- examples/camera-virtual-camera/README.md | 75 +++++++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/examples/camera-virtual-camera/README.md b/examples/camera-virtual-camera/README.md index 72ba479cc..4391c41a1 100644 --- a/examples/camera-virtual-camera/README.md +++ b/examples/camera-virtual-camera/README.md @@ -71,9 +71,78 @@ in the frame counts: over one 1080p run the passthrough camera wrote 1325 frames while the inverted one wrote 523 — and both sinks' dropped-frame counters read zero, so neither was outrun. The effect is simply slower, because a 1920×1080 frame goes out to the host and back for it. -An effect that stays on the GPU — a compute kernel, or a graphics pass as -`examples/camera-python-effects` writes them — does not pay that, and the two -cameras keep pace. +An effect that stays on the GPU does not pay that — `ShaderEffect`, below, is +one. + +## One host, many looks + +`processors/shader_effect.py` is an effect written once. It takes a fragment +shader as config, and the shader *is* the look — the engine compiles it at +`setup()` and draws it over every frame as one fullscreen pass, so the pixels +never leave the GPU. Three looks ship in `processors/shaders/` to start from: +`grayscale.frag`, `vignette.frag` and `pixelate.frag`. + +```python +from processors.shader_effect import SHIPPED_SHADERS_DIRECTORY, ShaderEffect + +vignette = rt.add( + ShaderEffect, + config={"fragment_glsl": (SHIPPED_SHADERS_DIRECTORY / "vignette.frag").read_text()}, +) +``` + +A new look is a paragraph of GLSL, not a processor. The shader samples the +frame as a `sampler2D` named `upstream_frame` (or whatever +`sampled_input_binding_name` in the config says), may read `screen_uv` at +location 0 — 0..1 across the frame from the top left — and writes one colour: + +```glsl +#version 450 +layout(location = 0) in vec2 screen_uv; +layout(location = 0) out vec4 painted_colour; +layout(set = 0, binding = 0) uniform sampler2D upstream_frame; + +void main() { + vec4 source = texture(upstream_frame, screen_uv); + painted_colour = vec4(source.bgr, source.a); +} +``` + +A shader that does not compile never gets as far as a frame: the effect is +refused at `setup()` with the compiler's own diagnostic in `streamlib logs`, +and every other processor in the graph keeps running. + +The ports are the inverting effect's, so a look takes its place without +touching anything else. **Into the running app, over MCP**, that is one remove +and one add — `ShaderEffect` is importable as +`processors.shader_effect:ShaderEffect` from this directory, which is what +`add_processor` needs: + +1. `remove_processor` with the `InvertingEffect`'s id from `graph`; +2. `add_processor` with `{"type": "processors.shader_effect:ShaderEffect", + "config": {"fragment_glsl": "…"}}`; +3. `connect` the camera's `video` to its `video_from_upstream`, and its + `video_to_downstream` to the inverted sink's `video`. + +To stack a look on top of the inversion instead, the node's +`insert_processor_between_linked_processors` prompt splices one into the link +between the effect and its sink. Swapping one look for another is the same +remove and add again. The camera, both sinks and every other processor +keep running through it — only the second camera goes without new frames for +the moment the splice takes, under a second on a 640×480 feed — and at that +size the effect keeps the source's 30 fps. + +Each frame is copied device-to-device into a texture the effect owns before the +pass samples it: a camera publishes buffer-backed frames, and a draw binds +texture-backed ones. `cupy` does nothing in the module but that copy. + +The tests in `tests/` run each shipped look over a known pattern and check a +pixel against the same maths done on the CPU, and check that a broken shader is +refused. They boot an engine and build kernels, so they need an NVIDIA GPU: + +```bash +uv run pytest +``` ## Run it From 5b4438565e5cc06d91a5943e9a8d31e53ec1b771 Mon Sep 17 00:00:00 2001 From: Jonathan Fontanez Date: Sat, 12 Sep 2026 23:59:50 -0400 Subject: [PATCH 3/5] =?UTF-8?q?fix(examples):=20ShaderEffect=20review=20ro?= =?UTF-8?q?und=20=E2=80=94=20explicit=20names,=20fact-only=20comments,=20p?= =?UTF-8?q?roven=20README=20claims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #2216 Co-Authored-By: Claude Opus 5 --- examples/camera-virtual-camera/README.md | 12 +++++------- .../processors/shader_effect.py | 17 +++++++++-------- .../processors/shaders/pixelate.frag | 6 ++---- examples/camera-virtual-camera/pyproject.toml | 7 ++----- .../tests/test_shader_effect.py | 14 +++++++++----- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/examples/camera-virtual-camera/README.md b/examples/camera-virtual-camera/README.md index 4391c41a1..e31d70dea 100644 --- a/examples/camera-virtual-camera/README.md +++ b/examples/camera-virtual-camera/README.md @@ -124,13 +124,11 @@ and one add — `ShaderEffect` is importable as 3. `connect` the camera's `video` to its `video_from_upstream`, and its `video_to_downstream` to the inverted sink's `video`. -To stack a look on top of the inversion instead, the node's -`insert_processor_between_linked_processors` prompt splices one into the link -between the effect and its sink. Swapping one look for another is the same -remove and add again. The camera, both sinks and every other processor -keep running through it — only the second camera goes without new frames for -the moment the splice takes, under a second on a 640×480 feed — and at that -size the effect keeps the source's 30 fps. +Swapping one look for another is the same remove and add again. The camera, +both sinks and every other processor keep running through it — only the second +camera goes without new frames for the moment the splice takes, under a second +on a 640×480 feed — and the effect keeps the source's 30 fps, at 1920×1080 as +much as at 640×480. Each frame is copied device-to-device into a texture the effect owns before the pass samples it: a camera publishes buffer-backed frames, and a draw binds diff --git a/examples/camera-virtual-camera/processors/shader_effect.py b/examples/camera-virtual-camera/processors/shader_effect.py index 7040dac59..c0399676f 100644 --- a/examples/camera-virtual-camera/processors/shader_effect.py +++ b/examples/camera-virtual-camera/processors/shader_effect.py @@ -39,7 +39,7 @@ # One format end to end: the camera publishes RGBA8 and a `VirtualCameraSink` # samples RGBA8 on its way to the device's buffers. -TEXTURE_FORMAT = "rgba8_unorm" +LANDING_AND_RENDERED_FRAME_TEXTURE_FORMAT = "rgba8_unorm" # The texture each incoming frame lands in and the shader samples. SAMPLED_LANDING_TEXTURE_USAGE = ["texture_binding"] @@ -86,7 +86,7 @@ def __init__(self, config: ShaderEffectConfig) -> None: def setup(self, ctx: RuntimeContextFullAccess) -> None: try: self.graphics_kernel = ctx.gpu_full_access.create_graphics_kernel( - color_attachment_formats=[TEXTURE_FORMAT], + color_attachment_formats=[LANDING_AND_RENDERED_FRAME_TEXTURE_FORMAT], vertex_source=FULLSCREEN_TRIANGLE_VERTEX_GLSL, fragment_source=self.fragment_glsl, bindings={ @@ -94,8 +94,8 @@ def setup(self, ctx: RuntimeContextFullAccess) -> None: }, label="ShaderEffect", ) - except Exception as refusal: - raise ValueError( + except RuntimeError as refusal: + raise RuntimeError( f"ShaderEffect could not build its pass from `fragment_glsl` sampling " f"`{self.sampled_input_binding_name}` " f"(`sampled_input_binding_name`): {refusal}" @@ -103,10 +103,12 @@ def setup(self, ctx: RuntimeContextFullAccess) -> None: # Depth 1: the draw returns with the GPU work retired, and nothing # outside this processor ever names a landing texture. self.landing_texture_ring = ProcessorOutputTextureRing( - TEXTURE_FORMAT, SAMPLED_LANDING_TEXTURE_USAGE, depth=1 + LANDING_AND_RENDERED_FRAME_TEXTURE_FORMAT, + SAMPLED_LANDING_TEXTURE_USAGE, + depth=1, ) self.rendered_output_texture_ring = ProcessorOutputTextureRing( - TEXTURE_FORMAT, RENDERED_OUTPUT_TEXTURE_USAGE + LANDING_AND_RENDERED_FRAME_TEXTURE_FORMAT, RENDERED_OUTPUT_TEXTURE_USAGE ) def process(self, ctx: RuntimeContextLimitedAccess) -> None: @@ -119,8 +121,7 @@ def process(self, ctx: RuntimeContextLimitedAccess) -> None: # A camera publishes buffer-backed frames and a draw binds # texture-backed surfaces only, so each frame is copied device-to-device - # into a texture this processor owns. cupy does nothing here but that - # copy; the frame is a DLPack producer in its own right. + # into a texture this processor owns. landing_texture = self.landing_texture_ring.next_texture_for_this_frame( ctx.gpu_limited_access, frame.width, frame.height ) diff --git a/examples/camera-virtual-camera/processors/shaders/pixelate.frag b/examples/camera-virtual-camera/processors/shaders/pixelate.frag index 64c2a2d88..a184fff85 100644 --- a/examples/camera-virtual-camera/processors/shaders/pixelate.frag +++ b/examples/camera-virtual-camera/processors/shaders/pixelate.frag @@ -13,15 +13,13 @@ layout(set = 0, binding = 0) uniform sampler2D upstream_frame; const int CELL_SIZE_IN_PIXELS = 16; void main() { - ivec2 at = ivec2(gl_FragCoord.xy); - ivec2 cell_origin = (at / CELL_SIZE_IN_PIXELS) * CELL_SIZE_IN_PIXELS; + ivec2 fragment_pixel_coordinate = ivec2(gl_FragCoord.xy); + ivec2 cell_origin = (fragment_pixel_coordinate / CELL_SIZE_IN_PIXELS) * CELL_SIZE_IN_PIXELS; // Clamped because a cell hanging off the right or bottom edge has its // centre outside the frame. ivec2 cell_centre = min( cell_origin + CELL_SIZE_IN_PIXELS / 2, textureSize(upstream_frame, 0) - 1 ); - // texelFetch rather than texture(): the centre is an exact texel index, so - // there is nothing to filter. painted_colour = texelFetch(upstream_frame, cell_centre, 0); } diff --git a/examples/camera-virtual-camera/pyproject.toml b/examples/camera-virtual-camera/pyproject.toml index 34aeffbc7..85cf969b3 100644 --- a/examples/camera-virtual-camera/pyproject.toml +++ b/examples/camera-virtual-camera/pyproject.toml @@ -2,14 +2,11 @@ name = "camera-virtual-camera" version = "0.1.0" requires-python = ">=3.12" -# `ShaderEffect` takes its look as a config class, which the wheel constructs -# from 0.21.0; a node serves the MCP recipe that inserts an effect between two -# running processors from 0.22.1. +# 0.22.1 is the release `ShaderEffect`'s tests and live swap ran against. dependencies = [ "streamlib>=0.22.1", "numpy>=2.1", - # `ShaderEffect`'s landing copy, device-to-device. Any DLPack-speaking GPU - # array package would serve; cupy is the smallest one that does nothing else. + # `ShaderEffect`'s landing copy, device-to-device. "cupy-cuda13x>=14.2", ] diff --git a/examples/camera-virtual-camera/tests/test_shader_effect.py b/examples/camera-virtual-camera/tests/test_shader_effect.py index ec523da26..150d9d30e 100644 --- a/examples/camera-virtual-camera/tests/test_shader_effect.py +++ b/examples/camera-virtual-camera/tests/test_shader_effect.py @@ -59,14 +59,16 @@ def __init__(self, *arguments: str) -> None: start_new_session=True, ) self.output_lines: list[str] = [] - self._incoming: queue.Queue[str | None] = queue.Queue() + self._output_lines_pumped_from_the_app_pipe: queue.Queue[str | None] = ( + queue.Queue() + ) threading.Thread(target=self._pump_output, daemon=True).start() def _pump_output(self) -> None: assert self.process.stdout is not None for line in self.process.stdout: - self._incoming.put(line) - self._incoming.put(None) + self._output_lines_pumped_from_the_app_pipe.put(line) + self._output_lines_pumped_from_the_app_pipe.put(None) @property def output(self) -> str: @@ -81,7 +83,9 @@ def await_line( if remaining <= 0: raise AssertionError(f"no {what} within {timeout} s:\n{self.output}") try: - line = self._incoming.get(timeout=min(0.5, remaining)) + line = self._output_lines_pumped_from_the_app_pipe.get( + timeout=min(0.5, remaining) + ) except queue.Empty: continue if line is None: @@ -135,7 +139,7 @@ def assert_rendered_matches_the_reference( ) -> None: assert all( abs(rendered_channel - expected_channel) <= PIXEL_CHANNEL_TOLERANCE - for rendered_channel, expected_channel in zip(rendered, expected) + for rendered_channel, expected_channel in zip(rendered, expected, strict=True) ), f"{what}: rendered {rendered}, the CPU reference is {expected}" From e09811194aa29e1fe702188cfef6edee122f9461 Mon Sep 17 00:00:00 2001 From: Jonathan Fontanez Date: Sun, 13 Sep 2026 00:02:06 -0400 Subject: [PATCH 4/5] docs(examples): say which run each ShaderEffect rate figure came from Refs #2216 Co-Authored-By: Claude Opus 5 --- examples/camera-virtual-camera/README.md | 6 ++++-- examples/camera-virtual-camera/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/camera-virtual-camera/README.md b/examples/camera-virtual-camera/README.md index e31d70dea..f47395fb7 100644 --- a/examples/camera-virtual-camera/README.md +++ b/examples/camera-virtual-camera/README.md @@ -127,8 +127,10 @@ and one add — `ShaderEffect` is importable as Swapping one look for another is the same remove and add again. The camera, both sinks and every other processor keep running through it — only the second camera goes without new frames for the moment the splice takes, under a second -on a 640×480 feed — and the effect keeps the source's 30 fps, at 1920×1080 as -much as at 640×480. +on a 640×480 feed. + +The effect keeps its source's rate: 30 fps in that 640×480 showcase run, and +30 fps from a synthetic 1920×1080 source into a counting sink. Each frame is copied device-to-device into a texture the effect owns before the pass samples it: a camera publishes buffer-backed frames, and a draw binds diff --git a/examples/camera-virtual-camera/pyproject.toml b/examples/camera-virtual-camera/pyproject.toml index 85cf969b3..19980e850 100644 --- a/examples/camera-virtual-camera/pyproject.toml +++ b/examples/camera-virtual-camera/pyproject.toml @@ -2,7 +2,7 @@ name = "camera-virtual-camera" version = "0.1.0" requires-python = ">=3.12" -# 0.22.1 is the release `ShaderEffect`'s tests and live swap ran against. +# 0.22.1 is the release `ShaderEffect`'s tests pass against. dependencies = [ "streamlib>=0.22.1", "numpy>=2.1", From 1878a8a82ea4107d0e8410e9da2f7701c2c31b43 Mon Sep 17 00:00:00 2001 From: Jonathan Fontanez Date: Sun, 13 Sep 2026 09:06:28 -0400 Subject: [PATCH 5/5] fix(examples): suppress Ruff A004 on the line ShaderEffect imports streamlib's input decorator Refs #2216 Co-Authored-By: Claude Opus 5 --- examples/camera-virtual-camera/processors/shader_effect.py | 2 +- .../tests/shader_effect_test_processors.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/camera-virtual-camera/processors/shader_effect.py b/examples/camera-virtual-camera/processors/shader_effect.py index c0399676f..c5b511aa1 100644 --- a/examples/camera-virtual-camera/processors/shader_effect.py +++ b/examples/camera-virtual-camera/processors/shader_effect.py @@ -26,7 +26,7 @@ RuntimeContextFullAccess, RuntimeContextLimitedAccess, VideoFrame, - input, + input, # noqa: A004 — streamlib's port decorator output, processor, ) diff --git a/examples/camera-virtual-camera/tests/shader_effect_test_processors.py b/examples/camera-virtual-camera/tests/shader_effect_test_processors.py index bee0b33b2..f67d63cac 100644 --- a/examples/camera-virtual-camera/tests/shader_effect_test_processors.py +++ b/examples/camera-virtual-camera/tests/shader_effect_test_processors.py @@ -17,7 +17,7 @@ RuntimeContextLimitedAccess, VideoFrame, clock, - input, + input, # noqa: A004 — streamlib's port decorator log, output, processor,