From b08188e8f1be782f9f5ab07ce362ed3a725529de Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Thu, 11 Jun 2026 15:39:16 +0100 Subject: [PATCH 1/3] Document gsplat fragment shader customization and relighting --- .../building/custom-shaders/fragment.md | 97 +++++++++++++++++++ .../building/custom-shaders/index.md | 60 ++++++++++++ .../vertex.md} | 21 +--- .../gaussian-splatting/building/relighting.md | 86 ++++++++++++++++ .../work-buffer-rendering.md | 2 + sidebars.js | 14 ++- 6 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md create mode 100644 docs/user-manual/gaussian-splatting/building/custom-shaders/index.md rename docs/user-manual/gaussian-splatting/building/{custom-shaders.md => custom-shaders/vertex.md} (76%) create mode 100644 docs/user-manual/gaussian-splatting/building/relighting.md diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md new file mode 100644 index 00000000000..307ceadbd19 --- /dev/null +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md @@ -0,0 +1,97 @@ +--- +title: Fragment Stage +description: "Customize the final Gaussian splat fragment color with the gsplatModifyPS shader chunk: per-pixel color modification, GLSL/WGSL, and available shader inputs." +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The `gsplatModifyPS` chunk customizes the final splat color in the fragment stage. It runs **once per covered pixel**, so effects can vary smoothly across a splat's footprint — something the per-splat vertex stage cannot do. + +**View Live Example** - The [Relighting](/user-manual/gaussian-splatting/building/relighting) technique is built on this hook. + + + +## Overridable Function + +The chunk overrides a single function: + + + + +```glsl +void modifySplatColor(vec2 gaussianUV, inout vec4 color); +``` + + + + +```wgsl +fn modifySplatColor(gaussianUV: vec2f, color: ptr); +``` + + + + +It is called in the forward pass after the gaussian falloff and opacity dither have been evaluated, just before the color is premultiplied and output: + +- `gaussianUV` — the fragment's position within the gaussian footprint: `(0,0)` at the splat center, length 1 at the edge where the splat is clipped. `dot(gaussianUV, gaussianUV)` gives the normalized squared radius used by the falloff. +- `color` — `rgb` is the splat color, `a` is the final fragment alpha. Both can be modified; alpha changes affect the blending weight, enabling custom falloffs or per-pixel fades. + +## Available Inputs + +Inside the chunk you can also use: + +- `gl_FragCoord` (GLSL) / `pcPosition` (WGSL) — the fragment's framebuffer position in pixels +- `uScreenSize` — engine-provided `vec4` uniform: `xy` = render target size, `zw` = inverse size +- Your own uniforms and textures, declared in the chunk and driven via material parameters + +## Example + +This chunk samples a screen-aligned texture at the fragment's own screen position and modulates the splat color by it — the core of the [Relighting](/user-manual/gaussian-splatting/building/relighting) technique: + + + + +```glsl +uniform sampler2D uTintMap; +uniform vec4 uScreenSize; + +void modifySplatColor(vec2 gaussianUV, inout vec4 color) { + vec3 tint = textureLod(uTintMap, gl_FragCoord.xy * uScreenSize.zw, 0.0).rgb; + color.rgb *= tint; +} +``` + + + + +```wgsl +var uTintMap: texture_2d; +var uTintMapSampler: sampler; +uniform uScreenSize: vec4f; + +fn modifySplatColor(gaussianUV: vec2f, color: ptr) { + let tint = textureSampleLevel(uTintMap, uTintMapSampler, pcPosition.xy * uniform.uScreenSize.zw, 0.0).rgb; + *color = vec4f((*color).rgb * tint, (*color).a); +} +``` + + + + +Apply it the same way as the vertex chunk, using the `gsplatModifyPS` key: + +```javascript +const sceneMat = app.scene.gsplat.material; + +sceneMat.getShaderChunks('glsl').set('gsplatModifyPS', glslFragShader); +sceneMat.getShaderChunks('wgsl').set('gsplatModifyPS', wgslFragShader); +sceneMat.setParameter('uTintMap', tintTexture); +sceneMat.update(); +``` + +## See Also + +- [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — move, scale and tint splats +- [Relighting](/user-manual/gaussian-splatting/building/relighting) — light splats using a proxy mesh, built on this hook diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md new file mode 100644 index 00000000000..cb6bc472026 --- /dev/null +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md @@ -0,0 +1,60 @@ +--- +title: Custom Shaders +description: "Customize Gaussian splat rendering with shader chunks on the scene gsplat material: vertex and fragment stage hooks, how to choose between them, and how to apply them." +--- + +The PlayCanvas Engine lets you customize how Gaussian Splats are rendered by overriding shader chunks. The chunks are set on the scene-wide gsplat material ([`app.scene.gsplat.material`](https://api.playcanvas.com/engine/classes/GSplatParams.html#material)), so a single custom shader applies to **all** splats in the scene. + +There are two customization points, one per shader stage: + +| Chunk | Stage | Runs | Purpose | +| --- | --- | --- | --- | +| [`gsplatModifyVS`](/user-manual/gaussian-splatting/building/custom-shaders/vertex) | Vertex | Once per splat | Modify splat position, rotation, scale, color and opacity | +| [`gsplatModifyPS`](/user-manual/gaussian-splatting/building/custom-shaders/fragment) | Fragment | Once per covered pixel | Modify the final color and alpha of each splat fragment | + +## Choosing a Stage + +**Use the [vertex stage](/user-manual/gaussian-splatting/building/custom-shaders/vertex)** for anything that is uniform across a splat: moving, scaling, rotating, hiding splats, or tinting them based on their position. It runs once per splat, so it is the cheaper option and the only one that can change splat geometry. + +**Use the [fragment stage](/user-manual/gaussian-splatting/building/custom-shaders/fragment)** when the effect needs to vary *across* a splat's footprint — for example when sampling a texture. It runs once per covered fragment, so it costs more on heavily overlapping splats. + +The two stages can be combined freely — implement either or both. + +## Applying Chunks + +Both chunks follow the same pattern: set the chunk source for each shader language (GLSL covers WebGL, WGSL covers WebGPU), then update the material to recompile: + +```javascript +const sceneMat = app.scene.gsplat.material; + +sceneMat.getShaderChunks('glsl').set('gsplatModifyVS', glslChunk); +sceneMat.getShaderChunks('wgsl').set('gsplatModifyVS', wgslChunk); +sceneMat.update(); +``` + +Custom uniforms declared by your chunks are driven through material parameters each frame: + +```javascript +app.on('update', (dt) => { + sceneMat.setParameter('uTime', currentTime); + sceneMat.update(); +}); +``` + +## Removing Chunks + +To revert to default rendering, delete the chunk override and update the material: + +```javascript +const sceneMat = app.scene.gsplat.material; +sceneMat.getShaderChunks('glsl').delete('gsplatModifyVS'); +sceneMat.getShaderChunks('wgsl').delete('gsplatModifyVS'); +sceneMat.update(); +``` + +## See Also + +- [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — move, scale and tint splats +- [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — per-pixel color modification +- [Relighting](/user-manual/gaussian-splatting/building/relighting) — light splats using a proxy mesh, built on the fragment hook +- [Work Buffer Rendering](/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering) — customize the global render pass that draws the sorted splats diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md similarity index 76% rename from docs/user-manual/gaussian-splatting/building/custom-shaders.md rename to docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md index 8bae1ff3da1..5ed647634a7 100644 --- a/docs/user-manual/gaussian-splatting/building/custom-shaders.md +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md @@ -1,12 +1,12 @@ --- -title: Custom Shaders -description: "Customize Gaussian splat rendering with the gsplatModifyVS shader chunk on the scene gsplat material: overridable functions, GLSL/WGSL, and a live example." +title: Vertex Stage +description: "Customize Gaussian splat position, rotation, scale and color with the gsplatModifyVS shader chunk: overridable functions, GLSL/WGSL, and a live example." --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -The PlayCanvas Engine lets you customize how Gaussian Splats are rendered by overriding the `gsplatModifyVS` shader chunk. The chunk is set on the scene-wide gsplat material ([`app.scene.gsplat.material`](https://api.playcanvas.com/engine/classes/GSplatParams.html#material)), so a single custom shader applies to **all** splats in the scene. +The `gsplatModifyVS` chunk customizes splats in the vertex stage. It runs **once per splat**, making it the right place to move, rotate, scale, hide or tint whole splats. **View Live Example** - See shader chunk customization in action with animated splats. @@ -14,7 +14,7 @@ The PlayCanvas Engine lets you customize how Gaussian Splats are rendered by ove ## Overridable Functions -The `gsplatModifyVS` chunk lets you override three functions in the splat vertex stage: +The `gsplatModifyVS` chunk lets you override three functions: | Function | Purpose | | --- | --- | @@ -121,17 +121,6 @@ app.on('update', (dt) => { }); ``` -## Removing a Custom Shader - -To revert to default rendering, delete the chunk override and update the material: - -```javascript -const sceneMat = app.scene.gsplat.material; -sceneMat.getShaderChunks('glsl').delete('gsplatModifyVS'); -sceneMat.getShaderChunks('wgsl').delete('gsplatModifyVS'); -sceneMat.update(); -``` - ## See Also -- [Work Buffer Rendering](/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering) — customize the global render pass that draws the sorted splats +- [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — per-pixel color modification diff --git a/docs/user-manual/gaussian-splatting/building/relighting.md b/docs/user-manual/gaussian-splatting/building/relighting.md new file mode 100644 index 00000000000..376827dd927 --- /dev/null +++ b/docs/user-manual/gaussian-splatting/building/relighting.md @@ -0,0 +1,86 @@ +--- +title: Relighting +description: "Relight Gaussian splat scenes with standard lights using a proxy mesh and the GsplatRelighting script: setup, lights, shadows and tuning." +--- + +Gaussian splat scenes are captured with their lighting baked in. Relighting lets you change that lighting at runtime — add a sun with soft shadows, place point lights, or swap the environment — by lighting a **proxy mesh** of the scene with standard lights and transferring that lighting onto the splats per pixel. + +**View Live Example** - A splat scene relit by an HDRI environment, a PCSS directional light and shadow-casting omni lights. + + + +## How It Works + +1. A simplified mesh approximating the splat scene (for example reconstructed from the splats, or the photogrammetry mesh) is placed on a dedicated layer, together with the lights that should light it. +2. The `GsplatRelighting` script renders that layer from a camera matching the main camera into an offscreen texture: lit mesh color in RGB, and a mesh coverage mask in alpha. +3. A [fragment stage](/user-manual/gaussian-splatting/building/custom-shaders/fragment) shader chunk modulates each splat fragment by the lighting sampled at its own screen position. Because the texture is screen-aligned with the main camera, no reprojection is needed and the lighting transfers per pixel. + +Splats not covered by the proxy mesh (such as the sky) are left untinted, or scaled by a separate background multiplier so they can follow the environment exposure. + +## Setup + +The technique is packaged in the [`GsplatRelighting`](https://github.com/playcanvas/engine/blob/main/scripts/esm/gsplat/gsplat-relighting.mjs) script. Attach it to the entity holding your main camera: + +```javascript +import { GsplatRelighting } from 'playcanvas/scripts/esm/gsplat/gsplat-relighting.mjs'; + +camera.addComponent('script'); +const relighting = camera.script.create(GsplatRelighting, { + properties: { + blend: 0.5, + brightness: 1 + } +}); +``` + +Place the proxy mesh on the relighting layer, with a material configured to write the coverage mask: + +```javascript +const meshMaterial = new pc.StandardMaterial(); +meshMaterial.diffuse = new pc.Color(0.5, 0.5, 0.5); +relighting.configureMaterial(meshMaterial); + +const meshEntity = meshAsset.resource.instantiateRenderEntity(); +meshEntity.findComponents('render').forEach((render) => { + render.layers = [relighting.layer.id]; + render.meshInstances.forEach((meshInstance) => { + meshInstance.material = meshMaterial; + }); +}); +app.root.addChild(meshEntity); +``` + +Then add lights to the same layer — any standard lights work, including image based lighting, shadow-casting directional and omni lights: + +```javascript +const light = new pc.Entity('sun'); +light.addComponent('light', { + type: 'directional', + layers: [relighting.layer.id], + castShadows: true +}); +app.root.addChild(light); +``` + +## Script Attributes + +| Attribute | Default | Purpose | +| --- | --- | --- | +| `blend` | 1 | How much the mesh lighting affects the splats (0 = original splats, 1 = fully modulated) | +| `brightness` | 2 | Brightness of the lighting when tinting; 2 compensates the 0.5 gray albedo of the proxy material | +| `background` | 1 | Multiplier for splats not covered by the mesh (e.g. the sky), letting them follow environment exposure | +| `textureScale` | 1 | Resolution of the lighting texture relative to the back buffer | +| `layerName` | `'Relighting'` | Name of the layer for the proxy mesh and lights (created if missing) | +| `priority` | -1 | Priority of the internal camera; keep below the main camera so the texture renders first | + +The lighting texture is available as `relighting.texture` (HDR where supported), for debugging or further processing. + +## Notes + +- The proxy mesh quality directly drives the result: where its silhouette diverges from the splat surface, lighting boundaries land in the wrong place. A closer-matching mesh is the main quality lever. +- Lights that do not move can use [`SHADOWUPDATE_THISFRAME`](https://api.playcanvas.com/engine/variables/SHADOWUPDATE_THISFRAME.html) to render their shadow maps once instead of every frame — the live example does this for its omni lights. + +## See Also + +- [Custom Shaders](/user-manual/gaussian-splatting/building/custom-shaders) — the shader chunk system this is built on +- [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — the per-pixel hook used to apply the lighting diff --git a/docs/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering.md b/docs/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering.md index d1ec4ec96bc..2ec9ea0dd4a 100644 --- a/docs/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering.md +++ b/docs/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering.md @@ -82,6 +82,8 @@ Your modifier code must implement three functions: `modifySplatCenter` always executes first. You can use it to sample extra streams and store values in global variables, or execute code shared between the three functions. +In addition to this vertex-stage modifier, the final splat color can also be customized per pixel using the `gsplatModifyPS` fragment chunk — see [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment). + ### Removing the Modifier To remove customization and restore default rendering: diff --git a/sidebars.js b/sidebars.js index 25566ec27e9..62a4a4e13d5 100644 --- a/sidebars.js +++ b/sidebars.js @@ -910,7 +910,19 @@ const sidebars = { 'user-manual/gaussian-splatting/building/picking', 'user-manual/gaussian-splatting/building/shadows', 'user-manual/gaussian-splatting/building/fisheye', - 'user-manual/gaussian-splatting/building/custom-shaders', + { + type: 'category', + label: 'Custom Shaders', + link: { + type: 'doc', + id: 'user-manual/gaussian-splatting/building/custom-shaders/index', + }, + items: [ + 'user-manual/gaussian-splatting/building/custom-shaders/vertex', + 'user-manual/gaussian-splatting/building/custom-shaders/fragment', + ], + }, + 'user-manual/gaussian-splatting/building/relighting', { type: 'category', label: 'Procedural Splats', From 24afdca20723c5886bbd4259f169f5ee1823c69a Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Thu, 11 Jun 2026 20:02:40 +0100 Subject: [PATCH 2/3] Feature the shader rings example on the Fragment Stage page --- .../building/custom-shaders/fragment.md | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md index 307ceadbd19..37f8278a837 100644 --- a/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md @@ -8,9 +8,9 @@ import TabItem from '@theme/TabItem'; The `gsplatModifyPS` chunk customizes the final splat color in the fragment stage. It runs **once per covered pixel**, so effects can vary smoothly across a splat's footprint — something the per-splat vertex stage cannot do. -**View Live Example** - The [Relighting](/user-manual/gaussian-splatting/building/relighting) technique is built on this hook. +**View Live Example** - Each splat rendered as a ring of its own color, with a highlight wave. - + ## Overridable Function @@ -48,18 +48,25 @@ Inside the chunk you can also use: ## Example -This chunk samples a screen-aligned texture at the fragment's own screen position and modulates the splat color by it — the core of the [Relighting](/user-manual/gaussian-splatting/building/relighting) technique: +This is the chunk used by the live example above. It renders each splat as a ring of its own color: `gaussianUV` provides the position within the splat footprint, and `fwidth` converts the requested ring width from pixels into footprint units, keeping the ring a constant screen-space width at any zoom: ```glsl -uniform sampler2D uTintMap; -uniform vec4 uScreenSize; +uniform float uRingWidth; +uniform float uRingAlpha; void modifySplatColor(vec2 gaussianUV, inout vec4 color) { - vec3 tint = textureLod(uTintMap, gl_FragCoord.xy * uScreenSize.zw, 0.0).rgb; - color.rgb *= tint; + // distance from the splat center: 0 at center, 1 at the clipping edge + float radius = length(gaussianUV); + + // ring of constant screen-space width at the splat edge - fwidth gives the + // change of radius per screen pixel, converting pixels to radius units + float radiusPerPixel = fwidth(radius); + float innerEdge = 1.0 - uRingWidth * radiusPerPixel; + float ring = smoothstep(innerEdge - radiusPerPixel, innerEdge, radius); + color.a = ring * uRingAlpha; } ``` @@ -67,30 +74,39 @@ void modifySplatColor(vec2 gaussianUV, inout vec4 color) { ```wgsl -var uTintMap: texture_2d; -var uTintMapSampler: sampler; -uniform uScreenSize: vec4f; +uniform uRingWidth: f32; +uniform uRingAlpha: f32; fn modifySplatColor(gaussianUV: vec2f, color: ptr) { - let tint = textureSampleLevel(uTintMap, uTintMapSampler, pcPosition.xy * uniform.uScreenSize.zw, 0.0).rgb; - *color = vec4f((*color).rgb * tint, (*color).a); + // distance from the splat center: 0 at center, 1 at the clipping edge + let radius = length(gaussianUV); + + // ring of constant screen-space width at the splat edge - fwidth gives the + // change of radius per screen pixel, converting pixels to radius units + let radiusPerPixel = fwidth(radius); + let innerEdge = 1.0 - uniform.uRingWidth * radiusPerPixel; + let ring = smoothstep(innerEdge - radiusPerPixel, innerEdge, radius); + *color = vec4f((*color).rgb, ring * uniform.uRingAlpha); } ``` -Apply it the same way as the vertex chunk, using the `gsplatModifyPS` key: +Apply it the same way as the vertex chunk, using the `gsplatModifyPS` key, and drive the uniforms via material parameters: ```javascript const sceneMat = app.scene.gsplat.material; sceneMat.getShaderChunks('glsl').set('gsplatModifyPS', glslFragShader); sceneMat.getShaderChunks('wgsl').set('gsplatModifyPS', wgslFragShader); -sceneMat.setParameter('uTintMap', tintTexture); +sceneMat.setParameter('uRingWidth', 1); +sceneMat.setParameter('uRingAlpha', 0.25); sceneMat.update(); ``` +For an effect that samples a screen-aligned texture at each fragment's screen position, see [Relighting](/user-manual/gaussian-splatting/building/relighting) — it modulates splats by the lighting of a proxy mesh rendered to an offscreen texture. + ## See Also - [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — move, scale and tint splats From 3e5051dd3bd3ab7dde35b166123668e36c371146 Mon Sep 17 00:00:00 2001 From: Martin Valigursky Date: Fri, 12 Jun 2026 12:40:46 +0100 Subject: [PATCH 3/3] Document gsplat varying streams --- .../building/custom-shaders/fragment.md | 1 + .../building/custom-shaders/index.md | 3 +- .../building/custom-shaders/varyings.md | 189 ++++++++++++++++++ .../building/custom-shaders/vertex.md | 1 + sidebars.js | 1 + 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 docs/user-manual/gaussian-splatting/building/custom-shaders/varyings.md diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md index 37f8278a837..0dd5cc9562d 100644 --- a/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/fragment.md @@ -110,4 +110,5 @@ For an effect that samples a screen-aligned texture at each fragment's screen po ## See Also - [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — move, scale and tint splats +- [Varying Streams](/user-manual/gaussian-splatting/building/custom-shaders/varyings) — read per-splat values written by the vertex stage - [Relighting](/user-manual/gaussian-splatting/building/relighting) — light splats using a proxy mesh, built on this hook diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md index cb6bc472026..e3c0456f4f0 100644 --- a/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/index.md @@ -18,7 +18,7 @@ There are two customization points, one per shader stage: **Use the [fragment stage](/user-manual/gaussian-splatting/building/custom-shaders/fragment)** when the effect needs to vary *across* a splat's footprint — for example when sampling a texture. It runs once per covered fragment, so it costs more on heavily overlapping splats. -The two stages can be combined freely — implement either or both. +The two stages can be combined freely — implement either or both. Per-splat values can also be passed from the vertex stage to the fragment stage using [varying streams](/user-manual/gaussian-splatting/building/custom-shaders/varyings) — for example to classify a splat once and pay per-pixel cost only where needed. ## Applying Chunks @@ -56,5 +56,6 @@ sceneMat.update(); - [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — move, scale and tint splats - [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — per-pixel color modification +- [Varying Streams](/user-manual/gaussian-splatting/building/custom-shaders/varyings) — pass per-splat data from the vertex stage to the fragment stage - [Relighting](/user-manual/gaussian-splatting/building/relighting) — light splats using a proxy mesh, built on the fragment hook - [Work Buffer Rendering](/user-manual/gaussian-splatting/rendering-architecture/work-buffer-rendering) — customize the global render pass that draws the sorted splats diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/varyings.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/varyings.md new file mode 100644 index 00000000000..91deb16a9b8 --- /dev/null +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/varyings.md @@ -0,0 +1,189 @@ +--- +title: Varying Streams +description: "Pass per-splat data from the gsplat vertex stage to the fragment stage using custom varying streams: API, generated set/get functions, and a live clipping example." +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Varying streams pass custom per-splat data from the [vertex stage](/user-manual/gaussian-splatting/building/custom-shaders/vertex) to the [fragment stage](/user-manual/gaussian-splatting/building/custom-shaders/fragment). A value is computed **once per splat** in the `gsplatModifyVS` chunk and read by every fragment of that splat in the `gsplatModifyPS` chunk. + +The typical use is classification: decide something about a splat once, then let the fragment stage pay per-pixel cost only where needed. + +**View Live Example** - Splats clipped by an animated box, with per-pixel clipping only on splats intersecting the box surface. + + + +## Adding Streams + +Streams are managed via [`app.scene.gsplat.varyings`](https://api.playcanvas.com/engine/classes/GSplatParams.html#varyings): + +```javascript +app.scene.gsplat.varyings.add([ + { name: 'clipState', type: pc.TYPE_UINT32, components: 1 } +]); + +// later, to remove +app.scene.gsplat.varyings.remove(['clipState']); +``` + +Supported types are `TYPE_FLOAT32`, `TYPE_INT32` and `TYPE_UINT32`, with 1 to 4 components. + +For each stream, two functions are generated and made available to your shader chunks: + +| Function | Available in | Purpose | +| --- | --- | --- | +| `set(value)` | `gsplatModifyVS` | Write the per-splat value (runs once per splat) | +| `get()` | `gsplatModifyPS` | Read the per-splat value for the current fragment | + +Adding or removing streams rebuilds the gsplat shaders, so configure them at startup rather than toggling them at runtime. + +## Example + +The live example above clips splats by an animated world-space box. The vertex stage classifies each splat against the box once per splat: splats fully inside are clipped entirely, splats fully outside set a flag so their fragments skip all work, and only splats intersecting the box surface run the per-pixel test. + +**1. Write the per-splat value in the vertex stage chunk:** + + + + +```glsl +uniform vec3 uClipCenter; +uniform vec3 uClipHalf; + +void modifySplatCenter(inout vec3 center) { +} + +void modifySplatRotationScale(vec3 originalCenter, vec3 modifiedCenter, inout vec4 rotation, inout vec3 scale) { + // signed distance of the splat center from the clipping box surface (negative inside) + vec3 d = abs(modifiedCenter - uClipCenter) - uClipHalf; + float sdf = length(max(d, vec3(0.0))) + min(max(d.x, max(d.y, d.z)), 0.0); + + // conservative splat radius + float radius = 2.0 * gsplatGetSizeFromScale(scale); + + if (sdf < -radius) { + // fully inside the box - clip the whole splat + scale = vec3(0.0); + setClipState(1u); + } else if (sdf > radius) { + // fully outside the box - no per-pixel clipping needed + setClipState(1u); + } else { + // intersects the box surface - clip per pixel in the fragment shader + setClipState(0u); + } +} + +void modifySplatColor(vec3 center, inout vec4 color) { +} +``` + + + + +```wgsl +uniform uClipCenter: vec3f; +uniform uClipHalf: vec3f; + +fn modifySplatCenter(center: ptr) { +} + +fn modifySplatRotationScale(originalCenter: vec3f, modifiedCenter: vec3f, rotation: ptr, scale: ptr) { + // signed distance of the splat center from the clipping box surface (negative inside) + let d = abs(modifiedCenter - uniform.uClipCenter) - uniform.uClipHalf; + let sdf = length(max(d, vec3f(0.0))) + min(max(d.x, max(d.y, d.z)), 0.0); + + // conservative splat radius + let radius = 2.0 * gsplatGetSizeFromScale(*scale); + + if (sdf < -radius) { + // fully inside the box - clip the whole splat + *scale = vec3f(0.0); + setClipState(1u); + } else if (sdf > radius) { + // fully outside the box - no per-pixel clipping needed + setClipState(1u); + } else { + // intersects the box surface - clip per pixel in the fragment shader + setClipState(0u); + } +} + +fn modifySplatColor(center: vec3f, color: ptr) { +} +``` + + + + +**2. Read it in the fragment stage chunk** and early-out before the expensive per-pixel work: + + + + +```glsl +uniform vec3 uClipCenter; +uniform vec3 uClipHalf; +uniform mat4 uInvViewProj; +uniform vec4 uScreenSize; + +void modifySplatColor(vec2 gaussianUV, inout vec4 color) { + // splats fully inside or outside the box were already resolved per splat in the vertex stage + if (getClipState() == 1u) return; + + // reconstruct the world position of this fragment (on the splat's depth plane) + vec3 ndc = vec3(gl_FragCoord.xy * uScreenSize.zw, gl_FragCoord.z) * 2.0 - 1.0; + vec4 world = uInvViewProj * vec4(ndc, 1.0); + vec3 worldPos = world.xyz / world.w; + + // clip fragments inside the box + vec3 d = abs(worldPos - uClipCenter) - uClipHalf; + if (max(d.x, max(d.y, d.z)) < 0.0) { + color.a = 0.0; + } +} +``` + + + + +```wgsl +uniform uClipCenter: vec3f; +uniform uClipHalf: vec3f; +uniform uInvViewProj: mat4x4f; +uniform uScreenSize: vec4f; + +fn modifySplatColor(gaussianUV: vec2f, color: ptr) { + // splats fully inside or outside the box were already resolved per splat in the vertex stage + if (getClipState() == 1u) { + return; + } + + // reconstruct the world position of this fragment (on the splat's depth plane) + let uv = pcPosition.xy * uniform.uScreenSize.zw; + let ndc = vec3f(uv.x * 2.0 - 1.0, (1.0 - uv.y) * 2.0 - 1.0, pcPosition.z * 2.0 - 1.0); + let world = uniform.uInvViewProj * vec4f(ndc, 1.0); + let worldPos = world.xyz / world.w; + + // clip fragments inside the box + let d = abs(worldPos - uniform.uClipCenter) - uniform.uClipHalf; + if (max(d.x, max(d.y, d.z)) < 0.0) { + *color = vec4f((*color).rgb, 0.0); + } +} +``` + + + + +Both chunks are applied to the scene gsplat material as usual, using the `gsplatModifyVS` and `gsplatModifyPS` keys. + +## Memory Considerations + +On some platforms each component is stored in per-splat video memory, so its size scales with the number of rendered splats. Keep the data as compact as possible - prefer fewer components, and consider bit-packing multiple small values into a single uint component instead of using separate streams. + +## See Also + +- [Vertex Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/vertex) — where the values are written +- [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — where the values are read diff --git a/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md b/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md index 5ed647634a7..4b2a4cd07d9 100644 --- a/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md +++ b/docs/user-manual/gaussian-splatting/building/custom-shaders/vertex.md @@ -124,3 +124,4 @@ app.on('update', (dt) => { ## See Also - [Fragment Stage Customization](/user-manual/gaussian-splatting/building/custom-shaders/fragment) — per-pixel color modification +- [Varying Streams](/user-manual/gaussian-splatting/building/custom-shaders/varyings) — pass per-splat values computed here to the fragment stage diff --git a/sidebars.js b/sidebars.js index 62a4a4e13d5..e6e211e4ef2 100644 --- a/sidebars.js +++ b/sidebars.js @@ -920,6 +920,7 @@ const sidebars = { items: [ 'user-manual/gaussian-splatting/building/custom-shaders/vertex', 'user-manual/gaussian-splatting/building/custom-shaders/fragment', + 'user-manual/gaussian-splatting/building/custom-shaders/varyings', ], }, 'user-manual/gaussian-splatting/building/relighting',