diff --git a/CHANGELOG.md b/CHANGELOG.md index 0175530..6a66a7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,16 @@ iOS changes are in the [iOS changelog](packages/splatkit-ios/distribution/CHANGE ## Unreleased +### Added + +- `RenderPolicySupport::rasterMask` in the shared engine, listing the raster strategies a backend builds. +- `RenderPolicy::lodSplatLimit` in the shared engine: a live cap on selected hierarchy splats, 0 for the loaded capacity. +- `buildCollider` in splat-core makes a walk-mode collider from a world's splats, a port of PlayCanvas splat-transform's collision voxel passes, with `encodeGlb` and `tools/splat_collider` to write it as a `.glb`. +- Shared engine stats count frames the display showed when a renderer reports presentation times, with a 95th percentile frame time, a 1% low and dropped frames; Metal reports them, Vulkan still counts submitted frames. + ### Changed +- Walk mode refuses steps onto a floor more than 0.35 m higher, looking 0.25 m ahead, so it climbs stairs and steps over door tracks but no longer climbs counters, chairs or tables whose top the hip probe passes over, and slides along them when walked into at an angle. - The [React Native example](apps/react-native/README.md) replaces the React Native dev app: a template React Native 0.87.1 app that installs `@splatkit/react-native` from npm and runs on Android and iOS. ## [0.1.0-alpha07] - 2026-09-16 diff --git a/Package.swift b/Package.swift index 0e8a247..023f92c 100644 --- a/Package.swift +++ b/Package.swift @@ -8,8 +8,8 @@ let package = Package( targets: [ .binaryTarget( name: "SplatKitCore", - url: "https://github.com/Xget7/splatkit-ios/releases/download/v0.1.0-alpha.3/SplatKitCore.xcframework.zip", - checksum: "2258de61db98721528a805bdfefbc6b764aada311bda6149b3abcf98d5194906" + url: "https://github.com/Xget7/splatkit-ios/releases/download/v0.1.0-alpha.4/SplatKitCore.xcframework.zip", + checksum: "7b92ec52cbcd1f42bfc6ab31d16befd4b8134ee1c7f0a71b26789a4cbcf4f5c5" ), .target( name: "SplatKit", diff --git a/README.md b/README.md index 70347c7..5cd5298 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Experimental alpha: APIs and quality/performance tradeoffs are still evolving. Android: API 29+, Vulkan 1.1, arm64-v8a. The GPU path additionally checks subgroup and memory limits. -Maven Central has `io.github.xget7:splatkit-android:0.1.0-alpha07`, with the render policy, `onWorldFrameReady` and 16 KB alignment; see the [Android releases](https://github.com/Xget7/splatkit-android/releases). +Maven Central has `io.github.xget7:splatkit-android:0.1.0-alpha08`, with host-driven walking, the render policy and `onWorldFrameReady`; see the [Android releases](https://github.com/Xget7/splatkit-android/releases). To build current source: ```sh @@ -17,7 +17,7 @@ cd apps/android-dev ./gradlew :splatkit:assembleRelease :splatkit:testDebugUnitTest ``` -iOS: add [splatkit-ios](https://github.com/Xget7/splatkit-ios) to Swift Package Manager, version `0.1.0-alpha.3`. +iOS: add [splatkit-ios](https://github.com/Xget7/splatkit-ios) to Swift Package Manager, version `0.1.0-alpha.4`. Use the native view, forward lifecycle and load worlds asynchronously; see each SDK's README for examples. ## Implemented scope @@ -28,7 +28,7 @@ Use the native view, forward lifecycle and load worlds asynchronously; see each | Offline `.lodsplat` and GPU hierarchical selection | Yes | Yes | | 16-bit quantized depth / two radix passes | Per-view policy approximation | Per-view policy approximation | | SH degrees 0–3, walk/fly, touch, motion, loaded/drawn stats | Yes | Yes | -| Hybrid compute screen tiles | Experimental | Not implemented | +| Hybrid compute screen tiles | Experimental per-view opt-in, for dense close-up scenes | Not implemented | | Per-view render policy and capabilities | Yes | Yes | | React Native policy prop and events | iPhone 17 Pro validated | Mi 9 validated | diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index 41a1310..cbc1e18 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -97,6 +97,33 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +============================================================================ +splat-transform (PlayCanvas), whose collision voxel passes splat-core's +collider builder ports +============================================================================ + +MIT License + +Copyright (c) 2011-2026 PlayCanvas Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + ============================================================================ zstd (Meta Platforms), taken under BSD-3-Clause ============================================================================ diff --git a/apps/ios-dev/SplatKitDev/ContentView.swift b/apps/ios-dev/SplatKitDev/ContentView.swift index a27ab87..6dd09aa 100644 --- a/apps/ios-dev/SplatKitDev/ContentView.swift +++ b/apps/ios-dev/SplatKitDev/ContentView.swift @@ -22,18 +22,23 @@ struct ContentView: View { .frame(maxWidth: .infinity, maxHeight: .infinity) .accessibilityIdentifier("splat.preparing") } - hud -// VStack { -// Spacer() -// HStack(alignment: .bottom) { -// Joystick { forward, right in -// session.view.setWalkVelocity(forward: forward * walkSpeed, right: right * walkSpeed) -// } -// Spacer() -// controls -// } -// .padding(24) -// } + if session.scenePrepared && (args.bool("hud") ?? true) { + overlay + } + // The SDK ships no walking control; this is the app's own, over the SDK's view. + if session.walking { + VStack { + Spacer() + HStack(alignment: .bottom) { + Joystick { forward, right in + session.view.setWalkVelocity(forward: forward * walkSpeed, right: right * walkSpeed) + } + Spacer() + controls + } + .padding(24) + } + } } .onAppear(perform: start) .onDisappear { UIApplication.shared.isIdleTimerDisabled = false } @@ -47,32 +52,26 @@ struct ContentView: View { } } - private var hud: some View { - let s = session.stats - return VStack(alignment: .leading, spacing: 2) { - Text(session.gpu) - Text(session.status) - Text(String(format: "%.0f fps frame %.1f ms gpu %.1f ms sort %.0f ms", s.fps, s.frameMillis, s.gpuMillis, s.sortMillis)) - Text("\(s.loadedSplatCount) loaded \(s.drawnSplatCount) drawn") - if s.computeTileCount + s.hardwareTileCount > 0 { - Text("tiles \(s.computeTileCount) compute / \(s.hardwareTileCount) hardware") - .accessibilityIdentifier("splat.tiles") - Text("compute: \(s.nonemptyComputeTileCount) with splats") - } else { - Text("tiles: unavailable") - .accessibilityIdentifier("splat.tiles") + private var shotSelection: Binding { + Binding(get: { session.shot }, set: { if let shot = $0 { session.shot = shot } }) + } + + private var overlay: some View { + VStack(alignment: .leading) { + RenderStatsCard(session: session) + Spacer() + VStack(spacing: 6) { + if session.orbit != nil { + CapsulePicker(items: CameraShot.allCases, selection: shotSelection) { $0.title } + } + CapsulePicker(items: QualityTier.allCases, selection: $session.quality) { $0.title } } - Text("\(s.walking ? "walk" : "fly") \(s.motion ? "gyro" : "touch")") - if !captureMessage.isEmpty { Text(captureMessage) } + .frame(maxWidth: .infinity) } - .font(.system(size: 12, weight: .medium, design: .monospaced)) - .foregroundColor(.white) - .padding(8) - .background(Color.black.opacity(0.4)) - .cornerRadius(6) - .padding(.top, 54) - .padding(.leading, 12) - .allowsHitTesting(false) + .padding(.horizontal, 16) + .padding(.top, 4) + .padding(.bottom, 8) + .transition(.opacity) } private var controls: some View { @@ -103,6 +102,11 @@ struct ContentView: View { view.motionToggleEnabled = !issMap view.maxShDegree = args.int("sh") ?? 1 if let v = args.int("shdraw") { view.shDegree = v } + // Explicit renderer switches win; otherwise start at a quality level, High by default. + let customQuality = ["scale", "shdraw", "depth-key-bits", "min-pixel-radius", "lod-error-pixels", "lod-splat-limit"].contains(where: args.has) + if !customQuality { + session.quality = args.string("quality").flatMap(QualityTier.init(rawValue:)) ?? .high + } if let v = args.int("budget") { view.splatBudget = v } if let v = args.int("residency") { view.residencyBudget = v } if let v = args.float("margin") { view.cullMarginDegrees = v } @@ -121,11 +125,9 @@ struct ContentView: View { let url = LaunchArgs.resolve(collider) { view.loadCollider(file: url) } - // A 45 m radius gives a closer inspection view without changing the field of view. + // Broadside to the truss, far enough to clear the solar arrays. if issMap { - view.lookAt(from: SIMD3(0, -2, 43), - target: SIMD3(0, -2, -2), - up: (args.bool("orbit-horizontal") ?? true) ? SIMD3(0, 1, 0) : SIMD3(1, 0, 0)) + view.lookAt(from: SIMD3(0, 35, 128), target: SIMD3(0, -2, -2), up: SIMD3(0, 1, 0)) } else { view.cameraPose = CameraPose(x: 0, y: 0, z: 0, yaw: 0, pitch: 0) } @@ -143,11 +145,12 @@ struct ContentView: View { session.motion = view.isMotionEnabled if orbitEnabled { var settings = OrbitPath.Settings(pivot: SIMD3(0, -2, -2)) - settings.horizontal = args.bool("orbit-horizontal") ?? true if let radius = args.float("radius"), radius.isFinite, radius > 0 { settings.radius = radius } if let speed = args.float("speed"), speed.isFinite { settings.degreesPerSecond = speed } if let start = args.float("start"), start.isFinite { settings.startDegrees = start } - let orbit = OrbitPath(view: view, settings: settings) + let shot = args.string("shot").flatMap(CameraShot.init(rawValue:)) ?? .orbit + session.shot = shot + let orbit = OrbitPath(view: view, settings: settings, shot: shot) session.orbit = orbit } if let v = args.float("walk") { view.setWalkVelocity(forward: v, right: 0) } diff --git a/apps/ios-dev/SplatKitDev/LaunchArgs.swift b/apps/ios-dev/SplatKitDev/LaunchArgs.swift index c43fe25..d5d9602 100644 --- a/apps/ios-dev/SplatKitDev/LaunchArgs.swift +++ b/apps/ios-dev/SplatKitDev/LaunchArgs.swift @@ -7,9 +7,9 @@ import Foundation /// `--linear`, `--gyro <0|1>`, `--pose x,y,z,yaw,pitch`, `--walk `, /// `--benchmark [seconds]`, `--capture ` (writes Documents/capture.png). /// The default world is kitchen_500k.spz. ISS stays in Documents; use --world iss_10M.spz. -/// ISS orbits at a constant 45 m radius and 4 degrees/s; --radius, --speed and --start override it. -/// Its long X axis stays horizontal in screen space, without changing the orbit plane. -/// --orbit-horizontal 0 restores the previous vertical framing for benchmark comparisons. +/// ISS runs a camera shot (--shot overview|orbit|detail|flyby|tour, default orbit) with +Y up. +/// The orbit shot circles at 135 m and 5 degrees/s from azimuth 20; --radius (at least 100), +/// --speed and --start override it. /// Its starting pose is prepared during loading; motion starts only after the first /// successful GPU frame. Loading stays covered while the renderer prepares that frame. /// --orbit 0, --pose, --walk or --benchmark disable the automatic orbit and enable touch look. @@ -21,17 +21,24 @@ import Foundation /// --depth-key-bits <16|32> selects linear camera-depth quantization and two radix /// passes (16), or the unchanged four-pass ordering (32, default). Restart to change. /// Quantization can change transparency ordering within a bin; no splats are removed. -/// --tile-raster 1 selects bounded hybrid 16x16 compute compositing, with T <= 0.0001. +/// --lod-error-pixels sets the hierarchy refinement threshold (default 1; lower draws more). +/// --lod-splat-limit caps the splats a frame selects, 0 for the loaded budget; detail thins evenly. +/// --tile-raster 1 sets the view's raster policy to hybrid: bounded 16x16 compute compositing, +/// with T <= 0.0001. It helps only where many large splats overlap; ISS orbits run slower. /// Hardware fallback retains its existing 254/255 opacity coverage mask. /// Dense tiles (>512 candidates) and tiles touched by large footprints (>16 tiles) /// use hardware completion, never truncated lists. Scratch is capped at 128 MiB. +/// --quality picks the starting level (default high) unless --scale, +/// --shdraw, --depth-key-bits or --min-pixel-radius set the renderer directly. +/// --shot picks the ISS camera shot (default orbit). +/// --hud 0 hides the stats card and quality picker, for screen recordings. /// --keep-awake 1 disables idle screen locking only while this dev view is active. /// --resource-monitor 1 logs process footprint, remaining process allowance, Metal /// allocation and thermal state at 2 Hz, and pauses on memory/thermal pressure. /// --memory-limit-mib lowers its conservative 2800 MiB process-footprint guard. /// --run-seconds enables monitoring and pauses that long after the first world frame. /// These dev-only guards stop future frames, not allocations or GPU work in flight. -/// Default 0 retains hardware rasterization. A GPU error stops submission until renderer recreation. +/// A GPU error stops submission until renderer recreation. struct LaunchArgs { let values: [String: String] diff --git a/apps/ios-dev/SplatKitDev/OrbitPath.swift b/apps/ios-dev/SplatKitDev/OrbitPath.swift index c52dd96..55fd137 100644 --- a/apps/ios-dev/SplatKitDev/OrbitPath.swift +++ b/apps/ios-dev/SplatKitDev/OrbitPath.swift @@ -1,14 +1,65 @@ import QuartzCore import simd -/// Constant-distance inspection orbit. The camera moves; its field of view never changes. +/// Camera shots around the station. Every shot keeps the station's own up axis (+Y) up, so +/// the camera never rolls; they differ in the path the eye follows, its height and pace. +enum CameraShot: String, CaseIterable, Identifiable { + case overview, orbit, detail, flyby, tour + + var id: String { rawValue } + + var title: String { + switch self { + case .overview: "Overview" + case .orbit: "Orbit" + case .detail: "Detail" + case .flyby: "Flyby" + case .tour: "Tour" + } + } +} + +/// Constant field of view; the camera moves. The station's plan lies in the XZ plane, with the +/// long truss on X and the modules on Z. The eye circles the vertical axis on an ellipse +/// whose semi-axes are X and Z, so a close pass can hug the modules and swing wide of the +/// solar arrays at the truss ends. Switching shots eases every parameter over a few seconds, +/// so the camera never jumps. final class OrbitPath { struct Settings { var pivot: SIMD3 - var radius: Float = 45 - var degreesPerSecond: Float = 4 - var startDegrees: Float = 90 - var horizontal: Bool = true + /// Orbit shot radius, at least `minimumRadius`. + var radius: Float = 135 + var degreesPerSecond: Float = 5 + /// Azimuth about +Y; 0 puts the camera on +Z, broadside to the truss. + var startDegrees: Float = 20 + } + + /// Any circle this wide clears the arrays, whose far corners are 88 m from the pivot. + static let minimumRadius: Float = 100 + + /// One instant of a shot. + private struct Pose { + /// Semi-axes of the eye's ellipse along the truss (X) and across it (Z). + var alongTruss: Float + var acrossTruss: Float + /// Degrees above the station's plane. + var elevation: Float + var degreesPerSecond: Float + /// Offset of the look target along the truss. + var drift: Float + + static func circle(_ radius: Float, elevation: Float, degreesPerSecond: Float) -> Pose { + Pose(alongTruss: radius, acrossTruss: radius, elevation: elevation, + degreesPerSecond: degreesPerSecond, drift: 0) + } + + static func mix(_ a: Pose, _ b: Pose, _ t: Float) -> Pose { + Pose(alongTruss: simd_mix(a.alongTruss, b.alongTruss, t), + acrossTruss: simd_mix(a.acrossTruss, b.acrossTruss, t), + elevation: simd_mix(a.elevation, b.elevation, t), + degreesPerSecond: simd_mix(a.degreesPerSecond, b.degreesPerSecond, t), + drift: simd_mix(a.drift, b.drift, t)) + } } // CADisplayLink retains its target. A weak proxy lets the session release this path. @@ -17,20 +68,35 @@ final class OrbitPath { @objc func tick(_ link: CADisplayLink) { owner?.tick(link) } } + private static let transitionSeconds: Float = 3 + /// Tour order and how long each leg holds before easing into the next. + private static let tourLegs: [(CameraShot, Float)] = [(.overview, 20), (.orbit, 24), (.detail, 36), (.flyby, 30)] + private let view: SplatMetalView private let settings: Settings private let target = TickTarget() private var link: CADisplayLink? private var lastTimestamp: CFTimeInterval? - private var angle: Double + /// Azimuth in radians, advanced by the blended pace so a change of speed never jumps. + private var azimuth: Float + /// Seconds of motion, paused while the path is not running. + private var clock: Float = 0 + private(set) var shot: CameraShot + private var from: Pose + private var shotStarted: Float = 0 - init(view: SplatMetalView, settings: Settings) { + init(view: SplatMetalView, settings: Settings, shot: CameraShot = .orbit) { self.view = view + var settings = settings + settings.radius = max(settings.radius, Self.minimumRadius) self.settings = settings - angle = Double(settings.startDegrees) * .pi / 180 + self.shot = shot + azimuth = settings.startDegrees * .pi / 180 + from = .circle(settings.radius, elevation: 16, degreesPerSecond: settings.degreesPerSecond) target.owner = self - NSLog("SplatOrbit: axis=X horizontal=%d radius=%.2f speed=%.2f start=%.2f", - settings.horizontal ? 1 : 0, settings.radius, settings.degreesPerSecond, settings.startDegrees) + from = pose(of: shot, at: 0) + NSLog("SplatOrbit: shot=%@ radius=%.2f speed=%.2f start=%.2f", shot.rawValue, + settings.radius, settings.degreesPerSecond, settings.startDegrees) // Prepare the requested starting pose without advancing time during loading. applyPose() } @@ -53,23 +119,80 @@ final class OrbitPath { lastTimestamp = nil } + /// Eases from wherever the camera is now into the new shot. + func setShot(_ next: CameraShot) { + guard next != shot else { return } + from = current() + shot = next + shotStarted = clock + NSLog("SplatOrbit: shot=%@", next.rawValue) + } + private func tick(_ link: CADisplayLink) { if let previous = lastTimestamp { - angle += (link.timestamp - previous) * Double(settings.degreesPerSecond) * .pi / 180 - angle = angle.truncatingRemainder(dividingBy: 2 * .pi) + // A stalled main thread must not teleport the camera. + let dt = min(Float(link.timestamp - previous), 0.1) + clock += dt + azimuth += dt * current().degreesPerSecond * .pi / 180 + azimuth = azimuth.truncatingRemainder(dividingBy: 2 * .pi) } lastTimestamp = link.timestamp applyPose() } + /// The blended pose: the previous one eased into the shot's own motion. + private func current() -> Pose { + let target = pose(of: shot, at: clock - shotStarted) + return Pose.mix(from, target, Self.ease((clock - shotStarted) / Self.transitionSeconds)) + } + + private static func ease(_ t: Float) -> Float { + let t = min(max(t, 0), 1) + return t * t * (3 - 2 * t) + } + + /// Where a shot wants the camera `elapsed` seconds after it started. + private func pose(of shot: CameraShot, at elapsed: Float) -> Pose { + let wave = { (period: Float) in sin(elapsed * 2 * .pi / period) } + switch shot { + case .overview: + // High and far: the whole plan, arrays included, turning slowly. + return .circle(160, elevation: 38, degreesPerSecond: 4) + case .orbit: + return .circle(settings.radius, elevation: 16, degreesPerSecond: settings.degreesPerSecond) + case .detail: + // 64 m across the truss hugs the modules; 86 m along it clears the arrays' boxes + // (|X| 30-66 m, Z -28-45 m) at every azimuth. The eye bobs and the target drifts. + return Pose(alongTruss: 86, acrossTruss: 64, elevation: 10 + 8 * wave(50), + degreesPerSecond: 3, drift: 14 * wave(70)) + case .flyby: + // Faster, breathing in and out while dipping below the plane and climbing over it. + let radius = 125 + 25 * wave(30) + return .circle(radius, elevation: 10 + 25 * wave(26), degreesPerSecond: 8) + case .tour: + let loop = Self.tourLegs.reduce(0) { $0 + $1.1 } + var t = elapsed.truncatingRemainder(dividingBy: loop) + for (index, leg) in Self.tourLegs.enumerated() { + if t < leg.1 { + let pose = self.pose(of: leg.0, at: t) + // Ease the last seconds of a leg into the next leg's opening pose. + let next = Self.tourLegs[(index + 1) % Self.tourLegs.count].0 + let blend = Self.ease((t - (leg.1 - Self.transitionSeconds)) / Self.transitionSeconds) + return Pose.mix(pose, self.pose(of: next, at: 0), blend) + } + t -= leg.1 + } + return self.pose(of: .orbit, at: 0) + } + } + private func applyPose() { - // Keep the same YZ orbit around the ISS's long X axis. A tangent up vector - // rolls the camera 90 degrees, keeping that axis horizontal at every angle. - // Unlike fixed world-Y up, it never becomes parallel to the viewing ray. - let outward = SIMD3(0, Float(cos(angle)), Float(sin(angle))) - let axis = SIMD3(1, 0, 0) - let up = settings.horizontal ? simd_cross(outward, axis) : axis - view.lookAt(from: settings.pivot + settings.radius * outward, - target: settings.pivot, up: up) + let pose = current() + let x = pose.alongTruss * sin(azimuth) + let z = pose.acrossTruss * cos(azimuth) + let elevation = pose.elevation * .pi / 180 + let eye = SIMD3(x * cos(elevation), simd_length(SIMD2(x, z)) * sin(elevation), z * cos(elevation)) + let focus = settings.pivot + SIMD3(pose.drift, 0, 0) + view.lookAt(from: settings.pivot + eye, target: focus, up: SIMD3(0, 1, 0)) } } diff --git a/apps/ios-dev/SplatKitDev/QualityTier.swift b/apps/ios-dev/SplatKitDev/QualityTier.swift new file mode 100644 index 0000000..7730881 --- /dev/null +++ b/apps/ios-dev/SplatKitDev/QualityTier.swift @@ -0,0 +1,77 @@ +import Foundation + +/// Live quality levels. Each changes only settings that apply without reloading the world. +/// The splat limit is what moves the frame rate on a large hierarchy world; the error +/// threshold is the finest detail a level asks for when the limit leaves room. +enum QualityTier: String, CaseIterable, Identifiable { + case ultra, high, balanced, fast + + var id: String { rawValue } + + var title: String { + switch self { + case .ultra: "Ultra" + case .high: "High" + case .balanced: "Balanced" + case .fast: "Fast" + } + } + + /// Fraction of the view's resolution the splats are drawn at. + var renderScale: Float { + switch self { + case .ultra, .high: 1 + case .balanced: 0.9 + case .fast: 0.75 + } + } + + /// Requested harmonics; the view caps it at what the world carries. + var shDegree: Int { self == .fast ? 0 : 3 } + + /// Screen error, in pixels, a hierarchy node may cover before it splits into its + /// children. Lower draws more splats, up to the budget the world was loaded with. + var lodErrorPixels: Float { + switch self { + case .ultra: 0.4 + case .high: 0.6 + case .balanced: 1 + case .fast: 1.6 + } + } + + /// Most hierarchy splats a frame selects, 0 for everything the world loaded with. + var lodSplatLimit: UInt32 { + switch self { + case .ultra: 0 + case .high: 2_800_000 + case .balanced: 2_000_000 + case .fast: 1_300_000 + } + } + + /// 32-bit keys keep exact depth order; 16-bit halves the radix passes. + var sortDepth: UInt32 { self == .ultra ? 32 : 16 } + + /// Smallest splat footprint kept, in pixels; lower keeps more fine splats. + /// Applies under --metal-culling 1. + var subpixelThreshold: Float { + switch self { + case .ultra: 0.2 + case .high: 0.3 + case .balanced: 0.5 + case .fast: 0.8 + } + } + + func apply(to view: SplatMetalView) { + view.renderScale = renderScale + view.shDegree = shDegree + var policy = view.renderPolicy + policy.sortDepth = sortDepth + policy.lodErrorPixels = lodErrorPixels + policy.lodSplatLimit = lodSplatLimit + policy.subpixelThreshold = subpixelThreshold + view.renderPolicy = policy + } +} diff --git a/apps/ios-dev/SplatKitDev/RenderOverlay.swift b/apps/ios-dev/SplatKitDev/RenderOverlay.swift new file mode 100644 index 0000000..23a3c44 --- /dev/null +++ b/apps/ios-dev/SplatKitDev/RenderOverlay.swift @@ -0,0 +1,171 @@ +import SwiftUI + +/// The live render card: frame rate with its recent history, then what the frame costs. +/// Frame rate and tails count frames the display showed, not frames submitted. +struct RenderStatsCard: View { + @ObservedObject var session: SplatSession + + var body: some View { + let s = session.stats + // The engine skips frames while nothing moves: no frame shown is a still view, not 0 fps. + let idle = session.isIdle + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .center, spacing: 8) { + VStack(alignment: .leading, spacing: 0) { + HStack(alignment: .firstTextBaseline, spacing: 4) { + // Digits roll between readings; Idle replaces them outright, and no other + // text in the card animates, so an old value never shows through a new one. + Text(idle ? "Idle" : String(format: "%.0f", s.fps)) + .font(.system(size: 22, weight: .semibold, design: .rounded)) + .monospacedDigit() + .contentTransition(.numericText()) + .animation(.easeOut(duration: 0.25), value: s.fps) + .id(idle) + .transition(.identity) + if !idle { + Text("FPS") + .font(.system(size: 9, weight: .bold, design: .rounded)) + .foregroundStyle(.secondary) + } + } + HStack(spacing: 5) { + Circle().fill(idle ? Color.gray : fpsColor(s.fps)).frame(width: 5, height: 5) + Text(idle ? "no redraw" : String(format: "%.1f ms", s.frameMillis)) + .font(.system(size: 9, weight: .medium)) + .monospacedDigit() + .foregroundStyle(.secondary) + } + } + FpsSparkline(samples: session.fpsHistory) + .frame(width: 56, height: 24) + } + + Divider().overlay(Color.white.opacity(0.15)) + + VStack(alignment: .leading, spacing: 3) { + StatRow(label: "p95 / 1% low", value: s.presentTiming + ? String(format: "%.0f ms / %.0f", s.frameMillisP95, s.lowFps) : "-") + StatRow(label: "Drawn", value: compact(s.drawnSplatCount)) + StatRow(label: "Scene", value: compact(s.loadedSplatCount)) + StatRow(label: "GPU/sort", value: String(format: "%.0f/%.0f ms", s.gpuMillis, s.sortMillis)) + StatRow(label: "Res", value: session.resolutionText) + StatRow(label: "Pipe", value: session.pipelineText) + StatRow(label: "Thermal", value: session.thermalText) + } + } + .foregroundStyle(.white) + .padding(.horizontal, 9) + .padding(.vertical, 7) + .frame(width: 148, alignment: .leading) + .background(.ultraThinMaterial.opacity(0.75), in: RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay(RoundedRectangle(cornerRadius: 12, style: .continuous).strokeBorder(Color.white.opacity(0.1))) + .environment(\.colorScheme, .dark) + .allowsHitTesting(false) + .accessibilityIdentifier("splat.stats") + } + + private func fpsColor(_ fps: Float) -> Color { + fps >= 30 ? .green : fps >= 20 ? .yellow : .red + } + + /// 2,022,194 reads as 2.02M; the card is too narrow for full counts. + private func compact(_ value: Int) -> String { + value >= 1_000_000 ? String(format: "%.2fM", Double(value) / 1_000_000) + : value >= 1_000 ? String(format: "%.0fK", Double(value) / 1_000) : "\(value)" + } +} + +private struct StatRow: View { + let label: String + let value: String + + var body: some View { + HStack(spacing: 4) { + Text(label) + .foregroundStyle(.secondary) + Spacer(minLength: 4) + Text(value) + .monospacedDigit() + .lineLimit(1) + .minimumScaleFactor(0.7) + } + .font(.system(size: 9.5, weight: .medium)) + } +} + +/// Recent frame rate, scaled to 60 FPS, with a faint 30 FPS guide. +private struct FpsSparkline: View { + let samples: [Float] + + var body: some View { + GeometryReader { geometry in + let size = geometry.size + let point = { (index: Int, fps: Float) -> CGPoint in + let x = samples.count > 1 ? size.width * CGFloat(index) / CGFloat(samples.count - 1) : 0 + let y = size.height * (1 - CGFloat(min(max(fps, 0), 60) / 60)) + return CGPoint(x: x, y: y) + } + ZStack { + Path { path in + path.move(to: CGPoint(x: 0, y: size.height / 2)) + path.addLine(to: CGPoint(x: size.width, y: size.height / 2)) + } + .stroke(Color.white.opacity(0.15), style: StrokeStyle(lineWidth: 1, dash: [3, 3])) + if samples.count > 1 { + Path { path in + path.move(to: CGPoint(x: 0, y: size.height)) + for (index, fps) in samples.enumerated() { path.addLine(to: point(index, fps)) } + path.addLine(to: CGPoint(x: size.width, y: size.height)) + path.closeSubpath() + } + .fill(LinearGradient(colors: [Color.green.opacity(0.35), Color.green.opacity(0)], + startPoint: .top, endPoint: .bottom)) + Path { path in + for (index, fps) in samples.enumerated() { + index == 0 ? path.move(to: point(index, fps)) : path.addLine(to: point(index, fps)) + } + } + .stroke(Color.green, style: StrokeStyle(lineWidth: 1.5, lineCap: .round, lineJoin: .round)) + } + } + } + } +} + +/// A row of choices in one glass capsule; the selection slides between them. +struct CapsulePicker: View { + let items: [Item] + @Binding var selection: Item? + let title: (Item) -> String + @Namespace private var highlight + + var body: some View { + HStack(spacing: 2) { + ForEach(items) { item in + let selected = item == selection + Button { + withAnimation(.spring(response: 0.32, dampingFraction: 0.82)) { selection = item } + } label: { + Text(title(item)) + .font(.system(size: 11, weight: .semibold, design: .rounded)) + .foregroundStyle(selected ? Color.black : Color.white) + .padding(.horizontal, 10) + .padding(.vertical, 5) + .background { + if selected { + Capsule().fill(Color.white).matchedGeometryEffect(id: "highlight", in: highlight) + } + } + .contentShape(Capsule()) + } + .buttonStyle(.plain) + .accessibilityIdentifier("picker.\(title(item).lowercased())") + } + } + .padding(3) + .background(.ultraThinMaterial.opacity(0.75), in: Capsule()) + .overlay(Capsule().strokeBorder(Color.white.opacity(0.12))) + .environment(\.colorScheme, .dark) + .sensoryFeedback(.selection, trigger: selection) + } +} diff --git a/apps/ios-dev/SplatKitDev/SplatKitDevApp.swift b/apps/ios-dev/SplatKitDev/SplatKitDevApp.swift index f3586f7..ef732e4 100644 --- a/apps/ios-dev/SplatKitDev/SplatKitDevApp.swift +++ b/apps/ios-dev/SplatKitDev/SplatKitDevApp.swift @@ -3,18 +3,15 @@ import SwiftUI @main struct SplatKitDevApp: App { init() { - // Configure the internal culling experiment and tile raster before SplatSession - // creates its renderer. The threshold and sort key width are per-view policy now. + // Configure the internal culling experiment before SplatSession creates its renderer. + // The threshold, sort key width and raster strategy are per-view policy now. let enabled = LaunchArgs().bool("metal-culling") ?? false setenv("SPLATKIT_METAL_CULLING_EXPERIMENT", enabled ? "1" : "0", 1) - let tileRaster = LaunchArgs().bool("tile-raster") ?? false - setenv("SPLATKIT_METAL_TILE_RASTER", tileRaster ? "1" : "0", 1) } var body: some Scene { WindowGroup { ContentView() - .ignoresSafeArea() .statusBarHidden() } } diff --git a/apps/ios-dev/SplatKitDev/SplatViewHost.swift b/apps/ios-dev/SplatKitDev/SplatViewHost.swift index d1797f1..3d98eb5 100644 --- a/apps/ios-dev/SplatKitDev/SplatViewHost.swift +++ b/apps/ios-dev/SplatKitDev/SplatViewHost.swift @@ -15,6 +15,14 @@ final class SplatSession: ObservableObject { @Published var gpu = "" @Published var scenePrepared = false @Published var loadingFailed = false + /// The last 30 seconds of frame rate, sampled with the stats. + @Published var fpsHistory: [Float] = [] + @Published var quality: QualityTier? { + didSet { quality?.apply(to: view) } + } + @Published var shot: CameraShot = .orbit { + didSet { orbit?.setShot(shot) } + } private var timer: Timer? private var delegateBox: Delegate? private let monitorResources: Bool @@ -46,6 +54,9 @@ final class SplatSession: ObservableObject { policy.subpixelThreshold = radius } if args.string("depth-key-bits") == "16" { policy.sortDepth = 16 } + if let pixels = args.float("lod-error-pixels"), pixels.isFinite, pixels > 0 { policy.lodErrorPixels = pixels } + if let limit = args.int("lod-splat-limit"), limit >= 0 { policy.lodSplatLimit = UInt32(limit) } + if args.bool("tile-raster") ?? false { policy.raster = 2 } view.renderPolicy = policy // BOOL fields of the C structs import as ObjCBool. let applied = view.renderPolicy, caps = view.deviceCapabilities @@ -58,7 +69,12 @@ final class SplatSession: ObservableObject { if monitorResources { memoryWarningObserver = NotificationCenter.default.addObserver( forName: UIApplication.didReceiveMemoryWarningNotification, object: nil, queue: .main - ) { [weak self] _ in self?.stopRun("memory-warning") } + ) { [weak self] _ in + // Loading a large world warns at its transient peak and then settles, so a + // warning only samples now; the footprint and headroom limits decide. + NSLog("SplatMemoryWarning: sampling resources") + self?.sampleResources() + } } } @@ -81,11 +97,19 @@ final class SplatSession: ObservableObject { stopPolling() } + /// A prepared scene showed no frame in the last window: the engine skips redraws while + /// nothing moves, and the render loop, which publishes the stats, is still running. + var isIdle: Bool { scenePrepared && stats.fps == 0 } + func startPolling() { timer?.invalidate() timer = Timer.scheduledTimer(withTimeInterval: 0.5, repeats: true) { [weak self] _ in guard let self else { return } self.stats = self.view.readStats() + if self.scenePrepared && !self.isIdle { + self.fpsHistory.append(self.stats.fps) + if self.fpsHistory.count > 60 { self.fpsHistory.removeFirst(self.fpsHistory.count - 60) } + } self.motion = self.view.isMotionEnabled if self.monitorResources { self.sampleResources() } } @@ -134,6 +158,30 @@ final class SplatSession: ObservableObject { NSLog("SplatRunStopped: reason=%@", reason) } + var resolutionText: String { + let scale = view.window?.screen.nativeScale ?? UIScreen.main.nativeScale + let width = Int((view.bounds.width * scale * CGFloat(view.renderScale)).rounded()) + let height = Int((view.bounds.height * scale * CGFloat(view.renderScale)).rounded()) + return "\(width)×\(height)" + } + + var pipelineText: String { + let policy = view.renderPolicy + let sh = min(view.shDegree, view.maxShDegree) + let limit = policy.lodSplatLimit == 0 ? "all" : String(format: "%.1fM", Double(policy.lodSplatLimit) / 1_000_000) + return "SH\(sh) \(policy.sortDepth)b \(limit)" + } + + var thermalText: String { + switch ProcessInfo.processInfo.thermalState { + case .nominal: "Nominal" + case .fair: "Fair" + case .serious: "Serious" + case .critical: "Critical" + @unknown default: "Unknown" + } + } + private final class Delegate: SplatViewDelegate { weak var session: SplatSession? init(session: SplatSession) { self.session = session } diff --git a/apps/react-native/App.tsx b/apps/react-native/App.tsx index 426a4a1..37794dc 100644 --- a/apps/react-native/App.tsx +++ b/apps/react-native/App.tsx @@ -1,73 +1,126 @@ /** - * SplatKit React Native example: one full-screen SplatKitView. - * Drag with one finger to look around and with two fingers to walk. + * SplatKit React Native example: one full-screen SplatKitView you can walk through. + * + * Drag anywhere to look around and push the thumb stick to walk once the collider is + * ready. The SDK draws no walking control and no HUD: the stick, the stats card and the + * quality picker below are this app's own, and they drive the view through props and + * SplatKitCommands. * * @format */ -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useMemo, useRef, useState } from 'react'; import { StatusBar, StyleSheet, Text, View } from 'react-native'; import { SafeAreaProvider, useSafeAreaInsets, } from 'react-native-safe-area-context'; import { - DeviceCapabilities, + ColliderPhase, + PolicyPhase, + QualityPreset, SplatKitBuilder, + SplatKitCommands, SplatKitView, SplatKitViewProps, + WorldPhase, + conservativeCapabilities, nativeCapabilitiesFromEvent, + optionalTimingMillis, toNativePolicyProp, toNativeViewProps, } from '@splatkit/react-native'; +import Hud, { RenderStats } from './Hud'; +import Joystick from './Joystick'; type EventOf< - K extends 'onCapabilities' | 'onWorldEvent' | 'onStats' | 'onPolicyEvent', + K extends + | 'onCapabilities' + | 'onWorldEvent' + | 'onStats' + | 'onPolicyEvent' + | 'onColliderEvent', > = Parameters>[0]; -// Conservative limits for the first build only; each engine reports its own in onCapabilities. -const INITIAL_CAPABILITIES: DeviceCapabilities = { - limits: { - maxLodCapacitySplats: 1_000_000, - minResidencyCapacitySplats: 100_000, - maxResidencyCapacitySplats: 1_000_000, - }, - supportsComputeTiles: false, - supportsHiZOcclusion: false, - supportsSubgroups: false, - maxTextureDimension: 4096, -}; +/** The first policy revision; each capabilities report and preset change takes the next. */ +const FIRST_REVISION = 1; + +/** Meters per second at the rim of the stick: an unhurried walk. */ +const WALK_SPEED = 1.4; + +/** The walker: eye height, shoulder width and the rise it can climb, in meters. */ +const WALKER = { eyeHeight: 1.5, bodyRadius: 0.35, stepHeight: 0.35 } as const; + +const WORLD_REQUEST = 'world'; +const COLLIDER_REQUEST = 'collider'; + +/** + * The example opens at the sharpest preset: a phone renders a scene this size comfortably, + * and anything coarser leaves distant geometry visibly soft. The picker moves it down. + */ +const INITIAL_PRESET: QualityPreset = QualityPreset.highEnd; + +/** + * How far a LOD node may drift from the full-detail scene, in pixels of the drawn frame. + * The presets choose a value per tier; this app asks for less than any of them, because + * the far half of a captured room is where the error shows and a phone can afford it. + */ +const LOD_ERROR_PIXELS = 0.5; type Props = Readonly<{ // Set by MainActivity on Android and SceneDelegate on iOS; see README.md. worldPath: string; + colliderPath: string; }>; -function App({ worldPath }: Props) { +function App(props: Props) { return ( - + ); } -function Splat({ worldPath }: Props) { +function Splat({ worldPath, colliderPath }: Props) { const insets = useSafeAreaInsets(); - // Every new configuration needs a new policy revision. + const view = useRef>(null); + // Every new configuration needs a new policy revision. The engine reports its own limits in onCapabilities, which only arrive once it exists: + // the first world request is built against the limits every adapter accepts. const [capabilities, setCapabilities] = useState({ - value: INITIAL_CAPABILITIES, - revision: 1, + value: conservativeCapabilities, + revision: FIRST_REVISION, }); - const [status, setStatus] = useState('Loading world'); + const [preset, setPreset] = useState(INITIAL_PRESET); + const [status, setStatus] = useState('Loading world'); + const [walking, setWalking] = useState(false); + const [stats, setStats] = useState(null); const configuration = useMemo( () => new SplatKitBuilder() - .withWorld({ requestId: 'world', filePath: worldPath, maxShDegree: 3 }) - .withPreset('balanced') + .withWorld({ + requestId: WORLD_REQUEST, + filePath: worldPath, + maxShDegree: 3, + }) + .withPreset(preset) + .withPerformance({ lodErrorPixels: LOD_ERROR_PIXELS }) .build(capabilities.value), - [worldPath, capabilities.value], + [worldPath, preset, capabilities.value], + ); + + // The LOD and residency budgets are read once, while the engine builds the world, so a + // raised budget only reaches it under a new request. The first build guesses conservative + // limits and the engine's real ones arrive later, so the id carries the budgets it was + // built with: it changes exactly when a reload would pick something up, and never else. + const { lodCapacitySplats, residencyCapacitySplats } = configuration.world; + const world = useMemo( + () => ({ + ...configuration.world, + requestId: `${WORLD_REQUEST}-${lodCapacitySplats}-${residencyCapacitySplats}`, + }), + [configuration.world, lodCapacitySplats, residencyCapacitySplats], ); const onCapabilities = useCallback((event: EventOf<'onCapabilities'>) => { @@ -78,44 +131,100 @@ function Splat({ worldPath }: Props) { const onWorldEvent = useCallback( (event: EventOf<'onWorldEvent'>) => { const { phase, loadedSplats, message } = event.nativeEvent; + if (phase === WorldPhase.failed) { + setStatus(`Could not load ${worldPath}: ${message}`); + return; + } + // A large world takes seconds to decode, build and upload, and the first frame comes + // later still. The status line carries the load until then and the HUD takes over. setStatus( - phase === 'failed' - ? `Could not load ${worldPath}: ${message}` - : `${loadedSplats.toLocaleString()} splats ${phase}`, + phase === WorldPhase.frameReady + ? null + : `${loadedSplats.toLocaleString()} splats uploaded`, ); }, [worldPath], ); + const onColliderEvent = useCallback((event: EventOf<'onColliderEvent'>) => { + const { phase, message } = event.nativeEvent; + setWalking(phase === ColliderPhase.ready); + if (phase === ColliderPhase.failed) setStatus(`Collider failed: ${message}`); + }, []); + const onStats = useCallback((event: EventOf<'onStats'>) => { - const { drawnSplats, gpuMillis, gpuTimingAvailable } = event.nativeEvent; - const gpu = gpuTimingAvailable ? `, GPU ${gpuMillis.toFixed(1)} ms` : ''; - setStatus(`${drawnSplats.toLocaleString()} splats drawn${gpu}`); + const { + loadedSplats, + drawnSplats, + frameMillis, + frameTimingAvailable, + gpuMillis, + gpuTimingAvailable, + sortMillis, + sortTimingAvailable, + } = event.nativeEvent; + setStats({ + loadedSplats, + drawnSplats, + frameMillis: optionalTimingMillis(frameTimingAvailable, frameMillis), + gpuMillis: optionalTimingMillis(gpuTimingAvailable, gpuMillis), + sortMillis: optionalTimingMillis(sortTimingAvailable, sortMillis), + }); }, []); const onPolicyEvent = useCallback((event: EventOf<'onPolicyEvent'>) => { const { phase, message } = event.nativeEvent; - if (phase === 'rejected') + if (phase === PolicyPhase.rejected) console.warn(`SplatKit policy rejected: ${message}`); }, []); + // Straight to the native view, so the stick moves the camera with no React commit. + const onStick = useCallback((forward: number, right: number) => { + const target = view.current; + if (target) + SplatKitCommands.setWalkVelocity( + target, + forward * WALK_SPEED, + right * WALK_SPEED, + ); + }, []); + return ( - - {status} - + + {walking && ( + + + + )} + {status !== null && ( + + {status} + + )} ); } @@ -125,6 +234,10 @@ const styles = StyleSheet.create({ flex: 1, backgroundColor: 'black', }, + stick: { + position: 'absolute', + left: 28, + }, status: { position: 'absolute', left: 16, diff --git a/apps/react-native/Hud.tsx b/apps/react-native/Hud.tsx new file mode 100644 index 0000000..2d9b0a8 --- /dev/null +++ b/apps/react-native/Hud.tsx @@ -0,0 +1,230 @@ +/** + * The render HUD: a stats card and a quality picker, over the SplatKitView. + * + * Everything here is the app's own UI. The SDK reports numbers through onStats and + * onCapabilities and draws none of this itself. + * + * @format + */ + +import { memo } from 'react'; +import { Pressable, StyleSheet, Text, View } from 'react-native'; +import { QualityPreset, qualityPresets } from '@splatkit/react-native'; + +export type RenderStats = Readonly<{ + loadedSplats: number; + drawnSplats: number; + frameMillis: number | null; + gpuMillis: number | null; + sortMillis: number | null; +}>; + +const PRESET_TITLES: Readonly> = Object.freeze({ + [QualityPreset.highEnd]: 'Max', + [QualityPreset.high]: 'High', + [QualityPreset.balanced]: 'Balanced', + [QualityPreset.performance]: 'Fast', +}); + +/** Above this the frame rate reads as smooth, below the lower one as a problem. */ +const SMOOTH_FPS = 50; +const ROUGH_FPS = 25; + +function compact(value: number): string { + if (value >= 1e6) return `${(value / 1e6).toFixed(1)}M`; + if (value >= 1e3) return `${Math.round(value / 1e3)}k`; + return `${value}`; +} + +function millis(value: number | null): string { + return value === null ? '-' : `${value.toFixed(0)} ms`; +} + +function Row({ label, value }: { label: string; value: string }) { + return ( + + {label} + {value} + + ); +} + +function StatsCard({ stats }: { stats: RenderStats | null }) { + // The engine skips frames while nothing moves, so no frame time is a still view, not 0 fps. + const frameMillis = + stats !== null && stats.frameMillis !== null && stats.frameMillis > 0 + ? stats.frameMillis + : null; + const fps = frameMillis === null ? null : 1000 / frameMillis; + const pace = + fps === null + ? styles.dotIdle + : fps >= SMOOTH_FPS + ? styles.dotSmooth + : fps >= ROUGH_FPS + ? styles.dotRough + : styles.dotSlow; + return ( + + + {fps === null ? 'Idle' : fps.toFixed(0)} + {fps !== null && FPS} + + + + + {frameMillis === null ? 'no redraw' : millis(frameMillis)} + + + + + + + + ); +} + +type Props = Readonly<{ + stats: RenderStats | null; + preset: QualityPreset; + onPreset: (preset: QualityPreset) => void; + top: number; + bottom: number; +}>; + +function Hud({ stats, preset, onPreset, top, bottom }: Props) { + return ( + <> + + + + + {qualityPresets.map(value => { + const active = value === preset; + return ( + onPreset(value)} + style={[styles.preset, active && styles.presetActive]} + > + + {PRESET_TITLES[value]} + + + ); + })} + + + ); +} + +const styles = StyleSheet.create({ + cardSlot: { + position: 'absolute', + left: 16, + }, + card: { + width: 150, + paddingHorizontal: 10, + paddingVertical: 8, + borderRadius: 12, + backgroundColor: 'rgba(17, 17, 20, 0.62)', + borderWidth: StyleSheet.hairlineWidth, + borderColor: 'rgba(255, 255, 255, 0.12)', + }, + headline: { + flexDirection: 'row', + alignItems: 'center', + gap: 5, + }, + fps: { + color: 'white', + fontSize: 22, + fontWeight: '600', + fontVariant: ['tabular-nums'], + }, + fpsUnit: { + color: 'rgba(255, 255, 255, 0.55)', + fontSize: 9, + fontWeight: '700', + }, + dot: { + width: 5, + height: 5, + borderRadius: 2.5, + }, + dotIdle: { + backgroundColor: '#9ca3af', + }, + dotSmooth: { + backgroundColor: '#4ade80', + }, + dotRough: { + backgroundColor: '#facc15', + }, + dotSlow: { + backgroundColor: '#f87171', + }, + frame: { + color: 'rgba(255, 255, 255, 0.55)', + fontSize: 9, + fontWeight: '500', + fontVariant: ['tabular-nums'], + }, + divider: { + height: StyleSheet.hairlineWidth, + backgroundColor: 'rgba(255, 255, 255, 0.15)', + marginVertical: 7, + }, + row: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + paddingVertical: 1, + }, + rowLabel: { + color: 'rgba(255, 255, 255, 0.55)', + fontSize: 10, + }, + rowValue: { + color: 'white', + fontSize: 10, + fontWeight: '600', + fontVariant: ['tabular-nums'], + }, + presets: { + position: 'absolute', + left: 0, + right: 0, + flexDirection: 'row', + justifyContent: 'center', + gap: 6, + }, + preset: { + paddingHorizontal: 12, + paddingVertical: 6, + borderRadius: 999, + backgroundColor: 'rgba(17, 17, 20, 0.62)', + borderWidth: StyleSheet.hairlineWidth, + borderColor: 'rgba(255, 255, 255, 0.12)', + }, + presetActive: { + backgroundColor: 'rgba(255, 255, 255, 0.92)', + borderColor: 'transparent', + }, + presetText: { + color: 'rgba(255, 255, 255, 0.75)', + fontSize: 11, + fontWeight: '600', + }, + presetTextActive: { + color: '#111114', + }, +}); + +export default memo(Hud); diff --git a/apps/react-native/Joystick.tsx b/apps/react-native/Joystick.tsx new file mode 100644 index 0000000..cf28931 --- /dev/null +++ b/apps/react-native/Joystick.tsx @@ -0,0 +1,86 @@ +/** + * A thumb stick. Reports a direction in [-1, 1] on each axis, y positive upwards, while + * held, and (0, 0) once on release. + * + * PanResponder, not React state, drives the value: the knob is animated and the direction + * goes out on every touch move, so nothing re-renders while walking. + * + * @format + */ + +import { useMemo, useRef } from 'react'; +import { Animated, PanResponder, StyleSheet, View } from 'react-native'; + +const RADIUS = 62; +const KNOB = 52; + +type Props = Readonly<{ + onChange: (forward: number, right: number) => void; +}>; + +function Joystick({ onChange }: Props) { + const knob = useRef(new Animated.ValueXY({ x: 0, y: 0 })).current; + const change = useRef(onChange); + change.current = onChange; + + const responder = useMemo( + () => + PanResponder.create({ + onStartShouldSetPanResponder: () => true, + onMoveShouldSetPanResponder: () => true, + onPanResponderMove: (_event, gesture) => { + let { dx, dy } = gesture; + const length = Math.hypot(dx, dy); + if (length > RADIUS) { + dx = (dx / length) * RADIUS; + dy = (dy / length) * RADIUS; + } + knob.setValue({ x: dx, y: dy }); + change.current(-dy / RADIUS, dx / RADIUS); + }, + onPanResponderRelease: () => { + Animated.spring(knob, { + toValue: { x: 0, y: 0 }, + useNativeDriver: true, + speed: 24, + bounciness: 6, + }).start(); + change.current(0, 0); + }, + onPanResponderTerminate: () => { + knob.setValue({ x: 0, y: 0 }); + change.current(0, 0); + }, + }), + [knob], + ); + + return ( + + + + ); +} + +const styles = StyleSheet.create({ + base: { + width: RADIUS * 2, + height: RADIUS * 2, + borderRadius: RADIUS, + backgroundColor: 'rgba(255, 255, 255, 0.12)', + borderWidth: StyleSheet.hairlineWidth, + borderColor: 'rgba(255, 255, 255, 0.25)', + alignItems: 'center', + justifyContent: 'center', + }, + knob: { + width: KNOB, + height: KNOB, + borderRadius: KNOB / 2, + backgroundColor: 'rgba(255, 255, 255, 0.55)', + }, +}); + +export default Joystick; diff --git a/apps/react-native/README.md b/apps/react-native/README.md index 9c3f28b..d484553 100644 --- a/apps/react-native/README.md +++ b/apps/react-native/README.md @@ -1,7 +1,18 @@ # SplatKit React Native example -A React Native 0.87.1 app, made with `npx @react-native-community/cli init`, that installs [`@splatkit/react-native`](../../packages/react-native-splatkit/README.md) from npm and renders one full-screen `SplatKitView`. -Drag with one finger to look around and with two fingers to walk. +A React Native 0.87.1 app, made with `npx @react-native-community/cli init`, that renders one full-screen `SplatKitView` you can walk through. +Drag anywhere to look around, and double tap to toggle the gyroscope. +Once the collider is ready a thumb stick appears; it is this app's own control, in [`Joystick.tsx`](Joystick.tsx), and it drives the view through `SplatKitCommands.setWalkVelocity`. +The SDK draws no walking UI of its own. + +Inside this monorepo the app installs [`@splatkit/react-native`](../../packages/react-native-splatkit/README.md) from `../../packages`, and builds the Android SDK from source, so the example always exercises the current API. +Outside it, `npm install @splatkit/react-native` is the only change. + +Linking the package rather than unpacking it costs the example one extra piece of Metro config, in [`metro.config.js`](metro.config.js). +The package keeps React and React Native as devDependencies, so from the linked directory Metro resolves them to the package's own `node_modules` and the bundle ends up with two copies of each. +Two copies of React Native means two view config registries, and `SplatKitView` registers in the one the renderer does not read, which fails at render with `View config getter callback for component 'SplatKitView' must be a function`. +The `resolveRequest` hook pins both module names to this app. +An app that installs the package from npm needs none of this. ## Setup from zero in your own app @@ -17,7 +28,7 @@ Then match what this app changes from the template: - `android/gradle.properties`: `reactNativeArchitectures=arm64-v8a`. - `ios/Podfile`: `platform :ios, '17.0'`, and the Xcode deployment target 17.0; then `cd ios && pod install`. - `ios/SplatKitExample/AppDelegate.swift` and `Info.plist`: a scene delegate, which iOS 26 requires. -- A world file path for `withWorld`; here `MainActivity.kt` and `SceneDelegate` pass it as the `worldPath` initial prop. +- A world file path for `withWorld`, and a collider path for the `collider` prop; here `MainActivity.kt` and `SceneDelegate` pass both as the `worldPath` and `colliderPath` initial props. - [`App.tsx`](App.tsx): build the configuration, mount `SplatKitView` and rebuild when `onCapabilities` arrives. ## Run this app @@ -34,6 +45,7 @@ Android, on an arm64 device with Vulkan 1.1: ```sh npm run android adb push world.spz /sdcard/Android/data/com.splatkit.example/files/world.spz +adb push collider.glb /sdcard/Android/data/com.splatkit.example/files/collider.glb ``` iOS, on a device with iOS 17 or newer; set your signing team in Xcode first: @@ -43,7 +55,10 @@ cd ios && pod install && cd .. npm run ios -- --device xcrun devicectl device copy to --device --domain-type appDataContainer \ --domain-identifier com.splatkit.example --source world.spz --destination Documents/world.spz +xcrun devicectl device copy to --device --domain-type appDataContainer \ + --domain-identifier com.splatkit.example --source collider.glb --destination Documents/collider.glb ``` Restart the app after copying a world. +The collider is optional: without one the world still renders and looks around, and the thumb stick stays hidden. The status line shows the load, drawn splats and GPU time, or why the world failed to load. diff --git a/apps/react-native/android/app/src/main/java/com/splatkit/example/MainActivity.kt b/apps/react-native/android/app/src/main/java/com/splatkit/example/MainActivity.kt index ea7b68d..053e5b3 100644 --- a/apps/react-native/android/app/src/main/java/com/splatkit/example/MainActivity.kt +++ b/apps/react-native/android/app/src/main/java/com/splatkit/example/MainActivity.kt @@ -16,6 +16,10 @@ class MainActivity : ReactActivity() { override fun createReactActivityDelegate(): ReactActivityDelegate = object : DefaultReactActivityDelegate(this, mainComponentName, fabricEnabled) { override fun getLaunchOptions(): Bundle = - Bundle().apply { putString("worldPath", File(getExternalFilesDir(null), "world.spz").path) } + Bundle().apply { + val files = getExternalFilesDir(null) + putString("worldPath", File(files, "world.spz").path) + putString("colliderPath", File(files, "collider.glb").path) + } } } diff --git a/apps/react-native/android/settings.gradle b/apps/react-native/android/settings.gradle index 253de63..9961825 100644 --- a/apps/react-native/android/settings.gradle +++ b/apps/react-native/android/settings.gradle @@ -1,6 +1,23 @@ -pluginManagement { includeBuild("../node_modules/@react-native/gradle-plugin") } +pluginManagement { + includeBuild("../node_modules/@react-native/gradle-plugin") + repositories { + google() + mavenCentral() + gradlePluginPortal() + } + // Building the SplatKit Android SDK from source below brings its own plugins in too. + plugins { id('com.vanniktech.maven.publish') version '0.34.0' } +} plugins { id("com.facebook.react.settings") } extensions.configure(com.facebook.react.ReactSettingsExtension){ ex -> ex.autolinkLibrariesFromCommand() } rootProject.name = 'com.splatkit.example' include ':app' includeBuild('../node_modules/@react-native/gradle-plugin') + +// This example lives in the SplatKit monorepo, so it builds the Android SDK from source +// instead of the Maven release; @splatkit/react-native picks up :splatkit when it exists. +def splatkitAndroid = file('../../../packages/splatkit-android') +if (splatkitAndroid.isDirectory()) { + include ':splatkit' + project(':splatkit').projectDir = splatkitAndroid +} diff --git a/apps/react-native/ios/Podfile.lock b/apps/react-native/ios/Podfile.lock index 3d832d9..cef1dd0 100644 --- a/apps/react-native/ios/Podfile.lock +++ b/apps/react-native/ios/Podfile.lock @@ -1968,94 +1968,94 @@ EXTERNAL SOURCES: :path: build/rncore-facades/Yoga SPEC CHECKSUMS: - boost: 8502bfdc46e2804408af2e163049ece181009af4 - DoubleConversion: 501a575c40c0b50c89685762dd3e415e6bb26fb0 - fast_float: 7c925d1cebf8328d4018d061f49377e9f887e3b4 - FBLazyVector: e08a27053b503eeb756ea4bd75d570d2f2522a1b - fmt: ba0886e864ebb21d14c98b2e12e164b8002f5b93 - glog: 3cf7cb46f743c4bc758ae3e6d9c15bd1b4cfd6d4 + boost: 2a9d6e1d9873337543f08d9dbc007b288f4e113b + DoubleConversion: e85e4faac763226bfd1cc4c6a98d8355149b27a5 + fast_float: 908e5ee35999f3d91af4b962ac951acac591866c + FBLazyVector: 1f950f55bf9d0f1130874f80ff0932945b97e2a8 + fmt: c7776a2a4b1d7ce88b809a1e4d1f7add0f2369df + glog: b4427c3119645da795ff09985ba9da909c940c96 hermes-engine: 63c5db7d529c36335fada6582fd293ae2a105d77 - RCT-Folly: 82921e585c0400eda46fac1068297aa9dfb035e4 - RCTDeprecation: 5cc865306ce4974b7a57ddbc575fd1f3fef41957 - RCTRequired: ea6f2c41f2670961ce9a48a4bf17a190dd53e493 + RCT-Folly: d173e541eeb78cad5c81011c1bbf62d5b86b9778 + RCTDeprecation: 1dce662386078170e119b89c1600db1b6e3d42f6 + RCTRequired: b5a9168b680baa2cda4f6b4764d10b6e58368b25 RCTSwiftUI: 78ca6a66d8413ab78e24a133674b40b8c472144f - RCTSwiftUIWrapper: d547076b3b5889f57e81c543ed5253b4e7bbdc63 + RCTSwiftUIWrapper: 7ddb997cbba2447cdab73a65f6d019a30d47e352 RCTTypeSafety: 189f0320246cf5879b25268ad027c47f44df071c React: 2de853bffdce9d79608a256892cadfdd42124fb1 - React-bridging: ad588ebf664b5339b159f7bb1076a52967a5e433 + React-bridging: 36710fbb84391e0c1ca07392e855a73f328058aa React-callinvoker: 33c0c2c46a808f55ef5a2927fa373977a334ea3e - React-Core: 48305724ff82e5922efc52bf8d0e713fbf84be5a - React-Core-prebuilt: cd44efd12c9bfad37c194372a6d3a18a6a6d5c00 - React-CoreModules: a190415306d55fbe58fb3d8eb045c0e46c5c8437 - React-cxxreact: 2ed666fa84fbcda80286da0c2fd6ad8636c7e9d0 + React-Core: a85e5b125475d5d744af9614165ddb0944b1f86d + React-Core-prebuilt: d02240839df49bdfe03148fae0a647aaf7288d53 + React-CoreModules: 6bc28ab7a1b0450fe502b38b1ba6323db9f2a472 + React-cxxreact: cc133718b2f46a384b4ffafda73c87e8de23eab2 React-cxxstableapi: 1e0ad8a5ecb7f2f5440c012798cf20bec6341c1f React-debug: 32e0e1c05c42cbf517be7c5e0dc823a23efc9034 - React-defaultsnativemodule: 74475d2452c818f9ebc3513b2df900bedd18c396 - React-domnativemodule: a425b8b6acbb47a7b6779c1e334bd8fc236f67b0 - React-Fabric: 005469f5cb798c3047bacb43cc560d95eee19171 - React-FabricComponents: 171963d034307ae4e020ef18993b92e856c1f8b3 - React-FabricImage: 459f14722cf17b2bf74f98b68da2fc7de4af51e9 - React-featureflags: 0eb89858b7b57ef8f3b695f9129164d55e6c5e4d - React-featureflagsnativemodule: 73fe3a75cf31e35dfcedc7c3e4b2b2c2240202c4 - React-graphics: b0ab8b47fc0f07e29d1da0792cff0d658f913ea8 - React-hermes: f1043e16098d7e31cb55faa28494d9c3dec782f3 - React-idlecallbacksnativemodule: 6eb47bb954639dc29839ccb472032b730d9fa3ab - React-ImageManager: c101d51571137c332e6b7fce83e6d12d8493cd52 - React-intersectionobservernativemodule: fa76007b27a150c43125b21ba3a0f76847426c03 - React-jserrorhandler: e5d7f718ed444fa040297a654ff0ca229907e54f - React-jsi: cd9cf66cfffee23062dbd4b7ceca27f6294adfc8 - React-jsiexecutor: 834be45f4c635f93cc5c0137f5db62d5297b7d7f - React-jsinspector: 17b99007b14ee0588c775ad8fbae828353bc4dae - React-jsinspectorcdp: f56181a316a781334bbd7f061e4aa0f8f94b9f0c - React-jsinspectornetwork: 7ca0da8af486f5444a3f7dc81a1cafe13ca23572 - React-jsinspectortracing: 49a963331332d1c0abef67cab184e80f6bdcd341 - React-jsitooling: d85d39d2633572a39a5b045c9d589d64187de600 - React-jsitracing: 810e823cc6d83c091d45468967558d1eace28922 - React-logger: 8ecf7242442a7e222a636d24ac9c56ea948fd455 - React-Mapbuffer: 295203fc648a43acf7fba609ddf573aa9fca849f - React-microtasksnativemodule: 269f81deee27a2a9bedb6b2d1088c1ae93de9bd3 - React-mutationobservernativemodule: 877e9f730e0d8175c2e952901b55c170fc3d372f - react-native-safe-area-context: b3e507b0dbfb44dfadb29f7a31e884ac13789723 - React-NativeModulesApple: adb1dd8361dff80e9f6652fb3fdbf033cfbe2a89 - React-networking: c00b437c4408f0640ba0d4158357133488521eca + React-defaultsnativemodule: 4078c2f78144b3e3fd71d5f5eeb8223bf1d42d53 + React-domnativemodule: 2d2e689dd512119220daff132d5d6055097537ed + React-Fabric: d8c9b71b92283a54e3e5147fcfa33a27ce214c65 + React-FabricComponents: dcb7fd6b6ea264da44e31ac732c83376831d75ac + React-FabricImage: 795156884ef44f95ec1a1f7d0597fe692125a248 + React-featureflags: 0e5c334efeb10c0daf014b72b6eda0b7919c2639 + React-featureflagsnativemodule: a330849ba11eded156bc383de5d905bf7480645f + React-graphics: d7db9e8ee93d6fb8f699f68206e62281bf1c7158 + React-hermes: d07a60a7b642f06c5dd65998998677074cd7e2d7 + React-idlecallbacksnativemodule: 6202a6e76ea5116af3e3707904692e2650d1b829 + React-ImageManager: a2017335b0c95df628bfa4bdd1563523eb2f3ac9 + React-intersectionobservernativemodule: 7756a02c5c69834c671e8ffb8e293ce7849d88e8 + React-jserrorhandler: 93233d7474fa474c5c4b5fa4483e3dcf8b15144d + React-jsi: ef4818b4098ffe8ca8e2ff414615c8d5a3fde3bf + React-jsiexecutor: ae34be528c2c002e9e7e515889f020117c0dce23 + React-jsinspector: d493f06b774c10ce02a3502c21f6d894a3a2ca88 + React-jsinspectorcdp: 309621906dfda2ee1b52df582e234800e406b444 + React-jsinspectornetwork: 0deadf9bbfdf0d223fbb7383e5bd52af6c573397 + React-jsinspectortracing: 19d2cc0872a14f2f84d4bfa5283497681c98c334 + React-jsitooling: 243c8c7738e83d1186520946f2c928d04f891fc7 + React-jsitracing: 042598d0629dcb96d4f281ae4f90961785a51a52 + React-logger: 17fba47610dd5a3821fe2825b3c88fdbe9e07da4 + React-Mapbuffer: fedd8e34e26194cf70bd66e77c49170e22d88b82 + React-microtasksnativemodule: aa3ea1db91e208694042bd8af0bda65202f4a68d + React-mutationobservernativemodule: 2dcf0e333fe6621c3ffea838fcc48d54435724a1 + react-native-safe-area-context: d31381d291323aff31330587eaac543dabd0e258 + React-NativeModulesApple: 885f4cd7626a1ffe20e0dcd23ef8528f415795d3 + React-networking: 31c21d3a8ea93c859706f876d27578782b05d250 React-oscompat: 3175a287dd7175d1a20fdce5048a4b9e11f4b915 - React-perflogger: 50bffa76f6e8d101e1e324d06cbbbf1fd9afcf06 - React-performancecdpmetrics: a5a874381b84b596e344e25666596ba6d44b961d - React-performancetimeline: bacec680535024bc6a5a7dc646d5a62b46c6a2fd + React-perflogger: f42acd1aef5eda2157ac473b7e4f562b097eeffe + React-performancecdpmetrics: f74ad4c95809026ac2a1648782231ee3d2ff98d9 + React-performancetimeline: 2ccd8f1cdaee196c86f849fb4d8ba2e5ade15428 React-RCTActionSheet: eae121cc5c15015e5adf24fe863f2a31895a2411 - React-RCTAnimatedModuleProvider: 24721e9467537b981b1a4b96ebf384a9324c80c5 - React-RCTAnimation: 8d988bbf7585d660d79027933a3c4976a26a8496 - React-RCTAppDelegate: da272f46f8f23ce42ddf5eeeca740af5cd503a2c - React-RCTBlob: e0b7c51bec9b327e34eebb356af5fb5504a401b7 - React-RCTFabric: 2d3c788f9cac0cf0be2f6db39d2df2e576a2a6d6 - React-RCTFBReactNativeSpec: 9f977b0764f1f1cd97dd4d02d30a6fcd8608c517 - React-RCTImage: 5c0afa67274b21518491935a895d427c6530c7a6 - React-RCTLinking: bf7d1245f38e5df7432fad0c18caf4d171c2832c - React-RCTNetwork: 0f1ca5f4182b133a62248983a100a202275937de - React-RCTRuntime: 7adbf03a4ac5d66fc0953d4fbc7f93425662dc82 - React-RCTSettings: 7c7387e52b838cfeb49c8c8049223a72b2ebce88 - React-RCTText: 10d157fe52d742fcdec5adae67e161b530fc7a57 - React-RCTVibration: a19a6e696173d014e9228fa16b003f9afa1b14dc + React-RCTAnimatedModuleProvider: c66bdce02929e5c8e9b158256d59904943c09043 + React-RCTAnimation: 1ed2b0d9e955b26896eefb7eed0e402e6801d1a2 + React-RCTAppDelegate: 1f50b1d642a0152bd7f3d07ac0074cb40c8668a9 + React-RCTBlob: 7e7aa3a79aa98fb9351a3eaf4bc405242b45ab5b + React-RCTFabric: d68fb0b8998151bf37bd06aaeb681af40b11348d + React-RCTFBReactNativeSpec: 5e6b60d5132631ebf4db89ba7db78d1ab5783290 + React-RCTImage: ed521f108a7396a54421a512606138bb8abe2486 + React-RCTLinking: de1c5bf6cdc0e369c9f48f661ff31b05ac98fda2 + React-RCTNetwork: 320d4b1090631c7481c68a77f07f44f8d49dd7d2 + React-RCTRuntime: 79cc5abf548209975237bcd97153cab0bbf818da + React-RCTSettings: f0538b65b944ae9b2858f7a871cd87ec1bd8a9f8 + React-RCTText: 9e02a4520b1c8199bb376712d7df810a593ebea1 + React-RCTVibration: 9d32bdf22b2cb00352aec28fa6a8138514771b4e React-rendererconsistency: 24e61e6a927fbcbc676e2a88e5d219c1b5b144e7 - React-renderercss: 20ba84f23fab261df5e18c04d486e4669df9d33e - React-rendererdebug: 914e9d02aa71fef5af72d6642a0979865ecfe8d4 - React-RuntimeApple: 7ebd7c54e08cfe2c30188248c03e507c10729721 - React-RuntimeCore: 31f5796d73bfbca978d2e23a956a2cf131082108 - React-runtimeexecutor: 740750b1a196add2e2a591a3154cb580eb9af336 - React-RuntimeHermes: 384cebd1e7ed03329b2de18add97154fa4204a4c - React-runtimescheduler: 39b8dae97bda24f0d80bff480aadeee852b9eb22 - React-timing: 8ff0fb7aebbf0b4605a3057418670632e329cc5a - React-utils: d77852fe2b8259da760f21a68819d48038f00de3 - React-viewtransitionnativemodule: 1bc6bb59029e578e62cb1a818ff4a4d548071a66 - React-webperformancenativemodule: 10cf05e9bcf2f28a4b1c498d580e21ba3cc114f1 - ReactAppDependencyProvider: 3949ea6b2df0d96295128aae79f068cb6c5f81dc - ReactCodegen: 13d2ebfe2d8f1200052509b4fc4abf7a6825a7fc - ReactCommon: 749169e12aa3461a3df3656a00e48dbe0456b2d6 + React-renderercss: 1fe071fa7f7c5273c68d56ff0b083c36ff954471 + React-rendererdebug: 85c54a64999e907fece60817b562eb30a029a7d9 + React-RuntimeApple: ae702863a426edb565e280dc48a3c983e7cfcdd2 + React-RuntimeCore: 9869bde14a20ca689d1d0ef6b0f242c6f199e874 + React-runtimeexecutor: fc15be8a3b0e44fb85dc0823f779420a5c2c81c0 + React-RuntimeHermes: 00ec926c2b80b2a648dd50f7e849e2192c8818c6 + React-runtimescheduler: aa08fa7b8ba4b24cc8924a14970bfd167abb994e + React-timing: 1a12f3691f98b53f3bed92dc2c3be63abc0185b0 + React-utils: 579b27fc3ca9ba76306920f91b03eb94b7d5cf11 + React-viewtransitionnativemodule: b9181c55775121e80ac46df11e01661231154c0c + React-webperformancenativemodule: 519253bbcaf05552edb2bd8f5ee38720ee49b9db + ReactAppDependencyProvider: 531dee09fa5ac21ef8a236a0c4767c3b2f9db977 + ReactCodegen: 35b96d81c5528b5774f73e938ce4be8eb879d7bd + ReactCommon: 7d2d036b7250e66dce71d19f47160d4b31fbb92a ReactNativeDependencies: 94c922c39e0e6150ee9ad05e2ca1946e0ef55783 - SocketRocket: 37aec555668fb852ec12e3c0de59a86ca58f0871 - SplatKitReactNative: dd151ba03b14156f41b94a5afe75e5b88cd57c79 - Yoga: d1c536142c5ff8ec8cd856ab2a7c227a1d875c8e + SocketRocket: d57b1af80c029e12a75cd9b218287c67381e4593 + SplatKitReactNative: 5aa04eb04a1be7f0149fca20facbd2ee2ba153d4 + Yoga: 58b502c52b14ad3c4468805d461fc37d275c2a42 PODFILE CHECKSUM: 9b0ece2bec34696702f6b30137c4b3da4e1af38a -COCOAPODS: 1.17.0 +COCOAPODS: 1.16.2 diff --git a/apps/react-native/ios/SplatKitExample/AppDelegate.swift b/apps/react-native/ios/SplatKitExample/AppDelegate.swift index 3a77e86..47608a2 100644 --- a/apps/react-native/ios/SplatKitExample/AppDelegate.swift +++ b/apps/react-native/ios/SplatKitExample/AppDelegate.swift @@ -41,7 +41,10 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { factory.startReactNative( withModuleName: "SplatKitExample", in: window, - initialProperties: ["worldPath": documents.appendingPathComponent("world.spz").path], + initialProperties: [ + "worldPath": documents.appendingPathComponent("world.spz").path, + "colliderPath": documents.appendingPathComponent("collider.glb").path, + ], launchOptions: nil ) } diff --git a/apps/react-native/metro.config.js b/apps/react-native/metro.config.js index 2a0a21c..112f4f9 100644 --- a/apps/react-native/metro.config.js +++ b/apps/react-native/metro.config.js @@ -1,11 +1,45 @@ +const path = require('node:path'); const { getDefaultConfig, mergeConfig } = require('@react-native/metro-config'); +// This example lives in the SplatKit monorepo and installs the package from ../../packages, +// so npm links it instead of unpacking it. Metro only follows a symlink out of the project +// when the target is a watch folder. +const sdk = path.resolve(__dirname, '../../packages/react-native-splatkit'); +const modules = path.resolve(__dirname, 'node_modules'); + +// The SDK keeps React and React Native as devDependencies, so from its own directory Metro +// resolves them to its node_modules and the bundle ends up with two copies. Two copies means +// two view config registries: the SDK registers SplatKitView in its own and the renderer, on +// the app's copy, then renders a component it has never heard of. Pin both to this app. +const single = new Set(['react', 'react-native']); +// Resolution starts at the requesting file and walks up, so a request answered as if it came +// from this file lands in the app's node_modules wherever it was made. Rewriting the request +// to an absolute path would do the same but skip the package's own exports map, which is how +// react-native/asset-registry and every other subpath import finds its file. +const here = path.join(__dirname, 'index.js'); + /** * Metro configuration * https://reactnative.dev/docs/metro * * @type {import('@react-native/metro-config').MetroConfig} */ -const config = {}; +const config = { + watchFolders: [sdk], + resolver: { + nodeModulesPaths: [modules], + resolveRequest: (context, moduleName, platform) => { + const scope = moduleName.split('/')[0]; + if (single.has(scope)) { + return context.resolveRequest( + { ...context, originModulePath: here }, + moduleName, + platform, + ); + } + return context.resolveRequest(context, moduleName, platform); + }, + }, +}; module.exports = mergeConfig(getDefaultConfig(__dirname), config); diff --git a/apps/react-native/package-lock.json b/apps/react-native/package-lock.json index 764e894..4048c5c 100644 --- a/apps/react-native/package-lock.json +++ b/apps/react-native/package-lock.json @@ -8,7 +8,7 @@ "name": "splatkit-react-native-example", "version": "0.0.1", "dependencies": { - "@splatkit/react-native": "0.1.0-alpha.1", + "@splatkit/react-native": "file:../../packages/react-native-splatkit", "react": "19.2.3", "react-native": "0.87.1", "react-native-safe-area-context": "^5.5.2" @@ -33,6 +33,24 @@ "node": ">= 22.11.0" } }, + "../../packages/react-native-splatkit": { + "version": "0.1.0-alpha.1", + "license": "MIT", + "devDependencies": { + "@react-native/codegen": "0.87.1", + "@types/react": "19.2.14", + "react": "19.2.4", + "react-native": "0.87.1", + "typescript": "5.9.3" + }, + "engines": { + "node": "^22.13.0 || ^24.3.0 || >=26.0.0" + }, + "peerDependencies": { + "react": "^19.2.3", + "react-native": "~0.87.1" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -2813,17 +2831,8 @@ "license": "MIT" }, "node_modules/@splatkit/react-native": { - "version": "0.1.0-alpha.1", - "resolved": "https://registry.npmjs.org/@splatkit/react-native/-/react-native-0.1.0-alpha.1.tgz", - "integrity": "sha512-EQhSeXxbK5hjyURGkqbFrKAPEefbqtvnGkHQ+OJoBCuSMVihdV+l+Qp4IRx6mxv665zjJiPtKv9fC60J5LmPvQ==", - "license": "MIT", - "engines": { - "node": "^22.13.0 || ^24.3.0 || >=26.0.0" - }, - "peerDependencies": { - "react": "^19.2.3", - "react-native": "~0.87.1" - } + "resolved": "../../packages/react-native-splatkit", + "link": true }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", @@ -9692,7 +9701,7 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/apps/react-native/package.json b/apps/react-native/package.json index 6f01bca..0dd52f9 100644 --- a/apps/react-native/package.json +++ b/apps/react-native/package.json @@ -10,7 +10,7 @@ "start": "react-native start" }, "dependencies": { - "@splatkit/react-native": "0.1.0-alpha.1", + "@splatkit/react-native": "file:../../packages/react-native-splatkit", "react": "19.2.3", "react-native": "0.87.1", "react-native-safe-area-context": "^5.5.2" diff --git a/packages/react-native-splatkit/CHANGELOG.md b/packages/react-native-splatkit/CHANGELOG.md index 86ccbc4..9bf7269 100644 --- a/packages/react-native-splatkit/CHANGELOG.md +++ b/packages/react-native-splatkit/CHANGELOG.md @@ -18,6 +18,12 @@ This release replaces the previous `react-native-splatkit` binding, which bound - Standalone build wiring for installing outside the SplatKit repository: the Android module applies the `com.facebook.react` Gradle plugin for autolinking and Codegen and depends on `io.github.xget7:splatkit-android` from Maven Central by default, and `SplatKitReactNative.podspec` vendors a `SplatKitCore.xcframework` fetched and checksum-verified at `npm prepack`. - `peerDependencies` on `react` and `react-native` 0.87, and a `publish.yml` workflow that publishes a `v*` tag to npm. - One-finger look and two-finger walk on iOS, matching the Android SDK view's touch input. +- `policyRasterMask` in `onCapabilities` and `rasterStrategies` in the native policy support, so `computeTile` on iOS reports a fallback instead of an applied raster. + +### Changed + +- Every preset rasterizes in hardware; `high` and `balanced` requested hybrid tiles and `highEnd` compute tiles. + `withPerformance({raster: 'hybrid'})` opts into iOS screen tiles, which only help where many large translucent splats overlap. ### Fixed diff --git a/packages/react-native-splatkit/README.md b/packages/react-native-splatkit/README.md index ade44ad..1ff0f76 100644 --- a/packages/react-native-splatkit/README.md +++ b/packages/react-native-splatkit/README.md @@ -1,87 +1,334 @@ # SplatKit React Native -Fabric view for the SplatKit Metal and Vulkan renderers, with one renderer policy contract across JS and both native adapters. -Experimental alpha: APIs change before 1.0. -New Architecture only, on React Native 0.87, Android minSdk 29 arm64-v8a and iOS 17. -A fresh React Native 0.87.1 app ran the packed package on an iPhone 17 Pro; Android runs in the SplatKit RN dev app on a Mi 9. -One finger drags to look and two fingers walk. -Changes are listed in the [changelog](CHANGELOG.md). +`@splatkit/react-native` renders 3D Gaussian splat scenes natively on Metal (iOS) and Vulkan (Android) through one Fabric component, `SplatKitView`, and turns a collider mesh into a walkable, collidable scene. -## Install +[![npm version](https://img.shields.io/npm/v/@splatkit/react-native.svg)](https://www.npmjs.com/package/@splatkit/react-native) +[![license: MIT](https://img.shields.io/npm/l/@splatkit/react-native.svg)](LICENSE) +[![platform: iOS | Android](https://img.shields.io/badge/platform-iOS%20%7C%20Android-lightgrey.svg)](#requirements) + + + +> **Experimental alpha.** APIs change before 1.0. +> The package has been validated on an iPhone 17 Pro and a Mi 9 (Adreno 640); no performance numbers are published beyond that. + +## Features + +- Native renderer, not a WebGL or JS port: Metal on iOS, Vulkan on Android, behind one Fabric view. +- New Architecture (Fabric) only, driven by `@react-native/codegen`. +- Loads `.spz`, `.ply` and `.lodsplat` files, and tiled worlds that stream progressively, straight from an absolute local file path (no JS byte transport). +- LOD selection and residency streaming with host-tunable splat-count budgets. +- A single, versioned per-view render policy that both native adapters validate and report back through `onCapabilities` and `onPolicyEvent`. +- Walk-mode collision: load a collider GLB and the camera becomes a character that stands on floors, is stopped by walls and climbs steps. +- Host-driven navigation: the SDK draws no walking or look UI, and imperative commands drive the camera at touch rate with no React commit per frame. +- Throttled stats (`onStats`, at most 2 Hz) and camera pose (`onCameraPose`, on your own interval) events, so hosts can build a HUD or a minimap without flooding the bridge. + +## Requirements + +| | Minimum | +| --- | --- | +| React Native | `0.87.x` (peer dependency `~0.87.1`) | +| React | `^19.2.3` | +| Architecture | New Architecture (Fabric) enabled; `SplatKitView` has no legacy-bridge fallback | +| iOS | iOS 17.0+, Xcode with CocoaPods, a device or simulator slice matching `SplatKitCore.xcframework` | +| Android | API 29+ (`minSdkVersion 29`), Vulkan 1.1, `arm64-v8a` only | +| Node | `^22.13.0`, `^24.3.0`, or `>=26.0.0` | + +## Installation ```sh npm install @splatkit/react-native cd ios && pod install ``` -Android autolinks through the app's `com.facebook.react` Gradle plugin, which also runs Codegen, and pulls `io.github.xget7:splatkit-android` from Maven Central. -iOS vendors `SplatKitCore.xcframework` from the matching `splatkit-ios` release, checksum-verified when the npm package is packed. -Set `SPLATKIT_IOS_XCFRAMEWORK_PATH` to a local framework build before `npm pack` to test unreleased SDK changes. -iOS 26 terminates apps that skip the UIScene lifecycle, so hosts built from the React Native 0.87 template need a scene delegate. -The [example app](https://github.com/Xget7/splatkit-android/tree/main/apps/react-native) lists every change from the template, starting from zero. +- **iOS**: the podspec vendors `SplatKitCore.xcframework`, fetched and checksum-verified when the npm package is packed. + To test an unreleased native build, set `SPLATKIT_IOS_XCFRAMEWORK_PATH` to a local framework before `npm pack`. + iOS 26 terminates apps that skip the UIScene lifecycle, so a React Native 0.87 template app needs a scene delegate; see the [example app](https://github.com/Xget7/splatkit-android/tree/main/apps/react-native). +- **Android**: the module autolinks through the app's `com.facebook.react` Gradle plugin, which also runs Fabric Codegen, and pulls `io.github.xget7:splatkit-android` from Maven Central. + Set `minSdkVersion = 29` and `reactNativeArchitectures=arm64-v8a` in the host app. + +## Quick start + +```tsx +import {SplatKitBuilder, SplatKitView, toNativeViewProps} from '@splatkit/react-native'; + +// Conservative placeholder limits for the very first frame; each engine reports its +// real limits in onCapabilities, and the configuration should be rebuilt from those. +const INITIAL_CAPABILITIES = { + limits: { + maxLodCapacitySplats: 1_000_000, + minResidencyCapacitySplats: 100_000, + maxResidencyCapacitySplats: 1_000_000, + }, + supportsComputeTiles: false, + supportsHiZOcclusion: false, + supportsSubgroups: false, + maxTextureDimension: 4096, +}; + +function Scene({worldPath}: {worldPath: string}) { + const configuration = new SplatKitBuilder() + .withWorld({requestId: 'lobby', filePath: worldPath, maxShDegree: 3}) + .withPreset('balanced') + .build(INITIAL_CAPABILITIES); + + return ; +} +``` + +`filePath` must be an absolute, readable local path, never a URL: the view never fetches or receives splat bytes over the bridge. +One finger drag looks around by default (`touchLookEnabled`), and there is no walking control until you add one; see [Walking](#walking). + +### Getting a world file onto a device + +Worlds are not bundled with the app; push or copy them onto the device's sandbox and point `filePath` at that path. + +Android, to the app's external files directory: + +```sh +adb push world.spz /sdcard/Android/data//files/world.spz +``` -## Use +iOS, to the app's Documents directory on a connected device: + +```sh +xcrun devicectl device copy to --device --domain-type appDataContainer \ + --domain-identifier --source world.spz --destination Documents/world.spz +``` + +Restart the app after copying a new world file. + +## Walking + +The SDK deliberately ships no walking or look UI: `SplatKitView` handles only a one-finger drag to look and a double tap to toggle the gyroscope, and every other control is the host's own. + +1. Load a collider mesh through the `collider` prop: `{requestId, filePath}`, an absolute path to a collider GLB. +2. Wait for `onColliderEvent` with `phase: 'ready'` before you show walking controls; `phase: 'failed'` carries `errorCode` and `message`. +3. Tune the walker's shape with the `character` prop: `{eyeHeight, bodyRadius, stepHeight}`, in meters, applied immediately and to any collider loaded later. +4. Drive the camera with `SplatKitCommands`, imported from the package root. + Commands go straight to the native view, bypassing React's render and commit cycle, so a joystick or a look pad can drive the camera at touch rate. + +```tsx +import {useRef} from 'react'; +import {PanResponder, View} from 'react-native'; +import {SplatKitCommands, SplatKitView} from '@splatkit/react-native'; + +const WALK_SPEED = 1.4; // meters per second at full stick deflection +const RADIUS = 62; + +function Joystick({viewRef}: {viewRef: React.RefObject>}) { + const responder = useRef( + PanResponder.create({ + onStartShouldSetPanResponder: () => true, + onPanResponderMove: (_event, gesture) => { + const forward = Math.max(-1, Math.min(1, -gesture.dy / RADIUS)) * WALK_SPEED; + const right = Math.max(-1, Math.min(1, gesture.dx / RADIUS)) * WALK_SPEED; + const target = viewRef.current; + if (target) SplatKitCommands.setWalkVelocity(target, forward, right); + }, + onPanResponderRelease: () => { + const target = viewRef.current; + if (target) SplatKitCommands.setWalkVelocity(target, 0, 0); + }, + }), + ).current; + + return ; +} + +// const viewRef = useRef(null); +// +// {walking && } +``` + +This mirrors the thumb stick in the [example app](https://github.com/Xget7/splatkit-android/tree/main/apps/react-native): the SDK never renders it, the host owns it entirely, and it disappears whenever `onColliderEvent` has not reported `ready`. + +## Performance and quality + +`SplatKitBuilder` resolves a requested policy against host and native limits into a `SplatKitConfiguration` with `world`, `render` and `performance` (`requested`, `effective`, `diagnostics`): ```ts -import {SplatKitBuilder, SplatKitView, toNativePolicyProp, toNativeViewProps} from '@splatkit/react-native'; - -const config = new SplatKitBuilder() - .withWorld({ - requestId: 'lobby-1', - filePath: '/absolute/path/lobby.spz', - maxShDegree: 3, - }) +import {SplatKitBuilder, nativeCapabilitiesFromEvent, toNativePolicyProp, toNativeViewProps} from '@splatkit/react-native'; + +const configuration = new SplatKitBuilder() + .withWorld({requestId: 'lobby-1', filePath: '/absolute/path/lobby.spz', maxShDegree: 3}) .withPreset('high') .withPerformance({lodBudgetSplats: 2_500_000}) - .build(hostCapabilities); + .build(capabilities); // a DeviceCapabilities snapshot, e.g. from nativeCapabilitiesFromEvent -hostLogger.warn(config.performance.diagnostics); -const props = toNativeViewProps(config); -const policy = toNativePolicyProp(config, revision); -// +const props = toNativeViewProps(configuration); +const policy = toNativePolicyProp(configuration, revision); +// ``` -`build()` preserves the complete requested policy and separately reports effective values and warnings. -Effective means resolved Fabric props, not an acknowledgment of native applied state; `onPolicyEvent` is that acknowledgment. -The bridge expresses `world`, `paused`, `renderScale`, `shDegree` and the versioned renderer `policy`. -LOD and residency capacities are carried inside `world`, so changing either requires a new world request. -Render scale, draw SH degree and both splat capacities have one authority in `withPerformance()`. -`withWorld()` accepts world identity, file path and load-time maximum SH degree, while `withRender()` accepts only `paused`. -This is an intentional pre-1.0 change from passing capacities to `withWorld()` or scale and SH to `withRender()`. - -Presets are provisional SDK defaults and install complete values: - -| Preset | Render scale | SH degree | LOD budget | Residency budget | -| --- | ---: | ---: | ---: | ---: | -| `highEnd` | 1.25 | 3 | 4,000,000 | 4,000,000 | -| `high` | 1.0 | 3 | 3,000,000 | 3,000,000 | -| `balanced` | 0.85 | 2 | 2,000,000 | 2,000,000 | -| `performance` | 0.65 | 1 | 1,000,000 | 1,000,000 | - -Calling `withPreset()` replaces the current policy with that preset. -Calling `withPerformance()` applies manual overrides to the current preset and marks the request `manual`. -Budgets count splats, including each splat's loaded SH payload, and are not byte or memory guarantees. -Host-supplied limits can clamp valid budget requests and the world's maximum loaded SH degree can cap draw SH. -Every fallback is reported in `diagnostics`. -Malformed numbers, invalid enum values and unknown options fail instead of degrading. - -Raster strategy, tile size, LOD error, thresholds, culling, early termination and sort depth travel in the `policy` prop. -Give each changed policy a new positive Int32 revision; 0 or less means no policy. -An effective policy field holds a value only when native capabilities say the adapter applies it. -Other fields stay `null` and warn `native-option-fallback`, or `native-support-unknown` before capabilities are known, when the request differs from the native default. -`targetFps` is optional, always resolves to `null` and does not enable dynamic quality. -`nativeCapabilitiesFromEvent()` turns `onCapabilities` into the snapshot `build()` accepts. -That event arrives once per engine, and each world load creates one, so build the first configuration with host limits and rebuild when it arrives. -Even when a snapshot identifies a possible fallback, the bridge does not claim the native renderer applied it. - -## Native adapters - -Native applies each new revision, and re-applies the current one to every new engine, then reports it in `onPolicyEvent`. -`applied` means the whole request landed, `warning` means a field fell back and `rejected` means the previous policy stayed. -Every event carries the effective policy; a rejection's `errorCode` is `INVALID_POLICY` or `POLICY_PREPARATION_FAILED`. - -Both adapters decode off the render thread, tag world events with the load that produced them and drop stale events, and emit the shared failure codes `INVALID_REQUEST`, `WORLD_LOAD_FAILED` and `GPU_UNAVAILABLE`. -Both re-validate the policy prop: an unknown raster or sort depth is invalid, never a silent default. -The iOS adapter throttles stats snapshots to 2 Hz and releases its engine and display link on recycle, invalidate and background. -The Android adapter isolates each accepted replacement in its own engine. -Vulkan applies sort depth and the sub-pixel threshold with GPU visibility; Metal applies sort depth with GPU sort and the sub-pixel threshold only under tight culling. +- `withWorld()` accepts world identity, file path and load-time maximum SH degree only. +- `withRender()` accepts only `paused`. +- `withPreset(preset)` replaces the current policy with one of the presets below. +- `withPerformance(options)` overrides fields on top of the current preset and marks the policy `manual`. +- `build(capabilities)` clamps requested LOD and residency budgets to host limits, caps `shDegree` at the world's `maxShDegree`, and records every fallback in `performance.diagnostics` instead of silently degrading; malformed numbers, invalid enums and unknown options throw. + +### Presets + +Every preset rasterizes in `hardware`, with frustum culling and early termination enabled. + +| Preset | Render scale | SH degree | LOD budget | Residency budget | Tile size | LOD error px | Alpha threshold | Sub-pixel threshold | Hi-Z occlusion | Sort depth | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :---: | ---: | +| `highEnd` | 1.25 | 3 | 4,000,000 | 4,000,000 | 16 | 0.75 | 1/255 | 0.35 | Yes | 32 | +| `high` | 1.0 | 3 | 3,000,000 | 3,000,000 | 16 | 1.0 | 1/255 | 0.5 | Yes | 32 | +| `balanced` | 0.85 | 2 | 2,000,000 | 2,000,000 | 16 | 1.25 | 1/255 | 0.65 | No | 16 | +| `performance` | 0.65 | 1 | 1,000,000 | 1,000,000 | 8 | 2.0 | 2/255 | 1.0 | No | 16 | + +`withPerformance({raster: 'hybrid'})` opts into experimental screen-tile compositing, which only helps where many large translucent splats overlap one pixel (close-up interiors) and costs more on distant or sparse scenes. +Today only the iOS adapter builds `hybrid`; `computeTile` is not implemented by any adapter. +`onCapabilities.policyRasterMask` reports which raster strategies an adapter actually applies. + +### The `policy` prop and revisions + +`policy` carries the versioned renderer policy: `raster`, `tileSize`, `lodErrorPixels`, `alphaThreshold`, `subpixelThreshold`, `enableFrustumCulling`, `enableHiZOcclusion`, `enableEarlyTermination` and `sortDepth`, under a `revision`. +Native re-validates the whole policy and never trusts JS state; give every changed policy a new positive `revision`, since 0 or less means no policy. +Native re-applies the current revision to every new engine, which a world reload also creates, so expect `onPolicyEvent` again after a reload. + +`onCapabilities` fires once per engine, before its first policy event, and reports splat-count limits, GPU feature support and exactly which policy fields that adapter applies. +Pass its payload through `nativeCapabilitiesFromEvent()` to get the `DeviceCapabilities` that `build()` expects, and rebuild your configuration when it arrives. + +`onPolicyEvent` reports one of three phases: `applied` (the whole request landed), `warning` (one or more fields fell back; `message` explains why) or `rejected` (the previous policy stayed in effect, with `errorCode` `INVALID_POLICY` or `POLICY_PREPARATION_FAILED`). +Every `onPolicyEvent` carries the effective policy values, whichever phase it reports. + +### Splat-count knobs + +| Knob | Lives in | Effect | +| --- | --- | --- | +| `lodCapacitySplats` | `world` (via `withPerformance({lodBudgetSplats})`) | LOD tree budget; 0 disables load-time tree building. Changing it needs a new world load. | +| `residencyCapacitySplats` | `world` (via `withPerformance({residencyCapacitySplats})`) | Resident streaming splats, not a byte or memory guarantee. Changing it needs a new world load. | +| `maxShDegree` | `world` (`withWorld()`) | Load-time cap on stored spherical-harmonics degree; caps `shDegree` below. | +| `shDegree` | view prop (`withPerformance({shDegree})`) | Draw-time SH degree, 0-3, clamped to the world's `maxShDegree`. | +| `renderScale` | view prop (`withPerformance({renderScale})`) | Render resolution scale, `[0.1, 2]`. | + +LOD and residency capacities travel inside the `world` request, so changing either requires a new `requestId` and reload; render scale, draw SH degree and the rest of the policy update in place through the `policy` and view props. + +## API reference + +### `SplatKitView` props + +`SplatKitView` extends the standard React Native `ViewProps` (`style`, `pointerEvents`, ...). + +| Prop | Type | Default | Description | +| --- | --- | --- | --- | +| `world` | `WorldRequest` | - | Load transaction: `requestId`, absolute `filePath`, `maxShDegree`, `lodCapacitySplats`, `residencyCapacitySplats`. Omit to render nothing. | +| `collider` | `ColliderRequest` | - | `{requestId, filePath}`, the walk-mode counterpart of `world`; an empty `requestId` releases walk mode. | +| `character` | `Character` | - | `{eyeHeight, bodyRadius, stepHeight}`, the walker's shape in meters. | +| `paused` | `boolean` | `false` | Freezes rendering. | +| `renderScale` | `number` | `1` | Render resolution scale. | +| `shDegree` | `number` (0-3) | `3` | Draw-time spherical-harmonics degree. | +| `linearBlending` | `boolean` | `false` | Blends in linear light instead of the encoded space training used. | +| `cullMarginDegrees` | `number` | `10` | CPU fallback's angular culling margin in degrees; the GPU path uses projected bounds instead. | +| `motionEnabled` | `boolean` | `false` | Drives the camera with the gyroscope; ignored where the sensor is missing. | +| `touchLookEnabled` | `boolean` | `true` | Whether a one-finger drag on the view turns the camera. | +| `lookSensitivity` | `number` | `0.004` | Radians per point dragged to look. | +| `cameraPoseInterval` | `number` | `0` | Seconds between `onCameraPose` events; `0` never sends one. | +| `policy` | `NativeRenderPolicy` | - | The requested renderer policy; see [The `policy` prop and revisions](#the-policy-prop-and-revisions). | + +### Events + +| Event | Payload | Notes | +| --- | --- | --- | +| `onWorldEvent` | `{requestId, phase: 'uploaded' \| 'frameReady' \| 'failed', loadedSplats, errorCode, message}` | Fires per phase of a world load; `errorCode`/`message` are set only on `failed`. | +| `onColliderEvent` | `{requestId, phase: 'ready' \| 'failed', errorCode, message}` | Fires when a collider finishes loading or fails. | +| `onStats` | `{requestId, loadedSplats, drawnSplats, frameMillis, frameTimingAvailable, gpuMillis, gpuTimingAvailable, sortMillis, sortTimingAvailable}` | Throttled to at most 2 Hz by the adapter; a `*TimingAvailable` flag of `false` means its paired value is not meaningful. | +| `onCameraPose` | `{x, y, z, yaw, pitch}` | Throttled to `cameraPoseInterval`, in the world's frame, and sent only when the pose changed. | +| `onPolicyEvent` | `{revision, phase: 'applied' \| 'warning' \| 'rejected', errorCode, message, raster, tileSize, lodErrorPixels, alphaThreshold, subpixelThreshold, enableFrustumCulling, enableHiZOcclusion, enableEarlyTermination, sortDepth}` | One per policy application, including re-application to each new engine a world load creates. | +| `onCapabilities` | `{maxLodCapacitySplats, minResidencyCapacitySplats, maxResidencyCapacitySplats, supportsComputeTiles, supportsHiZOcclusion, supportsSubgroups, maxTextureDimension, policyRaster, policyRasterMask, policyTileSize, policyLodErrorPixels, policyAlphaThreshold, policySubpixelThreshold, policyEnableFrustumCulling, policyEnableHiZOcclusion, policyEnableEarlyTermination, policySortDepth}` | Emitted once per engine, before its first `onPolicyEvent`. | + +### Commands (`SplatKitCommands`) + +Exported from the package root; each command takes the `SplatKitView` ref as its first argument and returns nothing. + +| Command | Signature | Notes | +| --- | --- | --- | +| `setWalkVelocity` | `(ref, forward: number, right: number)` | Meters per second, held until called again; forward is where the camera looks. Requires walk mode to be `ready`. | +| `look` | `(ref, deltaYaw: number, deltaPitch: number)` | Radians. Pitch is clamped; ignored while the gyroscope drives the view. | +| `setCameraPose` | `(ref, x: number, y: number, z: number, yaw: number, pitch: number)` | Teleports the camera; while walking, it settles onto the floor under the new point. | + +### Contract types and validators (`src/contracts.ts`) + +| Export | Shape / signature | +| --- | --- | +| `SHDegree` | `0 \| 1 \| 2 \| 3` | +| `WorldRequest` | `{requestId, filePath, maxShDegree, lodCapacitySplats, residencyCapacitySplats}` | +| `RenderOptions` | `{paused, renderScale, shDegree}` | +| `ColliderRequest` | `{requestId, filePath}` | +| `Character` | `{eyeHeight, bodyRadius, stepHeight}` | +| `ColliderEvent` | `{requestId, phase, errorCode, message}` | +| `CameraPose` | `{x, y, z, yaw, pitch}` | +| `SplatLimits` | `{maxLodCapacitySplats, minResidencyCapacitySplats, maxResidencyCapacitySplats}` | +| `WorldEvent` | `{requestId, phase, loadedSplats, errorCode, message}` | +| `validateWorldRequest(request, limits)` | Throws on structural violations; does not touch the filesystem or the GPU. | +| `validateColliderRequest(request)` | Throws unless `requestId` is nonempty and `filePath` is absolute. | +| `validateCharacter(character)` | Throws unless the walker's shape is finite and internally consistent. | +| `validateRenderOptions(options)` | Throws unless `paused`, `renderScale` and `shDegree` are in range. | +| `optionalTimingMillis(available, value)` | Returns `value`, or `null` when `available` is `false` (zero is a valid measurement). | + +### Performance types and helpers (`src/performance.ts`) + +| Export | Kind | Purpose | +| --- | --- | --- | +| `SplatKitBuilder` | class | `withWorld()`, `withRender({paused})`, `withPerformance(options)`, `withPreset(preset)`, `build(capabilities)` -> `SplatKitConfiguration`. | +| `toNativeViewProps(configuration)` | function | Returns the `world`, `paused`, `renderScale`, `shDegree` view props. | +| `toNativePolicyProp(configuration, revision)` | function | Returns the `policy` prop for a positive `revision`. | +| `nativeCapabilitiesFromEvent(event)` | function | Converts an `onCapabilities` payload into a `DeviceCapabilities` snapshot for `build()`. | +| `classifyPolicyChange(previous, next)` | function | Classifies a `PerformancePolicy` change as `'none' \| 'nativePropUpdate' \| 'worldReload' \| 'unavailable'`. | + +Supporting types: `RasterStrategy` (`'hardware' \| 'computeTile' \| 'hybrid'`), `SortDepth` (`16 \| 32`), `PerformanceMode` (`'auto' \| 'manual'`), `QualityPreset` (`'highEnd' \| 'high' \| 'balanced' \| 'performance'`), `NativePolicySupport`, `DeviceCapabilities`, `NativeRenderPolicy`, `NativeCapabilitiesEvent`, `SplatKitWorldRequest`, `PerformancePolicy`, `PerformanceOptions`, `EffectivePerformancePolicy`, `PerformanceResolution`, `SplatKitConfiguration`, `PolicyChangeKind`. + +`PerformanceDiagnostic` (`{severity: 'warning', code, option, requested, fallback, message}`) explains every fallback `build()` makes: + +| `code` | Meaning | +| --- | --- | +| `limit-clamped` | The value was clamped to a host- or native-supplied limit. | +| `native-option-fallback` | Native capabilities say this field is not applied; it falls back to the native default. | +| `native-support-unknown` | Native capabilities have not arrived yet, so support for this field is unknown. | +| `capability-fallback-unavailable` | The requested raster strategy is unsupported; `hardware` is the candidate fallback. | +| `fabric-option-unavailable` | The option has no corresponding Fabric prop (`targetFps` only, which is a request only and never enables dynamic quality). | + +## Error codes + +Both native adapters share the same failure codes, so a host can branch on `errorCode` without checking platform: + +| Code | Reported on | Meaning | +| --- | --- | --- | +| `INVALID_REQUEST` | `onWorldEvent`, `onColliderEvent` | The `world` or `collider` request failed structural validation before native ever tried to load it. | +| `WORLD_LOAD_FAILED` | `onWorldEvent` | The world file could not be decoded or loaded. | +| `GPU_UNAVAILABLE` | `onWorldEvent` | The GPU backend could not be initialized (for example, Vulkan device creation failed). | +| `COLLIDER_LOAD_FAILED` | `onColliderEvent` | The collider GLB could not be loaded. | +| `INVALID_POLICY` | `onPolicyEvent` | The `policy` prop failed validation (for example, an unknown raster strategy or sort depth). | +| `POLICY_PREPARATION_FAILED` | `onPolicyEvent` | The policy passed validation but native failed to prepare it; the previous policy stays in effect. | + +## Troubleshooting + +**The view is blank or black.** +Check that `world` is set and that `onWorldEvent` ever fires; a view with no `world` prop renders nothing. +On a simulator or emulator, GPU support is often missing or partial: prefer a physical device, especially on Android, where an emulator without full Vulkan 1.1 support reports `GPU_UNAVAILABLE`. + +**The world fails to load (`onWorldEvent` with `phase: 'failed'`).** +`filePath` must be an absolute, local, readable path, not a URL, a `require()` asset or a content URI; the view does no fetching and no JS byte transport. +Confirm the file actually exists at that path in the app's sandbox (see [Getting a world file onto a device](#getting-a-world-file-onto-a-device)), and read `message` for the native loader's reason. + +**Nothing renders and the app crashes or logs a Fabric/Codegen error.** +`SplatKitView` is Fabric-only; confirm the New Architecture is enabled and that `npx react-native codegen` (or a full rebuild) picked up `SplatKitSpec`. + +**Simulator vs. device.** +The package has only been verified on physical devices, an iPhone 17 Pro and a Mi 9 (Adreno 640). +The iOS Simulator and Android emulators can differ in Metal/Vulkan feature support from real hardware; if `onCapabilities` reports unexpectedly low limits or `supportsComputeTiles`/`supportsHiZOcclusion` as `false`, try a device before filing an issue. + +## Example app + +[`apps/react-native`](https://github.com/Xget7/splatkit-android/tree/main/apps/react-native) is a React Native 0.87.1 app built from the community template that installs this package from npm, drags to look, double-taps to toggle the gyroscope, and drives a joystick through `SplatKitCommands` once a collider is ready. +Its README lists every change needed from a fresh template, including `minSdkVersion`, `arm64-v8a`, the iOS 17 deployment target and the scene delegate iOS 26 requires. + +## Contributing + +This package is exported from a monorepo; see [CONTRIBUTING.md](CONTRIBUTING.md) for how to build, test and send changes back. + +## License + +[MIT](LICENSE) diff --git a/packages/react-native-splatkit/SplatKitReactNative.podspec b/packages/react-native-splatkit/SplatKitReactNative.podspec index 3fce6ba..7900667 100644 --- a/packages/react-native-splatkit/SplatKitReactNative.podspec +++ b/packages/react-native-splatkit/SplatKitReactNative.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'SplatKitReactNative' - s.version = '0.1.0-alpha.1' + s.version = '0.1.0-alpha.2' s.summary = 'Fabric view adapter for SplatKit native renderers' s.homepage = 'https://github.com/Xget7/react-native-splatkit' s.license = { :type => 'MIT' } diff --git a/packages/react-native-splatkit/android/README.md b/packages/react-native-splatkit/android/README.md index 858ba6b..a948c73 100644 --- a/packages/react-native-splatkit/android/README.md +++ b/packages/react-native-splatkit/android/README.md @@ -3,7 +3,7 @@ `SplatKitPackage` registers `SplatKitView` through the generated Codegen delegate. The module needs compileSdk 36 or the app's `rootProject.ext` value, minSdk 29, Java 17 and arm64-v8a. It applies `com.facebook.react`, so the consuming app autolinks it and runs its Codegen. -It depends on `io.github.xget7:splatkit-android:0.1.0-alpha07`, or on `project(':splatkit')` when the build defines one, as in the SplatKit repository. +It depends on `io.github.xget7:splatkit-android:0.1.0-alpha08`, or on `project(':splatkit')` when the build defines one, as in the SplatKit repository. Host tests, from the SplatKit repository root after `npm ci` in the package: diff --git a/packages/react-native-splatkit/android/build.gradle b/packages/react-native-splatkit/android/build.gradle index be66a4a..aaebe1b 100644 --- a/packages/react-native-splatkit/android/build.gradle +++ b/packages/react-native-splatkit/android/build.gradle @@ -29,7 +29,7 @@ android { // :splatkit Gradle project when this module builds inside that repository (the RN dev app // and this package's own verification harness). Kept in one place, so bumping the release // this package targets only means changing this line. -def splatkitAndroidVersion = '0.1.0-alpha07' +def splatkitAndroidVersion = '0.1.0-alpha08' dependencies { // Version resolution comes from the React Native Gradle plugin applied above; a diff --git a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitView.kt b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitView.kt index 2d5d474..eaec5fa 100644 --- a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitView.kt +++ b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitView.kt @@ -11,6 +11,8 @@ import com.facebook.react.common.LifecycleState import com.facebook.react.uimanager.ThemedReactContext import com.facebook.react.uimanager.UIManagerHelper import com.facebook.react.uimanager.events.Event +import com.splatkit.CameraPose +import com.splatkit.CharacterSettings import com.splatkit.RenderPolicy import com.splatkit.RenderPolicyResolution import com.splatkit.SplatSurfaceView @@ -36,10 +38,22 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r private var paused = false private var renderScale = 1.0 private var shDegree = 3 + private var linearBlending = false + private var cullMarginDegrees = 10.0 private var displayChanged = false // The last valid host policy, applied to the current engine and every engine built after. private var policy: RevisionedPolicy? = null private var policyChanged = false + private var collider: ColliderRequest? = null + private var pendingCollider: ColliderRequest? = null + private var colliderChanged = false + private var character: CharacterSettings? = null + private var characterChanged = false + private var motionEnabled = false + private var touchLookEnabled = true + private var lookSensitivity = 0.004 + private var cameraPoseIntervalMillis = 0L + private var navigationChanged = false init { reactContext.addLifecycleEventListener(this) } @@ -59,6 +73,54 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r internal fun invalidRequest(message: String) = emitWorld("", "failed", 0, "INVALID_REQUEST", message) + internal fun setCollider(value: ColliderRequest?) { + if (dropped) return + if (value != null && value != collider) { + try { + value.validate() + } catch (error: IllegalArgumentException) { + emitCollider(value.requestId, "failed", "INVALID_REQUEST", error.message.orEmpty()) + return + } + } + pendingCollider = value + colliderChanged = value != collider + } + + internal fun invalidCollider(message: String) = emitCollider("", "failed", "INVALID_REQUEST", message) + + internal fun setCharacter(value: CharacterSettings?) { + if (dropped || value == character) return + character = value + characterChanged = value != null + } + + fun setMotionEnabled(value: Boolean) { motionEnabled = value; navigationChanged = true } + fun setTouchLookEnabled(value: Boolean) { touchLookEnabled = value; navigationChanged = true } + fun setLookSensitivity(value: Double) { + require(value.isFinite() && value > 0) { "lookSensitivity must be finite and positive" } + lookSensitivity = value + navigationChanged = true + } + fun setCameraPoseInterval(seconds: Double) { + require(seconds.isFinite() && seconds >= 0) { "cameraPoseInterval must be finite and not negative" } + cameraPoseIntervalMillis = (seconds * 1000).toLong() + navigationChanged = true + } + + fun walk(forward: Double, right: Double) { + nativeView?.setWalkVelocity(forward.toFloat(), right.toFloat()) + } + + fun look(deltaYaw: Double, deltaPitch: Double) { + nativeView?.look(deltaYaw.toFloat(), deltaPitch.toFloat()) + } + + fun teleport(x: Double, y: Double, z: Double, yaw: Double, pitch: Double) { + nativeView?.cameraPose = + CameraPose(x.toFloat(), y.toFloat(), z.toFloat(), yaw.toFloat(), pitch.toFloat()) + } + fun setPaused(value: Boolean) { paused = value; updateRunning() } fun setRenderScale(value: Double) { require(value.isFinite() && value in 0.1..2.0) { "renderScale must be in 0.1..2" } @@ -70,6 +132,12 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r shDegree = value displayChanged = true } + fun setLinearBlending(value: Boolean) { linearBlending = value; displayChanged = true } + fun setCullMarginDegrees(value: Double) { + require(value.isFinite() && value in 0.0..80.0) { "cullMarginDegrees must be in 0..80" } + cullMarginDegrees = value + displayChanged = true + } /** * Stores the host policy; [commitProps] applies it to the current engine, and every engine @@ -102,12 +170,42 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r } if (displayChanged) { displayChanged = false - nativeView?.let { - it.renderScale = renderScale.toFloat() - it.shDegree = shDegree - } + nativeView?.let(::applyDisplay) } if (policyChanged) nativeView?.takeIf { it.isAvailable }?.let(::applyPolicy) + if (navigationChanged) { + navigationChanged = false + nativeView?.let(::applyNavigation) + } + if (characterChanged) { + characterChanged = false + character?.let { nativeView?.setCharacter(it) } + } + if (colliderChanged) { + colliderChanged = false + collider = pendingCollider + loadCollider() + } + } + + private fun applyDisplay(view: SplatSurfaceView) { + view.renderScale = renderScale.toFloat() + view.shDegree = shDegree + view.linearBlending = linearBlending + view.cullMarginDegrees = cullMarginDegrees.toFloat() + } + + private fun applyNavigation(view: SplatSurfaceView) { + view.setMotionEnabled(motionEnabled) + view.touchLookEnabled = touchLookEnabled + view.lookSensitivity = lookSensitivity.toFloat() + view.cameraPoseIntervalMillis = cameraPoseIntervalMillis + } + + private fun loadCollider() { + val view = nativeView ?: return + val request = collider ?: return + view.loadCollider(File(request.filePath)) } private fun applyPolicy(view: SplatSurfaceView) { @@ -126,8 +224,7 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r val token = session.generation val view = SplatSurfaceView(reactContext) nativeView = view - view.renderScale = renderScale.toFloat() - view.shDegree = shDegree + applyDisplay(view) view.listener = object : SplatSurfaceView.Listener { private fun outcome(phase: String, count: Int = 0, message: String = "") { if (dropped || !session.accept(token, phase)) return @@ -136,6 +233,25 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r override fun onWorldReady(splatCount: Int) = outcome("uploaded", splatCount) override fun onWorldFrameReady(splatCount: Int) = outcome("frameReady", splatCount) override fun onWorldFailed(message: String) = outcome("failed", message = message) + override fun onColliderReady() { + if (!dropped && token == session.generation) { + emitCollider(collider?.requestId.orEmpty(), "ready", "", "") + } + } + override fun onColliderFailed(message: String) { + if (!dropped && token == session.generation) { + emitCollider(collider?.requestId.orEmpty(), "failed", "COLLIDER_LOAD_FAILED", message) + } + } + } + view.cameraPoseListener = { pose -> + if (!dropped && token == session.generation) { + emit("topCameraPose", Arguments.createMap().apply { + putDouble("x", pose.x.toDouble()); putDouble("y", pose.y.toDouble()) + putDouble("z", pose.z.toDouble()); putDouble("yaw", pose.yaw.toDouble()) + putDouble("pitch", pose.pitch.toDouble()) + }) + } } addView(view, LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)) // Fabric lays this view out, but not native children added after mount: a later world's @@ -149,7 +265,13 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r emitCapabilities(view) // Queued on the render thread ahead of the load, so the decode observes the policy. applyPolicy(view) + applyNavigation(view) + navigationChanged = false + character?.let(view::setCharacter) + characterChanged = false view.loadWorld(File(world.filePath), world.maxShDegree, world.lodCapacitySplats, world.residencyCapacitySplats) + // This engine is new, so walk mode has to be rebuilt on it. + loadCollider() updateRunning() } @@ -158,6 +280,7 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r main.removeCallbacks(statsTick) nativeView?.let { it.listener = null + it.cameraPoseListener = null // Remove first so SurfaceHolder detaches before the render thread shuts down. removeView(it) it.release() @@ -184,10 +307,15 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r putString("requestId", world.requestId) putDouble("loadedSplats", stats.loadedSplatCount.toDouble()) putDouble("drawnSplats", stats.drawnSplatCount.toDouble()) - // The SDK does not expose timing validity bits. Do not infer availability from zero. - putDouble("frameMillis", 0.0); putBoolean("frameTimingAvailable", false) - putDouble("gpuMillis", 0.0); putBoolean("gpuTimingAvailable", false) - putDouble("sortMillis", 0.0); putBoolean("sortTimingAvailable", false) + // Zero is the SDK's own "not measured": the GPU and sort times are zero + // without timestamp queries, and the frame time is zero while nothing + // redraws. A host reads those as no measurement, which is what they are. + putDouble("frameMillis", stats.frameMillis.toDouble()) + putBoolean("frameTimingAvailable", stats.frameMillis > 0f) + putDouble("gpuMillis", stats.gpuMillis.toDouble()) + putBoolean("gpuTimingAvailable", stats.gpuMillis > 0f) + putDouble("sortMillis", stats.sortMillis.toDouble()) + putBoolean("sortTimingAvailable", stats.sortMillis > 0f) }) } main.postDelayed(this, 500) @@ -201,6 +329,13 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r }) } + private fun emitCollider(id: String, phase: String, code: String, message: String) { + emit("topColliderEvent", Arguments.createMap().apply { + putString("requestId", id); putString("phase", phase) + putString("errorCode", code); putString("message", message) + }) + } + /** Native limits, features and accepted policy; emitted once per engine. */ private fun emitCapabilities(view: SplatSurfaceView) { val caps = view.deviceCapabilities ?: return @@ -213,6 +348,8 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r putBoolean("supportsSubgroups", caps.supportsSubgroups) putInt("maxTextureDimension", caps.maxTextureDimension) putBoolean("policyRaster", caps.policy.raster) + // Vulkan applies no raster choice; the Kotlin support type has no strategy mask yet. + putInt("policyRasterMask", if (caps.policy.raster) 0b111 else 0) putBoolean("policyTileSize", caps.policy.tileSize) putBoolean("policyLodErrorPixels", caps.policy.lodErrorPixels) putBoolean("policyAlphaThreshold", caps.policy.alphaThreshold) @@ -265,5 +402,8 @@ class SplatKitView(private val reactContext: ThemedReactContext) : FrameLayout(r request = null pendingRequest = null policy = null + collider = null + pendingCollider = null + character = null } } diff --git a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitViewManager.kt b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitViewManager.kt index c785722..2eb76b8 100644 --- a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitViewManager.kt +++ b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/SplatKitViewManager.kt @@ -1,6 +1,7 @@ package com.splatkit.reactnative import com.facebook.react.bridge.ReadableMap +import com.splatkit.CharacterSettings import com.facebook.react.uimanager.SimpleViewManager import com.facebook.react.uimanager.ThemedReactContext import com.facebook.react.uimanager.ViewManagerDelegate @@ -30,6 +31,38 @@ class SplatKitViewManager : SimpleViewManager(), SplatKitViewManag view.invalidRequest(error.message ?: "Malformed world request") } } + override fun setLinearBlending(view: SplatKitView, value: Boolean) = view.setLinearBlending(value) + override fun setCullMarginDegrees(view: SplatKitView, value: Double) = view.setCullMarginDegrees(value) + override fun setMotionEnabled(view: SplatKitView, value: Boolean) = view.setMotionEnabled(value) + override fun setTouchLookEnabled(view: SplatKitView, value: Boolean) = view.setTouchLookEnabled(value) + override fun setLookSensitivity(view: SplatKitView, value: Double) = view.setLookSensitivity(value) + override fun setCameraPoseInterval(view: SplatKitView, value: Double) = view.setCameraPoseInterval(value) + override fun setCollider(view: SplatKitView, value: ReadableMap?) { + if (value == null) { view.setCollider(null); return } + try { + view.setCollider(ColliderRequest( + value.getString("requestId") ?: "", value.getString("filePath") ?: "")) + } catch (error: RuntimeException) { + view.invalidCollider(error.message ?: "Malformed collider request") + } + } + override fun setCharacter(view: SplatKitView, value: ReadableMap?) { + if (value == null) { view.setCharacter(null); return } + try { + view.setCharacter(CharacterSettings( + value.getDouble("eyeHeight").toFloat(), value.getDouble("bodyRadius").toFloat(), + value.getDouble("stepHeight").toFloat())) + } catch (error: RuntimeException) { + view.setCharacter(null) + } + } + override fun setWalkVelocity(view: SplatKitView, forward: Double, right: Double) = + view.walk(forward, right) + override fun look(view: SplatKitView, deltaYaw: Double, deltaPitch: Double) = + view.look(deltaYaw, deltaPitch) + override fun setCameraPose( + view: SplatKitView, x: Double, y: Double, z: Double, yaw: Double, pitch: Double, + ) = view.teleport(x, y, z, yaw, pitch) override fun setPolicy(view: SplatKitView, value: ReadableMap?) { if (value == null) { view.setPolicy(null); return } try { @@ -49,6 +82,8 @@ class SplatKitViewManager : SimpleViewManager(), SplatKitViewManag "topStats" to mapOf("registrationName" to "onStats"), "topPolicyEvent" to mapOf("registrationName" to "onPolicyEvent"), "topCapabilities" to mapOf("registrationName" to "onCapabilities"), + "topColliderEvent" to mapOf("registrationName" to "onColliderEvent"), + "topCameraPose" to mapOf("registrationName" to "onCameraPose"), ) override fun onDropViewInstance(view: SplatKitView) { view.dispose(); super.onDropViewInstance(view) } } diff --git a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/WorldSession.kt b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/WorldSession.kt index 359f67f..af09144 100644 --- a/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/WorldSession.kt +++ b/packages/react-native-splatkit/android/src/main/java/com/splatkit/reactnative/WorldSession.kt @@ -20,6 +20,15 @@ internal data class WorldRequest( } } +internal data class ColliderRequest(val requestId: String, val filePath: String) { + fun validate() { + require(requestId.isNotBlank()) { "requestId must be nonempty" } + require(filePath.startsWith('/') && !filePath.startsWith("//") && + !filePath.contains('\u0000') && filePath != "/") { "filePath must be an absolute local file" } + require(File(filePath).isFile && File(filePath).canRead()) { "filePath must name a readable file" } + } +} + /** One engine per transaction. Tokens suppress callbacks queued before replacement/release. */ internal class WorldSession { var generation = 0L diff --git a/packages/react-native-splatkit/ios/SplatKitRNView.h b/packages/react-native-splatkit/ios/SplatKitRNView.h index ab89c99..32b8e9c 100644 --- a/packages/react-native-splatkit/ios/SplatKitRNView.h +++ b/packages/react-native-splatkit/ios/SplatKitRNView.h @@ -16,12 +16,29 @@ typedef void (^SplatKitRNEventBlock)(NSDictionary *event); @property(nonatomic, assign) BOOL paused; @property(nonatomic, assign) CGFloat renderScale; @property(nonatomic, assign) NSInteger shDegree; +@property(nonatomic, assign) BOOL linearBlending; +@property(nonatomic, assign) CGFloat cullMarginDegrees; +@property(nonatomic, assign) BOOL touchLookEnabled; +@property(nonatomic, assign) CGFloat lookSensitivity; +@property(nonatomic, assign) BOOL motionEnabled; +/// Seconds between camera pose events; 0 sends none. +@property(nonatomic, assign) CFTimeInterval cameraPoseInterval; @property(nonatomic, copy, nullable) SplatKitRNEventBlock worldEvent; @property(nonatomic, copy, nullable) SplatKitRNEventBlock statsEvent; @property(nonatomic, copy, nullable) SplatKitRNEventBlock policyEvent; @property(nonatomic, copy, nullable) SplatKitRNEventBlock capabilitiesEvent; +@property(nonatomic, copy, nullable) SplatKitRNEventBlock colliderEvent; +@property(nonatomic, copy, nullable) SplatKitRNEventBlock cameraPoseEvent; - (void)loadWorld:(NSString *)path requestId:(NSString *)requestId maxShDegree:(NSInteger)maxShDegree lodCapacity:(NSInteger)lodCapacity residencyCapacity:(NSInteger)residencyCapacity; +/// Loads a collider GLB and enables walk mode when it is ready. An empty path releases it. +- (void)loadCollider:(NSString *)path requestId:(NSString *)requestId; +/// The walker's shape, applied at once and to a collider loaded later. NO when a value is +/// not a walkable one, and then the previous settings stay. +- (BOOL)setCharacter:(SKCharacterSettings)character; +- (void)setWalkVelocityForward:(float)forward right:(float)right; +- (void)lookWithDeltaYaw:(float)deltaYaw deltaPitch:(float)deltaPitch; +- (void)setPose:(SKCameraPose)pose; /// Remembers the host policy without applying it; a revision of 0 or less forgets it. Every /// engine a later `loadWorld:` creates applies the remembered policy before decoding. - (void)setPolicy:(SKRenderPolicy)policy revision:(NSInteger)revision; diff --git a/packages/react-native-splatkit/ios/SplatKitRNView.mm b/packages/react-native-splatkit/ios/SplatKitRNView.mm index 97fdfeb..1aa59cf 100644 --- a/packages/react-native-splatkit/ios/SplatKitRNView.mm +++ b/packages/react-native-splatkit/ios/SplatKitRNView.mm @@ -1,4 +1,5 @@ #import "SplatKitRNView.h" +#import #import #include @@ -6,15 +7,15 @@ /// Stable failure codes shared with the Android adapter, so hosts can branch on them. static NSString *const kErrorInvalidRequest = @"INVALID_REQUEST"; static NSString *const kErrorWorldLoadFailed = @"WORLD_LOAD_FAILED"; +static NSString *const kErrorColliderLoadFailed = @"COLLIDER_LOAD_FAILED"; static NSString *const kErrorGpuUnavailable = @"GPU_UNAVAILABLE"; static NSString *const kErrorInvalidPolicy = @"INVALID_POLICY"; static NSString *const kErrorPolicyPreparationFailed = @"POLICY_PREPARATION_FAILED"; /// Stats snapshots leave the render thread at most twice a second. static const CFTimeInterval kStatsInterval = 0.5; -/// SplatMetalView's and the Android view's touch sensitivities: radians and meters per point dragged. -static const float kLookSensitivity = 0.004f; -static const float kWalkSensitivity = 0.01f; +/// SplatMetalView's touch sensitivity: radians per point dragged. +static const CGFloat kLookSensitivity = 0.004; @class _SplatKitRNLinkProxy; @@ -25,6 +26,11 @@ @interface SplatKitRNView () @property(nonatomic, getter=isAttached) BOOL attached; @property(nonatomic, getter=isAppActive) BOOL appActive; @property(nonatomic) CFTimeInterval lastStatsAt; +@property(nonatomic) CFTimeInterval lastPoseAt; +@property(nonatomic, strong, nullable) CMMotionManager *motionManager; +@property(nonatomic, strong, nullable) NSOperationQueue *motionQueue; +@property(nonatomic, copy, nullable) NSString *colliderRequestId; +@property(nonatomic, copy, nullable) NSString *colliderPath; - (void)draw:(CADisplayLink *)link; @end @@ -51,6 +57,10 @@ @implementation SplatKitRNView { SKRenderPolicy _requestedPolicy; NSInteger _policyRevision; BOOL _hasPolicy; + SKCharacterSettings _character; + BOOL _hasCharacter; + SKCameraPose _lastPose; + BOOL _hasLastPose; } + (Class)layerClass { return CAMetalLayer.class; } @@ -62,18 +72,17 @@ - (instancetype)initWithFrame:(CGRect)frame { self.appActive = YES; self.renderScale = 1; self.shDegree = 3; + self.cullMarginDegrees = 10; + self.touchLookEnabled = YES; + self.lookSensitivity = kLookSensitivity; _generation = 0; _loaderQueue = dispatch_queue_create("com.splatkit.rn.loader", DISPATCH_QUEUE_SERIAL); self.linkProxy = [[_SplatKitRNLinkProxy alloc] init]; self.linkProxy.target = self; - // One finger looks around and two fingers walk, like the SDK views. + // Looking is the only touch the view handles; walking comes from the host's own control. UIPanGestureRecognizer *look = [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(onLook:)]; look.maximumNumberOfTouches = 1; [self addGestureRecognizer:look]; - UIPanGestureRecognizer *walk = [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(onWalk:)]; - walk.minimumNumberOfTouches = 2; - walk.maximumNumberOfTouches = 2; - [self addGestureRecognizer:walk]; NSNotificationCenter *center = NSNotificationCenter.defaultCenter; [center addObserver:self selector:@selector(appDidEnterBackground) name:UIApplicationDidEnterBackgroundNotification object:nil]; @@ -105,19 +114,109 @@ - (void)setShDegree:(NSInteger)value { [_engine setShDegree:(int)_shDegree]; } -// MARK: - Gestures +- (void)setLinearBlending:(BOOL)value { + _linearBlending = value; + [_engine setLinearBlending:value]; +} + +- (void)setCullMarginDegrees:(CGFloat)value { + _cullMarginDegrees = MIN(MAX(value, 0), 80); + [_engine setCullMargin:(float)_cullMarginDegrees]; +} + +// MARK: - Navigation -// The engine renders on the main thread here, so gestures drive it directly. +// The engine renders on the main thread here, so input drives it directly. - (void)onLook:(UIPanGestureRecognizer *)gesture { const CGPoint d = [gesture translationInView:self]; - [_engine lookWithDeltaYaw:-(float)d.x * kLookSensitivity deltaPitch:-(float)d.y * kLookSensitivity]; [gesture setTranslation:CGPointZero inView:self]; + if (!self.touchLookEnabled) return; + [_engine lookWithDeltaYaw:-(float)(d.x * self.lookSensitivity) + deltaPitch:-(float)(d.y * self.lookSensitivity)]; } -- (void)onWalk:(UIPanGestureRecognizer *)gesture { - const CGPoint d = [gesture translationInView:self]; - [_engine walkForward:-(float)d.y * kWalkSensitivity right:(float)d.x * kWalkSensitivity]; - [gesture setTranslation:CGPointZero inView:self]; +- (void)setWalkVelocityForward:(float)forward right:(float)right { + [_engine setVelocityForward:forward right:right]; +} + +- (void)lookWithDeltaYaw:(float)deltaYaw deltaPitch:(float)deltaPitch { + [_engine lookWithDeltaYaw:deltaYaw deltaPitch:deltaPitch]; +} + +- (void)setPose:(SKCameraPose)pose { + _engine.cameraPose = pose; +} + +- (BOOL)setCharacter:(SKCharacterSettings)character { + _character = character; + _hasCharacter = YES; + return _engine == nil ? YES : [_engine setCharacter:character]; +} + +// MARK: - Motion + +- (void)setMotionEnabled:(BOOL)enabled { + _motionEnabled = enabled; + [_engine setMotionEnabled:enabled]; + if (enabled) { + [self startMotion]; + } else { + [self stopMotion]; + } +} + +- (void)startMotion { + if (!self.motionEnabled || !self.isAttached || !self.isAppActive) return; + if (self.motionManager == nil) { + self.motionManager = [[CMMotionManager alloc] init]; + self.motionManager.deviceMotionUpdateInterval = 1.0 / 60.0; + self.motionQueue = [[NSOperationQueue alloc] init]; + self.motionQueue.name = @"com.splatkit.rn.motion"; + self.motionQueue.maxConcurrentOperationCount = 1; + } + CMMotionManager *manager = self.motionManager; + if (!manager.isDeviceMotionAvailable || manager.isDeviceMotionActive) return; + __weak SplatKitRNView *weakSelf = self; + [manager startDeviceMotionUpdatesUsingReferenceFrame:CMAttitudeReferenceFrameXArbitraryCorrectedZVertical + toQueue:self.motionQueue + withHandler:^(CMDeviceMotion *motion, NSError *error) { + if (motion == nil) return; + const CMRotationMatrix m = motion.attitude.rotationMatrix; + dispatch_async(dispatch_get_main_queue(), ^{ + SplatKitRNView *view = weakSelf; + if (view == nil || !view.motionEnabled) return; + [view applyAttitude:m]; + }); + }]; +} + +- (void)stopMotion { + [self.motionManager stopDeviceMotionUpdates]; +} + +/// CMRotationMatrix maps reference to device; the engine wants its transpose, with the +/// device axes turned so "x right, y up on screen" holds in the interface orientation. +- (void)applyAttitude:(CMRotationMatrix)m { + const float x[3] = {(float)m.m11, (float)m.m12, (float)m.m13}; + const float y[3] = {(float)m.m21, (float)m.m22, (float)m.m23}; + const float z[3] = {(float)m.m31, (float)m.m32, (float)m.m33}; + float right[3], up[3]; + switch (self.window.windowScene.interfaceOrientation) { + case UIInterfaceOrientationLandscapeRight: + for (int i = 0; i < 3; i++) { right[i] = y[i]; up[i] = -x[i]; } + break; + case UIInterfaceOrientationLandscapeLeft: + for (int i = 0; i < 3; i++) { right[i] = -y[i]; up[i] = x[i]; } + break; + case UIInterfaceOrientationPortraitUpsideDown: + for (int i = 0; i < 3; i++) { right[i] = -x[i]; up[i] = -y[i]; } + break; + default: + for (int i = 0; i < 3; i++) { right[i] = x[i]; up[i] = y[i]; } + break; + } + const float rowMajor[9] = {right[0], up[0], z[0], right[1], up[1], z[1], right[2], up[2], z[2]}; + [_engine setAttitude:rowMajor]; } // MARK: - Policy and capabilities @@ -177,6 +276,7 @@ - (void)emitCapabilities { @"supportsSubgroups": @(caps.supportsSubgroups), @"maxTextureDimension": @(caps.maxTextureDimension), @"policyRaster": @(caps.policy.raster), + @"policyRasterMask": @(caps.policy.raster ? (caps.policy.rasterMask != 0 ? caps.policy.rasterMask : 7u) : 0u), @"policyTileSize": @(caps.policy.tileSize), @"policyLodErrorPixels": @(caps.policy.lodErrorPixels), @"policyAlphaThreshold": @(caps.policy.alphaThreshold), @@ -194,6 +294,7 @@ - (void)didMoveToWindow { self.attached = YES; [self startDisplayLink]; [self attachLayer]; + [self startMotion]; } else { [self detach]; } @@ -236,31 +337,39 @@ - (void)updateDisplayLink { - (void)appDidEnterBackground { self.appActive = NO; [self updateDisplayLink]; + [self stopMotion]; } - (void)appWillEnterForeground { self.appActive = YES; [self updateDisplayLink]; + [self startMotion]; } - (void)detach { self.attached = NO; [self stopDisplayLink]; + [self stopMotion]; [_engine setLayer:nil]; } - (void)dispose { [self stopDisplayLink]; + [self stopMotion]; _generation.fetch_add(1, std::memory_order_acq_rel); [_engine setLayer:nil]; _engine = nil; self.requestId = nil; + _hasLastPose = NO; } - (void)recycle { [self dispose]; _hasPolicy = NO; _policyRevision = 0; + _hasCharacter = NO; + self.colliderPath = nil; + self.colliderRequestId = nil; } // MARK: - Loading @@ -293,6 +402,10 @@ - (void)loadWorld:(NSString *)path requestId:(NSString *)requestId maxShDegree:( [engine setResidencyBudget:(int)MAX(residencyCapacity, 1)]; [engine setRenderScale:(float)self.renderScale]; [engine setShDegree:(int)self.shDegree]; + [engine setLinearBlending:self.linearBlending]; + [engine setCullMargin:(float)self.cullMarginDegrees]; + [engine setMotionEnabled:self.motionEnabled]; + if (_hasCharacter) [engine setCharacter:_character]; // Report the new engine's capabilities once, then apply the host policy to it before the // decode is enqueued. Same order as the Android adapter. [self emitCapabilities]; @@ -318,6 +431,44 @@ - (void)loadWorld:(NSString *)path requestId:(NSString *)requestId maxShDegree:( if (generation != view->_generation.load(std::memory_order_acquire)) return; [engine loadWorldFile:path]; }); + + // The world's engine is new, so walk mode has to be rebuilt on it. + if (self.colliderPath.length > 0) [self enqueueColliderOnGeneration:generation]; +} + +- (void)loadCollider:(NSString *)path requestId:(NSString *)requestId { + if (requestId.length == 0 || path.length == 0) { + self.colliderPath = nil; + self.colliderRequestId = nil; + return; + } + if (![self isLocalPath:path]) { + [self emitCollider:requestId phase:@"failed" code:kErrorInvalidRequest + message:@"invalid collider request"]; + return; + } + self.colliderPath = [path copy]; + self.colliderRequestId = [requestId copy]; + if (_engine == nil) return; + [self enqueueColliderOnGeneration:_generation.load(std::memory_order_acquire)]; +} + +- (void)enqueueColliderOnGeneration:(uint64_t)generation { + SKSplatEngine *engine = _engine; + NSString *path = self.colliderPath; + if (engine == nil || path.length == 0) return; + __weak SplatKitRNView *weakSelf = self; + dispatch_async(_loaderQueue, ^{ + SplatKitRNView *view = weakSelf; + if (view == nil) return; + if (generation != view->_generation.load(std::memory_order_acquire)) return; + [engine loadColliderFile:path]; + }); +} + +- (BOOL)isLocalPath:(NSString *)path { + return [path hasPrefix:@"/"] && ![path hasPrefix:@"//"] && ![path isEqualToString:@"/"] && + [path rangeOfString:@"\0"].location == NSNotFound; } - (BOOL)validateRequestId:(NSString *)requestId path:(NSString *)path maxShDegree:(NSInteger)maxShDegree { @@ -333,6 +484,12 @@ - (BOOL)validateRequestId:(NSString *)requestId path:(NSString *)path maxShDegre - (void)deliverWorldEvent:(SKSplatEvent)event message:(NSString *)message count:(uint32_t)count requestId:(NSString *)requestId { + if (event == SKSplatEventColliderReady || event == SKSplatEventColliderFailed) { + const BOOL ready = event == SKSplatEventColliderReady; + [self emitCollider:self.colliderRequestId phase:ready ? @"ready" : @"failed" + code:ready ? @"" : kErrorColliderLoadFailed message:message]; + return; + } if (self.worldEvent == nil) return; NSString *phase = event == SKSplatEventWorldReady ? @"uploaded" : event == SKSplatEventWorldFrameReady ? @"frameReady" : @"failed"; @@ -341,6 +498,13 @@ - (void)deliverWorldEvent:(SKSplatEvent)event message:(NSString *)message count: @"errorCode": code, @"message": message ?: @""}); } +- (void)emitCollider:(NSString *)requestId phase:(NSString *)phase code:(NSString *)code + message:(NSString *)message { + if (self.colliderEvent == nil) return; + self.colliderEvent(@{@"requestId": requestId ?: @"", @"phase": phase, @"errorCode": code, + @"message": message ?: @""}); +} + - (void)emitWorld:(NSString *)requestId phase:(NSString *)phase count:(uint32_t)count code:(NSString *)code message:(NSString *)message { if (self.worldEvent == nil) return; @@ -352,8 +516,19 @@ - (void)draw:(CADisplayLink *)link { SKSplatEngine *engine = _engine; if (engine == nil) return; [engine render:(int64_t)(link.timestamp * 1000000000.0)]; - if (self.statsEvent == nil) return; const CFTimeInterval now = CACurrentMediaTime(); + if (self.cameraPoseEvent != nil && self.cameraPoseInterval > 0 && + now - self.lastPoseAt >= self.cameraPoseInterval) { + self.lastPoseAt = now; + const SKCameraPose pose = engine.cameraPose; + if (!_hasLastPose || memcmp(&pose, &_lastPose, sizeof(pose)) != 0) { + _lastPose = pose; + _hasLastPose = YES; + self.cameraPoseEvent(@{@"x": @(pose.x), @"y": @(pose.y), @"z": @(pose.z), + @"yaw": @(pose.yaw), @"pitch": @(pose.pitch)}); + } + } + if (self.statsEvent == nil) return; if (now - self.lastStatsAt < kStatsInterval) return; self.lastStatsAt = now; SKSplatStats stats = engine.stats; diff --git a/packages/react-native-splatkit/ios/SplatKitViewComponentView.mm b/packages/react-native-splatkit/ios/SplatKitViewComponentView.mm index 1584a31..214fa51 100644 --- a/packages/react-native-splatkit/ios/SplatKitViewComponentView.mm +++ b/packages/react-native-splatkit/ios/SplatKitViewComponentView.mm @@ -15,6 +15,7 @@ @implementation SplatKitViewComponentView { // finalizeUpdates; otherwise the capability and policy events of the first engine are dropped. BOOL _worldChanged; BOOL _policyChanged; + BOOL _colliderChanged; } + (ComponentDescriptorProvider)componentDescriptorProvider { @@ -24,6 +25,11 @@ + (ComponentDescriptorProvider)componentDescriptorProvider { - (instancetype)initWithFrame:(CGRect)frame { if ((self = [super initWithFrame:frame])) { + // Until the first updateProps the view answers from these, and the base class reads the + // concrete type to tell a configured subclass from a plain view: a debug build asserts on + // the ViewProps its own constructor leaves behind. + static const auto defaultProps = std::make_shared(); + _props = defaultProps; _splatView = [[SplatKitRNView alloc] initWithFrame:self.bounds]; [self addSubview:_splatView]; _splatView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; @@ -79,6 +85,25 @@ - (void)updateEventEmitter:(const EventEmitter::Shared &)eventEmitter { value.sortDepth = [event[@"sortDepth"] intValue]; emitter->onPolicyEvent(value); }; + _splatView.colliderEvent = ^(NSDictionary *event) { + if (!emitter) return; + SplatKitViewEventEmitter::OnColliderEvent value; + value.requestId = [event[@"requestId"] UTF8String]; + value.phase = [event[@"phase"] isEqualToString:@"ready"] + ? SplatKitViewEventEmitter::OnColliderEventPhase::Ready + : SplatKitViewEventEmitter::OnColliderEventPhase::Failed; + value.errorCode = [event[@"errorCode"] UTF8String]; + value.message = [event[@"message"] UTF8String]; + emitter->onColliderEvent(value); + }; + _splatView.cameraPoseEvent = ^(NSDictionary *event) { + if (!emitter) return; + SplatKitViewEventEmitter::OnCameraPose value; + value.x = [event[@"x"] doubleValue]; value.y = [event[@"y"] doubleValue]; + value.z = [event[@"z"] doubleValue]; value.yaw = [event[@"yaw"] doubleValue]; + value.pitch = [event[@"pitch"] doubleValue]; + emitter->onCameraPose(value); + }; _splatView.capabilitiesEvent = ^(NSDictionary *event) { if (!emitter) return; SplatKitViewEventEmitter::OnCapabilities value; @@ -90,6 +115,7 @@ - (void)updateEventEmitter:(const EventEmitter::Shared &)eventEmitter { value.supportsSubgroups = [event[@"supportsSubgroups"] boolValue]; value.maxTextureDimension = [event[@"maxTextureDimension"] intValue]; value.policyRaster = [event[@"policyRaster"] boolValue]; + value.policyRasterMask = [event[@"policyRasterMask"] intValue]; value.policyTileSize = [event[@"policyTileSize"] boolValue]; value.policyLodErrorPixels = [event[@"policyLodErrorPixels"] boolValue]; value.policyAlphaThreshold = [event[@"policyAlphaThreshold"] boolValue]; @@ -107,9 +133,30 @@ - (void)updateProps:(const Props::Shared &)props oldProps:(const Props::Shared & auto next = *std::static_pointer_cast(props); auto previous = oldProps ? std::static_pointer_cast(oldProps) : nullptr; _splatView.paused = next.paused; _splatView.renderScale = next.renderScale; _splatView.shDegree = next.shDegree; + _splatView.linearBlending = next.linearBlending; + _splatView.cullMarginDegrees = next.cullMarginDegrees; + _splatView.touchLookEnabled = next.touchLookEnabled; + _splatView.lookSensitivity = next.lookSensitivity; + _splatView.cameraPoseInterval = next.cameraPoseInterval; + if (!previous || next.motionEnabled != previous->motionEnabled) { + _splatView.motionEnabled = next.motionEnabled; + } + // A zero eye height is an absent character prop; the native default stands. + if (next.character.eyeHeight > 0 && + (!previous || next.character.eyeHeight != previous->character.eyeHeight || + next.character.bodyRadius != previous->character.bodyRadius || + next.character.stepHeight != previous->character.stepHeight)) { + SKCharacterSettings character{}; + character.eyeHeight = static_cast(next.character.eyeHeight); + character.bodyRadius = static_cast(next.character.bodyRadius); + character.stepHeight = static_cast(next.character.stepHeight); + [_splatView setCharacter:character]; + } // Codegen gives an absent policy revision 0; like Android, 0 or less means no policy. if (!previous || next.policy.revision != previous->policy.revision) { - SKRenderPolicy policy; + // Value initialized: the props carry no lodSplatLimit, and an indeterminate one starves + // the selector to that many splats. Zero is the engine's own default, meaning no limit. + SKRenderPolicy policy{}; policy.raster = static_cast(next.policy.raster); policy.tileSize = static_cast(next.policy.tileSize); policy.lodErrorPixels = static_cast(next.policy.lodErrorPixels); @@ -123,13 +170,22 @@ - (void)updateProps:(const Props::Shared &)props oldProps:(const Props::Shared & _policyChanged = YES; } if (!previous || next.world.requestId != previous->world.requestId) _worldChanged = YES; + if (!previous || next.collider.requestId != previous->collider.requestId) _colliderChanged = YES; } - (void)finalizeUpdates:(RNComponentViewUpdateMask)updateMask { [super finalizeUpdates:updateMask]; const BOOL worldChanged = _worldChanged, policyChanged = _policyChanged; + const BOOL colliderChanged = _colliderChanged; _worldChanged = NO; _policyChanged = NO; + _colliderChanged = NO; + // Before the world: a load rebuilds walk mode on the engine it creates. + if (colliderChanged) { + const auto &collider = std::static_pointer_cast(_props)->collider; + [_splatView loadCollider:[NSString stringWithUTF8String:collider.filePath.c_str()] + requestId:[NSString stringWithUTF8String:collider.requestId.c_str()]]; + } if (worldChanged) { const auto &world = std::static_pointer_cast(_props)->world; if (world.requestId.empty()) { @@ -147,14 +203,30 @@ - (void)finalizeUpdates:(RNComponentViewUpdateMask)updateMask { if (policyChanged) [_splatView applyStoredPolicy]; } +- (void)handleCommand:(const NSString *)commandName args:(const NSArray *)args { + if ([commandName isEqualToString:@"setWalkVelocity"] && args.count == 2) { + [_splatView setWalkVelocityForward:[args[0] floatValue] right:[args[1] floatValue]]; + } else if ([commandName isEqualToString:@"look"] && args.count == 2) { + [_splatView lookWithDeltaYaw:[args[0] floatValue] deltaPitch:[args[1] floatValue]]; + } else if ([commandName isEqualToString:@"setCameraPose"] && args.count == 5) { + SKCameraPose pose; + pose.x = [args[0] floatValue]; pose.y = [args[1] floatValue]; pose.z = [args[2] floatValue]; + pose.yaw = [args[3] floatValue]; pose.pitch = [args[4] floatValue]; + [_splatView setPose:pose]; + } +} + - (void)prepareForRecycle { [super prepareForRecycle]; _worldChanged = NO; _policyChanged = NO; + _colliderChanged = NO; _splatView.worldEvent = nil; _splatView.statsEvent = nil; _splatView.policyEvent = nil; _splatView.capabilitiesEvent = nil; + _splatView.colliderEvent = nil; + _splatView.cameraPoseEvent = nil; // A recycled view serves another component next; keep no engine, world or policy. [_splatView recycle]; } @@ -163,6 +235,8 @@ - (void)invalidate { _splatView.statsEvent = nil; _splatView.policyEvent = nil; _splatView.capabilitiesEvent = nil; + _splatView.colliderEvent = nil; + _splatView.cameraPoseEvent = nil; [_splatView dispose]; [super invalidate]; } diff --git a/packages/react-native-splatkit/package.json b/packages/react-native-splatkit/package.json index 57a10cb..2fa2ac7 100644 --- a/packages/react-native-splatkit/package.json +++ b/packages/react-native-splatkit/package.json @@ -1,6 +1,6 @@ { "name": "@splatkit/react-native", - "version": "0.1.0-alpha.1", + "version": "0.1.0-alpha.2", "description": "SplatKit Fabric view with native Metal and Vulkan adapters", "repository": { "type": "git", "url": "https://github.com/Xget7/react-native-splatkit.git" }, "license": "MIT", diff --git a/packages/react-native-splatkit/scripts/codegen.cjs b/packages/react-native-splatkit/scripts/codegen.cjs index c115468..a5e1985 100644 --- a/packages/react-native-splatkit/scripts/codegen.cjs +++ b/packages/react-native-splatkit/scripts/codegen.cjs @@ -19,9 +19,23 @@ const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8')); const spec = schema.modules.SplatKitView?.components?.SplatKitView; assert.ok(spec, 'Codegen must discover SplatKitView'); assert.deepEqual(spec.props.map(prop => prop.name).sort(), - ['paused', 'policy', 'renderScale', 'shDegree', 'world']); + ['cameraPoseInterval', 'character', 'collider', 'cullMarginDegrees', 'linearBlending', + 'lookSensitivity', 'motionEnabled', 'paused', 'policy', 'renderScale', 'shDegree', + 'touchLookEnabled', 'world']); assert.deepEqual(spec.events.map(event => event.name).sort(), - ['onCapabilities', 'onPolicyEvent', 'onStats', 'onWorldEvent']); + ['onCameraPose', 'onCapabilities', 'onColliderEvent', 'onPolicyEvent', 'onStats', 'onWorldEvent']); +assert.deepEqual(spec.commands.map(command => command.name).sort(), + ['look', 'setCameraPose', 'setWalkVelocity']); +assert.deepEqual(spec.commands.find(command => command.name === 'setWalkVelocity') + .typeAnnotation.params.map(param => [param.name, param.typeAnnotation.type]), + [['forward', 'DoubleTypeAnnotation'], ['right', 'DoubleTypeAnnotation']]); +const collider = spec.props.find(prop => prop.name === 'collider').typeAnnotation.properties; +assert.deepEqual(collider.map(prop => prop.name).sort(), ['filePath', 'requestId']); +const character = spec.props.find(prop => prop.name === 'character').typeAnnotation.properties; +assert.deepEqual(character.map(prop => prop.name).sort(), + ['bodyRadius', 'eyeHeight', 'stepHeight']); +const pose = spec.events.find(event => event.name === 'onCameraPose').typeAnnotation.argument.properties; +assert.deepEqual(pose.map(prop => prop.name).sort(), ['pitch', 'x', 'y', 'yaw', 'z']); const world = spec.props.find(prop => prop.name === 'world').typeAnnotation.properties; assert.deepEqual(world.map(prop => prop.name).sort(), ['filePath', 'lodCapacitySplats', 'maxShDegree', 'requestId', 'residencyCapacitySplats']); diff --git a/packages/react-native-splatkit/scripts/ios-xcframework.json b/packages/react-native-splatkit/scripts/ios-xcframework.json index 85d56d4..5481703 100644 --- a/packages/react-native-splatkit/scripts/ios-xcframework.json +++ b/packages/react-native-splatkit/scripts/ios-xcframework.json @@ -1,5 +1,5 @@ { "_comment": "The splatkit-ios release scripts/fetch-ios-xcframework.cjs downloads at npm prepack. Update after cutting a new splatkit-ios release; sha256 comes from swift package compute-checksum (scripts/package-ios.sh), which is also the SPM checksum.", - "version": "0.1.0-alpha.3", - "sha256": "2258de61db98721528a805bdfefbc6b764aada311bda6149b3abcf98d5194906" + "version": "0.1.0-alpha.4", + "sha256": "7b92ec52cbcd1f42bfc6ab31d16befd4b8134ee1c7f0a71b26789a4cbcf4f5c5" } diff --git a/packages/react-native-splatkit/src/contracts.ts b/packages/react-native-splatkit/src/contracts.ts index 6ce641d..7ba3046 100644 --- a/packages/react-native-splatkit/src/contracts.ts +++ b/packages/react-native-splatkit/src/contracts.ts @@ -19,6 +19,46 @@ export type RenderOptions = Readonly<{ shDegree: SHDegree; }>; +/** Immutable collider transaction. An empty requestId releases walk mode. */ +export type ColliderRequest = Readonly<{ + requestId: string; + /** Absolute, readable local path to a collider GLB. */ + filePath: string; +}>; + +/** The walker's shape in walk mode, in meters. */ +export type Character = Readonly<{ + eyeHeight: number; + bodyRadius: number; + stepHeight: number; +}>; + +/** + * What a collider load reports. Compare against these rather than the strings: the native + * adapters and the codegen spec spell the same values, and the test suite keeps them equal. + */ +export const ColliderPhase = { + /** Walk mode is on and the walker stands on the collider. */ + ready: 'ready', + failed: 'failed', +} as const; +export type ColliderPhase = (typeof ColliderPhase)[keyof typeof ColliderPhase]; + +export type ColliderEvent = Readonly<{ + requestId: string; + phase: ColliderPhase; + errorCode: string; + message: string; +}>; + +export type CameraPose = Readonly<{ + x: number; + y: number; + z: number; + yaw: number; + pitch: number; +}>; + /** Values must come from the native adapter, never a JS device-name lookup. */ export type SplatLimits = Readonly<{ maxLodCapacitySplats: number; @@ -26,9 +66,19 @@ export type SplatLimits = Readonly<{ maxResidencyCapacitySplats: number; }>; +/** What a world load reports. See {@link ColliderPhase} on comparing against constants. */ +export const WorldPhase = { + /** Decoded and on the GPU; nothing has been drawn with it yet. */ + uploaded: 'uploaded', + /** The first frame containing the world has been presented. */ + frameReady: 'frameReady', + failed: 'failed', +} as const; +export type WorldPhase = (typeof WorldPhase)[keyof typeof WorldPhase]; + export type WorldEvent = Readonly<{ requestId: string; - phase: 'uploaded' | 'frameReady' | 'failed'; + phase: WorldPhase; loadedSplats: number; /** Empty on success; adapters must supply a stable code on failure. */ errorCode: string; @@ -46,9 +96,7 @@ export function validateWorldRequest(request: WorldRequest, limits: SplatLimits) if (typeof request.requestId !== 'string' || request.requestId.trim().length === 0) { throw new TypeError('requestId must be a nonempty string'); } - if (typeof request.filePath !== 'string' || !request.filePath.startsWith('/') || - request.filePath.startsWith('//') || request.filePath.includes('\0') || - request.filePath === '/') { + if (!localPath(request.filePath)) { throw new TypeError('filePath must be an absolute local file path, not a URL'); } integer('maxLodCapacitySplats', limits.maxLodCapacitySplats, 0, 0x7fffffff); @@ -61,6 +109,34 @@ export function validateWorldRequest(request: WorldRequest, limits: SplatLimits) limits.minResidencyCapacitySplats, limits.maxResidencyCapacitySplats); } +function localPath(value: unknown): value is string { + return typeof value === 'string' && value.startsWith('/') && !value.startsWith('//') && + value !== '/' && !value.includes('\0'); +} + +export function validateColliderRequest(request: ColliderRequest): void { + if (typeof request.requestId !== 'string' || request.requestId.trim().length === 0) { + throw new TypeError('requestId must be a nonempty string'); + } + if (!localPath(request.filePath)) { + throw new TypeError('filePath must be an absolute local file path, not a URL'); + } +} + +/** Native refuses these too; failing here names the field instead of keeping the old walker. */ +export function validateCharacter(character: Character): void { + const {eyeHeight, bodyRadius, stepHeight} = character; + if (!Number.isFinite(eyeHeight) || eyeHeight <= 0 || eyeHeight > 100) { + throw new RangeError('eyeHeight must be finite and in (0, 100]'); + } + if (!Number.isFinite(bodyRadius) || bodyRadius < 0 || bodyRadius >= eyeHeight) { + throw new RangeError('bodyRadius must be finite, not negative and under eyeHeight'); + } + if (!Number.isFinite(stepHeight) || stepHeight < 0 || stepHeight >= eyeHeight) { + throw new RangeError('stepHeight must be finite, not negative and under eyeHeight'); + } +} + export function validateRenderOptions(options: RenderOptions): void { if (typeof options.paused !== 'boolean') throw new TypeError('paused must be boolean'); if (!Number.isFinite(options.renderScale) || options.renderScale < 0.1 || options.renderScale > 2) { diff --git a/packages/react-native-splatkit/src/index.ts b/packages/react-native-splatkit/src/index.ts index 7822002..94ed797 100644 --- a/packages/react-native-splatkit/src/index.ts +++ b/packages/react-native-splatkit/src/index.ts @@ -1,5 +1,13 @@ // SplatKitView is the Codegen host component; build its props with toNativeViewProps and toNativePolicyProp. +import {Commands} from './specs/SplatViewNativeComponent'; + export * from './contracts'; export * from './performance'; export {default as SplatKitView} from './specs/SplatViewNativeComponent'; -export type {NativeProps as SplatKitViewProps} from './specs/SplatViewNativeComponent'; +export type { + NativeProps as SplatKitViewProps, + NativeCommands as SplatKitCommandSet, +} from './specs/SplatViewNativeComponent'; + +/** Imperative navigation commands for a mounted SplatKitView ref. */ +export const SplatKitCommands = Commands; diff --git a/packages/react-native-splatkit/src/performance.ts b/packages/react-native-splatkit/src/performance.ts index 845a1a2..65011d3 100644 --- a/packages/react-native-splatkit/src/performance.ts +++ b/packages/react-native-splatkit/src/performance.ts @@ -2,14 +2,41 @@ import type {RenderOptions, SHDegree, SplatLimits, WorldRequest} from './contrac import {validateRenderOptions, validateWorldRequest} from './contracts'; import type {NativeProps} from './specs/SplatViewNativeComponent'; -export type RasterStrategy = 'hardware' | 'computeTile' | 'hybrid'; +/** + * `hardware` is the default and the fastest choice for most scenes. `hybrid` composites screen + * tiles in compute and suits close-up views where many large translucent splats overlap each + * pixel; on distant or sparse scenes it adds GPU work and lowers the frame rate. It is + * experimental and iOS-only today. `computeTile` is not implemented by any adapter yet. + */ +export const RasterStrategy = { + hardware: 'hardware', + computeTile: 'computeTile', + hybrid: 'hybrid', +} as const; +export type RasterStrategy = (typeof RasterStrategy)[keyof typeof RasterStrategy]; + export type SortDepth = 16 | 32; -export type PerformanceMode = 'auto' | 'manual'; -export type QualityPreset = 'highEnd' | 'high' | 'balanced' | 'performance'; + +export const PerformanceMode = { auto: 'auto', manual: 'manual' } as const; +export type PerformanceMode = (typeof PerformanceMode)[keyof typeof PerformanceMode]; + +/** Ordered by decreasing detail; `build` resolves each against the device's own limits. */ +export const QualityPreset = { + highEnd: 'highEnd', + high: 'high', + balanced: 'balanced', + performance: 'performance', +} as const; +export type QualityPreset = (typeof QualityPreset)[keyof typeof QualityPreset]; +export const qualityPresets: readonly QualityPreset[] = Object.freeze([ + QualityPreset.highEnd, QualityPreset.high, QualityPreset.balanced, QualityPreset.performance, +]); /** Which policy fields one native adapter applies; the rest fall back natively. */ export type NativePolicySupport = Readonly<{ raster: boolean; + /** The strategies native applies when `raster` is true. */ + rasterStrategies: readonly RasterStrategy[]; tileSize: boolean; lodErrorPixels: boolean; alphaThreshold: boolean; @@ -31,6 +58,27 @@ export type DeviceCapabilities = Readonly<{ policy?: NativePolicySupport; }>; +/** + * What to build the first world request against, before an engine has reported its own. + * + * Capabilities arrive with the engine a world load creates, so the first request has to be + * built against a guess, and a guess above what the adapter accepts is rejected outright: + * no engine, no capabilities, and a host with no way to learn what it got wrong. These are + * the narrowest limits any shipped backend reports, so every adapter accepts a request + * built against them. Rebuild on `onCapabilities` to reach the limits the device really has. + */ +export const conservativeCapabilities: DeviceCapabilities = Object.freeze({ + limits: Object.freeze({ + maxLodCapacitySplats: 2_200_000, + minResidencyCapacitySplats: 100_000, + maxResidencyCapacitySplats: 8_000_000, + }), + supportsComputeTiles: false, + supportsHiZOcclusion: false, + supportsSubgroups: false, + maxTextureDimension: 4096, +}); + /** The policy as it crosses the Fabric boundary; enum values are the numbers native uses. */ export type NativeRenderPolicy = Readonly<{ revision: number; @@ -55,6 +103,7 @@ export type NativeCapabilitiesEvent = Readonly<{ supportsSubgroups: boolean; maxTextureDimension: number; policyRaster: boolean; + policyRasterMask: number; policyTileSize: boolean; policyLodErrorPixels: boolean; policyAlphaThreshold: boolean; @@ -133,7 +182,25 @@ export type SplatKitConfiguration = Readonly<{ performance: PerformanceResolution; }>; -export type PolicyChangeKind = 'none' | 'nativePropUpdate' | 'worldReload' | 'unavailable'; +/** What applying a changed policy costs. Compare against these, not the strings. */ +export const PolicyChangeKind = { + none: 'none', + /** New Fabric props reach the live engine; nothing reloads. */ + nativePropUpdate: 'nativePropUpdate', + /** A load-time budget moved: only a fresh world request picks it up. */ + worldReload: 'worldReload', + unavailable: 'unavailable', +} as const; +export type PolicyChangeKind = (typeof PolicyChangeKind)[keyof typeof PolicyChangeKind]; + +/** What one policy application reported. */ +export const PolicyPhase = { + applied: 'applied', + /** A field fell back; the message joins the reasons. */ + warning: 'warning', + rejected: 'rejected', +} as const; +export type PolicyPhase = (typeof PolicyPhase)[keyof typeof PolicyPhase]; const loadKeys = new Set([ 'lodBudgetSplats', 'residencyCapacitySplats', @@ -154,12 +221,12 @@ export function classifyPolicyChange( let propUpdate = false; for (const key of Object.keys(previous) as (keyof PerformancePolicy)[]) { if (previous[key] === next[key] || key === 'mode' || key === 'preset') continue; - if (loadKeys.has(key)) return 'worldReload'; + if (loadKeys.has(key)) return PolicyChangeKind.worldReload; if (propKeys.has(key)) propUpdate = true; else unavailable = true; } - if (propUpdate) return 'nativePropUpdate'; - return unavailable ? 'unavailable' : 'none'; + if (propUpdate) return PolicyChangeKind.nativePropUpdate; + return unavailable ? PolicyChangeKind.unavailable : PolicyChangeKind.none; } /** Converts resolved values to the complete current Fabric prop surface. */ @@ -214,6 +281,8 @@ export function nativeCapabilitiesFromEvent(event: NativeCapabilitiesEvent): Dev maxTextureDimension: event.maxTextureDimension, policy: Object.freeze({ raster: event.policyRaster, + rasterStrategies: Object.freeze((Object.keys(rasterWire) as RasterStrategy[]) + .filter(strategy => (event.policyRasterMask & (1 << rasterWire[strategy])) !== 0)), tileSize: event.policyTileSize, lodErrorPixels: event.policyLodErrorPixels, alphaThreshold: event.policyAlphaThreshold, @@ -234,18 +303,20 @@ const nativeDefaults = { } as const; type PresetValues = Omit; +// Every preset rasterizes in hardware; screen tiles only pay off on some scenes, so they are +// an explicit withPerformance({raster: 'hybrid'}) choice. const presets: Readonly> = { - highEnd: {raster: 'computeTile', tileSize: 16, lodErrorPixels: 0.75, + highEnd: {raster: 'hardware', tileSize: 16, lodErrorPixels: 0.75, lodBudgetSplats: 4_000_000, alphaThreshold: 1 / 255, subpixelThreshold: 0.35, enableFrustumCulling: true, enableHiZOcclusion: true, enableEarlyTermination: true, sortDepth: 32, renderScale: 1.25, shDegree: 3, residencyCapacitySplats: 4_000_000, targetFps: null}, - high: {raster: 'hybrid', tileSize: 16, lodErrorPixels: 1, + high: {raster: 'hardware', tileSize: 16, lodErrorPixels: 1, lodBudgetSplats: 3_000_000, alphaThreshold: 1 / 255, subpixelThreshold: 0.5, enableFrustumCulling: true, enableHiZOcclusion: true, enableEarlyTermination: true, sortDepth: 32, renderScale: 1, shDegree: 3, residencyCapacitySplats: 3_000_000, targetFps: null}, - balanced: {raster: 'hybrid', tileSize: 16, lodErrorPixels: 1.25, + balanced: {raster: 'hardware', tileSize: 16, lodErrorPixels: 1.25, lodBudgetSplats: 2_000_000, alphaThreshold: 1 / 255, subpixelThreshold: 0.65, enableFrustumCulling: true, enableHiZOcclusion: false, enableEarlyTermination: true, sortDepth: 16, renderScale: 0.85, shDegree: 2, @@ -412,7 +483,10 @@ export class SplatKitBuilder { const fields = effectiveFields as Record<(typeof policyPropKeys)[number], PerformancePolicy[keyof PerformancePolicy] | null>; for (const option of policyPropKeys) { - if (support[option]) fields[option] = requested[option]; + const applied = option === 'raster' + ? support.raster && support.rasterStrategies.includes(requested.raster) + : support[option]; + if (applied) fields[option] = requested[option]; else if (requested[option] !== nativeDefaults[option]) { warn('native-option-fallback', option, requested[option], null, `${option} fell back to the native default; read onPolicyEvent for the applied value`); diff --git a/packages/react-native-splatkit/src/specs/SplatViewNativeComponent.ts b/packages/react-native-splatkit/src/specs/SplatViewNativeComponent.ts index 2f3c20d..029916f 100644 --- a/packages/react-native-splatkit/src/specs/SplatViewNativeComponent.ts +++ b/packages/react-native-splatkit/src/specs/SplatViewNativeComponent.ts @@ -1,5 +1,5 @@ import type {CodegenTypes, HostComponent, ViewProps} from 'react-native'; -import {codegenNativeComponent} from 'react-native'; +import {codegenNativeComponent, codegenNativeCommands} from 'react-native'; // Keep these wire records local: Codegen does not resolve arbitrary imported aliases. type NativeWorldRequest = Readonly<{ @@ -10,6 +10,37 @@ type NativeWorldRequest = Readonly<{ residencyCapacitySplats: CodegenTypes.Int32; }>; +/** Immutable collider transaction, the walk-mode counterpart of a world request. */ +type NativeColliderRequest = Readonly<{ + /** Unique for each replacement; empty releases walk mode. */ + requestId: string; + /** Absolute, readable local path to a collider GLB. */ + filePath: string; +}>; + +type NativeColliderEvent = Readonly<{ + requestId: string; + phase: 'ready' | 'failed'; + errorCode: string; + message: string; +}>; + +/** The walker's shape in walk mode, in meters. Zero eyeHeight means the native default. */ +type NativeCharacter = Readonly<{ + eyeHeight: CodegenTypes.Double; + bodyRadius: CodegenTypes.Double; + stepHeight: CodegenTypes.Double; +}>; + +/** Where the camera is and where it looks: meters and radians, in the world's frame. */ +type NativeCameraPoseEvent = Readonly<{ + x: CodegenTypes.Double; + y: CodegenTypes.Double; + z: CodegenTypes.Double; + yaw: CodegenTypes.Double; + pitch: CodegenTypes.Double; +}>; + type NativeWorldEvent = Readonly<{ requestId: string; phase: 'uploaded' | 'frameReady' | 'failed'; @@ -80,6 +111,8 @@ type NativeCapabilitiesEvent = Readonly<{ supportsSubgroups: boolean; maxTextureDimension: CodegenTypes.Int32; policyRaster: boolean; + /** Raster strategies native applies: bit 0 hardware, bit 1 computeTile, bit 2 hybrid. */ + policyRasterMask: CodegenTypes.Int32; policyTileSize: boolean; policyLodErrorPixels: boolean; policyAlphaThreshold: boolean; @@ -92,17 +125,70 @@ type NativeCapabilitiesEvent = Readonly<{ export interface NativeProps extends ViewProps { world?: NativeWorldRequest; + /** Enables walk mode when it is ready; an empty requestId releases it. */ + collider?: NativeColliderRequest; + /** The walker's shape, applied at once and to a collider loaded later. */ + character?: NativeCharacter; paused?: CodegenTypes.WithDefault; renderScale?: CodegenTypes.WithDefault; shDegree?: CodegenTypes.WithDefault; + /** Blend in linear light instead of the encoded space the training used. */ + linearBlending?: CodegenTypes.WithDefault; + /** CPU fallback's angular culling margin in degrees; the GPU path uses projected bounds. */ + cullMarginDegrees?: CodegenTypes.WithDefault; + /** Drives the camera with the gyroscope. Ignored where the sensor is missing. */ + motionEnabled?: CodegenTypes.WithDefault; + /** Whether a drag on the view turns the camera. Off when the host looks with its own control. */ + touchLookEnabled?: CodegenTypes.WithDefault; + /** Radians per point dragged to look. */ + lookSensitivity?: CodegenTypes.WithDefault; + /** Seconds between onCameraPose events; 0, the default, never sends one. */ + cameraPoseInterval?: CodegenTypes.WithDefault; /** The resolved renderer policy; native re-validates and reports the effective one. */ policy?: NativeRenderPolicy; onWorldEvent?: CodegenTypes.DirectEventHandler; /** Adapter must throttle snapshots to at most 2 Hz; no per-frame JS callbacks. */ onStats?: CodegenTypes.DirectEventHandler; + onColliderEvent?: CodegenTypes.DirectEventHandler; + /** Throttled to cameraPoseInterval, and sent only when the pose changed. */ + onCameraPose?: CodegenTypes.DirectEventHandler; onPolicyEvent?: CodegenTypes.DirectEventHandler; /** Emitted once per engine, which each world load creates, before its first policy event. */ onCapabilities?: CodegenTypes.DirectEventHandler; } -export default codegenNativeComponent('SplatKitView') as HostComponent; +type ComponentType = HostComponent; + +/** + * Imperative navigation, for the host's own controls: a joystick or a look pad drives the + * camera at touch rate without a React commit per frame. + */ +export interface NativeCommands { + /** Meters per second until called again with zeros. Forward is where the camera looks. */ + setWalkVelocity: ( + viewRef: React.ComponentRef, + forward: CodegenTypes.Double, + right: CodegenTypes.Double, + ) => void; + /** Radians. Pitch is clamped, and ignored while the gyroscope drives the view. */ + look: ( + viewRef: React.ComponentRef, + deltaYaw: CodegenTypes.Double, + deltaPitch: CodegenTypes.Double, + ) => void; + /** Teleports; while walking the camera settles on the floor under the new point. */ + setCameraPose: ( + viewRef: React.ComponentRef, + x: CodegenTypes.Double, + y: CodegenTypes.Double, + z: CodegenTypes.Double, + yaw: CodegenTypes.Double, + pitch: CodegenTypes.Double, + ) => void; +} + +export const Commands: NativeCommands = codegenNativeCommands({ + supportedCommands: ['setWalkVelocity', 'look', 'setCameraPose'], +}); + +export default codegenNativeComponent('SplatKitView') as ComponentType; diff --git a/packages/react-native-splatkit/tests/android-adapter.test.cjs b/packages/react-native-splatkit/tests/android-adapter.test.cjs index 2df67d4..5357739 100644 --- a/packages/react-native-splatkit/tests/android-adapter.test.cjs +++ b/packages/react-native-splatkit/tests/android-adapter.test.cjs @@ -17,9 +17,57 @@ test('Android adapter maps the versioned policy prop and its events', () => { assert.match(manager, /override fun onAfterUpdateTransaction[\s\S]*commitProps\(\)/); // The stored policy is applied to each engine a world builds, after capabilities and // before the load, and outcomes from a replaced engine are dropped. - assert.match(view, /emitCapabilities\(view\)\s*\/\/[^\n]*\n\s*applyPolicy\(view\)\s*view\.loadWorld/); + assert.match(view, /emitCapabilities\(view\)[\s\S]{0,400}?applyPolicy\(view\)[\s\S]{0,400}?view\.loadWorld/); assert.match(view, /token == session\.generation\) emitPolicy\(requested\.revision/); const prop = read('android/src/main/java/com/splatkit/reactnative/PolicyProp.kt'); assert.match(prop, /INVALID_POLICY/); assert.match(prop, /POLICY_PREPARATION_FAILED/); }); + +test('Android adapter exposes navigation as props, commands and collider events', () => { + const manager = read('android/src/main/java/com/splatkit/reactnative/SplatKitViewManager.kt'); + const view = read('android/src/main/java/com/splatkit/reactnative/SplatKitView.kt'); + for (const method of ['setCollider', 'setCharacter', 'setMotionEnabled', 'setTouchLookEnabled', + 'setLookSensitivity', 'setCameraPoseInterval', 'setWalkVelocity', 'look', 'setCameraPose']) { + assert.match(manager, new RegExp(`override fun ${method}\\(`), method); + } + assert.match(manager, /"topColliderEvent" to mapOf\("registrationName" to "onColliderEvent"\)/); + assert.match(manager, /"topCameraPose" to mapOf\("registrationName" to "onCameraPose"\)/); + // A world builds a new engine, so walk mode is rebuilt on it after the load is queued. + assert.match(view, /view\.loadWorld\([\s\S]*loadCollider\(\)/); + assert.match(view, /COLLIDER_LOAD_FAILED/); + // Commands reach the SDK view directly, with no React commit per frame. + assert.match(view, /fun walk\(forward: Double, right: Double\) \{\s*nativeView\?\.setWalkVelocity/); +}); + +// A request built against limits the adapter refuses is rejected before any engine exists, so +// no capabilities event follows and the host cannot learn what it got wrong: the first guess +// has to be one every adapter accepts. +test('conservativeCapabilities fit the ranges the Android adapter accepts', () => { + const kotlin = fs.readFileSync( + path.join(root, 'android/src/main/java/com/splatkit/reactnative/WorldSession.kt'), 'utf8'); + const number = text => Number(text.replace(/_/g, '')); + const lod = kotlin.match(/lodCapacitySplats in (\d[\d_]*)\.\.(\d[\d_]*)/); + const residency = kotlin.match(/residencyCapacitySplats in (\d[\d_]*)\.\.(\d[\d_]*)/); + assert.ok(lod && residency, 'WorldSession must state both accepted ranges'); + + const {limits} = require('../build/performance.js').conservativeCapabilities; + assert.ok(limits.maxLodCapacitySplats <= number(lod[2]), + `maxLodCapacitySplats ${limits.maxLodCapacitySplats} exceeds the adapter's ${lod[2]}`); + assert.ok(limits.minResidencyCapacitySplats >= number(residency[1]), + 'minResidencyCapacitySplats falls below the adapter minimum'); + assert.ok(limits.maxResidencyCapacitySplats <= number(residency[2]), + 'maxResidencyCapacitySplats exceeds the adapter maximum'); +}); + +// The adapter reported every timing as unavailable, so a HUD on Android could never show a +// frame rate or a GPU time even while the engine was measuring both. +test('Android adapter reports the timings the SDK measures', () => { + const kotlin = fs.readFileSync( + path.join(root, 'android/src/main/java/com/splatkit/reactnative/SplatKitView.kt'), 'utf8'); + for (const field of ['frame', 'gpu', 'sort']) { + assert.match(kotlin, new RegExp(`putDouble\\("${field}Millis", stats\\.${field}Millis`)); + assert.match(kotlin, + new RegExp(`putBoolean\\("${field}TimingAvailable", stats\\.${field}Millis > 0f\\)`)); + } +}); diff --git a/packages/react-native-splatkit/tests/contracts.test.cjs b/packages/react-native-splatkit/tests/contracts.test.cjs index 6b9637e..1f338a8 100644 --- a/packages/react-native-splatkit/tests/contracts.test.cjs +++ b/packages/react-native-splatkit/tests/contracts.test.cjs @@ -69,3 +69,15 @@ test('unavailable is null, while a measured zero remains zero', () => { assert.equal(optionalTimingMillis(true, 15), 15); for (const value of [NaN, Infinity, -1]) assert.equal(optionalTimingMillis(true, value), null); }); + +test('collider requests and character settings are validated before they reach native', () => { + const {validateColliderRequest, validateCharacter} = require('../build/contracts.js'); + validateColliderRequest({requestId: 'c1', filePath: '/tmp/room.glb'}); + assert.throws(() => validateColliderRequest({requestId: ' ', filePath: '/tmp/room.glb'}), TypeError); + assert.throws(() => validateColliderRequest({requestId: 'c1', filePath: 'https://x/room.glb'}), TypeError); + validateCharacter({eyeHeight: 1.5, bodyRadius: 0.35, stepHeight: 0.35}); + assert.throws(() => validateCharacter({eyeHeight: 0, bodyRadius: 0.35, stepHeight: 0.35}), RangeError); + // A body wider than the walker is tall, or a step it could never reach, is not a walker. + assert.throws(() => validateCharacter({eyeHeight: 1.5, bodyRadius: 2, stepHeight: 0.35}), RangeError); + assert.throws(() => validateCharacter({eyeHeight: 1.5, bodyRadius: 0.35, stepHeight: 1.5}), RangeError); +}); diff --git a/packages/react-native-splatkit/tests/ios-adapter.test.cjs b/packages/react-native-splatkit/tests/ios-adapter.test.cjs index 65728f0..17fb06f 100644 --- a/packages/react-native-splatkit/tests/ios-adapter.test.cjs +++ b/packages/react-native-splatkit/tests/ios-adapter.test.cjs @@ -52,3 +52,39 @@ test('iOS adapter applies the versioned policy prop and reports it by revision', assert.match(native, /INVALID_POLICY/); assert.match(native, /POLICY_PREPARATION_FAILED/); }); + +test('iOS adapter exposes navigation as props, commands and collider events', () => { + const mm = fs.readFileSync(path.join(root, 'ios/SplatKitViewComponentView.mm'), 'utf8'); + const native = fs.readFileSync(path.join(root, 'ios/SplatKitRNView.mm'), 'utf8'); + // Looking is the only touch the view keeps; walking is the host's own control. + assert.match(native, /UIPanGestureRecognizer/); + assert.doesNotMatch(native, /minimumNumberOfTouches/); + assert.match(native, /setWalkVelocityForward:/); + assert.match(native, /COLLIDER_LOAD_FAILED/); + // The collider is re-queued on the engine each world load creates. + assert.match(native, /loadWorldFile[\s\S]*enqueueColliderOnGeneration/); + assert.match(native, /startDeviceMotionUpdatesUsingReferenceFrame/); + assert.match(mm, /handleCommand[\s\S]*setWalkVelocity[\s\S]*look[\s\S]*setCameraPose/); + assert.match(mm, /onColliderEvent/); + assert.match(mm, /onCameraPose/); + // Walk mode is rebuilt before the world load that replaces the engine. + assert.match(mm, /if \(colliderChanged\)[\s\S]*if \(worldChanged\)/); +}); + +// The props carry no lodSplatLimit, so an indeterminate one reached the selector and starved +// it to that many splats: the world rendered or vanished depending on the stack. +test('iOS adapter value initializes the C structs it fills from props', () => { + const mm = fs.readFileSync(path.join(root, 'ios/SplatKitViewComponentView.mm'), 'utf8'); + for (const type of ['SKRenderPolicy', 'SKCharacterSettings']) { + assert.match(mm, new RegExp(`${type} \\w+\\{\\}`), `${type} must be value initialized`); + assert.doesNotMatch(mm, new RegExp(`${type} \\w+;`), `${type} must not be left indeterminate`); + } +}); + +// A debug build asserts that a component view carries its own props type from construction, +// so a missing default aborted the app on the first commit while release builds ran. +test('iOS component view seeds _props with its own default props', () => { + const mm = fs.readFileSync(path.join(root, 'ios/SplatKitViewComponentView.mm'), 'utf8'); + assert.match(mm, /initWithFrame:[\s\S]*std::make_shared\(\)/); + assert.match(mm, /initWithFrame:[\s\S]*_props = defaultProps;/); +}); diff --git a/packages/react-native-splatkit/tests/performance.test.cjs b/packages/react-native-splatkit/tests/performance.test.cjs index ee5e46a..ec4fb78 100644 --- a/packages/react-native-splatkit/tests/performance.test.cjs +++ b/packages/react-native-splatkit/tests/performance.test.cjs @@ -194,13 +194,13 @@ test('classifies only operations represented by the Fabric contract', () => { assert.equal(classifyPolicyChange(policy, {...policy, lodBudgetSplats: 1_000_000}), 'worldReload'); // Renderer policy fields travel in the versioned policy prop, not a world reload. - assert.equal(classifyPolicyChange(policy, {...policy, raster: 'hardware'}), + assert.equal(classifyPolicyChange(policy, {...policy, raster: 'hybrid'}), 'nativePropUpdate'); assert.equal(classifyPolicyChange(policy, {...policy, sortDepth: 32}), 'nativePropUpdate'); assert.equal(classifyPolicyChange(policy, {...policy, targetFps: 60}), 'unavailable'); }); -const nativeCapabilities = nativeCapabilitiesFromEvent({ +const nativeCapabilitiesEvent = Object.freeze({ maxLodCapacitySplats: 2_200_000, minResidencyCapacitySplats: 100_000, maxResidencyCapacitySplats: 8_000_000, @@ -209,6 +209,7 @@ const nativeCapabilities = nativeCapabilitiesFromEvent({ supportsSubgroups: true, maxTextureDimension: 16_384, policyRaster: false, + policyRasterMask: 0, policyTileSize: false, policyLodErrorPixels: false, policyAlphaThreshold: false, @@ -218,6 +219,7 @@ const nativeCapabilities = nativeCapabilitiesFromEvent({ policyEnableEarlyTermination: false, policySortDepth: true, }); +const nativeCapabilities = nativeCapabilitiesFromEvent(nativeCapabilitiesEvent); test('native capabilities resolve the requested policy against what the adapter accepts', () => { const config = new SplatKitBuilder().withWorld(world).withPerformance({ @@ -241,6 +243,28 @@ test('native capabilities resolve the requested policy against what the adapter } }); +test('every preset rasterizes in hardware; hybrid tiles are an explicit choice', () => { + for (const preset of ['highEnd', 'high', 'balanced', 'performance']) { + const config = build(builder => builder.withPreset(preset)); + assert.equal(config.performance.requested.raster, 'hardware', preset); + assert(!config.performance.diagnostics.some(item => item.option === 'raster'), preset); + } +}); + +test('a raster mask applies only the strategies native builds', () => { + const metal = nativeCapabilitiesFromEvent({...nativeCapabilitiesEvent, policyRaster: true, + policyRasterMask: 0b101}); + assert.deepEqual(metal.policy.rasterStrategies, ['hardware', 'hybrid']); + const hybrid = new SplatKitBuilder().withWorld(world).withPerformance({raster: 'hybrid'}) + .build(metal); + assert.equal(hybrid.performance.effective.raster, 'hybrid'); + const computeTile = new SplatKitBuilder().withWorld(world) + .withPerformance({raster: 'computeTile'}).build(metal); + assert.equal(computeTile.performance.effective.raster, null); + assert(computeTile.performance.diagnostics.some(item => + item.option === 'raster' && item.code === 'native-option-fallback')); +}); + test('builds the versioned native policy prop from the requested values', () => { const config = build(builder => builder.withPerformance({ raster: 'computeTile', diff --git a/packages/splat-core/CMakeLists.txt b/packages/splat-core/CMakeLists.txt index 0d49490..60a7e2e 100644 --- a/packages/splat-core/CMakeLists.txt +++ b/packages/splat-core/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_POSITION_INDEPENDENT_CODE ON) option(SPLAT_CORE_BUILD_TESTS "Build splat_core unit tests" ${PROJECT_IS_TOP_LEVEL}) -option(SPLAT_CORE_BUILD_TOOLS "Build ply2spz, splat-tile and splat_lod_build" ${PROJECT_IS_TOP_LEVEL}) +option(SPLAT_CORE_BUILD_TOOLS "Build ply2spz, splat-tile, splat_lod_build and splat_collider" ${PROJECT_IS_TOP_LEVEL}) # "thread" or "address" (address implies undefined). Off by default; CI runs both. set(SPLAT_CORE_SANITIZE "" CACHE STRING "Sanitizer for splat_core and its tests") @@ -27,8 +27,10 @@ add_library(splat_core STATIC src/formats/SplatDecoder.cpp src/formats/SpzDecoder.cpp src/formats/GlbDecoder.cpp + src/formats/GlbEncoder.cpp src/navigation/Collider.cpp src/navigation/CharacterController.cpp + src/navigation/ColliderBuilder.cpp src/sorting/DistanceSorter.cpp src/sorting/AsyncSorter.cpp src/sorting/SpatialOrder.cpp diff --git a/packages/splat-core/README.md b/packages/splat-core/README.md index ad78831..90242ab 100644 --- a/packages/splat-core/README.md +++ b/packages/splat-core/README.md @@ -7,13 +7,13 @@ No graphics, no platform APIs, no React Native. | Domain | Responsibility | Public headers | |---|---|---| -| Formats | Decode `.spz` worlds and `.glb` colliders into library-owned types in the internal frame | `splat/formats/*.h`, `splat/core/*.h` | +| Formats | Decode `.spz` worlds and `.glb` colliders into library-owned types in the internal frame, and encode `.glb` colliders | `splat/formats/*.h`, `splat/core/*.h` | | Loading | Map files, decode and prepare worlds for a render thread | `splat/io/*.h`, `splat/loading/*.h` | | Level of detail | Load-time trees, offline `.lodsplat` files and budgeted selection | `splat/lod/*.h` | | Tiles | Tilesets, tile loading and residency-bounded streaming | `splat/tiles/*.h` | | Sorting | Distance order on a background thread, spatial reorder and visibility planning | `splat/sorting/*.h` | | Math | Column major matrices, vectors and frusta shared by every renderer | `splat/math/*.h` | -| Navigation | Collider grid, raycast, character controller | `splat/navigation/*.h` | +| Navigation | Collider grid, raycast, character controller, colliders built from splats | `splat/navigation/*.h` | | Diagnostics | Timing summaries with percentiles | `splat/diagnostics/*.h` | ## Build and test @@ -41,3 +41,19 @@ Neither tool prunes unless asked; the default output is the whole scene. `tools/splat-tile` takes the same `--sh` and `--prune-alpha` before partitioning a scene into streamed tiles. `tools/splat_lod_build` writes an offline `.lodsplat` hierarchy; the [iOS README](../splatkit-ios/README.md) shows its options. The coordinates are written as they are; the reference 3DGS frame is what the decoder assumes for a file without a frame tag, so the scene stands upright. + +## Generating a collider + +Walk mode needs a collider. +For a world shipped without one, `buildCollider` (`splat/navigation/ColliderBuilder.h`) makes it from the splats: a C++ port of the voxel collision passes of [PlayCanvas splat-transform](https://github.com/playcanvas/splat-transform). +Splat opacity is summed into 5 cm voxels, the space outside an enclosed scene is filled, and a 1.6 m walker box is flood-filled from the origin; what it cannot reach becomes solid, so floaters and the far side of walls are gone. +A surface net meshes the result, with vertices moved to where the splats are. +`tools/splat_collider` writes it as a `.glb` in the World Labs frame: + +``` +build/tools/splat_collider kitchen.spz kitchen_collider.glb +``` + +`--voxel`, `--solid-opacity`, `--exterior-fill`, `--floor-fill` (outdoor ground), `--capsule-height`, `--capsule-radius` and `--seed x,y,z` match the options in the header; `--capsule-height 0` keeps every surface. +`--compare reference.glb` reports floor, wall and walk-reach differences against another collider. +The World Labs kitchen (500k splats) builds in about a second on an M-series Mac: its floor is within 1 cm of the collider shipped with it, and the walker reaches no space that collider does not have. diff --git a/packages/splat-core/include/splat/formats/GlbEncoder.h b/packages/splat-core/include/splat/formats/GlbEncoder.h new file mode 100644 index 0000000..13be82b --- /dev/null +++ b/packages/splat-core/include/splat/formats/GlbEncoder.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +#include "splat/core/CoordinateFrame.h" +#include "splat/formats/TriangleMesh.h" + +namespace splat { + +struct GlbEncodeOptions { + // Frame the file is written in. The default matches the splats of a World Labs export, so + // `decodeGlb` with its default options reads the mesh back unchanged. + CoordinateFrame targetFrame = kWorldLabsFrame; +}; + +// Encodes an internal-frame (RUB) mesh as a binary glTF (.glb) with one node, one mesh and +// one indexed triangle primitive: float positions with their bounds and uint32 indices. +std::vector encodeGlb(const TriangleMesh& mesh, const GlbEncodeOptions& options = {}); + +} // namespace splat diff --git a/packages/splat-core/include/splat/navigation/CharacterController.h b/packages/splat-core/include/splat/navigation/CharacterController.h index 63dc78b..8f397bf 100644 --- a/packages/splat-core/include/splat/navigation/CharacterController.h +++ b/packages/splat-core/include/splat/navigation/CharacterController.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "splat/math/Vec3.h" #include "splat/navigation/Collider.h" @@ -12,22 +14,36 @@ struct CharacterSettings { float floorProbeUp = 0.5f; // the floor ray starts this far above the feet; when // that is under the floor, the ray from the eye counts float floorProbeDown = 4.0f; + // The highest rise walked onto, within stepLookAhead of each step: a stair's riser or a + // doorstep, not a chair seat. Shorter than a stair's tread. + float stepHeight = 0.35f; + float stepLookAhead = 0.25f; + // What is stepped over rather than onto: a door track or a sill up to this high over the + // floor and this wide, with a floor at most stepHeight higher past it. + float stepOverHeight = 0.6f; + float stepOverWidth = 0.3f; float snapRate = 12.0f; // per second; eye eases toward floor + eyeHeight }; -// A walking eye: moves on the XZ plane, slides along walls, snaps to the floor and -// refuses to step where the collider has no floor, which makes the collider the -// boundary of the world. Pure state and math; the camera reads `position()`. +// A walking eye: moves on the XZ plane, slides along walls and furniture, snaps to the floor, +// climbs stairs, steps over door tracks and refuses to step where the collider has no floor or +// rises higher, which makes the collider the boundary of the world. Pure state and math; the camera +// reads `position()`. class CharacterController { public: explicit CharacterController(const Collider& collider, CharacterSettings settings = {}); Vec3 position() const { return position_; } - void setPosition(Vec3 p) { position_ = p; } + void setPosition(Vec3 p) { + position_ = p; + stepOver_.reset(); + } const CharacterSettings& settings() const { return settings_; } + // Keeps the position; the next update eases the eye to the new height. + void setSettings(CharacterSettings settings) { settings_ = settings; } // Applies a horizontal step. Vertical components of `delta` are ignored. - // Returns true when the step (or part of it) was taken. + // Returns true when the step, part of it or a slide was taken. bool move(Vec3 delta); // Per frame gravity: eases the eye toward floor + eyeHeight when a floor is below. @@ -37,9 +53,35 @@ class CharacterController { std::optional floorBelow(Vec3 at) const; private: + // A step over a track: the floor past it, which the eye follows meanwhile, and where and + // which way the step over began. + struct StepOver { + float floor; + Vec3 from; + Vec3 direction; + }; + struct Step { + Vec3 next; + std::optional stepOver; + }; + + // Where a horizontal step of `d` ends, after walls, or nothing when it is refused. + std::optional step(Vec3 d) const; + void take(const Step& step); + // The floor stood on: past a track being stepped over, the floor beyond it. + std::optional standingFloor() const; + const Collider& collider_; CharacterSettings settings_; Vec3 position_; + std::optional stepOver_; }; +// An eye position the walker can stand at and leave, nearest `from` in the horizontal plane +// and, in a column of floors, nearest it in height. A collider bounds the world, so a camera +// that starts outside it, or in the solid between two floors, has no floor to stand on and +// every step is refused; this finds it one. Nothing when the collider has no standing room. +std::optional findStandingSpot(const Collider& collider, const CharacterSettings& settings, + Vec3 from); + } // namespace splat diff --git a/packages/splat-core/include/splat/navigation/ColliderBuilder.h b/packages/splat-core/include/splat/navigation/ColliderBuilder.h new file mode 100644 index 0000000..6a15f4b --- /dev/null +++ b/packages/splat-core/include/splat/navigation/ColliderBuilder.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include "splat/core/Result.h" +#include "splat/formats/SplatCloud.h" +#include "splat/formats/TriangleMesh.h" +#include "splat/math/Vec3.h" + +namespace splat { + +struct ColliderBuildOptions { + // Edge of a voxel in meters. Grows when the world would need more than `maxVoxels`. + float voxelSize = 0.05f; + std::size_t maxVoxels = 64'000'000; + // Opacity a voxel must accumulate to be solid, in (0, 1). + float solidOpacity = 0.1f; + // Positions outside these quantiles on any axis are ignored when sizing the grid, so a + // few distant splats do not stretch it. + float boundsQuantile = 0.0001f; + + // A point inside the walkable space. World Labs worlds are captured from their origin. + Vec3 seed{0, 0, 0}; + // Interior scenes: gaps narrower than twice this many meters in the shell around the + // seed are sealed, and everything outside the shell becomes solid. 0 disables it. + float exteriorFillRadius = 1.6f; + // Outdoor scenes: every column is solid from the bottom up to its first surface, with + // holes narrower than twice this many meters closed first. Negative disables it. + float floorFillRadius = -1.0f; + // The walker, as a vertical box: space it can reach from `seed` stays empty and all other + // space becomes solid, which removes floaters and unreachable pockets. Both round to whole + // voxels. Height 0 disables it. + float capsuleHeight = 1.6f; + float capsuleRadius = 0.2f; +}; + +struct ColliderBuildReport { + float voxelSize = 0; + uint32_t dims[3] = {0, 0, 0}; + std::size_t splatsUsed = 0; + std::size_t solidVoxels = 0; + // Whether each pass ran. The exterior fill is skipped when the seed is not enclosed. + bool exteriorFilled = false; + bool floorFilled = false; + bool carved = false; +}; + +// Builds a walk-mode collider from the splats alone, for worlds shipped without one. +// A port of the voxel pipeline of PlayCanvas splat-transform (MIT): splat densities are +// summed per voxel and a voxel whose opacity reaches `solidOpacity` is solid; isolated voxels +// are cleaned; the exterior is sealed, columns are floor-filled and the walkable space is +// carved from `seed` when enabled; a surface net extracts the triangles, in the cloud's frame. +// Returns `corrupt` when there are no opaque splats, nothing solid remains, or the carve is +// enabled and the walker's box fits nowhere near `seed`. +Result buildCollider(const SplatCloud& cloud, + const ColliderBuildOptions& options = {}, + ColliderBuildReport* report = nullptr); + +} // namespace splat diff --git a/packages/splat-core/src/formats/GlbEncoder.cpp b/packages/splat-core/src/formats/GlbEncoder.cpp new file mode 100644 index 0000000..9c3e1c2 --- /dev/null +++ b/packages/splat-core/src/formats/GlbEncoder.cpp @@ -0,0 +1,101 @@ +#include "splat/formats/GlbEncoder.h" + +#include +#include +#include +#include + +#include + +namespace splat { +namespace { + +constexpr uint32_t kGlbMagic = 0x46546C67; // "glTF" +constexpr uint32_t kChunkJson = 0x4E4F534A; +constexpr uint32_t kChunkBin = 0x004E4942; + +void putU32(std::vector& out, uint32_t v) { + const auto* p = reinterpret_cast(&v); + out.insert(out.end(), p, p + 4); +} + +void putBytes(std::vector& out, const void* data, std::size_t size) { + const auto* p = static_cast(data); + out.insert(out.end(), p, p + size); +} + +} // namespace + +std::vector encodeGlb(const TriangleMesh& mesh, const GlbEncodeOptions& options) { + // RUB -> RDF negates Y and Z: a rotation, so triangle winding is unchanged. + const float sign[3] = {1, options.targetFrame == CoordinateFrame::rdf ? -1.0f : 1.0f, + options.targetFrame == CoordinateFrame::rdf ? -1.0f : 1.0f}; + std::vector positions(mesh.positions.size()); + float lo[3]; + float hi[3]; + std::fill(lo, lo + 3, std::numeric_limits::max()); + std::fill(hi, hi + 3, std::numeric_limits::lowest()); + for (std::size_t i = 0; i < positions.size(); ++i) { + const std::size_t axis = i % 3; + positions[i] = mesh.positions[i] * sign[axis]; + lo[axis] = std::min(lo[axis], positions[i]); + hi[axis] = std::max(hi[axis], positions[i]); + } + const std::size_t vertices = mesh.vertexCount(); + if (vertices == 0) { + std::fill(lo, lo + 3, 0.0f); + std::fill(hi, hi + 3, 0.0f); + } + + const std::size_t positionBytes = positions.size() * sizeof(float); + const std::size_t indexBytes = mesh.indices.size() * sizeof(uint32_t); + using Json = nlohmann::json; + const Json doc = { + {"asset", {{"version", "2.0"}, {"generator", "SplatKit collider builder"}}}, + {"scene", 0}, + {"scenes", Json::array({{{"nodes", Json::array({0})}}})}, + {"nodes", Json::array({{{"mesh", 0}}})}, + {"meshes", + Json::array( + {{{"primitives", + Json::array({{{"attributes", {{"POSITION", 0}}}, {"indices", 1}, {"mode", 4}}})}}})}, + {"buffers", Json::array({{{"byteLength", positionBytes + indexBytes}}})}, + {"bufferViews", + Json::array( + {{{"buffer", 0}, {"byteOffset", 0}, {"byteLength", positionBytes}, {"target", 34962}}, + {{"buffer", 0}, + {"byteOffset", positionBytes}, + {"byteLength", indexBytes}, + {"target", 34963}}})}, + {"accessors", Json::array({{{"bufferView", 0}, + {"componentType", 5126}, + {"count", vertices}, + {"type", "VEC3"}, + {"min", {lo[0], lo[1], lo[2]}}, + {"max", {hi[0], hi[1], hi[2]}}}, + {{"bufferView", 1}, + {"componentType", 5125}, + {"count", mesh.indices.size()}, + {"type", "SCALAR"}}})}, + }; + std::string json = doc.dump(); + while (json.size() % 4 != 0) json += ' '; + // Floats and uint32 indices keep the BIN chunk four-byte aligned, as GLB requires. + const std::size_t binBytes = positionBytes + indexBytes; + + std::vector glb; + glb.reserve(12 + 8 + json.size() + 8 + binBytes); + putU32(glb, kGlbMagic); + putU32(glb, 2); + putU32(glb, static_cast(12 + 8 + json.size() + 8 + binBytes)); + putU32(glb, static_cast(json.size())); + putU32(glb, kChunkJson); + putBytes(glb, json.data(), json.size()); + putU32(glb, static_cast(binBytes)); + putU32(glb, kChunkBin); + putBytes(glb, positions.data(), positionBytes); + putBytes(glb, mesh.indices.data(), indexBytes); + return glb; +} + +} // namespace splat diff --git a/packages/splat-core/src/lod/LodFile.cpp b/packages/splat-core/src/lod/LodFile.cpp index bbdee1d..d966aac 100644 --- a/packages/splat-core/src/lod/LodFile.cpp +++ b/packages/splat-core/src/lod/LodFile.cpp @@ -14,7 +14,11 @@ namespace { constexpr std::array kMagic{'L', 'O', 'D', 'S', 'P', 'L', 'A', 'T'}; constexpr size_t kHeader = 64; constexpr size_t kMaxNodes = 20000000; -constexpr uint32_t kMaxDepth = 32; +// The grid builder's finest level is floored at a 2^20 fraction of the extent, so at the +// default base of 1.5 it spans about 35 levels and a real world reaches the mid thirties. +// The cap only bounds a corrupt file: the BFS order checks below already forbid cycles, and +// no renderer descends the tree recursively. It stays inside the byte depths are counted in. +constexpr uint32_t kMaxDepth = 64; Error corrupt(const char* message) { return {ErrorCode::corrupt, message}; } diff --git a/packages/splat-core/src/navigation/CharacterController.cpp b/packages/splat-core/src/navigation/CharacterController.cpp index c25316a..dab6081 100644 --- a/packages/splat-core/src/navigation/CharacterController.cpp +++ b/packages/splat-core/src/navigation/CharacterController.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace splat { @@ -25,9 +26,59 @@ std::optional CharacterController::floorBelow(Vec3 at) const { } bool CharacterController::move(Vec3 delta) { - Vec3 d{delta.x, 0, delta.z}; + const Vec3 d{delta.x, 0, delta.z}; const float len = length(d); if (len < 1e-5f) return false; + if (const auto next = step(d)) { + take(*next); + return true; + } + // Furniture under the hip probe refuses a step instead of stopping it at a wall, so slide + // along it here: take the nearest walkable heading to either side, at the speed of the push + // along it. Walkable on both sides at once, the push is head on and the walker stays put. + const Vec3 dir = d / len; + constexpr float kSlideStep = 15.0f * 3.14159265f / 180; + for (int i = 1; i < 6; ++i) { + const float angle = static_cast(i) * kSlideStep; + const float c = std::cos(angle); + const float s = std::sin(angle); + const auto left = step(Vec3{dir.x * c + dir.z * s, 0, dir.z * c - dir.x * s} * (len * c)); + const auto right = step(Vec3{dir.x * c - dir.z * s, 0, dir.z * c + dir.x * s} * (len * c)); + if (left && right) return false; + if (left || right) { + take(left ? *left : *right); + return true; + } + } + return false; +} + +void CharacterController::take(const Step& step) { + const Vec3 d = step.next - position_; + position_ = step.next; + if (step.stepOver) { + if (!stepOver_) stepOver_ = step.stepOver; + return; + } + if (!stepOver_) return; + // The step over ends on the floor past the track, or when the walker turns back or has + // walked past where the floor was measured without reaching it. + const auto floor = floorBelow(position_); + const Vec3 from = position_ - stepOver_->from; + const float along = dot(Vec3{from.x, 0, from.z}, stepOver_->direction); + if ((floor && std::abs(*floor - stepOver_->floor) < 0.05f) || dot(d, stepOver_->direction) <= 0 || + along > settings_.stepLookAhead + settings_.stepOverWidth) { + stepOver_.reset(); + } +} + +std::optional CharacterController::standingFloor() const { + if (stepOver_) return stepOver_->floor; + return floorBelow(position_); +} + +std::optional CharacterController::step(Vec3 d) const { + const float len = length(d); const Vec3 dir = d / len; // Probe at hip height so low furniture blocks too; slide along whatever we hit. @@ -51,20 +102,130 @@ bool CharacterController::move(Vec3 delta) { d += slide; } } - if (length(d) < 1e-5f) return false; + if (length(d) < 1e-5f) return std::nullopt; + const Vec3 heading = normalize(d); // The collider is also the boundary of the generated world: refuse steps with no floor. const Vec3 next = position_ + d; - if (!floorBelow(next)) return false; - position_ = next; - return true; + const auto nextFloor = floorBelow(next); + if (!nextFloor) return std::nullopt; + // Nor onto one higher than a step: past the edge of a chair seat or a counter under the hip + // probe, the floor probe finds its top. The floor a little ahead counts too, or a walk a + // frame at a time would climb any slope, a chair's rounded side included, in small rises. + const auto floor = standingFloor(); + if (!floor) return Step{next, std::nullopt}; + const float limit = *floor + settings_.stepHeight; + const auto ahead = floorBelow(next + heading * settings_.stepLookAhead); + if (std::max(*nextFloor, ahead.value_or(*nextFloor)) <= limit) return Step{next, std::nullopt}; + // A door track or a sill is stepped over: nothing in the way stands higher than + // stepOverHeight, and within stepOverWidth past the look ahead there is a floor to land on. + const float reach = settings_.stepLookAhead + settings_.stepOverWidth; + const auto landing = floorBelow(next + heading * reach); + if (!landing || *landing > limit) return std::nullopt; + constexpr float kSpacing = 0.02f; + const int samples = static_cast(reach / kSpacing); + for (int i = 0; i <= samples; ++i) { + const auto f = floorBelow(next + heading * (static_cast(i) * kSpacing)); + if (f && *f > *floor + settings_.stepOverHeight) return std::nullopt; + } + return Step{next, StepOver{*landing, next, heading}}; } void CharacterController::update(float dtSeconds) { - if (const auto floor = floorBelow(position_)) { + if (const auto floor = standingFloor()) { const float target = *floor + settings_.eyeHeight; position_.y += (target - position_.y) * std::min(1.0f, dtSeconds * settings_.snapRate); } } +namespace { + +constexpr int kColumnSurfaces = 16; +constexpr int kProbeDirections = 8; +// Enough for a room-sized collider at arm's length spacing; a larger one is searched coarsely +// rather than slowly, since the walker only needs somewhere to stand, not the closest spot. +constexpr int kMaxCandidates = 16384; + +// Every upward facing surface in the column at (x, z), top down. +void columnSurfaces(const Collider& collider, float x, float z, std::vector& out) { + out.clear(); + float y = collider.boundsMax().y + 0.01f; + for (int i = 0; i < kColumnSurfaces; ++i) { + const float span = y - collider.boundsMin().y + 0.02f; + if (span <= 0) break; + const auto hit = collider.raycast({x, y, z}, {0, -1, 0}, span); + if (!hit) break; + // The normal faces the ray, so a floor's points up. + if (hit->normal.y > 0.1f) out.push_back(hit->point.y); + y = hit->point.y - 0.01f; + } +} + +// How many of the compass directions a walker standing at `eye` can step in. +int openDirections(const Collider& collider, const CharacterSettings& settings, Vec3 eye) { + int open = 0; + for (int i = 0; i < kProbeDirections; ++i) { + const float angle = 2.0f * 3.14159265f * static_cast(i) / kProbeDirections; + CharacterController probe(collider, settings); + probe.setPosition(eye); + open += probe.move( + {std::cos(angle) * settings.bodyRadius, 0, std::sin(angle) * settings.bodyRadius}); + } + return open; +} + +} // namespace + +std::optional findStandingSpot(const Collider& collider, const CharacterSettings& settings, + Vec3 from) { + const Vec3 lo = collider.boundsMin(); + const Vec3 hi = collider.boundsMax(); + if (lo.x > hi.x || lo.z > hi.z) return std::nullopt; + const float width = hi.x - lo.x; + const float depth = hi.z - lo.z; + float spacing = std::max(2.0f * settings.bodyRadius, 0.1f); + // Rings out to the far corner, coarser than arm's length only if the collider is vast. + const float reach = std::sqrt(width * width + depth * depth); + const auto ringsFor = [&](float step) { return static_cast(reach / step) + 1; }; + while (ringsFor(spacing) * ringsFor(spacing) * 4 > kMaxCandidates) spacing *= 2.0f; + + // A camera standing over the collider names a place, and the search keeps it. One outside + // names none, so the middle of the collider is the better start than the edge nearest it: + // the walker arrives in the space rather than pressed against its boundary. + const bool over = from.x >= lo.x && from.x <= hi.x && from.z >= lo.z && from.z <= hi.z; + const Vec3 start = over ? from : Vec3{(lo.x + hi.x) * 0.5f, from.y, (lo.z + hi.z) * 0.5f}; + std::vector surfaces; + std::optional fallback; + int fallbackOpen = 0; + for (int ring = 0; ring <= ringsFor(spacing); ++ring) { + for (int ix = -ring; ix <= ring; ++ix) + for (int iz = -ring; iz <= ring; ++iz) { + // Only the new perimeter: the inside of the square was searched by earlier rings. + if (ring > 0 && std::abs(ix) != ring && std::abs(iz) != ring) continue; + const float x = start.x + static_cast(ix) * spacing; + const float z = start.z + static_cast(iz) * spacing; + if (x < lo.x || x > hi.x || z < lo.z || z > hi.z) continue; + columnSurfaces(collider, x, z, surfaces); + // Nearest floor in height first, so a teleport lands on the storey it aimed at. + std::sort(surfaces.begin(), surfaces.end(), [&](float a, float b) { + return std::abs(a + settings.eyeHeight - from.y) < + std::abs(b + settings.eyeHeight - from.y); + }); + for (const float floor : surfaces) { + const Vec3 eye{x, floor + settings.eyeHeight, z}; + const int open = openDirections(collider, settings, eye); + if (open == kProbeDirections) return eye; + if (open > fallbackOpen) { + fallbackOpen = open; + fallback = eye; + } + } + } + // A spot with room on every side beats a nearer one that is half boxed in, but only + // within the ring that found it: no need to search the whole collider for a better one. + if (fallback && ring > 0) return fallback; + } + return fallback; +} + } // namespace splat diff --git a/packages/splat-core/src/navigation/ColliderBuilder.cpp b/packages/splat-core/src/navigation/ColliderBuilder.cpp new file mode 100644 index 0000000..9db8585 --- /dev/null +++ b/packages/splat-core/src/navigation/ColliderBuilder.cpp @@ -0,0 +1,546 @@ +// The voxel passes follow PlayCanvas splat-transform (MIT, see THIRD_PARTY_LICENSES.txt): +// src/lib/gpu/gpu-voxelization.ts, voxel/block-cleanup.ts, fill-exterior.ts, fill-floor.ts +// and carve.ts. That implementation runs on sparse 4x4x4 block grids on the GPU; this one +// runs the same passes on a dense CPU grid. +#include "splat/navigation/ColliderBuilder.h" + +#include +#include +#include +#include +#include +#include + +namespace splat { +namespace { + +// A dense voxel grid. Voxel (x, y, z) spans origin + (x, y, z) * size to one voxel more. +struct Grid { + std::array origin{}; + float size = 0; + std::array n{}; + + std::size_t count() const { + return static_cast(n[0]) * static_cast(n[1]) * + static_cast(n[2]); + } + std::size_t index(int x, int y, int z) const { + return (static_cast(z) * static_cast(n[1]) + + static_cast(y)) * + static_cast(n[0]) + + static_cast(x); + } + std::size_t stride(int axis) const { + return axis == 0 ? 1 + : axis == 1 ? static_cast(n[0]) + : static_cast(n[0]) * static_cast(n[1]); + } + bool contains(int x, int y, int z) const { + return x >= 0 && y >= 0 && z >= 0 && x < n[0] && y < n[1] && z < n[2]; + } + std::array voxelOf(Vec3 p) const { + return {static_cast(std::floor((p.x - origin[0]) / size)), + static_cast(std::floor((p.y - origin[1]) / size)), + static_cast(std::floor((p.z - origin[2]) / size))}; + } +}; + +using Mask = std::vector; + +float quantile(std::vector values, float q) { + const auto at = + static_cast(std::clamp(q, 0.0f, 1.0f) * static_cast(values.size() - 1)); + std::nth_element(values.begin(), values.begin() + static_cast(at), values.end()); + return values[at]; +} + +// Inverse of a symmetric 3x3 stored as xx, xy, xz, yy, yz, zz. False unless it is positive +// definite, so a distance through the inverse is never negative. +bool invertPositiveDefinite(const std::array& m, std::array* inv) { + const double a = m[0]; + const double b = m[1]; + const double c = m[2]; + const double d = m[3]; + const double e = m[4]; + const double f = m[5]; + const double c00 = d * f - e * e; + const double c01 = c * e - b * f; + const double c02 = b * e - c * d; + const double det = a * c00 + b * c01 + c * c02; + if (a <= 0 || a * d - b * b <= 0 || !(det > 1e-300)) return false; + const double s = 1.0 / det; + *inv = {c00 * s, c01 * s, c02 * s, (a * f - c * c) * s, (b * c - a * e) * s, (a * d - b * b) * s}; + return true; +} + +// Separable box dilation: a voxel is set when any voxel within `radius` along each axis is. +// Outside the grid counts as empty. +Mask dilate(const Grid& g, const Mask& source, std::array radius) { + Mask current = source; + Mask next(current.size()); + std::vector prefix; + for (int axis = 0; axis < 3; ++axis) { + const int r = radius[static_cast(axis)]; + if (r <= 0) continue; + const int length = g.n[static_cast(axis)]; + const std::size_t step = g.stride(axis); + prefix.resize(static_cast(length) + 1); + const auto a = static_cast((axis + 1) % 3); + const auto b = static_cast((axis + 2) % 3); + std::array p{}; + for (p[a] = 0; p[a] < g.n[a]; ++p[a]) { + for (p[b] = 0; p[b] < g.n[b]; ++p[b]) { + p[static_cast(axis)] = 0; + const std::size_t start = g.index(p[0], p[1], p[2]); + prefix[0] = 0; + for (std::size_t i = 0; i < static_cast(length); ++i) { + prefix[i + 1] = prefix[i] + current[start + i * step]; + } + for (int i = 0; i < length; ++i) { + const auto from = static_cast(std::max(0, i - r)); + const auto to = static_cast(std::min(length, i + r + 1)); + next[start + static_cast(i) * step] = prefix[to] > prefix[from] ? 1 : 0; + } + } + } + current.swap(next); + } + return current; +} + +// 6-connected flood fill from `seeds` through voxels that are not `blocked`. +Mask flood(const Grid& g, const Mask& blocked, const std::vector& seeds) { + Mask visited(blocked.size(), 0); + std::vector stack; + const auto visit = [&](std::size_t j) { + if (!blocked[j] && !visited[j]) { + visited[j] = 1; + stack.push_back(j); + } + }; + for (const std::size_t s : seeds) visit(s); + const std::size_t sy = g.stride(1); + const std::size_t sz = g.stride(2); + while (!stack.empty()) { + const std::size_t i = stack.back(); + stack.pop_back(); + const auto x = static_cast(i % sy); + const auto y = static_cast((i / sy) % static_cast(g.n[1])); + const auto z = static_cast(i / sz); + if (x > 0) visit(i - 1); + if (x + 1 < g.n[0]) visit(i + 1); + if (y > 0) visit(i - sy); + if (y + 1 < g.n[1]) visit(i + sy); + if (z > 0) visit(i - sz); + if (z + 1 < g.n[2]) visit(i + sz); + } + return visited; +} + +// block-cleanup.ts: drops solid voxels with no solid 6-neighbour and fills empty voxels whose +// six neighbours are all solid. Outside the grid counts as empty. +void cleanup(const Grid& g, Mask& solid) { + Mask next = solid; + for (int z = 0; z < g.n[2]; ++z) { + for (int y = 0; y < g.n[1]; ++y) { + for (int x = 0; x < g.n[0]; ++x) { + const auto at = [&](int dx, int dy, int dz) { + return g.contains(x + dx, y + dy, z + dz) ? solid[g.index(x + dx, y + dy, z + dz)] : 0; + }; + const int neighbours = + at(-1, 0, 0) + at(1, 0, 0) + at(0, -1, 0) + at(0, 1, 0) + at(0, 0, -1) + at(0, 0, 1); + const std::size_t i = g.index(x, y, z); + if (solid[i] && neighbours == 0) next[i] = 0; + if (!solid[i] && neighbours == 6) next[i] = 1; + } + } + } + solid.swap(next); +} + +// The voxel holding `p` when it is free, else the nearest free one within `reach` voxels +// (Chebyshev distance), as in splat-transform's findNearestFreeCell. +std::optional nearestFree(const Grid& g, const Mask& blocked, Vec3 p, int reach) { + const auto s = g.voxelOf(p); + for (int r = 0; r <= reach; ++r) { + for (int dz = -r; dz <= r; ++dz) { + for (int dy = -r; dy <= r; ++dy) { + for (int dx = -r; dx <= r; ++dx) { + if (std::max({std::abs(dx), std::abs(dy), std::abs(dz)}) != r) continue; + const int x = s[0] + dx; + const int y = s[1] + dy; + const int z = s[2] + dz; + if (g.contains(x, y, z) && !blocked[g.index(x, y, z)]) return g.index(x, y, z); + } + } + } + } + return std::nullopt; +} + +// fill-exterior.ts: with gaps narrower than 2r + 1 voxels sealed, floods the space outside +// from the grid faces and turns what it reaches, grown back by r, solid. Skipped when that +// reaches the seed, because then the seed is not enclosed. +bool fillExterior(const Grid& g, Mask& solid, int r, Vec3 seed) { + const auto seedVoxel = nearestFree(g, solid, seed, r); + if (!seedVoxel) return false; + const Mask sealed = dilate(g, solid, {r, r, r}); + std::vector faces; + for (int z = 0; z < g.n[2]; ++z) { + for (int y = 0; y < g.n[1]; ++y) { + for (int x = 0; x < g.n[0]; ++x) { + if (x == 0 || y == 0 || z == 0 || x + 1 == g.n[0] || y + 1 == g.n[1] || z + 1 == g.n[2]) { + faces.push_back(g.index(x, y, z)); + } + } + } + } + const Mask outside = flood(g, sealed, faces); + const Mask grown = dilate(g, outside, {r, r, r}); + // splat-transform checks `outside` at the seed, but a seed within r of a surface is sealed + // itself, so an open scene would be filled over it. The grown outside reaches the free + // voxel nearest the seed exactly when no surface separates them. + if (grown[*seedVoxel]) return false; + for (std::size_t i = 0; i < solid.size(); ++i) solid[i] |= grown[i]; + return true; +} + +// fill-floor.ts: with holes narrower than 2r + 1 voxels closed in XZ, every column is empty +// from the bottom up to its first surface; that space, grown back by r in XZ, becomes solid. +void fillFloor(const Grid& g, Mask& solid, int r) { + const Mask closed = r > 0 ? dilate(g, solid, {r, 0, r}) : solid; + Mask under(solid.size(), 0); + for (int z = 0; z < g.n[2]; ++z) { + for (int x = 0; x < g.n[0]; ++x) { + for (int y = 0; y < g.n[1] && !closed[g.index(x, y, z)]; ++y) under[g.index(x, y, z)] = 1; + } + } + const Mask grown = r > 0 ? dilate(g, under, {r, 0, r}) : under; + for (std::size_t i = 0; i < solid.size(); ++i) solid[i] |= grown[i]; +} + +// carve.ts: the walker's box fits wherever the solid, grown by the box's half size, is empty. +// The placements reachable from the seed (or the nearest free voxel to it), grown back by the +// half size, are the space the walker sweeps; all other space becomes solid. +bool carve(const Grid& g, Mask& solid, int radius, int halfHeight, Vec3 seed) { + const Mask blocked = dilate(g, solid, {radius, halfHeight, radius}); + const auto start = nearestFree(g, blocked, seed, 2 * std::max(radius, halfHeight)); + if (!start) return false; + const Mask swept = dilate(g, flood(g, blocked, {*start}), {radius, halfHeight, radius}); + for (std::size_t i = 0; i < solid.size(); ++i) solid[i] = swept[i] ? 0 : 1; + return true; +} + +// Calls `add(voxel, weight, position)` for every voxel within three sigmas of each splat, with +// the splat's opacity times exp(-d^2 / 2), d being the Mahalanobis distance from its center +// to the nearest point of the voxel. +template +void forEachFootprint(const Grid& g, const SplatCloud& cloud, const std::vector& used, + Add add) { + const float inv = 1.0f / g.size; + for (const uint32_t i : used) { + const float* c = &cloud.covariances[static_cast(i) * 6]; + // A flat splat has a singular covariance, and float rounding leaves a needle's slightly + // indefinite, which would put voxels at a negative distance with infinite density. A + // thickness of a millionth of its size, and at least a micrometer, fixes both. + const double flat = 1e-12 + 1e-6 * (std::abs(c[0]) + std::abs(c[3]) + std::abs(c[5])); + const std::array cov{c[0] + flat, c[1], c[2], c[3] + flat, c[4], c[5] + flat}; + std::array q{}; + if (!invertPositiveDefinite(cov, &q)) continue; + const float* p = &cloud.positions[static_cast(i) * 3]; + const double alpha = std::min(cloud.alphas[i], 1.0f); + int from[3]; + int to[3]; + bool outside = false; + for (std::size_t axis = 0; axis < 3; ++axis) { + const double extent = 3.0 * std::sqrt(std::max(0.0, cov[axis == 0 ? 0 : axis == 1 ? 3 : 5])); + const double at = (p[axis] - g.origin[axis]) * inv; + from[axis] = std::max(0, static_cast(std::floor(at - extent * inv))); + to[axis] = std::min(g.n[axis] - 1, static_cast(std::floor(at + extent * inv))); + outside = outside || from[axis] > to[axis]; + } + if (outside) continue; + for (int z = from[2]; z <= to[2]; ++z) { + const double z0 = g.origin[2] + z * g.size; + const double dz = std::clamp(static_cast(p[2]), z0, z0 + g.size) - p[2]; + for (int y = from[1]; y <= to[1]; ++y) { + const double y0 = g.origin[1] + y * g.size; + const double dy = std::clamp(static_cast(p[1]), y0, y0 + g.size) - p[1]; + const double yz = q[3] * dy * dy + 2 * q[4] * dy * dz + q[5] * dz * dz; + const std::size_t row = g.index(0, y, z); + for (int x = from[0]; x <= to[0]; ++x) { + const double x0 = g.origin[0] + x * g.size; + const double dx = std::clamp(static_cast(p[0]), x0, x0 + g.size) - p[0]; + const double d2 = q[0] * dx * dx + 2 * (q[1] * dx * dy + q[2] * dx * dz) + yz; + if (d2 < 18.0) add(row + static_cast(x), alpha * std::exp(-0.5 * d2), p); + } + } + } + } +} + +// Where the splats that made each solid voxel solid are, on average, by the weight each +// added. A voxel is solid wherever a splat reaches into it, so its faces can lie most of a +// voxel away from the surface; the surface net moves its vertices to these points instead. +// Sorted by voxel. +struct Anchors { + std::vector voxels; + std::vector> points; + + const std::array* find(std::size_t voxel) const { + const auto it = std::lower_bound(voxels.begin(), voxels.end(), voxel); + if (it == voxels.end() || *it != voxel) return nullptr; + return &points[static_cast(it - voxels.begin())]; + } +}; + +Anchors anchor(const Grid& g, const SplatCloud& cloud, const std::vector& used, + const Mask& solid) { + constexpr uint32_t kNone = UINT32_MAX; + std::vector slot(solid.size(), kNone); + Anchors anchors; + for (std::size_t i = 0; i < solid.size(); ++i) { + if (!solid[i]) continue; + slot[i] = static_cast(anchors.voxels.size()); + anchors.voxels.push_back(i); + } + std::vector> sums(anchors.voxels.size(), {0, 0, 0, 0}); + forEachFootprint(g, cloud, used, [&](std::size_t voxel, double weight, const float* p) { + if (slot[voxel] == kNone) return; + auto& sum = sums[slot[voxel]]; + for (std::size_t axis = 0; axis < 3; ++axis) sum[axis] += weight * p[axis]; + sum[3] += weight; + }); + // The cleanup fills a few voxels no splat reaches; they get no anchor. + std::size_t kept = 0; + anchors.points.resize(sums.size()); + for (std::size_t i = 0; i < sums.size(); ++i) { + if (!(sums[i][3] > 0)) continue; + anchors.voxels[kept] = anchors.voxels[i]; + for (std::size_t axis = 0; axis < 3; ++axis) { + anchors.points[kept][axis] = static_cast(sums[i][axis] / sums[i][3]); + } + ++kept; + } + anchors.voxels.resize(kept); + anchors.points.resize(kept); + return anchors; +} + +// Naive surface nets over the solid mask, the grid surrounded by `outside`. Cell +// (x, y, z) has the centers of voxels x - 1..x, y - 1..y, z - 1..z as corners; a mixed cell +// gets one vertex at the mean midpoint of its crossing edges, and every solid-empty pair of +// neighbouring voxels gets a quad joining the four cells around the edge between them. +TriangleMesh surfaceNet(const Grid& g, const Mask& solid, bool outside, const Anchors& anchors) { + const auto inside = [&](int x, int y, int z) { + return g.contains(x, y, z) ? solid[g.index(x, y, z)] != 0 : outside; + }; + const int cx = g.n[0] + 1; + const int cy = g.n[1] + 1; + const int cz = g.n[2] + 1; + const auto cell = [cx](int x, int y) { + return static_cast(y) * static_cast(cx) + static_cast(x); + }; + // Vertex ids of the cells in the previous layer (z - 1) and this one (z). + std::vector below(static_cast(cx) * static_cast(cy)); + std::vector layer(below.size()); + TriangleMesh mesh; + // The four cells are listed in a cycle around the edge; `flip` reverses the cycle so every + // face winds counter-clockwise seen from the empty side. + const auto quad = [&](uint32_t a, uint32_t b, uint32_t c, uint32_t d, bool flip) { + if (flip) std::swap(b, d); + for (const uint32_t v : {a, b, c, a, c, d}) mesh.indices.push_back(v); + }; + constexpr int kCorner[8][3] = {{0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {1, 1, 0}, + {0, 0, 1}, {1, 0, 1}, {0, 1, 1}, {1, 1, 1}}; + constexpr int kEdge[12][2] = {{0, 1}, {2, 3}, {4, 5}, {6, 7}, {0, 2}, {1, 3}, + {4, 6}, {5, 7}, {0, 4}, {1, 5}, {2, 6}, {3, 7}}; + for (int z = 0; z < cz; ++z) { + for (int y = 0; y < cy; ++y) { + for (int x = 0; x < cx; ++x) { + bool corner[8]; + int solidCorners = 0; + for (int k = 0; k < 8; ++k) { + corner[k] = inside(x - 1 + kCorner[k][0], y - 1 + kCorner[k][1], z - 1 + kCorner[k][2]); + solidCorners += corner[k] ? 1 : 0; + } + if (solidCorners == 0 || solidCorners == 8) continue; + layer[cell(x, y)] = static_cast(mesh.vertexCount()); + const int at[3] = {x, y, z}; + // The mean of the solid corners' anchors, kept within the voxels around the cell. + float sum[3] = {0, 0, 0}; + int anchored = 0; + for (int k = 0; k < 8; ++k) { + const int v[3] = {x - 1 + kCorner[k][0], y - 1 + kCorner[k][1], z - 1 + kCorner[k][2]}; + if (!corner[k] || !g.contains(v[0], v[1], v[2])) continue; + const auto* point = anchors.find(g.index(v[0], v[1], v[2])); + if (point == nullptr) continue; + for (std::size_t axis = 0; axis < 3; ++axis) sum[axis] += (*point)[axis]; + ++anchored; + } + if (anchored > 0) { + for (std::size_t axis = 0; axis < 3; ++axis) { + const float lo = g.origin[axis] + static_cast(at[axis] - 1) * g.size; + mesh.positions.push_back( + std::clamp(sum[axis] / static_cast(anchored), lo, lo + 2 * g.size)); + } + continue; + } + // Space the fills or the carve made solid has no splats: the midpoints of the edges + // that cross the surface, averaged. + int crossings = 0; + for (const auto& e : kEdge) { + if (corner[e[0]] == corner[e[1]]) continue; + for (int axis = 0; axis < 3; ++axis) { + sum[axis] += 0.5f * static_cast(kCorner[e[0]][axis] + kCorner[e[1]][axis]); + } + ++crossings; + } + for (std::size_t axis = 0; axis < 3; ++axis) { + // Corner k's voxel center sits at origin + (at - 0.5 + k) * size. + const float offset = sum[axis] / static_cast(crossings); + mesh.positions.push_back(g.origin[axis] + + (static_cast(at[axis]) - 0.5f + offset) * g.size); + } + } + } + // Pairs in voxel plane z - 1, along x and along y: their cells lie in layers z - 1 and z. + const int vz = z - 1; + if (vz >= 0 && vz < g.n[2]) { + for (int vy = 0; vy < g.n[1]; ++vy) { + for (int vx = -1; vx < g.n[0]; ++vx) { + const bool here = inside(vx, vy, vz); + if (here == inside(vx + 1, vy, vz)) continue; + quad(below[cell(vx + 1, vy)], layer[cell(vx + 1, vy)], layer[cell(vx + 1, vy + 1)], + below[cell(vx + 1, vy + 1)], here); + } + } + for (int vy = -1; vy < g.n[1]; ++vy) { + for (int vx = 0; vx < g.n[0]; ++vx) { + const bool here = inside(vx, vy, vz); + if (here == inside(vx, vy + 1, vz)) continue; + quad(below[cell(vx, vy + 1)], below[cell(vx + 1, vy + 1)], layer[cell(vx + 1, vy + 1)], + layer[cell(vx, vy + 1)], here); + } + } + } + // Pairs between voxel planes z - 1 and z: their four cells all lie in layer z. + for (int vy = 0; vy < g.n[1]; ++vy) { + for (int vx = 0; vx < g.n[0]; ++vx) { + const bool here = inside(vx, vy, z - 1); + if (here == inside(vx, vy, z)) continue; + quad(layer[cell(vx, vy)], layer[cell(vx + 1, vy)], layer[cell(vx + 1, vy + 1)], + layer[cell(vx, vy + 1)], !here); + } + } + below.swap(layer); + } + return mesh; +} + +} // namespace + +Result buildCollider(const SplatCloud& cloud, const ColliderBuildOptions& options, + ColliderBuildReport* report) { + if (!(options.voxelSize > 0) || options.solidOpacity <= 0 || options.solidOpacity >= 1 || + options.maxVoxels < 64 || !(options.exteriorFillRadius >= 0) || + !(options.capsuleHeight >= 0) || !(options.capsuleRadius >= 0) || + options.boundsQuantile < 0 || options.boundsQuantile >= 0.5f) { + return Error{ErrorCode::corrupt, "invalid collider build options"}; + } + const std::size_t count = cloud.count(); + if (cloud.covariances.size() < count * 6 || cloud.alphas.size() < count) { + return Error{ErrorCode::corrupt, "splat cloud is missing covariances or alphas"}; + } + std::vector used; + used.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + const float* p = &cloud.positions[i * 3]; + const float* c = &cloud.covariances[i * 6]; + if (!(cloud.alphas[i] > 0)) continue; + if (!std::isfinite(p[0] + p[1] + p[2]) || !std::isfinite(c[0] + c[3] + c[5])) continue; + used.push_back(static_cast(i)); + } + if (used.empty()) return Error{ErrorCode::corrupt, "no splats to build a collider from"}; + + // The grid spans the splats' robust bounds, padded with the empty space the fills need. + std::array lo{}; + std::array hi{}; + for (std::size_t axis = 0; axis < 3; ++axis) { + std::vector values; + values.reserve(used.size()); + for (const uint32_t i : used) values.push_back(cloud.positions[i * 3 + axis]); + lo[axis] = quantile(values, options.boundsQuantile); + hi[axis] = quantile(std::move(values), 1.0f - options.boundsQuantile); + } + const bool exterior = options.exteriorFillRadius > 0; + const bool floor = options.floorFillRadius >= 0; + Grid g; + g.size = options.voxelSize; + int exteriorVoxels = 0; + int floorVoxels = 0; + for (;;) { + exteriorVoxels = + exterior ? static_cast(std::ceil(options.exteriorFillRadius / g.size)) : 0; + floorVoxels = floor ? static_cast(std::ceil(options.floorFillRadius / g.size)) : 0; + // The exterior flood needs room to pass around the scene outside its r-voxel seal, so it + // gets 2r + 1 voxels where splat-transform pads r + 1. + const int padXZ = std::max(2 * exteriorVoxels, floorVoxels) + 1; + const int pad[3] = {padXZ, 2 * exteriorVoxels + 1, padXZ}; + for (std::size_t axis = 0; axis < 3; ++axis) { + g.n[axis] = static_cast(std::ceil((hi[axis] - lo[axis]) / g.size)) + 1 + 2 * pad[axis]; + g.origin[axis] = lo[axis] - static_cast(pad[axis]) * g.size; + } + if (g.count() <= options.maxVoxels) break; + const double over = static_cast(g.count()) / static_cast(options.maxVoxels); + g.size *= std::max(1.02f, static_cast(std::cbrt(over))); + } + + // gpu-voxelization.ts: each splat adds opacity * exp(-d^2 / 2) to a voxel, d being the + // Mahalanobis distance from its center to the nearest point of the voxel, so a splat + // thinner than a voxel still reaches every voxel it passes through. Beer-Lambert turns the + // summed density into opacity. + Mask solid(g.count()); + { + std::vector density(g.count(), 0.0f); + forEachFootprint(g, cloud, used, [&](std::size_t voxel, double weight, const float*) { + density[voxel] += static_cast(weight); + }); + const float solidDensity = -std::log1p(-options.solidOpacity); + for (std::size_t i = 0; i < solid.size(); ++i) solid[i] = density[i] >= solidDensity ? 1 : 0; + } + cleanup(g, solid); + const Anchors anchors = anchor(g, cloud, used, solid); + const bool exteriorFilled = exterior && fillExterior(g, solid, exteriorVoxels, options.seed); + if (floor) fillFloor(g, solid, floorVoxels); + const bool carved = + options.capsuleHeight > 0 && + carve(g, solid, static_cast(std::lround(options.capsuleRadius / g.size)), + static_cast(std::lround(options.capsuleHeight / (2 * g.size))), options.seed); + // Without the carve the mesh would wrap every surface and the grid's outer box, with the + // walker's start inside some solid. + if (options.capsuleHeight > 0 && !carved) { + return Error{ErrorCode::corrupt, "no room for the walker near the collider seed"}; + } + + std::size_t solidCount = 0; + for (const uint8_t v : solid) solidCount += v; + if (report != nullptr) { + report->voxelSize = g.size; + for (std::size_t axis = 0; axis < 3; ++axis) + report->dims[axis] = static_cast(g.n[axis]); + report->splatsUsed = used.size(); + report->solidVoxels = solidCount; + report->exteriorFilled = exteriorFilled; + report->floorFilled = floor; + report->carved = carved; + } + if (solidCount == 0 || solidCount == solid.size()) { + return Error{ErrorCode::corrupt, "the splats leave no surface to collide with"}; + } + // After a carve everything the walker cannot reach is solid, beyond the grid too, so only + // the walkable cavity is meshed rather than also the grid's outer box. + return surfaceNet(g, solid, carved, anchors); +} + +} // namespace splat diff --git a/packages/splat-core/tests/CMakeLists.txt b/packages/splat-core/tests/CMakeLists.txt index 14a2dcf..1517cfa 100644 --- a/packages/splat-core/tests/CMakeLists.txt +++ b/packages/splat-core/tests/CMakeLists.txt @@ -2,7 +2,9 @@ add_executable(splat_core_tests formats/SplatDecoderTest.cpp formats/SpzDecoderTest.cpp formats/GlbDecoderTest.cpp + formats/GlbEncoderTest.cpp navigation/ColliderTest.cpp + navigation/ColliderBuilderTest.cpp math/Mat4Test.cpp math/HalfTest.cpp math/SymmetricEigenTest.cpp diff --git a/packages/splat-core/tests/formats/GlbEncoderTest.cpp b/packages/splat-core/tests/formats/GlbEncoderTest.cpp new file mode 100644 index 0000000..b292d87 --- /dev/null +++ b/packages/splat-core/tests/formats/GlbEncoderTest.cpp @@ -0,0 +1,41 @@ +#include "splat/formats/GlbDecoder.h" +#include "splat/formats/GlbEncoder.h" + +#include + +namespace splat { +namespace { + +TriangleMesh twoTriangles() { + TriangleMesh mesh; + mesh.positions = {0, 0, 0, 1, 0, 0, 1, 2, -3, 0, 2, -3}; + mesh.indices = {0, 1, 2, 0, 2, 3}; + return mesh; +} + +TEST(GlbEncoder, RoundTripsThroughTheDecoderInEitherFrame) { + for (const CoordinateFrame frame : {CoordinateFrame::rdf, CoordinateFrame::rub}) { + const auto glb = encodeGlb(twoTriangles(), {frame}); + EXPECT_EQ(glb.size() % 4, 0u); + GlbDecodeOptions decode; + decode.sourceFrame = frame; + auto decoded = decodeGlb(glb.data(), glb.size(), decode); + ASSERT_TRUE(decoded.ok()) << decoded.error().message; + EXPECT_EQ(decoded.value().positions, twoTriangles().positions); + EXPECT_EQ(decoded.value().indices, twoTriangles().indices); + } +} + +TEST(GlbEncoder, WritesWorldLabsFrameByDefault) { + const auto glb = encodeGlb(twoTriangles()); + GlbDecodeOptions asIs; + asIs.sourceFrame = CoordinateFrame::rub; + auto raw = decodeGlb(glb.data(), glb.size(), asIs); + ASSERT_TRUE(raw.ok()) << raw.error().message; + // Y and Z are negated on disk. + EXPECT_FLOAT_EQ(raw.value().positions[7], -2); + EXPECT_FLOAT_EQ(raw.value().positions[8], 3); +} + +} // namespace +} // namespace splat diff --git a/packages/splat-core/tests/lod/LodTreeTest.cpp b/packages/splat-core/tests/lod/LodTreeTest.cpp index 0622799..413ccf5 100644 --- a/packages/splat-core/tests/lod/LodTreeTest.cpp +++ b/packages/splat-core/tests/lod/LodTreeTest.cpp @@ -4,6 +4,8 @@ #include #include +#include "splat/lod/LodFile.h" + #include using splat::buildLodTree; @@ -171,3 +173,21 @@ TEST(LodTree, LargeRandomCloudBuildsAConnectedTreeOfBoundedSize) { EXPECT_LE(out.size(), 5000u); EXPECT_GT(out.size(), 4000u); } + +// A wide scene made of fine splats needs more grid levels than the validator once allowed, +// and the builder must never produce a tree its own validator rejects. +TEST(LodTree, DeepHierarchyOverAWideSceneStaysValid) { + // Point i sits just inside the cell the origin occupies at level i, so it joins the cluster + // one level later than the point before it and the hierarchy gains a level per point. + std::vector centres{{0, 0, 0}}; + for (int i = 0; i < 40; ++i) + centres.push_back({0.9f * std::pow(1.5f, static_cast(i - 20)), 0, 0}); + const LodTree t = buildLodTree(cloudOf(centres, 1e-5f)); + + const auto valid = splat::validateLodTree(t); + ASSERT_TRUE(valid) << valid.error().message; + EXPECT_GT(valid.value(), 32u); + std::vector leaves; + collectLeaves(t, 0, leaves); + EXPECT_EQ(leaves.size(), centres.size()); +} diff --git a/packages/splat-core/tests/navigation/ColliderBuilderTest.cpp b/packages/splat-core/tests/navigation/ColliderBuilderTest.cpp new file mode 100644 index 0000000..b354f6e --- /dev/null +++ b/packages/splat-core/tests/navigation/ColliderBuilderTest.cpp @@ -0,0 +1,371 @@ +#include "splat/formats/SpzDecoder.h" +#include "splat/navigation/CharacterController.h" +#include "splat/navigation/Collider.h" +#include "splat/navigation/ColliderBuilder.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace splat { +namespace { + +// Flat splats tiling an axis-aligned rectangle, like a trained surface: 3 cm apart, 2 cm wide +// in the plane and 3 mm thick across it. The rectangle spans `fromA`..`toA` on the axis after +// `normalAxis` and `fromB`..`toB` on the one after that, wrapping from Z to X. +void addPlane(SplatCloud& cloud, int normalAxis, float offset, float fromA, float toA, float fromB, + float toB, float alpha = 0.9f) { + constexpr float kStep = 0.03f; + const int a = (normalAxis + 1) % 3; + const int b = (normalAxis + 2) % 3; + const int countA = static_cast(std::floor((toA - fromA) / kStep + 1e-3f)) + 1; + const int countB = static_cast(std::floor((toB - fromB) / kStep + 1e-3f)) + 1; + for (int i = 0; i < countA; ++i) { + for (int j = 0; j < countB; ++j) { + const float u = fromA + static_cast(i) * kStep; + const float v = fromB + static_cast(j) * kStep; + float p[3]; + p[normalAxis] = offset; + p[a] = u; + p[b] = v; + cloud.positions.insert(cloud.positions.end(), p, p + 3); + float variance[3]; + variance[normalAxis] = 0.003f * 0.003f; + variance[a] = variance[b] = 0.02f * 0.02f; + cloud.covariances.insert(cloud.covariances.end(), + {variance[0], 0, 0, variance[1], 0, variance[2]}); + cloud.colors.insert(cloud.colors.end(), {0.5f, 0.5f, 0.5f}); + cloud.alphas.push_back(alpha); + } + } +} + +// A 4 x 4 m floor at y = 0 with a 2 m wall at x = 1.5. +SplatCloud room() { + SplatCloud cloud; + addPlane(cloud, 1, 0.0f, -2, 2, -2, 2); + addPlane(cloud, 0, 1.5f, 0, 2, -2, 2); + return cloud; +} + +// Six times the signed volume enclosed by the mesh; positive when faces wind outward. +double signedVolume6(const TriangleMesh& mesh) { + double sum = 0; + for (std::size_t t = 0; t < mesh.indices.size(); t += 3) { + const float* p = &mesh.positions[mesh.indices[t] * 3]; + const float* q = &mesh.positions[mesh.indices[t + 1] * 3]; + const float* r = &mesh.positions[mesh.indices[t + 2] * 3]; + sum += p[0] * (q[1] * r[2] - q[2] * r[1]) - p[1] * (q[0] * r[2] - q[2] * r[0]) + + p[2] * (q[0] * r[1] - q[1] * r[0]); + } + return sum; +} + +TEST(ColliderBuilder, AFloorOfSplatsIsWalkableAtItsHeight) { + ColliderBuildReport report; + auto built = buildCollider(room(), {}, &report); + ASSERT_TRUE(built.ok()) << built.error().message; + EXPECT_FLOAT_EQ(report.voxelSize, 0.05f); + EXPECT_EQ(report.splatsUsed, room().count()); + // The room is open, so the exterior fill finds the seed from outside and stands down. + EXPECT_FALSE(report.exteriorFilled); + EXPECT_TRUE(report.carved); + const Collider collider(built.value()); + // Everywhere on the floor, away from its edges, a ray from eye height lands on it, where + // the splats are. + for (int i = 0; i <= 8; ++i) { + for (int j = 0; j <= 9; ++j) { + const float x = -1.6f + static_cast(i) * 0.35f; + const float z = -1.6f + static_cast(j) * 0.35f; + const auto hit = collider.raycast({x, 1.6f, z}, {0, -1, 0}, 3); + ASSERT_TRUE(hit) << x << ", " << z; + EXPECT_NEAR(hit->point.y, 0.0f, 0.01f) << x << ", " << z; + } + } +} + +TEST(ColliderBuilder, AWallOfSplatsBlocksAtItsPlace) { + auto built = buildCollider(room()); + ASSERT_TRUE(built.ok()) << built.error().message; + const Collider collider(built.value()); + const auto hit = collider.raycast({0, 1, 0.3f}, {1, 0, 0}, 3); + ASSERT_TRUE(hit); + EXPECT_NEAR(hit->point.x, 1.5f, 0.051f); + EXPECT_NEAR(std::abs(hit->normal.x), 1.0f, 0.2f); +} + +TEST(ColliderBuilder, AFaintNeedleSplatLeavesNoWall) { + // A long, needle-thin splat from Les Tanins, whose covariance float rounding leaves slightly + // indefinite. Taken as it is, the distance to it goes negative and its density infinite. + SplatCloud cloud = room(); + cloud.positions.insert(cloud.positions.end(), {0.0f, 0.8f, 0.0f}); + cloud.covariances.insert(cloud.covariances.end(), {0.105019063f, 0.142086759f, -0.0123928171f, + 0.277978182f, -0.0603894703f, 0.0236564223f}); + cloud.colors.insert(cloud.colors.end(), {0.5f, 0.5f, 0.5f}); + cloud.alphas.push_back(0.03f); + auto built = buildCollider(cloud); + ASSERT_TRUE(built.ok()) << built.error().message; + const Collider collider(built.value()); + for (int i = 0; i <= 8; ++i) { + for (int j = 0; j <= 8; ++j) { + const float x = -1.0f + static_cast(i) * 0.25f; + const float z = -1.0f + static_cast(j) * 0.25f; + const auto hit = collider.raycast({x, 1.6f, z}, {0, -1, 0}, 3); + ASSERT_TRUE(hit) << x << ", " << z; + EXPECT_NEAR(hit->point.y, 0.0f, 0.01f) << x << ", " << z; + } + } +} + +TEST(ColliderBuilder, TheSurfaceIsClosedAndWindsOutward) { + // A solid 0.6 m cube of round splats, voxelized as is. + SplatCloud cloud; + const auto at = [](int i) { return -0.3f + static_cast(i) * 0.03f; }; + for (int i = 0; i <= 20; ++i) { + for (int j = 0; j <= 20; ++j) { + for (int k = 0; k <= 20; ++k) { + cloud.positions.insert(cloud.positions.end(), {at(i), at(j), at(k)}); + cloud.covariances.insert(cloud.covariances.end(), {4e-4f, 0, 0, 4e-4f, 0, 4e-4f}); + cloud.colors.insert(cloud.colors.end(), {0.5f, 0.5f, 0.5f}); + cloud.alphas.push_back(0.9f); + } + } + } + ColliderBuildOptions options; + options.exteriorFillRadius = 0; + options.capsuleHeight = 0; + auto built = buildCollider(cloud, options); + ASSERT_TRUE(built.ok()) << built.error().message; + const TriangleMesh& mesh = built.value(); + // Every edge is shared by exactly two triangles, in opposite directions. + std::size_t unmatched = 0; + std::vector> edges; + for (std::size_t t = 0; t < mesh.indices.size(); t += 3) { + for (int e = 0; e < 3; ++e) { + edges.emplace_back(mesh.indices[t + e], mesh.indices[t + (e + 1) % 3]); + } + } + std::sort(edges.begin(), edges.end()); + for (const auto& [from, to] : edges) { + if (!std::binary_search(edges.begin(), edges.end(), std::make_pair(to, from))) ++unmatched; + } + EXPECT_EQ(unmatched, 0u); + // Outward, and near the outermost splat centers, 0.6 m apart: vertices move toward them + // from the faces of the solid voxels, which reach a voxel or two further, by up to a voxel. + const double volume = signedVolume6(mesh) / 6; + EXPECT_GT(volume, 0.6 * 0.6 * 0.6); + EXPECT_LT(volume, 0.7 * 0.7 * 0.7); +} + +// A closed 4 x 2.5 x 4 m room standing on y = 0. +SplatCloud closedRoom() { + SplatCloud cloud; + addPlane(cloud, 1, 0.0f, -2, 2, -2, 2); + addPlane(cloud, 1, 2.5f, -2, 2, -2, 2); + addPlane(cloud, 0, -2.0f, 0, 2.5f, -2, 2); + addPlane(cloud, 0, 2.0f, 0, 2.5f, -2, 2); + addPlane(cloud, 2, -2.0f, -2, 2, 0, 2.5f); + addPlane(cloud, 2, 2.0f, -2, 2, 0, 2.5f); + return cloud; +} + +TEST(ColliderBuilder, AnEnclosedRoomIsCarvedAroundTheWalker) { + // The seed inside, a faint haze across the room at 1.2 m and an opaque floater out of reach, + // 20 cm under the floor. + SplatCloud cloud = closedRoom(); + addPlane(cloud, 1, 1.2f, -2, 2, -2, 2, 0.002f); + addPlane(cloud, 1, -0.2f, -0.2f, 0.2f, -0.2f, 0.2f); + ColliderBuildOptions options; + options.seed = {0, 1.5f, 0}; + ColliderBuildReport report; + auto built = buildCollider(cloud, options, &report); + ASSERT_TRUE(built.ok()) << built.error().message; + EXPECT_TRUE(report.exteriorFilled); + EXPECT_TRUE(report.carved); + const Collider collider(built.value()); + for (int i = 0; i <= 6; ++i) { + const float x = -1.5f + static_cast(i) * 0.5f; + const auto floorHit = collider.raycast({x, 1.5f, 0.4f}, {0, -1, 0}, 3); + ASSERT_TRUE(floorHit) << x; + EXPECT_NEAR(floorHit->point.y, 0.0f, 0.051f) << x; + const auto ceiling = collider.raycast({x, 1.5f, 0.4f}, {0, 1, 0}, 3); + ASSERT_TRUE(ceiling) << x; + EXPECT_NEAR(ceiling->point.y, 2.5f, 0.051f) << x; + } + // The carved collider is only the cavity: nothing is left beyond the walls. + EXPECT_GE(collider.boundsMin().x, -2.1f); + EXPECT_LE(collider.boundsMax().y, 2.6f); +} + +// Whether the carve reaches the far half of the closed room, through a doorway of `width` in +// a wall across it at z = 0, from a seed in the near half. +bool carvesThroughDoorway(float width, float voxelSize) { + SplatCloud cloud = closedRoom(); + addPlane(cloud, 2, 0.0f, -2, -width / 2, 0, 2.5f); + addPlane(cloud, 2, 0.0f, width / 2, 2, 0, 2.5f); + ColliderBuildOptions options; + options.voxelSize = voxelSize; + options.seed = {0, 1.5f, -1}; + auto built = buildCollider(cloud, options); + EXPECT_TRUE(built.ok()) << built.error().message; + if (!built.ok()) return false; + return Collider(built.value()).raycast({0, 1.5f, 1}, {0, -1, 0}, 3).has_value(); +} + +TEST(ColliderBuilder, TheWalkerFitsThroughADoorwayAtLargerVoxels) { + // Voxels grow past 5 cm in large worlds. The walker's 20 cm radius rounds to whole voxels; + // rounding up made it 24 cm at 6 cm voxels, too wide for a 60 cm doorway. + EXPECT_TRUE(carvesThroughDoorway(0.6f, 0.06f)); + EXPECT_FALSE(carvesThroughDoorway(0.4f, 0.06f)); + EXPECT_TRUE(carvesThroughDoorway(0.6f, 0.05f)); + EXPECT_FALSE(carvesThroughDoorway(0.4f, 0.05f)); +} + +TEST(ColliderBuilder, AWalkerWithNoRoomAtTheSeedIsAnError) { + // Taller than the sealed room, so the box fits nowhere; a mesh of every surface would leave + // the walker inside solid. + ColliderBuildOptions options; + options.seed = {0, 1.5f, 0}; + options.capsuleHeight = 3; + auto built = buildCollider(closedRoom(), options); + ASSERT_FALSE(built.ok()); + EXPECT_EQ(built.error().code, ErrorCode::corrupt); + options.capsuleHeight = 2; + EXPECT_TRUE(buildCollider(closedRoom(), options).ok()); +} + +TEST(ColliderBuilder, FloorFillClosesHolesInTheGround) { + // Ground with a 50 cm hole, which a floor fill of 30 cm radius closes, and a stone a meter + // below one corner so the grid has space under the ground to fill. + SplatCloud ground; + addPlane(ground, 1, 0.0f, -2, 2, -2, 2); + addPlane(ground, 1, -1.0f, -1.9f, -1.8f, -1.9f, -1.8f); + SplatCloud holed; + for (std::size_t i = 0; i < ground.count(); ++i) { + const float* p = &ground.positions[i * 3]; + if (std::abs(p[0] - 0.5f) < 0.25f && std::abs(p[2] - 0.5f) < 0.25f) continue; + holed.positions.insert(holed.positions.end(), p, p + 3); + holed.covariances.insert(holed.covariances.end(), &ground.covariances[i * 6], + &ground.covariances[i * 6] + 6); + holed.colors.insert(holed.colors.end(), &ground.colors[i * 3], &ground.colors[i * 3] + 3); + holed.alphas.push_back(ground.alphas[i]); + } + ColliderBuildOptions options; + options.exteriorFillRadius = 0; + options.capsuleHeight = 0; + const auto depthAtHole = [&](const ColliderBuildOptions& o, ColliderBuildReport* report) { + auto built = buildCollider(holed, o, report); + EXPECT_TRUE(built.ok()); + const Collider collider(built.value()); + const auto hit = collider.raycast({0.5f, 1.6f, 0.5f}, {0, -1, 0}, 3); + return hit ? hit->point.y : -10.0f; + }; + ColliderBuildReport report; + EXPECT_LT(depthAtHole(options, &report), -0.05f); + EXPECT_FALSE(report.floorFilled); + options.floorFillRadius = 0.3f; + EXPECT_NEAR(depthAtHole(options, &report), 0.0f, 0.051f); + EXPECT_TRUE(report.floorFilled); +} + +TEST(ColliderBuilder, VoxelsGrowToStayWithinTheBudget) { + ColliderBuildOptions options; + options.maxVoxels = 200'000; + ColliderBuildReport report; + auto built = buildCollider(room(), options, &report); + ASSERT_TRUE(built.ok()) << built.error().message; + EXPECT_GT(report.voxelSize, 0.05f); + EXPECT_LE(std::size_t{report.dims[0]} * report.dims[1] * report.dims[2], options.maxVoxels); +} + +TEST(ColliderBuilder, NothingOpaqueIsAnError) { + SplatCloud cloud; + addPlane(cloud, 1, 0.0f, -1, 1, -1, 1, 0.0f); + auto built = buildCollider(cloud); + ASSERT_FALSE(built.ok()); + EXPECT_EQ(built.error().code, ErrorCode::corrupt); + EXPECT_FALSE(buildCollider(SplatCloud{}).ok()); + // Splats too faint to make any voxel solid. + SplatCloud faint; + addPlane(faint, 1, 0.0f, -1, 1, -1, 1, 0.001f); + EXPECT_FALSE(buildCollider(faint).ok()); +} + +// Opt-in integration test against a real World Labs export. +// Run with SPLAT_FIXTURES_DIR pointing at a folder containing kitchen_500k.spz. +TEST(ColliderBuilder, TheWorldLabsKitchenIsWalkableFromItsOrigin) { + const char* dir = std::getenv("SPLAT_FIXTURES_DIR"); + if (dir == nullptr) GTEST_SKIP() << "SPLAT_FIXTURES_DIR not set"; + std::ifstream file(std::string(dir) + "/kitchen_500k.spz", std::ios::binary); + ASSERT_TRUE(file.is_open()); + const std::vector bytes((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + auto cloud = decodeSpz(bytes.data(), bytes.size()); + ASSERT_TRUE(cloud.ok()) << cloud.error().message; + auto built = buildCollider(cloud.value()); + ASSERT_TRUE(built.ok()) << built.error().message; + const Collider collider(built.value()); + + // Walks the character in 10 cm steps from the origin, as far as it goes. + constexpr float kCell = 0.1f; + constexpr int kHalf = 60; + const auto cell = [](int x, int z) { + return static_cast(z + kHalf) * (2 * kHalf + 1) + + static_cast(x + kHalf); + }; + std::vector eye(cell(kHalf, kHalf) + 1, NAN); + CharacterController walker(collider); + walker.setPosition({0, 0, 0}); + walker.update(1); + eye[cell(0, 0)] = walker.position().y; + std::deque> queue{{0, 0}}; + std::size_t reached = 1; + while (!queue.empty()) { + const auto [x, z] = queue.front(); + queue.pop_front(); + for (const auto [dx, dz] : + {std::pair{1, 0}, std::pair{-1, 0}, std::pair{0, 1}, std::pair{0, -1}}) { + const int tx = x + dx; + const int tz = z + dz; + if (std::abs(tx) > kHalf || std::abs(tz) > kHalf || !std::isnan(eye[cell(tx, tz)])) continue; + walker.setPosition( + {static_cast(x) * kCell, eye[cell(x, z)], static_cast(z) * kCell}); + if (!walker.move({static_cast(dx) * kCell, 0, static_cast(dz) * kCell})) + continue; + walker.update(1); + const Vec3 at = walker.position(); + // A slide along a wall lands elsewhere; only a full step reaches the cell. + if (std::abs(at.x - static_cast(tx) * kCell) > 0.01f || + std::abs(at.z - static_cast(tz) * kCell) > 0.01f) { + continue; + } + eye[cell(tx, tz)] = at.y; + queue.emplace_back(tx, tz); + ++reached; + } + } + // The kitchen's floor, about a meter below the origin, not its counters: the collider + // shipped with the world gives 5.3 m2, and the walls keep the walk inside. + const float area = static_cast(reached) * kCell * kCell; + EXPECT_GT(area, 3.5f); + EXPECT_LT(area, 5.5f); + for (const float y : eye) { + if (!std::isnan(y)) EXPECT_NEAR(y, -1.0f + 1.5f, 0.25f); + } + // Nothing reaches the grid limits. + for (int i = -kHalf; i <= kHalf; ++i) { + EXPECT_TRUE(std::isnan(eye[cell(i, kHalf)]) && std::isnan(eye[cell(i, -kHalf)]) && + std::isnan(eye[cell(kHalf, i)]) && std::isnan(eye[cell(-kHalf, i)])); + } +} + +} // namespace +} // namespace splat diff --git a/packages/splat-core/tests/navigation/ColliderTest.cpp b/packages/splat-core/tests/navigation/ColliderTest.cpp index 68fb977..40db861 100644 --- a/packages/splat-core/tests/navigation/ColliderTest.cpp +++ b/packages/splat-core/tests/navigation/ColliderTest.cpp @@ -3,6 +3,7 @@ #include +#include #include namespace splat { @@ -135,5 +136,229 @@ TEST(CharacterController, DoesNotMistakeATableForTheFloorWhenStandingNormally) { EXPECT_NEAR(*p.floorBelow(p.position()), 0, 1e-4f); // the floor, not the table } +void addQuad(TriangleMesh& m, Vec3 a, Vec3 b, Vec3 c, Vec3 d) { + const auto base = static_cast(m.vertexCount()); + for (const Vec3 v : {a, b, c, d}) { + m.positions.push_back(v.x); + m.positions.push_back(v.y); + m.positions.push_back(v.z); + } + for (const uint32_t i : {0u, 1u, 2u, 0u, 2u, 3u}) m.indices.push_back(base + i); +} + +TEST(CharacterController, StepsUpAStepButNotOntoACounter) { + // A generated collider bounds the walkable space, so a counter is its front and its top with + // no floor under them. The hip ray passes over a 0.7 m top; past its edge the feet probe + // starts inside the counter and finds nothing below, and the probe from the eye finds the top. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1, 0.7f, -5}, {1, 0.7f, 5}, {1, 0, 5}); + addQuad(m, {1, 0.7f, -5}, {3, 0.7f, -5}, {3, 0.7f, 5}, {1, 0.7f, 5}); + // A 0.18 m stair on the floor side. + addQuad(m, {-3, 0.18f, -1}, {-2, 0.18f, -1}, {-2, 0.18f, 1}, {-3, 0.18f, 1}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0.5f, 1.5f, 0}); + EXPECT_FALSE(p.move({1, 0, 0})); + EXPECT_NEAR(p.position().x, 0.5f, 1e-6f); + // Nor a frame's worth at a time, from its edge. + p.setPosition({0.9f, 1.5f, 0}); + for (int i = 0; i < 100; ++i) p.move({0.02f, 0, 0}); + EXPECT_LT(p.position().x, 1.0f); + EXPECT_NEAR(p.position().y, 1.5f, 1e-4f); + p.setPosition({-1.5f, 1.5f, 0}); + EXPECT_TRUE(p.move({-1, 0, 0})); + p.update(1); + EXPECT_NEAR(p.position().y, 1.68f, 1e-4f); +} + +TEST(CharacterController, SlidesAlongACounterWalkedIntoAtAnAngle) { + // The counter of StepsUpAStepButNotOntoACounter: its top is under the hip probe, so no wall + // is hit, and a step toward it is refused. Walked into diagonally, the walk goes on along it. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1, 0.7f, -5}, {1, 0.7f, 5}, {1, 0, 5}); + addQuad(m, {1, 0.7f, -5}, {3, 0.7f, -5}, {3, 0.7f, 5}, {1, 0.7f, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0.5f, 1.5f, -2}); + for (int i = 0; i < 100; ++i) { + p.move({0.02f, 0, 0.02f}); + p.update(1.0f / 60); + } + EXPECT_LT(p.position().x, 1.0f); + EXPECT_GT(p.position().z, -1.0f); + EXPECT_NEAR(p.position().y, 1.5f, 1e-4f); + // Walked into head on, it stays put rather than drifting to either side. + const Vec3 before = p.position(); + for (int i = 0; i < 100; ++i) p.move({0.02f, 0, 0}); + EXPECT_NEAR(p.position().z, before.z, 1e-4f); + EXPECT_LT(p.position().x, 1.0f); +} + +TEST(CharacterController, ClimbsAStaircase) { + // Risers of 0.18 m and treads of 0.28 m, a slope of 33 degrees taken one step at a time. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + for (int i = 0; i < 8; ++i) { + const float x = 1 + static_cast(i) * 0.28f; + const float top = static_cast(i + 1) * 0.18f; + addQuad(m, {x, top - 0.18f, -5}, {x, top, -5}, {x, top, 5}, {x, top - 0.18f, 5}); + addQuad(m, {x, top, -5}, {x + 0.28f, top, -5}, {x + 0.28f, top, 5}, {x, top, 5}); + } + // A landing at the top. + addQuad(m, {3.24f, 1.44f, -5}, {6, 1.44f, -5}, {6, 1.44f, 5}, {3.24f, 1.44f, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0, 1.5f, 0}); + for (int i = 0; i < 200; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + } + for (int i = 0; i < 120; ++i) p.update(1.0f / 60); + EXPECT_NEAR(p.position().x, 4.0f, 1e-3f); + EXPECT_NEAR(p.position().y, 1.5f + 8 * 0.18f, 0.02f); +} + +TEST(CharacterController, WalksUpARampButNotUpASteepRise) { + // Beyond x = 1: for z < 0 a rise of 0.7 m over 0.3 m to a top the hip ray passes over, the + // front of a counter as splats leave it; for z > 0 a 15 degree ramp. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1.3f, 0.7f, -5}, {1.3f, 0.7f, 0}, {1, 0, 0}); + addQuad(m, {1.3f, 0.7f, -5}, {3, 0.7f, -5}, {3, 0.7f, 0}, {1.3f, 0.7f, 0}); + addQuad(m, {1, 0, 0}, {4, 0.8f, 0}, {4, 0.8f, 5}, {1, 0, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0, 1.5f, -2}); + for (int i = 0; i < 150; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + } + EXPECT_LT(p.position().x, 1.1f); + EXPECT_LT(p.position().y, 1.75f); + p.setPosition({0, 1.5f, 2}); + for (int i = 0; i < 150; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + } + EXPECT_NEAR(p.position().x, 3.0f, 1e-3f); +} + +TEST(CharacterController, DoesNotClimbOntoAChairAndFromThereATable) { + // A generated collider's chair: a 0.45 m seat block with no floor under it, beside a 0.75 m + // table top. Each rise is under the floor probe's 0.5 m, but a chair is no stair. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1, 0.45f, -5}, {1, 0.45f, 5}, {1, 0, 5}); + addQuad(m, {1, 0.45f, -5}, {1.5f, 0.45f, -5}, {1.5f, 0.45f, 5}, {1, 0.45f, 5}); + addQuad(m, {1.5f, 0.45f, -5}, {1.5f, 0.75f, -5}, {1.5f, 0.75f, 5}, {1.5f, 0.45f, 5}); + addQuad(m, {1.5f, 0.75f, -5}, {3, 0.75f, -5}, {3, 0.75f, 5}, {1.5f, 0.75f, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0, 1.5f, 0}); + for (int i = 0; i < 200; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + } + EXPECT_LT(p.position().x, 1.0f); + EXPECT_NEAR(p.position().y, 1.5f, 1e-3f); +} + +TEST(CharacterController, StepsThroughADoorwayOverItsTrack) { + // Les Tanins' garden door: from the paving a 0.29 m step up to the floor inside, with the + // sliding door's track, 0.18 m wide, standing 0.5 m over the paving in between. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1, 0.5f, -5}, {1, 0.5f, 5}, {1, 0, 5}); + addQuad(m, {1, 0.5f, -5}, {1.18f, 0.5f, -5}, {1.18f, 0.5f, 5}, {1, 0.5f, 5}); + addQuad(m, {1.18f, 0.5f, -5}, {1.18f, 0.29f, -5}, {1.18f, 0.29f, 5}, {1.18f, 0.5f, 5}); + addQuad(m, {1.18f, 0.29f, -5}, {6, 0.29f, -5}, {6, 0.29f, 5}, {1.18f, 0.29f, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0, 1.5f, 0}); + float highest = 0; + for (int i = 0; i < 200; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + highest = std::max(highest, p.position().y); + } + for (int i = 0; i < 120; ++i) p.update(1.0f / 60); + EXPECT_NEAR(p.position().x, 4.0f, 1e-3f); + EXPECT_NEAR(p.position().y, 1.79f, 1e-3f); + // The eye rises to the floor inside, not over the track. + EXPECT_LT(highest, 1.8f); +} + +TEST(CharacterController, DoesNotStepOverAPouf) { + // A 0.45 m pouf, 0.45 m across with the floor beyond it: wider than a track, higher than a + // step. Walking into it stops. + TriangleMesh m; + addQuad(m, {-5, 0, -5}, {1, 0, -5}, {1, 0, 5}, {-5, 0, 5}); + addQuad(m, {1, 0, -5}, {1, 0.45f, -5}, {1, 0.45f, 5}, {1, 0, 5}); + addQuad(m, {1, 0.45f, -5}, {1.45f, 0.45f, -5}, {1.45f, 0.45f, 5}, {1, 0.45f, 5}); + addQuad(m, {1.45f, 0.45f, -5}, {1.45f, 0, -5}, {1.45f, 0, 5}, {1.45f, 0.45f, 5}); + addQuad(m, {1.45f, 0, -5}, {6, 0, -5}, {6, 0, 5}, {1.45f, 0, 5}); + const Collider c(m); + CharacterController p(c); + p.setPosition({0, 1.5f, 0}); + for (int i = 0; i < 200; ++i) { + p.move({0.02f, 0, 0}); + p.update(1.0f / 60); + } + EXPECT_LT(p.position().x, 1.0f); + EXPECT_NEAR(p.position().y, 1.5f, 1e-3f); +} + +// A world's origin is its capture point, so a camera often starts where the collider has no +// floor. Every step would be refused there: the walker gets put on the nearest floor instead. +TEST(CharacterController, FindsAFloorWhenTheCameraStartsOffTheCollider) { + const Collider c(room()); + const CharacterSettings settings; + // Well outside the 10 x 10 m room and far above it. + const auto spot = findStandingSpot(c, settings, {40, 25, -40}); + ASSERT_TRUE(spot); + EXPECT_NEAR(spot->y, settings.eyeHeight, 1e-3f); + // A camera that is not over the collider names no place, so the walker arrives in the + // middle of the room rather than pressed against the edge nearest that camera. + EXPECT_LT(std::hypot(spot->x, spot->z), 2.0f); + + CharacterController walker(c, settings); + walker.setPosition(*spot); + ASSERT_TRUE(walker.floorBelow(*spot)); + // Not merely standing: it can leave, in every direction. + for (int i = 0; i < 8; ++i) { + const float angle = 2.0f * 3.14159265f * static_cast(i) / 8; + CharacterController probe(c, settings); + probe.setPosition(*spot); + EXPECT_TRUE(probe.move({std::cos(angle) * 0.3f, 0, std::sin(angle) * 0.3f})) << i; + } +} + +// A camera the host placed on a floor is left where it is. +TEST(CharacterController, KeepsAStartThatAlreadyHasAFloor) { + const Collider c(room()); + const CharacterSettings settings; + const Vec3 standing{1, settings.eyeHeight, 1}; + const auto spot = findStandingSpot(c, settings, standing); + ASSERT_TRUE(spot); + EXPECT_NEAR(spot->x, standing.x, 1e-3f); + EXPECT_NEAR(spot->z, standing.z, 1e-3f); + EXPECT_NEAR(spot->y, standing.y, 1e-3f); +} + +// Nothing to stand on: the caller keeps whatever it had rather than being teleported to junk. +TEST(CharacterController, FindsNothingInAColliderWithNoFloor) { + TriangleMesh wallOnly; + const auto base = static_cast(wallOnly.vertexCount()); + for (const Vec3 v : {Vec3{5, 0, -5}, Vec3{5, 3, -5}, Vec3{5, 3, 5}, Vec3{5, 0, 5}}) { + wallOnly.positions.push_back(v.x); + wallOnly.positions.push_back(v.y); + wallOnly.positions.push_back(v.z); + } + for (const uint32_t i : {0u, 1u, 2u, 0u, 2u, 3u}) wallOnly.indices.push_back(base + i); + EXPECT_FALSE(findStandingSpot(Collider(wallOnly), {}, {0, 0, 0})); +} + } // namespace } // namespace splat diff --git a/packages/splat-core/tools/CMakeLists.txt b/packages/splat-core/tools/CMakeLists.txt index a6123b8..cd52f25 100644 --- a/packages/splat-core/tools/CMakeLists.txt +++ b/packages/splat-core/tools/CMakeLists.txt @@ -14,3 +14,9 @@ target_link_libraries(splat-tile PRIVATE splat_core spz) target_compile_options(splat-tile PRIVATE $<$:-Wall -Wextra -Wpedantic -Werror> ) + +add_executable(splat_collider splat_collider.cpp) +target_link_libraries(splat_collider PRIVATE splat_core) +target_compile_options(splat_collider PRIVATE + $<$:-Wall -Wextra -Wpedantic -Werror> +) diff --git a/packages/splat-core/tools/splat_collider.cpp b/packages/splat-core/tools/splat_collider.cpp new file mode 100644 index 0000000..1569dc4 --- /dev/null +++ b/packages/splat-core/tools/splat_collider.cpp @@ -0,0 +1,347 @@ +// Builds a walk-mode collider (.glb) from a splat file, and optionally scores it against a +// reference collider: how much of the reference floor it covers, how far its floor height +// is off, and how far wall distances differ at hip height. `--map` writes both floor heights +// on a 5 cm grid for plotting. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "splat/formats/GlbDecoder.h" +#include "splat/formats/GlbEncoder.h" +#include "splat/formats/SplatDecoder.h" +#include "splat/io/MappedFile.h" +#include "splat/navigation/CharacterController.h" +#include "splat/navigation/Collider.h" +#include "splat/navigation/ColliderBuilder.h" + +namespace { + +// "x,y,z" in meters. +bool parseVec3(const std::string& text, splat::Vec3* out) { + float v[3]; + const char* at = text.c_str(); + for (int i = 0; i < 3; ++i) { + char* end = nullptr; + v[i] = std::strtof(at, &end); + if (end == at || !std::isfinite(v[i]) || *end != (i < 2 ? ',' : '\0')) return false; + at = end + 1; + } + *out = {v[0], v[1], v[2]}; + return true; +} + +bool parse(const std::string& text, float* value) { + // from_chars for float is missing from older Apple libc++. + char* end = nullptr; + *value = std::strtof(text.c_str(), &end); + return end == text.c_str() + text.size() && std::isfinite(*value); +} + +float percentile(std::vector values, float q) { + if (values.empty()) return NAN; + std::sort(values.begin(), values.end()); + return values[static_cast(q * static_cast(values.size() - 1))]; +} + +std::size_t cellIndex(int x, int z, int nx) { + return static_cast(z) * static_cast(nx) + static_cast(x); +} + +// Walks a character from `start` over a 10 cm grid, four directions at a time, and returns +// the eye height of every cell it can reach, NaN elsewhere. The grid spans `lo`..`hi` on X +// and Z. +std::vector reachable(const splat::Collider& collider, splat::Vec3 start, splat::Vec3 lo, + splat::Vec3 hi, int* nx, int* nz) { + constexpr float kCell = 0.1f; + *nx = static_cast((hi.x - lo.x) / kCell) + 1; + *nz = static_cast((hi.z - lo.z) / kCell) + 1; + std::vector eye(static_cast(*nx) * static_cast(*nz), NAN); + splat::CharacterController walker(collider); + const auto centre = [&](int x, int z) { + return splat::Vec3{lo.x + (static_cast(x) + 0.5f) * kCell, 0, + lo.z + (static_cast(z) + 0.5f) * kCell}; + }; + const int startX = static_cast(std::floor((start.x - lo.x) / kCell)); + const int startZ = static_cast(std::floor((start.z - lo.z) / kCell)); + if (startX < 0 || startZ < 0 || startX >= *nx || startZ >= *nz) return eye; + walker.setPosition(centre(startX, startZ) + splat::Vec3{0, start.y, 0}); + if (!walker.floorBelow(walker.position())) return eye; + walker.update(1); + std::deque> queue{{startX, startZ}}; + eye[cellIndex(startX, startZ, *nx)] = walker.position().y; + constexpr int kSteps[4][2] = {{1, 0}, {-1, 0}, {0, 1}, {0, -1}}; + while (!queue.empty()) { + const auto [x, z] = queue.front(); + queue.pop_front(); + for (const auto& step : kSteps) { + const int tx = x + step[0]; + const int tz = z + step[1]; + if (tx < 0 || tz < 0 || tx >= *nx || tz >= *nz) continue; + const auto index = cellIndex(tx, tz, *nx); + if (!std::isnan(eye[index])) continue; + splat::Vec3 from = centre(x, z); + from.y = eye[cellIndex(x, z, *nx)]; + walker.setPosition(from); + if (!walker.move({step[0] * kCell, 0, step[1] * kCell})) continue; + walker.update(1); + const splat::Vec3 to = centre(tx, tz); + const splat::Vec3 at = walker.position(); + // A slide along a wall lands elsewhere; only a full step reaches the cell. + if (std::abs(at.x - to.x) > 0.01f || std::abs(at.z - to.z) > 0.01f) continue; + eye[index] = at.y; + queue.emplace_back(tx, tz); + } + } + return eye; +} + +// Samples a grid over the reference's horizontal bounds. At each point where the reference +// has a floor below the eye (at the height of `start`, the seed), the candidate must have one +// too, at a similar height; from there, rays at hip height in eight directions must find walls +// at similar distances. Then both are walked from `start`. +void compare(const splat::Collider& candidate, const splat::Collider& reference, splat::Vec3 start, + const char* mapPath) { + const float kStep = mapPath != nullptr ? 0.05f : 0.2f; + constexpr float kProbeDown = 4.0f; + constexpr float kHip = 0.9f; + constexpr float kWallReach = 5.0f; + std::size_t floors = 0; + std::size_t covered = 0; + std::size_t extra = 0; + std::size_t samples = 0; + std::size_t wallRays = 0; + std::size_t wallMissing = 0; + std::vector floorError; + std::vector wallError; + const splat::Vec3 lo = reference.boundsMin(); + const splat::Vec3 hi = reference.boundsMax(); + FILE* map = mapPath != nullptr ? std::fopen(mapPath, "w") : nullptr; + if (map != nullptr) std::fprintf(map, "x,z,reference,candidate\n"); + const int columns = static_cast((hi.x - lo.x) / kStep); + const int rows = static_cast((hi.z - lo.z) / kStep); + for (int i = 0; i < columns; ++i) { + for (int j = 0; j < rows; ++j) { + const float x = lo.x + (static_cast(i) + 0.5f) * kStep; + const float z = lo.z + (static_cast(j) + 0.5f) * kStep; + ++samples; + const splat::Vec3 eye{x, start.y, z}; + const auto want = reference.raycast(eye, {0, -1, 0}, kProbeDown); + const auto got = candidate.raycast(eye, {0, -1, 0}, kProbeDown); + if (map != nullptr) { + std::fprintf(map, "%.3f,%.3f,%.3f,%.3f\n", x, z, want ? want->point.y : NAN, + got ? got->point.y : NAN); + } + if (!want) { + extra += got ? 1 : 0; + continue; + } + ++floors; + if (!got) continue; + ++covered; + floorError.push_back(got->point.y - want->point.y); + const splat::Vec3 hip{x, want->point.y + kHip, z}; + for (int d = 0; d < 8; ++d) { + const float angle = static_cast(d) * 3.14159265f / 4; + const splat::Vec3 dir{std::cos(angle), 0, std::sin(angle)}; + const auto wall = reference.raycast(hip, dir, kWallReach); + if (!wall) continue; + ++wallRays; + const auto seen = candidate.raycast(hip, dir, kWallReach); + if (!seen) { + ++wallMissing; + continue; + } + wallError.push_back(seen->distance - wall->distance); + } + } + } + if (map != nullptr) std::fclose(map); + + int nx = 0; + int nz = 0; + const auto want = reachable(reference, start, lo, hi, &nx, &nz); + const auto got = reachable(candidate, start, lo, hi, &nx, &nz); + std::size_t both = 0; + std::size_t onlyWant = 0; + std::size_t onlyGot = 0; + std::vector eyeError; + if (mapPath != nullptr) { + FILE* walk = std::fopen((std::string(mapPath) + ".walk.csv").c_str(), "w"); + std::fprintf(walk, "x,z,reference,candidate\n"); + for (int z = 0; z < nz; ++z) { + for (int x = 0; x < nx; ++x) { + const auto i = cellIndex(x, z, nx); + std::fprintf(walk, "%.3f,%.3f,%.3f,%.3f\n", lo.x + (x + 0.5f) * 0.1f, + lo.z + (z + 0.5f) * 0.1f, want[i], got[i]); + } + } + std::fclose(walk); + } + for (std::size_t i = 0; i < want.size(); ++i) { + const bool w = !std::isnan(want[i]); + const bool g = !std::isnan(got[i]); + both += w && g; + onlyWant += w && !g; + onlyGot += !w && g; + if (w && g) eyeError.push_back(std::abs(got[i] - want[i])); + } + std::vector floorAbs(floorError.size()); + std::vector wallAbs(wallError.size()); + std::transform(floorError.begin(), floorError.end(), floorAbs.begin(), + [](float v) { return std::abs(v); }); + std::transform(wallError.begin(), wallError.end(), wallAbs.begin(), + [](float v) { return std::abs(v); }); + std::printf("compare: %zu samples, reference floor at %zu\n", samples, floors); + std::printf(" floor coverage %.1f%%, floor where the reference has none %zu\n", + floors ? 100.0 * static_cast(covered) / static_cast(floors) : 0.0, + extra); + std::printf(" floor height error: median %+.3f m, |err| p50 %.3f p90 %.3f p99 %.3f m\n", + percentile(floorError, 0.5f), percentile(floorAbs, 0.5f), percentile(floorAbs, 0.9f), + percentile(floorAbs, 0.99f)); + std::printf( + " walls: %zu rays, %.1f%% missed, distance error median %+.3f m, |err| p50 %.3f p90 %.3f " + "m\n", + wallRays, + wallRays ? 100.0 * static_cast(wallMissing) / static_cast(wallRays) : 0.0, + percentile(wallError, 0.5f), percentile(wallAbs, 0.5f), percentile(wallAbs, 0.9f)); + std::printf( + " walking from the seed: reference reaches %.2f m2, candidate %.1f%% of it, plus %.2f m2 " + "more;\n" + " eye height |err| p50 %.3f p90 %.3f m\n", + static_cast(both + onlyWant) * 0.01, + both + onlyWant ? 100.0 * static_cast(both) / static_cast(both + onlyWant) + : 0.0, + static_cast(onlyGot) * 0.01, percentile(eyeError, 0.5f), percentile(eyeError, 0.9f)); +} + +std::optional readGlb(const char* path) { + auto mapped = splat::MappedFile::open(path); + if (!mapped) { + std::fprintf(stderr, "%s: %s\n", path, mapped.error().message.c_str()); + return std::nullopt; + } + auto mesh = splat::decodeGlb(mapped.value().data(), mapped.value().size()); + if (!mesh) { + std::fprintf(stderr, "%s: %s\n", path, mesh.error().message.c_str()); + return std::nullopt; + } + return std::move(mesh.value()); +} + +int usage() { + std::fprintf(stderr, + "usage: splat_collider input.spz output.glb [--voxel 0.05] [--max-voxels N]\n" + " [--solid-opacity 0.1] [--bounds-quantile 0.0001] [--seed 0,0,0]\n" + " [--exterior-fill 1.6] [--floor-fill -1] [--capsule-height 1.6]\n" + " [--capsule-radius 0.2] [--compare reference.glb [--map heights.csv]]\n"); + return 2; +} + +int run(int argc, char** argv) { + if (argc < 3 || (argc - 3) % 2 != 0) return usage(); + splat::ColliderBuildOptions options; + const char* reference = nullptr; + const char* mapPath = nullptr; + for (int i = 3; i < argc; i += 2) { + const std::string key(argv[i]); + if (key == "--compare") { + reference = argv[i + 1]; + continue; + } + if (key == "--map") { + mapPath = argv[i + 1]; + continue; + } + if (key == "--seed") { + if (!parseVec3(argv[i + 1], &options.seed)) return usage(); + continue; + } + float value = 0; + if (!parse(argv[i + 1], &value)) return usage(); + if (key == "--voxel" && value > 0) + options.voxelSize = value; + else if (key == "--max-voxels" && value >= 64) + options.maxVoxels = static_cast(value); + else if (key == "--solid-opacity" && value > 0 && value < 1) + options.solidOpacity = value; + else if (key == "--bounds-quantile" && value >= 0 && value < 0.5f) + options.boundsQuantile = value; + else if (key == "--exterior-fill" && value >= 0) + options.exteriorFillRadius = value; + else if (key == "--floor-fill") + options.floorFillRadius = value; + else if (key == "--capsule-height" && value >= 0) + options.capsuleHeight = value; + else if (key == "--capsule-radius" && value >= 0) + options.capsuleRadius = value; + else + return usage(); + } + + const auto start = std::chrono::steady_clock::now(); + const auto seconds = [&] { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + }; + auto cloud = [&]() -> splat::Result { + auto mapped = splat::MappedFile::open(argv[1]); + if (!mapped) return mapped.error(); + return splat::decodeSplatFile(mapped.value().data(), mapped.value().size()); + }(); + if (!cloud) { + std::fprintf(stderr, "%s\n", cloud.error().message.c_str()); + return 1; + } + const double decoded = seconds(); + splat::ColliderBuildReport report; + auto mesh = splat::buildCollider(cloud.value(), options, &report); + if (!mesh) { + std::fprintf(stderr, "%s\n", mesh.error().message.c_str()); + return 1; + } + std::printf("%zu splats (%zu used), %ux%ux%u voxels of %.3f m, %zu solid\n", + cloud.value().count(), report.splatsUsed, report.dims[0], report.dims[1], + report.dims[2], report.voxelSize, report.solidVoxels); + std::printf("exterior fill %s, floor fill %s, carve %s\n", + report.exteriorFilled ? "ran" : "skipped", report.floorFilled ? "ran" : "skipped", + report.carved ? "ran" : "skipped"); + std::printf("%zu triangles, %zu vertices; decode %.2f s, build %.2f s\n", + mesh.value().triangleCount(), mesh.value().vertexCount(), decoded, + seconds() - decoded); + const auto glb = splat::encodeGlb(mesh.value()); + std::ofstream out(argv[2], std::ios::binary); + out.write(reinterpret_cast(glb.data()), static_cast(glb.size())); + if (!out) { + std::fprintf(stderr, "cannot write %s\n", argv[2]); + return 1; + } + out.close(); + std::printf("wrote %.1f MB: %s\n", static_cast(glb.size()) / 1e6, argv[2]); + + if (reference != nullptr) { + const auto written = readGlb(argv[2]); + const auto wanted = readGlb(reference); + if (!written || !wanted) return 1; + std::printf("reference: %zu triangles\n", wanted->triangleCount()); + compare(splat::Collider(*written), splat::Collider(*wanted), options.seed, mapPath); + } + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + try { + return run(argc, argv); + } catch (const std::exception& e) { + std::fprintf(stderr, "splat_collider failed: %s\n", e.what()); + return 1; + } +} diff --git a/packages/splatkit-android/README.md b/packages/splatkit-android/README.md index 43c764a..a78d11a 100644 --- a/packages/splatkit-android/README.md +++ b/packages/splatkit-android/README.md @@ -7,12 +7,13 @@ An arm64 emulator can test functionality; it is not a phone performance measurem ## Install -Maven Central `0.1.0-alpha07` adds the render policy, `onWorldFrameReady`, the budgeted `loadWorld` and 16 KB alignment; see the [changelog](../../CHANGELOG.md) and [releases](https://github.com/Xget7/splatkit-android/releases). +Maven Central `0.1.0-alpha08` hands walking to the host: the view looks only, and `walkVelocity`, the character settings and the collider come from the app. +See the [changelog](../../CHANGELOG.md) and [releases](https://github.com/Xget7/splatkit-android/releases). Avoid alpha05, which corrupts Adreno sorting. ```kotlin dependencies { - implementation("io.github.xget7:splatkit-android:0.1.0-alpha07") + implementation("io.github.xget7:splatkit-android:0.1.0-alpha08") } ``` @@ -71,8 +72,12 @@ The [React Native package](../react-native-splatkit/README.md) maps its `policy` | `residencyBudget` | Resident splats for subsequent streamed worlds, separate from LOD selection. | | `cullMarginDegrees` | CPU fallback's angular margin; GPU visibility uses current-camera projected bounds. | | `linearBlending` | Optional linear-light blend; not the default trained-space compositing. | -| `setMotionEnabled`, `setWalkVelocity` | Gyroscope and continuous forward/right velocity. | -| `lookSensitivity`, `walkSensitivity` | Gesture tuning. | +| `setMotionEnabled`, `setWalkVelocity`, `walk` | Gyroscope, continuous forward/right velocity in meters per second, and a single step in meters. | +| `look(deltaYaw, deltaPitch)` | Turns the camera by radians, for a look pad or mouse. | +| `touchLookEnabled`, `lookSensitivity` | Whether a one-finger drag turns the camera, and radians per pixel dragged. | +| `motionToggleEnabled` | Whether a double tap toggles the gyroscope. | +| `setCharacter(CharacterSettings)`, `character` | The walker's shape in walk mode: `eyeHeight`, `bodyRadius`, `stepHeight`. | +| `cameraPoseIntervalMillis`, `cameraPoseListener` | Milliseconds between pose callbacks, and the callback itself; 0, the default, never reports. | | `isAvailable`, `gpuDescription` | Renderer availability and driver description. | | `applyRenderPolicy(policy)`, `renderPolicy`, `deviceCapabilities` | Per-instance renderer policy, re-validated on the render thread. Only `sortDepth` and `subpixelThreshold` apply, with GPU visibility; other fields fall back with a warning each. | | `readStats()` | FPS, frame/GPU/sort ms, loaded/drawn and screen-tile counts. Drawn counts and GPU ms describe the newest frame the GPU finished, also while nothing redraws, and are current when `onWorldFrameReady` fires. | diff --git a/packages/splatkit-android/build.gradle.kts b/packages/splatkit-android/build.gradle.kts index cea6550..126633d 100644 --- a/packages/splatkit-android/build.gradle.kts +++ b/packages/splatkit-android/build.gradle.kts @@ -46,7 +46,7 @@ dependencies { // publishing machine (ORG_GRADLE_PROJECT_mavenCentralUsername, mavenCentralPassword, // signingInMemoryKey, signingInMemoryKeyPassword); local builds need none of it. mavenPublishing { - coordinates("io.github.xget7", "splatkit-android", "0.1.0-alpha07") + coordinates("io.github.xget7", "splatkit-android", "0.1.0-alpha08") publishToMavenCentral(automaticRelease = true) if (project.findProperty("signingInMemoryKey") != null) signAllPublications() pom { diff --git a/packages/splatkit-android/src/main/cpp/jni/SplatKitJni.cpp b/packages/splatkit-android/src/main/cpp/jni/SplatKitJni.cpp index 16801f3..bbf530a 100644 --- a/packages/splatkit-android/src/main/cpp/jni/SplatKitJni.cpp +++ b/packages/splatkit-android/src/main/cpp/jni/SplatKitJni.cpp @@ -225,6 +225,17 @@ SPLATKIT_JNI(void, nativeSetVelocity) if (auto* engine = toEngine(handle)) engine->setVelocity(forward, right); } +SPLATKIT_JNI(jboolean, nativeSetCharacter) +(JNIEnv*, jobject, jlong handle, jfloat eyeHeight, jfloat bodyRadius, jfloat stepHeight) { + auto* engine = toEngine(handle); + if (engine == nullptr) return JNI_FALSE; + splat::CharacterSettings character = engine->character(); + character.eyeHeight = eyeHeight; + character.bodyRadius = bodyRadius; + character.stepHeight = stepHeight; + return engine->setCharacter(character) ? JNI_TRUE : JNI_FALSE; +} + // `rowMajor` is the 3x3 device to reference rotation as Android hands it out. SPLATKIT_JNI(void, nativeSetAttitude)(JNIEnv* env, jobject, jlong handle, jfloatArray rowMajor) { auto* engine = toEngine(handle); diff --git a/packages/splatkit-android/src/main/java/com/splatkit/CharacterSettings.kt b/packages/splatkit-android/src/main/java/com/splatkit/CharacterSettings.kt new file mode 100644 index 0000000..e7e6758 --- /dev/null +++ b/packages/splatkit-android/src/main/java/com/splatkit/CharacterSettings.kt @@ -0,0 +1,13 @@ +package com.splatkit + +/** + * The walker in walk mode, in meters. The defaults are a standing adult: the eye 1.5 m over + * the floor, 0.35 m kept from walls, and a 0.35 m rise walked onto, which climbs stairs and + * doorsteps but not chairs or counters. A step onto anything higher is refused and the walker + * slides along it instead. + */ +data class CharacterSettings( + val eyeHeight: Float = 1.5f, + val bodyRadius: Float = 0.35f, + val stepHeight: Float = 0.35f, +) diff --git a/packages/splatkit-android/src/main/java/com/splatkit/SplatSurfaceView.kt b/packages/splatkit-android/src/main/java/com/splatkit/SplatSurfaceView.kt index 03fbd48..604498f 100644 --- a/packages/splatkit-android/src/main/java/com/splatkit/SplatSurfaceView.kt +++ b/packages/splatkit-android/src/main/java/com/splatkit/SplatSurfaceView.kt @@ -1,6 +1,8 @@ package com.splatkit import android.content.Context +import android.os.Handler +import android.os.Looper import android.util.AttributeSet import android.util.Log import android.view.MotionEvent @@ -36,8 +38,11 @@ internal fun dispatchSplatEvent( * surface lifecycle: the engine gets the surface when Android creates it and gives it * back, synchronously, before Android destroys it. * - * Gestures: one finger drags the view (yaw, and pitch when the gyroscope is off); - * two fingers walk (up is forward, sideways strafes); a double tap toggles the gyroscope. + * Gestures: a finger drags the view to look (yaw, and pitch when the gyroscope is off) + * and a double tap toggles the gyroscope; both can be turned off. The view ships no + * walking control: the host draws its own, wherever it likes, and drives [setWalkVelocity] + * or [walk] from it. [com.splatkit.ui.JoystickView] is one such control, for a host that + * wants a ready-made one. */ class SplatSurfaceView @JvmOverloads constructor( context: Context, @@ -48,21 +53,60 @@ class SplatSurfaceView @JvmOverloads constructor( private val motion = MotionInput(context, renderThread.renderHandler) { renderThread.setAttitude(it) } private val touch = TouchInput(object : TouchInput.Listener { override fun onLook(deltaYaw: Float, deltaPitch: Float) = renderThread.look(deltaYaw, deltaPitch) - override fun onWalk(forward: Float, right: Float) = renderThread.walk(forward, right) - override fun onDoubleTap() = setMotionEnabled(!motionEnabled) + override fun onDoubleTap() { + if (motionToggleEnabled) setMotionEnabled(!motionEnabled) + } }) private var motionEnabled = false private var resumed = false + private val poseHandler = Handler(Looper.getMainLooper()) + private var lastPose: CameraPose? = null + private val reportPose = object : Runnable { + override fun run() { + val pose = cameraPose + if (pose != lastPose) { + lastPose = pose + cameraPoseListener?.invoke(pose) + } + poseHandler.postDelayed(this, cameraPoseIntervalMillis) + } + } /** Radians per pixel dragged. */ var lookSensitivity: Float get() = touch.lookSensitivity set(value) { touch.lookSensitivity = value } - /** Meters per pixel dragged with two fingers. */ - var walkSensitivity: Float - get() = touch.walkSensitivity - set(value) { touch.walkSensitivity = value } + /** + * Whether a drag on the view is allowed to turn the camera. A host that drives looking + * from its own control, and scripted tours, turn it off. + */ + var touchLookEnabled: Boolean + get() = touch.lookEnabled + set(value) { + touch.lookEnabled = value + if (!value) touch.letGo() + } + + /** Whether the double tap can toggle the gyroscope. */ + var motionToggleEnabled = true + + /** + * Where the camera is, on the main thread, at most this often in milliseconds, and only + * when it differs from the pose last delivered. Zero, the default, never reports. + */ + var cameraPoseIntervalMillis: Long = 0 + set(value) { + field = value.coerceAtLeast(0) + startPoseReports() + } + + /** Called with each pose [cameraPoseIntervalMillis] asks for. */ + var cameraPoseListener: ((CameraPose) -> Unit)? = null + set(value) { + field = value + startPoseReports() + } init { holder.addCallback(this) @@ -224,9 +268,39 @@ class SplatSurfaceView @JvmOverloads constructor( renderThread.setMaxShDegree(field) } - /** Walks continuously at the given speed in meters per second until called again with zeros. */ + /** + * Walks continuously at the given speed in meters per second until called again with + * zeros: what a joystick or a keyboard drives. Forward is where the camera looks, + * flattened onto the floor while walking; right strafes. + */ fun setWalkVelocity(forward: Float, right: Float) = renderThread.setVelocity(forward, right) + /** + * One step, in meters, for a host that integrates movement itself. The collider stops + * it at walls and the floor carries it, exactly as a velocity would. + */ + fun walk(forward: Float, right: Float) = renderThread.walk(forward, right) + + /** + * Turns the camera by these radians: what a look pad or a mouse drives. Pitch is + * clamped, and ignored while the gyroscope drives the view. + */ + fun look(deltaYaw: Float, deltaPitch: Float) = renderThread.look(deltaYaw, deltaPitch) + + /** + * The walker's shape in walk mode, applied at once and to a collider loaded later. + * False when a value is not a walkable one, and then the previous settings stay. + */ + fun setCharacter(settings: CharacterSettings): Boolean { + val accepted = renderThread.setCharacter(settings) + if (accepted) character = settings + return accepted + } + + /** The walker's shape in effect. */ + var character = CharacterSettings() + private set + /** * Runs a reproducible capture: the gyroscope goes off, the camera takes a fixed pose * and turns once over [seconds], then the frame time distribution is logged. @@ -275,20 +349,30 @@ class SplatSurfaceView @JvmOverloads constructor( motion.displayRotation = display?.rotation ?: 0 renderThread.resume() if (motionEnabled) motion.start() + startPoseReports() } fun pause() { resumed = false motion.stop() renderThread.pause() + poseHandler.removeCallbacks(reportPose) } fun release() { motion.stop() + poseHandler.removeCallbacks(reportPose) holder.removeCallback(this) renderThread.release() } + private fun startPoseReports() { + poseHandler.removeCallbacks(reportPose) + if (resumed && cameraPoseIntervalMillis > 0 && cameraPoseListener != null) { + poseHandler.postDelayed(reportPose, cameraPoseIntervalMillis) + } + } + override fun surfaceCreated(holder: SurfaceHolder) { motion.displayRotation = display?.rotation ?: 0 renderThread.surfaceCreated(holder.surface) diff --git a/packages/splatkit-android/src/main/java/com/splatkit/engine/RenderThread.kt b/packages/splatkit-android/src/main/java/com/splatkit/engine/RenderThread.kt index c7ee260..1bdd3c9 100644 --- a/packages/splatkit-android/src/main/java/com/splatkit/engine/RenderThread.kt +++ b/packages/splatkit-android/src/main/java/com/splatkit/engine/RenderThread.kt @@ -7,6 +7,7 @@ import android.util.Log import android.view.Choreographer import android.view.Surface import com.splatkit.CameraPose +import com.splatkit.CharacterSettings import com.splatkit.DeviceCapabilities import com.splatkit.RenderPolicy import com.splatkit.RenderPolicyResolution @@ -137,6 +138,14 @@ internal class RenderThread { fun walk(forward: Float, right: Float) = post { engine?.walk(forward, right) } fun setVelocity(forward: Float, right: Float) = post { engine?.setVelocity(forward, right) } fun setAttitude(rowMajor: FloatArray) = post { engine?.setAttitude(rowMajor) } + + /** Waits: the caller learns whether the settings were walkable. */ + fun setCharacter(settings: CharacterSettings): Boolean { + var accepted = false + runBlockingOnThread { accepted = engine?.setCharacter(settings) ?: false } + return accepted + } + fun setMotionEnabled(enabled: Boolean) = post { engine?.setMotionEnabled(enabled) } // Quality settings. diff --git a/packages/splatkit-android/src/main/java/com/splatkit/engine/SplatEngine.kt b/packages/splatkit-android/src/main/java/com/splatkit/engine/SplatEngine.kt index 8da0495..54856d3 100644 --- a/packages/splatkit-android/src/main/java/com/splatkit/engine/SplatEngine.kt +++ b/packages/splatkit-android/src/main/java/com/splatkit/engine/SplatEngine.kt @@ -3,6 +3,7 @@ package com.splatkit.engine import android.util.Log import android.view.Surface import com.splatkit.CameraPose +import com.splatkit.CharacterSettings import com.splatkit.DeviceCapabilities import com.splatkit.RenderPolicy import com.splatkit.SplatStats @@ -100,6 +101,10 @@ internal class SplatEngine { fun look(deltaYaw: Float, deltaPitch: Float) = nativeLook(handle, deltaYaw, deltaPitch) fun walk(forward: Float, right: Float) = nativeWalk(handle, forward, right) fun setVelocity(forward: Float, right: Float) = nativeSetVelocity(handle, forward, right) + /** False when a value is not a walkable one; the previous settings stay. */ + fun setCharacter(settings: CharacterSettings): Boolean = + nativeSetCharacter(handle, settings.eyeHeight, settings.bodyRadius, settings.stepHeight) + fun setAttitude(rowMajor: FloatArray) = nativeSetAttitude(handle, rowMajor) fun setMotionEnabled(enabled: Boolean) = nativeSetMotionEnabled(handle, enabled) @@ -183,6 +188,12 @@ internal class SplatEngine { private external fun nativeSetCameraPose(handle: Long, x: Float, y: Float, z: Float, yaw: Float, pitch: Float) /** Fills [out] (at least [POSE_FLOATS]) with x, y, z, yaw, pitch. */ private external fun nativeCameraPose(handle: Long, out: FloatArray) + private external fun nativeSetCharacter( + handle: Long, + eyeHeight: Float, + bodyRadius: Float, + stepHeight: Float, + ): Boolean private external fun nativeLook(handle: Long, deltaYaw: Float, deltaPitch: Float) private external fun nativeWalk(handle: Long, forward: Float, right: Float) private external fun nativeSetVelocity(handle: Long, forward: Float, right: Float) diff --git a/packages/splatkit-android/src/main/java/com/splatkit/input/TouchInput.kt b/packages/splatkit-android/src/main/java/com/splatkit/input/TouchInput.kt index dd73597..e346289 100644 --- a/packages/splatkit-android/src/main/java/com/splatkit/input/TouchInput.kt +++ b/packages/splatkit-android/src/main/java/com/splatkit/input/TouchInput.kt @@ -1,57 +1,72 @@ package com.splatkit.input import android.view.MotionEvent -import kotlin.math.abs /** - * The view's gestures: one finger drags the view (yaw, and pitch when the gyroscope is - * off), two fingers walk (up is forward, sideways strafes), and a double tap toggles the - * gyroscope. Pixels become radians and meters through the sensitivities. + * The view's own gestures, and the only touches the SDK handles itself: a finger drags to + * look (yaw, and pitch when the gyroscope is off), and a double tap toggles the gyroscope. + * Pixels become radians through [lookSensitivity]. + * + * Walking comes from the host through `setWalkVelocity` or `walk`, so its own controls, on + * its own views, keep every other touch. */ internal class TouchInput(private val listener: Listener) { interface Listener { fun onLook(deltaYaw: Float, deltaPitch: Float) - fun onWalk(forward: Float, right: Float) fun onDoubleTap() } /** Radians per pixel dragged. */ var lookSensitivity = 0.004f - /** Meters per pixel dragged with two fingers. */ - var walkSensitivity = 0.01f + /** Whether a drag is allowed to turn the camera. */ + var lookEnabled = true + // One finger leads the whole drag, by its id: a second one landing on the view neither + // jumps the camera nor takes over when the first lifts. + private var pointerId = MotionEvent.INVALID_POINTER_ID private var lastX = 0f private var lastY = 0f - private var lastPointerCount = 0 private var lastTapTime = 0L fun onTouchEvent(event: MotionEvent): Boolean { - val count = event.pointerCount - val x = (0 until count).sumOf { event.getX(it).toDouble() }.toFloat() / count - val y = (0 until count).sumOf { event.getY(it).toDouble() }.toFloat() / count when (event.actionMasked) { MotionEvent.ACTION_DOWN -> { val now = event.eventTime if (now - lastTapTime < DOUBLE_TAP_MILLIS) listener.onDoubleTap() lastTapTime = now + grab(event, 0) } - MotionEvent.ACTION_MOVE -> if (count == lastPointerCount) { - val dx = x - lastX - val dy = y - lastY - if (count == 1) { - listener.onLook(-dx * lookSensitivity, -dy * lookSensitivity) - } else if (abs(dx) + abs(dy) > 0f) { - listener.onWalk(-dy * walkSensitivity, dx * walkSensitivity) + MotionEvent.ACTION_MOVE -> { + val index = event.findPointerIndex(pointerId) + if (index >= 0) { + val x = event.getX(index) + val y = event.getY(index) + if (lookEnabled) { + listener.onLook((lastX - x) * lookSensitivity, (lastY - y) * lookSensitivity) + } + lastX = x + lastY = y } } + MotionEvent.ACTION_POINTER_UP -> + if (event.getPointerId(event.actionIndex) == pointerId) letGo() + MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> letGo() } - lastX = x - lastY = y - lastPointerCount = if (event.actionMasked == MotionEvent.ACTION_UP) 0 else count return true } + /** Lets go of the finger, when looking is turned off or the view goes away. */ + fun letGo() { + pointerId = MotionEvent.INVALID_POINTER_ID + } + + private fun grab(event: MotionEvent, index: Int) { + pointerId = event.getPointerId(index) + lastX = event.getX(index) + lastY = event.getY(index) + } + private companion object { const val DOUBLE_TAP_MILLIS = 300L } diff --git a/packages/splatkit-android/src/main/java/com/splatkit/ui/JoystickView.kt b/packages/splatkit-android/src/main/java/com/splatkit/ui/JoystickView.kt index b4f6478..3d02e6e 100644 --- a/packages/splatkit-android/src/main/java/com/splatkit/ui/JoystickView.kt +++ b/packages/splatkit-android/src/main/java/com/splatkit/ui/JoystickView.kt @@ -12,6 +12,10 @@ import kotlin.math.min /** * A thumb stick: a base ring with a knob that follows the finger and snaps back on release. * Reports a direction in [-1, 1] on each axis, y positive upwards, through [onMove]. + * + * Optional, and nothing in the SDK uses it: a host that wants walking lays this over its + * [com.splatkit.SplatSurfaceView] and feeds `setWalkVelocity(y * speed, x * speed)`. A host + * with its own controls ignores this class entirely. */ class JoystickView @JvmOverloads constructor( context: Context, diff --git a/packages/splatkit-engine/include/splatkit/camera/WalkCamera.h b/packages/splatkit-engine/include/splatkit/camera/WalkCamera.h index 61f9ecd..f40441e 100644 --- a/packages/splatkit-engine/include/splatkit/camera/WalkCamera.h +++ b/packages/splatkit-engine/include/splatkit/camera/WalkCamera.h @@ -23,6 +23,11 @@ class WalkCamera { void setCollider(std::unique_ptr collider); bool hasCollider() const { return collider_ != nullptr; } + // The walker's shape: eye height, body radius and what it climbs. Applies at once when + // walking, and to the walker a later collider creates. + void setCharacter(const splat::CharacterSettings& settings); + const splat::CharacterSettings& character() const { return character_; } + // Touch: radians. Yaw/pitch poses clamp pitch; look-at poses turn in screen axes. // Pitch input is ignored while motion is on. void look(float deltaYaw, float deltaPitch); @@ -56,6 +61,7 @@ class WalkCamera { private: std::unique_ptr collider_; std::unique_ptr player_; + splat::CharacterSettings character_; splat::Vec3 freePosition_; float yaw_ = 0; float pitch_ = 0; diff --git a/packages/splatkit-engine/include/splatkit/diagnostics/StatsPublisher.h b/packages/splatkit-engine/include/splatkit/diagnostics/StatsPublisher.h index ceec908..1c24c9f 100644 --- a/packages/splatkit-engine/include/splatkit/diagnostics/StatsPublisher.h +++ b/packages/splatkit-engine/include/splatkit/diagnostics/StatsPublisher.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include "splat/math/Vec3.h" @@ -11,10 +13,17 @@ namespace splatkit { // What a HUD shows. Readable from any thread, refreshed twice a second by the render loop. struct Stats { + // Frames per second over the last half second. With presentTiming, frames the display + // showed; without, frames the renderer submitted, which can exceed what reached the screen. float fps = 0; - float frameMillis = 0; // wall time between vsyncs, averaged over the window - float gpuMillis = 0; // GPU time of the last frame, from timestamp queries - float sortMillis = 0; // last completed sort + float frameMillis = 0; // 1000 / fps + // The fields below are zero unless the renderer reports when frames reached the display. + bool presentTiming = false; + float frameMillisP95 = 0; // 95th percentile display interval over the last 5 seconds + float lowFps = 0; // 1% low: the slowest 1% of those intervals, as a frame rate + uint32_t droppedFrames = 0; // submitted but never shown, in the last window + float gpuMillis = 0; // GPU time of the last frame, from timestamp queries + float sortMillis = 0; // last completed sort uint32_t splatCount = 0; bool walking = false; bool motion = false; @@ -57,6 +66,10 @@ class StatsPublisher { uint32_t hardwareTiles = 0; }; + // Display times, in nanoseconds on any one clock, of frames the display showed since the + // last call, oldest first, and how many submitted frames it never showed. Calling it, even + // empty, switches the frame rate to presented frames. Render thread, before `onFrame`. + void onPresented(const std::vector& times, uint32_t dropped); // Once per vsync, drawn or not. `sample` is called when the window closes. void onFrame(int64_t frameTimeNanos, bool rendered, const std::function& sample); // Publishes everything but the frame rate now, leaving the window open: stats read after @@ -72,9 +85,22 @@ class StatsPublisher { uint32_t windowFrames_ = 0; uint32_t windowsSinceLog_ = 0; bool lastLoggedIdle_ = false; + bool presentTiming_ = false; + uint32_t windowPresented_ = 0; + uint32_t windowDropped_ = 0; + int64_t lastPresent_ = 0; + struct Interval { + int64_t end = 0; + float millis = 0; + }; + std::deque intervals_; // display intervals of the last five seconds std::atomic fps_{0}; std::atomic frameMillis_{0}; + std::atomic presentTimingPublished_{false}; + std::atomic frameMillisP95_{0}; + std::atomic lowFps_{0}; + std::atomic droppedFrames_{0}; std::atomic gpuMillis_{0}; std::atomic sortMillis_{0}; std::atomic splats_{0}; diff --git a/packages/splatkit-engine/include/splatkit/engine/SplatEngine.h b/packages/splatkit-engine/include/splatkit/engine/SplatEngine.h index fd4efa0..5c65947 100644 --- a/packages/splatkit-engine/include/splatkit/engine/SplatEngine.h +++ b/packages/splatkit-engine/include/splatkit/engine/SplatEngine.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "splat/core/Result.h" #include "splat/loading/SplatWorldLoader.h" @@ -130,6 +131,9 @@ class SplatEngine { void setAttitude(const float rowMajor[9]) { camera_.setAttitude(rowMajor); } void setMotionEnabled(bool enabled) { camera_.setMotionEnabled(enabled); } void setVelocity(float forward, float right) { camera_.setVelocity(forward, right); } + // The walker's shape in walk mode. Unwalkable values are refused as a whole. + bool setCharacter(const splat::CharacterSettings& settings); + const splat::CharacterSettings& character() const { return camera_.character(); } // Draws the next frame even when nothing changed, for a renderer that has something // to do with it, such as a capture. @@ -184,6 +188,7 @@ class SplatEngine { splat::VisibilityPlanner planner_; Benchmark benchmark_; StatsPublisher stats_; + std::vector presentTimes_; std::atomic maxShDegree_{kMaxShDegree}; std::atomic residency_{2000000}; diff --git a/packages/splatkit-engine/include/splatkit/rendering/RenderPolicy.h b/packages/splatkit-engine/include/splatkit/rendering/RenderPolicy.h index 3e588e3..1d41574 100644 --- a/packages/splatkit-engine/include/splatkit/rendering/RenderPolicy.h +++ b/packages/splatkit-engine/include/splatkit/rendering/RenderPolicy.h @@ -28,6 +28,10 @@ struct RenderPolicy { uint32_t tileSize = 16; // Screen-space error a hierarchy node may cover before it is refined; > 0. float lodErrorPixels = 1.0f; + // Most hierarchy splats one frame may select, 0 for the capacity the world loaded with. + // Changes live. When the error threshold would select more, the backend raises the + // threshold evenly until the cut fits instead of truncating it in traversal order. + uint32_t lodSplatLimit = 0; // Opacity below which a splat contributes nothing; [0, 1]. Fixed at 1/255 today. float alphaThreshold = 1.0f / 255.0f; // Smallest source footprint, in pixels, kept for drawing; >= 0. @@ -45,12 +49,16 @@ struct RenderPolicySupport { bool raster = false; bool tileSize = false; bool lodErrorPixels = false; + bool lodSplatLimit = false; bool alphaThreshold = false; bool subpixelThreshold = false; bool enableFrustumCulling = false; bool enableHiZOcclusion = false; bool enableEarlyTermination = false; bool sortDepth = false; + // Accepted raster strategies as a bitmask over the enum values: bit 0 = hardware, + // bit 1 = computeTile, bit 2 = hybrid. Zero means all three. + uint32_t rasterMask = 0; // Accepted tile sizes as a bitmask over {8,16,32}: bit 0 = 8, bit 1 = 16, bit 2 = 32. // Zero means all three. uint32_t tileSizeMask = 0; @@ -71,7 +79,7 @@ struct SplatLimits { struct DeviceCapabilities { SplatLimits limits; - // Experimental hybrid screen tiles. False when the backend has no tile path. + // Experimental hybrid screen tiles can be requested. False when the backend has no tile path. bool supportsComputeTiles = false; // Conservative occlusion is not implemented anywhere yet. bool supportsHiZOcclusion = false; diff --git a/packages/splatkit-engine/include/splatkit/rendering/SplatRenderer.h b/packages/splatkit-engine/include/splatkit/rendering/SplatRenderer.h index f401916..15d9fa4 100644 --- a/packages/splatkit-engine/include/splatkit/rendering/SplatRenderer.h +++ b/packages/splatkit-engine/include/splatkit/rendering/SplatRenderer.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "splat/formats/SplatCloud.h" #include "splat/lod/LodTree.h" @@ -128,6 +129,14 @@ class SplatRenderer { virtual uint32_t lastSelectedCount() const { return 0; } virtual double lastSelectMillis() const { return 0; } virtual ScreenTileStats lastScreenTileStats() const { return {}; } + // True when the renderer learns when frames reach the display. It then moves into `times` + // the display times, in nanoseconds, of frames shown since the last call, oldest first, + // and returns how many submitted frames were never shown. + virtual bool reportsPresentTimes() const { return false; } + virtual uint32_t takePresentTimes(std::vector* times) { + times->clear(); + return 0; + } // GPU name and API version, for a HUD. virtual const std::string& deviceDescription() const = 0; }; diff --git a/packages/splatkit-engine/src/camera/WalkCamera.cpp b/packages/splatkit-engine/src/camera/WalkCamera.cpp index f32d491..8cd7959 100644 --- a/packages/splatkit-engine/src/camera/WalkCamera.cpp +++ b/packages/splatkit-engine/src/camera/WalkCamera.cpp @@ -21,13 +21,24 @@ void WalkCamera::setCollider(std::unique_ptr collider) { collider_ = std::move(collider); player_.reset(); if (collider_) { - player_ = std::make_unique(*collider_); - player_->setPosition(current); + player_ = std::make_unique(*collider_, character_); + // A camera sitting where the collider has no floor cannot take a single step. Worlds are + // captured about an arbitrary origin, so that is the common case, not a rare one: put the + // walker on the nearest floor rather than leave it frozen wherever the host left it. + const bool standing = player_->floorBelow(current).has_value(); + const auto spot = standing ? std::optional{current} + : splat::findStandingSpot(*collider_, character_, current); + player_->setPosition(spot.value_or(current)); } else { freePosition_ = current; } } +void WalkCamera::setCharacter(const splat::CharacterSettings& settings) { + character_ = settings; + if (player_) player_->setSettings(settings); +} + void WalkCamera::look(float deltaYaw, float deltaPitch) { if (scripted_ && !motion_) { // A look-at pose can carry roll or pass a pole. Keep that basis when touch diff --git a/packages/splatkit-engine/src/diagnostics/StatsPublisher.cpp b/packages/splatkit-engine/src/diagnostics/StatsPublisher.cpp index 9d530f3..aeb0801 100644 --- a/packages/splatkit-engine/src/diagnostics/StatsPublisher.cpp +++ b/packages/splatkit-engine/src/diagnostics/StatsPublisher.cpp @@ -1,16 +1,37 @@ #include "splatkit/diagnostics/StatsPublisher.h" +#include + #include "splatkit/Log.h" namespace splatkit { namespace { constexpr int64_t kWindowNanos = 500'000'000LL; +constexpr int64_t kHistoryNanos = 5'000'000'000LL; +// The engine skips frames while nothing moves; a gap this long is a still scene, not a hitch. +constexpr int64_t kIdleGapNanos = 1'000'000'000LL; constexpr uint32_t kWindowsPerLog = 4; constexpr auto kRelaxed = std::memory_order_relaxed; } // namespace +void StatsPublisher::onPresented(const std::vector& times, uint32_t dropped) { + presentTiming_ = true; + windowDropped_ += dropped; + for (const int64_t time : times) { + ++windowPresented_; + const int64_t gap = time - lastPresent_; + if (lastPresent_ != 0 && gap > 0 && gap < kIdleGapNanos) { + intervals_.push_back({time, static_cast(static_cast(gap) * 1e-6)}); + } + lastPresent_ = std::max(lastPresent_, time); + } + while (!intervals_.empty() && intervals_.front().end < lastPresent_ - kHistoryNanos) { + intervals_.pop_front(); + } +} + void StatsPublisher::onFrame(int64_t frameTimeNanos, bool rendered, const std::function& sample) { if (rendered) ++windowFrames_; @@ -19,25 +40,49 @@ void StatsPublisher::onFrame(int64_t frameTimeNanos, bool rendered, if (elapsed < kWindowNanos) return; const Sample s = sample(); - const auto fps = static_cast(windowFrames_ * 1e9 / static_cast(elapsed)); + const uint32_t frames = presentTiming_ ? windowPresented_ : windowFrames_; + const auto fps = static_cast(frames * 1e9 / static_cast(elapsed)); fps_.store(fps, kRelaxed); frameMillis_.store(fps > 0.0f ? 1000.0f / fps : 0.0f, kRelaxed); + float p95 = 0; + float lowFps = 0; + if (presentTiming_ && !intervals_.empty()) { + std::vector millis; + millis.reserve(intervals_.size()); + for (const Interval& interval : intervals_) millis.push_back(interval.millis); + std::sort(millis.begin(), millis.end()); + const std::size_t n = millis.size(); + p95 = millis[(n * 95) / 100]; + // Mean of the slowest 1%, at least one interval. + const std::size_t slowest = std::max(n / 100, 1); + double total = 0; + for (std::size_t i = n - slowest; i < n; ++i) total += millis[i]; + lowFps = static_cast(1000.0 * static_cast(slowest) / total); + } + presentTimingPublished_.store(presentTiming_, kRelaxed); + frameMillisP95_.store(p95, kRelaxed); + lowFps_.store(lowFps, kRelaxed); + droppedFrames_.store(windowDropped_, kRelaxed); publish(s); // An idle scene logs once, not every two seconds. - const bool idle = windowFrames_ == 0; + const bool idle = frames == 0; + const uint32_t dropped = windowDropped_; windowStart_ = frameTimeNanos; windowFrames_ = 0; + windowPresented_ = 0; + windowDropped_ = 0; if (++windowsSinceLog_ < kWindowsPerLog || (idle && lastLoggedIdle_)) return; windowsSinceLog_ = 0; lastLoggedIdle_ = idle; const CameraPose p = pose(); LOGI( "%.1f fps, gpu %.1f ms, sort %.1f ms, cull %.1f ms, select %.1f ms, %u drawn of %zu " - "selected of %u, pos %.2f %.2f %.2f, yaw %.2f pitch %.2f, %s%s", + "selected of %u, pos %.2f %.2f %.2f, yaw %.2f pitch %.2f, %s%s, %s frames, p95 %.1f ms, " + "1%% low %.1f fps, %u dropped", fps, s.gpuMillis, s.sortMillis, s.cullMillis, s.selectMillis, s.drawn, s.selected, s.gpuSplats, p.x, p.y, p.z, p.yaw, p.pitch, s.walking ? "walk" : "fly", - s.motion ? ", gyro" : ""); + s.motion ? ", gyro" : "", presentTiming_ ? "presented" : "submitted", p95, lowFps, dropped); } void StatsPublisher::publish(const Sample& s) { @@ -64,6 +109,10 @@ Stats StatsPublisher::stats() const { Stats s; s.fps = fps_.load(kRelaxed); s.frameMillis = frameMillis_.load(kRelaxed); + s.presentTiming = presentTimingPublished_.load(kRelaxed); + s.frameMillisP95 = frameMillisP95_.load(kRelaxed); + s.lowFps = lowFps_.load(kRelaxed); + s.droppedFrames = droppedFrames_.load(kRelaxed); s.gpuMillis = gpuMillis_.load(kRelaxed); s.sortMillis = sortMillis_.load(kRelaxed); s.splatCount = splats_.load(kRelaxed); diff --git a/packages/splatkit-engine/src/engine/SplatEngine.cpp b/packages/splatkit-engine/src/engine/SplatEngine.cpp index 6054dfc..bcc8daa 100644 --- a/packages/splatkit-engine/src/engine/SplatEngine.cpp +++ b/packages/splatkit-engine/src/engine/SplatEngine.cpp @@ -200,6 +200,24 @@ void SplatEngine::setCameraLookAt(splat::Vec3 position, splat::Vec3 target, spla publishPose(); } +bool SplatEngine::setCharacter(const splat::CharacterSettings& settings) { + const float values[] = {settings.eyeHeight, settings.bodyRadius, settings.hipHeight, + settings.floorProbeUp, settings.floorProbeDown, settings.stepHeight, + settings.stepLookAhead, settings.stepOverHeight, settings.stepOverWidth, + settings.snapRate}; + for (const float value : values) { + if (!std::isfinite(value) || value < 0) return false; + } + // A walker needs an eye over its hips, a floor ray that reaches and an eye that settles. + if (settings.eyeHeight <= 0 || settings.hipHeight >= settings.eyeHeight || + settings.floorProbeDown <= 0 || settings.snapRate <= 0) { + return false; + } + camera_.setCharacter(settings); + redrawNeeded_ = true; + return true; +} + void SplatEngine::publishPose() { stats_.publishPose(camera_.position(), camera_.yaw(), camera_.pitch()); } @@ -370,6 +388,10 @@ void SplatEngine::render(int64_t frameTimeNanos) { } const uint32_t generation = renderer_->generation(); if (generation != lastDrawnGeneration_) redrawNeeded_ = true; + if (renderer_->reportsPresentTimes()) { + const uint32_t dropped = renderer_->takePresentTimes(&presentTimes_); + stats_.onPresented(presentTimes_, dropped); + } const auto sampler = [this] { return sample(); }; if (!redrawNeeded_) { stats_.onFrame(frameTimeNanos, false, sampler); diff --git a/packages/splatkit-engine/src/rendering/RenderPolicy.cpp b/packages/splatkit-engine/src/rendering/RenderPolicy.cpp index a0dd160..2d3c254 100644 --- a/packages/splatkit-engine/src/rendering/RenderPolicy.cpp +++ b/packages/splatkit-engine/src/rendering/RenderPolicy.cpp @@ -10,6 +10,10 @@ bool finitePositive(float value) { return std::isfinite(value) && value > 0.0f; } +bool acceptedRaster(uint32_t mask, RasterStrategy strategy) { + return mask == 0 || (mask & (1u << static_cast(strategy))) != 0; +} + bool acceptedTileSize(uint32_t mask, uint32_t size) { if (mask == 0) return size == 8 || size == 16 || size == 32; const uint32_t bit = size == 8 ? 1u : size == 16 ? 2u : size == 32 ? 4u : 0u; @@ -85,7 +89,7 @@ RenderPolicyResolution resolveRenderPolicy(const RenderPolicy& requested, result.warnings.push_back({field, std::move(message)}); }; - if (support.raster) { + if (support.raster && acceptedRaster(support.rasterMask, requested.raster)) { result.effective.raster = requested.raster; } else if (requested.raster != result.effective.raster) { warn("raster", std::string("raster ") + toString(requested.raster) + @@ -114,6 +118,13 @@ RenderPolicyResolution resolveRenderPolicy(const RenderPolicy& requested, number(result.effective.lodErrorPixels)); } + if (support.lodSplatLimit) { + result.effective.lodSplatLimit = requested.lodSplatLimit; + } else if (requested.lodSplatLimit != result.effective.lodSplatLimit) { + warn("lodSplatLimit", "lodSplatLimit is not configurable on this backend; using " + + std::to_string(result.effective.lodSplatLimit)); + } + if (support.alphaThreshold) { result.effective.alphaThreshold = requested.alphaThreshold; } else if (requested.alphaThreshold != result.effective.alphaThreshold) { diff --git a/packages/splatkit-engine/tests/camera/WalkCameraTest.cpp b/packages/splatkit-engine/tests/camera/WalkCameraTest.cpp index 31eacfd..fc43f1b 100644 --- a/packages/splatkit-engine/tests/camera/WalkCameraTest.cpp +++ b/packages/splatkit-engine/tests/camera/WalkCameraTest.cpp @@ -1,6 +1,7 @@ #include "splatkit/camera/WalkCamera.h" #include +#include #include @@ -82,6 +83,44 @@ TEST(WalkCamera, VelocityMovesEveryUpdate) { EXPECT_NEAR(camera.position().z, -1.0f, 1e-5f); } +// A 10 x 10 m floor at y = 0, as two triangles. +splat::TriangleMesh floor() { + splat::TriangleMesh mesh; + mesh.positions = {-5, 0, -5, 5, 0, -5, 5, 0, 5, -5, 0, 5}; + mesh.indices = {0, 1, 2, 0, 2, 3}; + return mesh; +} + +// Settling: the eye eases toward the floor, so give it a second of updates. +void settle(WalkCamera& camera) { + for (int i = 0; i < 60; ++i) camera.update(1.0f / 60); +} + +TEST(WalkCamera, TheCharacterSetsTheEyeHeightOfTheWalkerThatFollows) { + WalkCamera camera; + splat::CharacterSettings settings; + settings.eyeHeight = 1.2f; + camera.setCharacter(settings); + camera.setPosition({0, 1.6f, 0}); + camera.setCollider(std::make_unique(floor())); + settle(camera); + EXPECT_NEAR(camera.position().y, 1.2f, 1e-3f); +} + +TEST(WalkCamera, TheCharacterChangesTheWalkerAlreadyOnItsFeet) { + WalkCamera camera; + camera.setPosition({0, 1.6f, 0}); + camera.setCollider(std::make_unique(floor())); + settle(camera); + EXPECT_NEAR(camera.position().y, 1.5f, 1e-3f); + splat::CharacterSettings settings; + settings.eyeHeight = 1.8f; + camera.setCharacter(settings); + EXPECT_NEAR(camera.position().x, 0.0f, 1e-5f); + settle(camera); + EXPECT_NEAR(camera.position().y, 1.8f, 1e-3f); +} + // A phone held upright facing north, in Android's East-North-Up frame: device x east, // device y up, device z south (out of the screen towards the user). The camera must // look north, which is -z in the engine's frame. diff --git a/packages/splatkit-engine/tests/diagnostics/StatsPublisherTest.cpp b/packages/splatkit-engine/tests/diagnostics/StatsPublisherTest.cpp index e05abdf..f7496af 100644 --- a/packages/splatkit-engine/tests/diagnostics/StatsPublisherTest.cpp +++ b/packages/splatkit-engine/tests/diagnostics/StatsPublisherTest.cpp @@ -1,5 +1,7 @@ #include +#include + #include "splatkit/diagnostics/StatsPublisher.h" namespace splatkit { @@ -43,5 +45,48 @@ TEST(StatsPublisher, PublishRefreshesCountsWithoutClosingTheWindow) { EXPECT_FLOAT_EQ(now.fps, 4.0f); } +// Submitting a frame is not showing it: once the renderer reports display times, the frame +// rate counts only frames that reached the screen. +TEST(StatsPublisher, PresentedFramesReplaceSubmittedOnesInTheFrameRate) { + StatsPublisher stats; + constexpr int64_t kFrame = 33'333'333LL; + stats.onFrame(kHalfSecond, true, [] { return counts(1, 1, 1); }); + EXPECT_FALSE(stats.stats().presentTiming); + + // Thirty submissions, but the display showed only every other one. + std::vector shown; + for (int i = 1; i <= 15; ++i) shown.push_back(kHalfSecond + 2 * i * kFrame); + for (int i = 1; i < 30; ++i) { + stats.onFrame(kHalfSecond + i * (kHalfSecond / 30), true, [] { return counts(1, 1, 1); }); + } + stats.onPresented(shown, 15); + stats.onFrame(2 * kHalfSecond, true, [] { return counts(1, 1, 1); }); + const Stats now = stats.stats(); + EXPECT_TRUE(now.presentTiming); + EXPECT_FLOAT_EQ(now.fps, 30.0f); + EXPECT_EQ(now.droppedFrames, 15u); + EXPECT_NEAR(now.frameMillisP95, 66.67f, 0.01f); + EXPECT_NEAR(now.lowFps, 15.0f, 0.01f); +} + +// The 1% low and p95 see a single hitch among steady frames; an idle gap is not a hitch. +TEST(StatsPublisher, TailsCatchAHitchButNotAStillScene) { + StatsPublisher stats; + constexpr int64_t kFrame = 16'666'667LL; + std::vector shown; + shown.reserve(201); + int64_t time = kHalfSecond; + for (int i = 0; i < 199; ++i) shown.push_back(time += kFrame); + shown.push_back(time += 100'000'000LL); // one 100 ms hitch + shown.push_back(time += 2'000'000'000LL); // two still seconds, then one frame + stats.onPresented(shown, 0); + stats.onFrame(kHalfSecond, false, [] { return counts(1, 1, 1); }); + stats.onFrame(time, false, [] { return counts(1, 1, 1); }); + const Stats now = stats.stats(); + EXPECT_NEAR(now.frameMillisP95, 16.67f, 0.01f); + // Fewer than 200 intervals: the slowest one is the 1%. + EXPECT_NEAR(now.lowFps, 10.0f, 0.01f); +} + } // namespace } // namespace splatkit diff --git a/packages/splatkit-engine/tests/rendering/RenderPolicyTest.cpp b/packages/splatkit-engine/tests/rendering/RenderPolicyTest.cpp index 6049751..c61f5de 100644 --- a/packages/splatkit-engine/tests/rendering/RenderPolicyTest.cpp +++ b/packages/splatkit-engine/tests/rendering/RenderPolicyTest.cpp @@ -99,6 +99,44 @@ TEST(RenderPolicy, BackendTileMaskRejectsAnOtherwiseValidSize) { EXPECT_EQ(resolved.warnings[0].field, "tileSize"); } +TEST(RenderPolicy, BackendRasterMaskRejectsAnOtherwiseValidStrategy) { + const RenderPolicy fallback{}; + RenderPolicySupport support = supportWith(fallback); + support.raster = true; + support.rasterMask = 0x1 | 0x4; // hardware and hybrid only + + RenderPolicy requested = fallback; + requested.raster = RasterStrategy::hybrid; + RenderPolicyResolution resolved = resolveRenderPolicy(requested, support); + ASSERT_TRUE(resolved.accepted) << resolved.error; + EXPECT_EQ(resolved.effective.raster, RasterStrategy::hybrid); + EXPECT_TRUE(resolved.warnings.empty()); + + requested.raster = RasterStrategy::computeTile; + resolved = resolveRenderPolicy(requested, support); + ASSERT_TRUE(resolved.accepted) << resolved.error; + EXPECT_EQ(resolved.effective.raster, RasterStrategy::hardware); + ASSERT_EQ(resolved.warnings.size(), 1u); + EXPECT_EQ(resolved.warnings[0].field, "raster"); +} + +TEST(RenderPolicy, SplatLimitAppliesOnlyWhereABackendSupportsIt) { + RenderPolicy requested; + requested.lodSplatLimit = 1'500'000; + + RenderPolicySupport support = supportWith(RenderPolicy{}); + RenderPolicyResolution resolved = resolveRenderPolicy(requested, support); + ASSERT_TRUE(resolved.accepted) << resolved.error; + EXPECT_EQ(resolved.effective.lodSplatLimit, 0u); + ASSERT_EQ(resolved.warnings.size(), 1u); + EXPECT_EQ(resolved.warnings[0].field, "lodSplatLimit"); + + support.lodSplatLimit = true; + resolved = resolveRenderPolicy(requested, support); + EXPECT_EQ(resolved.effective.lodSplatLimit, 1'500'000u); + EXPECT_TRUE(resolved.warnings.empty()); +} + TEST(RenderPolicy, BackendRangesClampSupportedFloats) { RenderPolicy fallback; fallback.lodErrorPixels = 1.0f; diff --git a/packages/splatkit-ios/README.md b/packages/splatkit-ios/README.md index a1549cb..1b51762 100644 --- a/packages/splatkit-ios/README.md +++ b/packages/splatkit-ios/README.md @@ -6,7 +6,7 @@ Requires iOS 17+ and Apple GPU family 7+ (A14/M1+); unsupported GPUs report unav ## Use it Swift Package Manager: add `https://github.com/Xget7/splatkit-ios`, product `SplatKit`, then `import SplatKit`. -Choose exact version `0.1.0-alpha.3`. +Choose exact version `0.1.0-alpha.4`. For source builds, build the static libraries with `scripts/build-ios.sh` from the repository root, then add to your target: @@ -44,15 +44,38 @@ Forward `resume()`, `pause()` and `release()` from the host's lifecycle; the lay | `linearBlending` | Blend in linear light instead of the encoded colour space | | `splatBudget`, `residencyBudget` | Most splats drawn per frame, most splats resident on the GPU; a tiled scene that fits the residency whole is fetched whole, so turning never meets a coarse stand-in | | `shDegree`, `maxShDegree` | Harmonics drawn, harmonics kept from the file | -| `setWalkVelocity(forward:right:)` | Continuous walking in meters per second | +| `setWalkVelocity(forward:right:)`, `walk(forward:right:)` | Continuous walking in meters per second, or a single step in meters; the host draws its own control and drives these | +| `look(deltaYaw:deltaPitch:)` | Turns the camera by radians, for a look pad or mouse | +| `setCharacter(_:)`, `character` | The walker's shape in walk mode: `CharacterSettings(eyeHeight:bodyRadius:stepHeight:)` | +| `touchLookEnabled`, `lookSensitivity` | Whether a one-finger drag turns the camera, and radians per point dragged | +| `motionToggleEnabled` | Whether a double tap anywhere in the view toggles the gyroscope | | `setMotionEnabled(_:)`, `isMotionEnabled` | Gyroscope driven camera | +| `cameraPoseInterval` | Seconds between `cameraPoseChanged` delegate calls; 0, the default, never | | `startBenchmark(seconds:)` | A reproducible turn with the frame time distribution logged | | `captureFrame(to:completion:)` | The next frame as a PNG | -| `renderPolicy`, `deviceCapabilities` | Per-view renderer policy, re-validated on the render thread; invalid or unpreparable requests keep the previous policy. Sort depth applies with GPU sort, the sub-pixel threshold only under the tight-culling experiment | +| `renderPolicy`, `deviceCapabilities` | Per-view renderer policy, re-validated on the render thread; invalid or unpreparable requests keep the previous policy. Sort depth, the raster strategy, the LOD error threshold and the LOD splat limit apply with GPU sort, the sub-pixel threshold only under the tight-culling experiment | | `readStats()`, `gpuDescription` | Frame, GPU and sort times, splats drawn, device name | -| `delegate` | World and collider outcomes, on the main thread | +| `delegate` | World, collider and camera pose outcomes, on the main thread | -Gestures: one finger looks, two fingers walk, a double tap toggles the gyroscope; `lookSensitivity` and `walkSensitivity` scale them. +## Raster strategy + +Hardware rasterization is the default and the fastest choice for most scenes. +Hybrid screen tiles are an experimental opt-in for scenes where many large translucent splats overlap each pixel, such as close-up interiors: + +```swift +var policy = splatView.renderPolicy +policy.raster = 2 // hybrid; 0 restores hardware +splatView.renderPolicy = policy +``` + +They composite 16×16 tiles in compute on top of the GPU's own tiling, so on distant or sparse scenes they only add work. +ISS at a 45 m orbit on an iPhone 17 Pro drew the same 1.96M splats at 22-23 FPS with tiles and 31 FPS without. +The first request builds the tile pipelines; switching back to hardware frees the tile scratch. +`computeTile` is not built and falls back to hardware with a warning. + +Touch: a one-finger drag looks around, and a double tap anywhere in the view toggles the gyroscope. +The view ships no walking control; the host draws its own, wherever it likes, and drives it with `setWalkVelocity` or `walk`. +`lookSensitivity` tunes the drag, and `touchLookEnabled` and `motionToggleEnabled` turn the two gestures off. ## Layout @@ -77,18 +100,19 @@ Colours blend in the encoded space by default, on a `bgra8Unorm` layer; `linearB Start motion after `splatView(_:worldFrameReady:)`, not upload-only `worldReady`. Keep the view attached/resumed while loading; readiness means GPU completion, not visual acceptance. -Dev-only switches; `--min-pixel-radius` and `--depth-key-bits` set the view's `renderPolicy`, the rest apply before renderer creation: +Dev-only switches; `--quality`, `--min-pixel-radius`, `--depth-key-bits`, `--lod-error-pixels`, `--lod-splat-limit` and `--tile-raster` set the view's `renderPolicy`, the rest apply before renderer creation: | Switch | Effect | |---|---| | `--metal-culling 1 --min-pixel-radius 1` | Covariance bounds, opacity/subpixel rejection | | `--depth-key-bits 16` | Two radix passes; uint32 storage unchanged; ties may shimmer | -| `--tile-raster 1` | 16×16 tiles; compute ≤512 candidates, dense/large-footprint tiles use hardware | -| `--budget 2200000` | LOD capacity, not guaranteed quality | -| `--orbit-horizontal 0` | Previous vertical framing for benchmark reproduction | +| `--tile-raster 1` | Hybrid raster: 16×16 tiles, compute ≤512 candidates, dense/large-footprint tiles use hardware | +| `--budget 4000000` | LOD capacity, at most 4M; not guaranteed quality | +| `--quality ultra` | Starting level of the on-screen picker: `ultra`, `high` (default), `balanced`, `fast` | +| `--shot tour` | ISS camera, +Y up: `overview`, `orbit` (default), `detail`, `flyby`, `tour` | | `--run-seconds 20` | Bounded run with resource monitoring | -Defaults: 32-bit sorting, tiles disabled. +Defaults: 32-bit sorting, hardware raster. Culling, LOD and depth quantization are approximations pending visual acceptance. Tile termination uses transmittance ≤0.0001; hardware geometry submission remains. Private allocations still consume unified memory. diff --git a/packages/splatkit-ios/Sources/SplatKit/RenderThread.swift b/packages/splatkit-ios/Sources/SplatKit/RenderThread.swift index fd458a4..fa9926f 100644 --- a/packages/splatkit-ios/Sources/SplatKit/RenderThread.swift +++ b/packages/splatkit-ios/Sources/SplatKit/RenderThread.swift @@ -147,12 +147,19 @@ final class RenderThread { post { [self] in engine?.look(withDeltaYaw: deltaYaw, deltaPitch: deltaPitch) } } + func setVelocity(_ forward: Float, _ right: Float) { + post { [self] in engine?.setVelocityForward(forward, right: right) } + } + func walk(_ forward: Float, _ right: Float) { post { [self] in engine?.walkForward(forward, right: right) } } - func setVelocity(_ forward: Float, _ right: Float) { - post { [self] in engine?.setVelocityForward(forward, right: right) } + /// Waits: the caller learns whether the settings were walkable. + func setCharacter(_ settings: SKCharacterSettings) -> Bool { + var accepted = false + sync { [self] in accepted = engine?.setCharacter(settings) ?? false } + return accepted } func setAttitude(_ rowMajor: [Float]) { diff --git a/packages/splatkit-ios/Sources/SplatKit/SplatMetalView.swift b/packages/splatkit-ios/Sources/SplatKit/SplatMetalView.swift index 97c3294..aea6497 100644 --- a/packages/splatkit-ios/Sources/SplatKit/SplatMetalView.swift +++ b/packages/splatkit-ios/Sources/SplatKit/SplatMetalView.swift @@ -23,10 +23,35 @@ public struct CameraPose: Equatable { } } +/// The walker in walk mode, in meters. The defaults are a standing adult: the eye 1.5 m over +/// the floor, 0.35 m kept from walls, and a 0.35 m rise walked onto, which climbs stairs and +/// doorsteps but not chairs or counters. A step onto anything higher is refused and the +/// walker slides along it instead. +public struct CharacterSettings: Equatable { + public var eyeHeight: Float + public var bodyRadius: Float + public var stepHeight: Float + + public init(eyeHeight: Float = 1.5, bodyRadius: Float = 0.35, stepHeight: Float = 0.35) { + self.eyeHeight = eyeHeight + self.bodyRadius = bodyRadius + self.stepHeight = stepHeight + } +} + /// A snapshot of what the engine is doing, refreshed twice a second. public struct SplatStats { + /// Frames per second over the last half second: frames the display showed when + /// `presentTiming` is true, otherwise frames submitted. public var fps: Float = 0 public var frameMillis: Float = 0 + public var presentTiming = false + /// 95th percentile display interval over the last 5 seconds; zero without present timing. + public var frameMillisP95: Float = 0 + /// Frame rate of the slowest 1% of display intervals over the last 5 seconds. + public var lowFps: Float = 0 + /// Submitted frames never shown in the last half second. + public var droppedFrames: Int = 0 /// GPU time of the last frame; zero until one completes. public var gpuMillis: Float = 0 public var sortMillis: Float = 0 @@ -61,6 +86,9 @@ public protocol SplatViewDelegate: AnyObject { /// Walk mode is on. func splatViewColliderReady(_ view: SplatMetalView) func splatView(_ view: SplatMetalView, colliderFailed message: String) + /// Where the camera ended up, at most once per `cameraPoseInterval` and only while it + /// moves. Off until that interval is set. + func splatView(_ view: SplatMetalView, cameraPoseChanged pose: CameraPose) } public extension SplatViewDelegate { @@ -69,6 +97,7 @@ public extension SplatViewDelegate { func splatView(_ view: SplatMetalView, worldFailed message: String) {} func splatViewColliderReady(_ view: SplatMetalView) {} func splatView(_ view: SplatMetalView, colliderFailed message: String) {} + func splatView(_ view: SplatMetalView, cameraPoseChanged pose: CameraPose) {} } /// A view that renders with SplatKit, on a CAMetalLayer of its own. @@ -77,8 +106,9 @@ public extension SplatViewDelegate { /// lifecycle: the engine gets the layer when the view is in a window and gives it back, /// synchronously, before the view leaves it. /// -/// Gestures: one finger drags the view (yaw, and pitch when the gyroscope is off); -/// two fingers walk (up is forward, sideways strafes); a double tap toggles the gyroscope. +/// Touch: a drag looks around (yaw, and pitch when the gyroscope is off) and a double tap +/// toggles the gyroscope; both can be turned off. The view ships no walking control: the +/// host draws its own, wherever it likes, and drives `setWalkVelocity` or `walk` from it. public final class SplatMetalView: UIView { public override class var layerClass: AnyClass { CAMetalLayer.self } @@ -89,15 +119,39 @@ public final class SplatMetalView: UIView { private var attached = false private var lastDrawableSize = CGSize.zero - /// Radians per point dragged. - public var lookSensitivity: Float = 0.004 - /// Meters per point dragged with two fingers. - public var walkSensitivity: Float = 0.01 - /// Whether a one-finger drag is allowed to move the camera. Scripted tours can disable it. - public var touchLookEnabled = true + private lazy var touchLook = TouchLook(view: self) { [weak self] yaw, pitch in + self?.renderThread.look(yaw, pitch) + } + private var poseTimer: Timer? + private var lastPose: CameraPose? + + /// Radians per point dragged to look. + public var lookSensitivity: Float { + get { touchLook.sensitivity } + set { touchLook.sensitivity = newValue } + } + /// Whether a drag on the view is allowed to turn the camera. A host that drives looking + /// from its own control, and scripted tours, turn it off. + public var touchLookEnabled: Bool { + get { touchLook.isEnabled } + set { + touchLook.isEnabled = newValue + if !newValue { touchLook.letGo() } + } + } /// Whether the double-tap gesture can toggle motion input. public var motionToggleEnabled = true + /// How often the delegate hears where the camera is, in seconds; 0, the default, never. + /// A pose is delivered only when it differs from the last one delivered. + public var cameraPoseInterval: TimeInterval = 0 { + didSet { + cameraPoseInterval = max(cameraPoseInterval, 0) + guard cameraPoseInterval != oldValue else { return } + startPoseTimer() + } + } + public weak var delegate: SplatViewDelegate? public override init(frame: CGRect) { @@ -124,15 +178,14 @@ public final class SplatMetalView: UIView { @unknown default: break } } - let look = UIPanGestureRecognizer(target: self, action: #selector(onLook(_:))) - look.maximumNumberOfTouches = 1 - addGestureRecognizer(look) - let walk = UIPanGestureRecognizer(target: self, action: #selector(onWalk(_:))) - walk.minimumNumberOfTouches = 2 - walk.maximumNumberOfTouches = 2 - addGestureRecognizer(walk) - let tap = UITapGestureRecognizer(target: self, action: #selector(onDoubleTap)) + isMultipleTouchEnabled = true + _ = touchLook + let tap = UITapGestureRecognizer(target: self, action: #selector(onDoubleTap(_:))) tap.numberOfTapsRequired = 2 + // The look drag keeps its touches while the tap is being recognized. + tap.cancelsTouchesInView = false + tap.delaysTouchesEnded = false + tap.delegate = touchLook addGestureRecognizer(tap) } @@ -229,11 +282,39 @@ public final class SplatMetalView: UIView { } } - /// Walks continuously at the given speed in meters per second until called again with zeros. + /// Walks continuously at the given speed in meters per second until called again with + /// zeros: what a joystick or a keyboard drives. Forward is where the camera looks, + /// flattened onto the floor while walking; right strafes. public func setWalkVelocity(forward: Float, right: Float) { renderThread.setVelocity(forward, right) } + /// One step, in meters, for a host that integrates movement itself. The collider stops + /// it at walls and the floor carries it, exactly as a velocity would. + public func walk(forward: Float, right: Float) { + renderThread.walk(forward, right) + } + + /// Turns the camera by these radians: what a look pad or a mouse drives. Pitch is + /// clamped, and ignored while the gyroscope drives the view. + public func look(deltaYaw: Float, deltaPitch: Float) { + renderThread.look(deltaYaw, deltaPitch) + } + + /// The walker's shape in walk mode, applied at once and to a collider loaded later. + /// False when a value is not a walkable one, and then the previous settings stay. + @discardableResult + public func setCharacter(_ settings: CharacterSettings) -> Bool { + let accepted = renderThread.setCharacter(SKCharacterSettings( + eyeHeight: settings.eyeHeight, bodyRadius: settings.bodyRadius, + stepHeight: settings.stepHeight)) + if accepted { character = settings } + return accepted + } + + /// The walker's shape in effect. + public private(set) var character = CharacterSettings() + /// Runs a reproducible capture: the gyroscope goes off, the camera takes a fixed pose /// and turns once over `seconds`, then the frame time distribution is logged. public func startBenchmark(seconds: Float = 10) { @@ -263,6 +344,10 @@ public final class SplatMetalView: UIView { guard let s = renderThread.stats() else { return stats } stats.fps = s.fps stats.frameMillis = s.frameMillis + stats.presentTiming = s.presentTiming.boolValue + stats.frameMillisP95 = s.frameMillisP95 + stats.lowFps = s.lowFps + stats.droppedFrames = Int(s.droppedFrames) stats.gpuMillis = s.gpuMillis stats.sortMillis = s.sortMillis stats.splatCount = Int(s.splatCount) @@ -289,20 +374,49 @@ public final class SplatMetalView: UIView { motion.interfaceOrientation = interfaceOrientation renderThread.resume() if motionEnabled { motion.start() } + startPoseTimer() } public func pause() { resumed = false motion.stop() renderThread.pause() + stopPoseTimer() } public func release() { motion.stop() + stopPoseTimer() detach() renderThread.release() } + // Camera pose reporting. + + private func startPoseTimer() { + stopPoseTimer() + guard resumed, cameraPoseInterval > 0 else { return } + let timer = Timer(timeInterval: cameraPoseInterval, repeats: true) { [weak self] _ in + self?.reportPose() + } + // Common modes: a pose keeps arriving while a host's own control is being dragged. + RunLoop.main.add(timer, forMode: .common) + poseTimer = timer + } + + private func stopPoseTimer() { + poseTimer?.invalidate() + poseTimer = nil + } + + private func reportPose() { + guard let delegate else { return } + let pose = cameraPose + guard pose != lastPose else { return } + lastPose = pose + delegate.splatView(self, cameraPoseChanged: pose) + } + // Layer lifecycle. private var metalLayer: CAMetalLayer { layer as! CAMetalLayer } @@ -351,6 +465,7 @@ public final class SplatMetalView: UIView { } private func detach() { + touchLook.letGo() guard attached else { return } attached = false renderThread.layerDetached() @@ -358,23 +473,7 @@ public final class SplatMetalView: UIView { // Gestures. - @objc private func onLook(_ g: UIPanGestureRecognizer) { - guard touchLookEnabled else { - g.setTranslation(.zero, in: self) - return - } - let d = g.translation(in: self) - renderThread.look(-Float(d.x) * lookSensitivity, -Float(d.y) * lookSensitivity) - g.setTranslation(.zero, in: self) - } - - @objc private func onWalk(_ g: UIPanGestureRecognizer) { - let d = g.translation(in: self) - renderThread.walk(-Float(d.y) * walkSensitivity, Float(d.x) * walkSensitivity) - g.setTranslation(.zero, in: self) - } - - @objc private func onDoubleTap() { + @objc private func onDoubleTap(_ g: UITapGestureRecognizer) { guard motionToggleEnabled else { return } setMotionEnabled(!motionEnabled) } diff --git a/packages/splatkit-ios/Sources/SplatKit/TouchLook.swift b/packages/splatkit-ios/Sources/SplatKit/TouchLook.swift new file mode 100644 index 0000000..dc06bd6 --- /dev/null +++ b/packages/splatkit-ios/Sources/SplatKit/TouchLook.swift @@ -0,0 +1,70 @@ +import UIKit +import UIKit.UIGestureRecognizerSubclass + +/// Turns the camera while a finger drags the view: the only touch the SDK handles itself. +/// Walking comes from the host through `setWalkVelocity`, so the host's own controls, on +/// its own views, keep every other touch. +/// +/// A gesture recognizer rather than the view's touch handlers: a host such as SwiftUI runs +/// its gestures over the whole hierarchy, and those take raw touches from the view. +final class TouchLook: UIGestureRecognizer, UIGestureRecognizerDelegate { + /// Radians per point dragged. + var sensitivity: Float = 0.004 + + private let onLook: (Float, Float) -> Void + private var touch: UITouch? + + init(view: UIView, onLook: @escaping (Float, Float) -> Void) { + self.onLook = onLook + super.init(target: nil, action: nil) + cancelsTouchesInView = false + delaysTouchesEnded = false + delegate = self + view.addGestureRecognizer(self) + } + + // The host's controls and the view's double tap recognize alongside this one. + func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer, + shouldRecognizeSimultaneouslyWith other: UIGestureRecognizer) -> Bool { + true + } + + override func touchesBegan(_ touches: Set, with event: UIEvent) { + guard touch == nil, let first = touches.first else { return } + touch = first + state = .began + } + + override func touchesMoved(_ touches: Set, with event: UIEvent) { + guard let view, let touch, touches.contains(touch) else { return } + let p = touch.location(in: view) + let q = touch.previousLocation(in: view) + onLook(-Float(p.x - q.x) * sensitivity, -Float(p.y - q.y) * sensitivity) + state = .changed + } + + override func touchesEnded(_ touches: Set, with event: UIEvent) { + lift(touches) + } + + override func touchesCancelled(_ touches: Set, with event: UIEvent) { + lift(touches) + } + + override func reset() { + super.reset() + touch = nil + } + + /// Lets go of the finger, when the view leaves the screen or looking is turned off. + func letGo() { + touch = nil + if state == .began || state == .changed { state = .ended } + } + + private func lift(_ touches: Set) { + guard let touch, touches.contains(touch) else { return } + self.touch = nil + state = (state == .began || state == .changed) ? .ended : .failed + } +} diff --git a/packages/splatkit-ios/Sources/SplatKitCore/engine/SKSplatEngine.mm b/packages/splatkit-ios/Sources/SplatKitCore/engine/SKSplatEngine.mm index 05115f2..0e32790 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/engine/SKSplatEngine.mm +++ b/packages/splatkit-ios/Sources/SplatKitCore/engine/SKSplatEngine.mm @@ -64,6 +64,7 @@ SKRenderPolicy fromCppPolicy(const splatkit::RenderPolicy& p) { out.enableHiZOcclusion = p.enableHiZOcclusion; out.enableEarlyTermination = p.enableEarlyTermination; out.sortDepth = static_cast(p.sortDepth); + out.lodSplatLimit = p.lodSplatLimit; return out; } @@ -78,6 +79,7 @@ bool toCppPolicy(SKRenderPolicy p, splatkit::RenderPolicy* out) { out->enableHiZOcclusion = p.enableHiZOcclusion == YES; out->enableEarlyTermination = p.enableEarlyTermination == YES; out->sortDepth = p.sortDepth == 16 ? splatkit::SortKeyBits::low16 : splatkit::SortKeyBits::full32; + out->lodSplatLimit = p.lodSplatLimit; return true; } @@ -92,11 +94,13 @@ SKRenderPolicySupport fromCppSupport(const splatkit::RenderPolicySupport& s) { out.enableHiZOcclusion = s.enableHiZOcclusion; out.enableEarlyTermination = s.enableEarlyTermination; out.sortDepth = s.sortDepth; + out.rasterMask = s.rasterMask; out.tileSizeMask = s.tileSizeMask; out.minLodErrorPixels = s.minLodErrorPixels; out.maxLodErrorPixels = s.maxLodErrorPixels; out.minSubpixelThreshold = s.minSubpixelThreshold; out.maxSubpixelThreshold = s.maxSubpixelThreshold; + out.lodSplatLimit = s.lodSplatLimit; return out; } @@ -192,7 +196,11 @@ - (SKSplatStats)stats { s.drawnSplatCount, s.computeTileCount, s.nonemptyComputeTileCount, - s.hardwareTileCount}; + s.hardwareTileCount, + s.presentTiming, + s.frameMillisP95, + s.lowFps, + s.droppedFrames}; } - (NSString*)gpuDescription { @@ -254,6 +262,19 @@ - (void)setVelocityForward:(float)forward right:(float)right { _engine->setVelocity(forward, right); } +- (BOOL)setCharacter:(SKCharacterSettings)settings { + splat::CharacterSettings character = _engine->character(); + character.eyeHeight = settings.eyeHeight; + character.bodyRadius = settings.bodyRadius; + character.stepHeight = settings.stepHeight; + return _engine->setCharacter(character) ? YES : NO; +} + +- (SKCharacterSettings)character { + const splat::CharacterSettings& character = _engine->character(); + return SKCharacterSettings{character.eyeHeight, character.bodyRadius, character.stepHeight}; +} + - (void)setAttitude:(const float*)rowMajor { _engine->setAttitude(rowMajor); } diff --git a/packages/splatkit-ios/Sources/SplatKitCore/include/SplatKit/SKSplatEngine.h b/packages/splatkit-ios/Sources/SplatKitCore/include/SplatKit/SKSplatEngine.h index 6bf1428..328548e 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/include/SplatKit/SKSplatEngine.h +++ b/packages/splatkit-ios/Sources/SplatKitCore/include/SplatKit/SKSplatEngine.h @@ -21,8 +21,18 @@ typedef struct { float z; } SKVec3; +/// The walker in walk mode, in meters. Defaults: 1.5 m eye, 0.35 m body, 0.35 m step, +/// which climbs stairs and doorsteps but not chairs or counters. +typedef struct { + float eyeHeight; + float bodyRadius; + /// The highest rise walked onto. Above it the step is refused and the walker slides. + float stepHeight; +} SKCharacterSettings; + /// A snapshot of what the engine is doing, refreshed twice a second. typedef struct { + /// Frames per second over the last half second. float fps; float frameMillis; float gpuMillis; @@ -34,10 +44,21 @@ typedef struct { uint32_t computeTileCount; uint32_t nonemptyComputeTileCount; uint32_t hardwareTileCount; + /// fps counts frames the display showed; without it, frames submitted. + BOOL presentTiming; + /// 95th percentile display interval over the last 5 seconds; zero without presentTiming. + float frameMillisP95; + /// 1% low frame rate over the last 5 seconds; zero without presentTiming. + float lowFps; + /// Submitted frames never shown in the last half second. + uint32_t droppedFrames; } SKSplatStats; /// The renderer-applicable policy. Mirror of the host contract: raster 0 hardware, /// 1 computeTile, 2 hybrid; tileSize 8/16/32; sortDepth 16/32. +/// Hardware is the default and the fastest choice for most scenes. Hybrid screen tiles are +/// an opt-in for scenes where many large translucent splats overlap each pixel, such as +/// close-up interiors; on distant or sparse scenes they add GPU work and lower the frame rate. typedef struct { uint32_t raster; uint32_t tileSize; @@ -48,6 +69,9 @@ typedef struct { BOOL enableHiZOcclusion; BOOL enableEarlyTermination; uint32_t sortDepth; + /// Most hierarchy splats one frame selects, 0 for the loaded capacity. Changes live; when + /// the error threshold would select more, detail thins evenly across the view. + uint32_t lodSplatLimit; } SKRenderPolicy; /// Which policy fields this backend can apply. A false field resolves to `fallback`. @@ -61,11 +85,14 @@ typedef struct { BOOL enableHiZOcclusion; BOOL enableEarlyTermination; BOOL sortDepth; + /// Accepted raster strategies: bit 0 hardware, bit 1 computeTile, bit 2 hybrid; 0 means all. + uint32_t rasterMask; uint32_t tileSizeMask; float minLodErrorPixels; float maxLodErrorPixels; float minSubpixelThreshold; float maxSubpixelThreshold; + BOOL lodSplatLimit; } SKRenderPolicySupport; /// Resource ceilings and device features. Values come from the native adapter. @@ -147,6 +174,10 @@ typedef NS_ENUM(NSInteger, SKSplatEvent) { - (void)lookAtFrom:(SKVec3)position target:(SKVec3)target up:(SKVec3)up; - (void)walkForward:(float)forward right:(float)right; - (void)setVelocityForward:(float)forward right:(float)right; +/// The walker's shape, applied at once in walk mode and to a collider loaded later. +/// NO when a value is not a walkable one; the previous settings stay. Render thread. +- (BOOL)setCharacter:(SKCharacterSettings)settings; +@property(nonatomic, readonly) SKCharacterSettings character; /// Device to reference rotation, row major 3x3, device axes x right, y up, z out of the /// screen, reference z up. - (void)setAttitude:(const float*)rowMajor; diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.h b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.h index daa7257..2de68b1 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.h +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.h @@ -1,6 +1,7 @@ #pragma once #import +#include #include #include "splat/lod/LodTree.h" @@ -11,6 +12,9 @@ namespace splatkit { // Create/upload while idle; encode and its consumers use the same command queue. class MetalLOD { public: + // Safety bound on the selected cut; buffers scale with the budget actually requested. + static constexpr uint32_t kMaxBudget = 4'000'000; + bool create(id device, id library); bool upload(id queue, const splat::LodTree& tree, uint32_t budget, float pixelLimit = 1.0f, float colorWeight = 4.0f, bool frustumCull = true); @@ -18,7 +22,16 @@ class MetalLOD { id indices() const { return indices_; } // State words 4/5 contain denied refinements/evaluated interior nodes. id count() const { return state_; } - uint32_t budget() const { return config_.budget; } + // Selection buffers are sized for this; the per-frame limit never exceeds it. + uint32_t budget() const { return capacity_; } + uint32_t limit() const { return config_.budget; } + // Most splats one selection may emit, 0 for the full capacity. Read at each encode. + void setSplatLimit(uint32_t splats) { + config_.budget = splats == 0 ? capacity_ : std::min(splats, capacity_); + } + // Screen-space error a node may cover before it refines. Read at each encode, so it + // applies from the next frame; callers change it between frames only. + void setPixelLimit(float pixels) { config_.pixelLimit = pixels; } private: struct Config { @@ -27,6 +40,7 @@ class MetalLOD { float colorWeight; uint32_t cull; } config_{}; + uint32_t capacity_ = 0; id device_ = nil; id initialize_ = nil, evaluate_ = nil, budget_ = nil; id compact_ = nil, allocate_ = nil, scatter_ = nil; diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.mm b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.mm index 4a4afbf..818d1ba 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.mm +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalLOD.mm @@ -33,7 +33,7 @@ if (!valid || capacity == 0 || !std::isfinite(pixelLimit) || pixelLimit < 0 || !std::isfinite(colorWeight) || colorWeight < 0) return false; - capacity = std::min({capacity, 2200000u, static_cast(tree.leafCount)}); + capacity = std::min({capacity, kMaxBudget, static_cast(tree.leafCount)}); splat::LodSelectionData compatibility; const auto* data = &tree.selection; if (data->clusters.empty()) { @@ -82,6 +82,7 @@ !groups_ || !blocks_ || !state_ || !frontier_[0] || !frontier_[1]) return false; depth_ = valid.value() + 1; + capacity_ = capacity; config_ = {capacity, pixelLimit, colorWeight, frustumCull ? 1u : 0u}; LOGI( "GPU LOD SSE: %zu interior clusters, %zu leaves, %u rounds, capacity %u, %.2f px, color %.2f", diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.h b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.h index 973ab9d..a276a29 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.h +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -84,12 +85,16 @@ class MetalSplatRenderer final : public SplatRenderer { double lastSelectMillis() const override { return gpuFailed_.load() ? 0 : lastSelectMillis_.load(); } + // The threshold the next selection runs at, at least the policy's. Render thread. + float lodPixels() const { return lodPixels_; } uint32_t lastLodLimitedCount() const { return lod_ ? lastLodLimitedCount_.load() : 0; } uint32_t lastLodEvaluatedCount() const { return lod_ ? lastLodEvaluatedCount_.load() : 0; } ScreenTileStats lastScreenTileStats() const override { if (gpuFailed_.load()) return {}; return {lastComputeTiles_.load(), lastNonemptyComputeTiles_.load(), lastHardwareTiles_.load()}; } + bool reportsPresentTimes() const override { return true; } + uint32_t takePresentTimes(std::vector* times) override; const std::string& deviceDescription() const override { return description_; } static constexpr int kMaxShDegree = 3; @@ -133,6 +138,13 @@ class MetalSplatRenderer final : public SplatRenderer { uint32_t generation_ = 0; uint64_t frame_ = 0; std::atomic lastGpuMillis_{0}; + // Filled by drawable presented handlers, which can outlive a frame's other state. + struct PresentLog { + std::mutex mutex; + std::vector times; // display times in nanoseconds, capped while nobody drains + uint32_t dropped = 0; + }; + std::shared_ptr presents_ = std::make_shared(); std::atomic completedWorldFrame_{false}; MetalVisibility visibility_; std::unique_ptr lod_; @@ -141,8 +153,18 @@ class MetalSplatRenderer final : public SplatRenderer { std::atomic lastLodLimitedCount_{0}, lastLodEvaluatedCount_{0}; std::atomic lastSelectMillis_{0}; float minPixelRadius_ = 0.5f; + float lodErrorPixels_ = 1.0f; + uint32_t lodSplatLimit_ = 0; + // The threshold selection runs at: the policy's, raised while a cut would exceed the splat + // limit so detail thins evenly instead of stopping wherever traversal ran out of room. + float lodPixels_ = 1.0f; + std::atomic lodReadbacks_{0}; // completed selections, so each adapts once + uint32_t lodAdaptedReadback_ = 0; + void adaptLodThreshold(); MetalRadixSort::KeyBits depthBits_ = MetalRadixSort::KeyBits::Full32; MetalTileRaster tileRaster_; + // Pipelines exist once a view first asks for hybrid tiles; computeRaster_ is the policy. + bool tileRasterReady_ = false; bool computeRaster_ = false; // A GPU error latches this renderer off. Never repeatedly resubmit failed work. std::atomic gpuFailed_{false}; diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.mm b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.mm index e94ad4b..bc707fb 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.mm +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalSplatRenderer.mm @@ -1,9 +1,12 @@ #include "rendering/MetalSplatRenderer.h" +#include + #include #include #include #include +#include #include #include "SplatShaderSource.h" @@ -62,14 +65,6 @@ "scratch", r->minPixelRadius_); if (!r->gpuSort_) LOGW("GPU sort unavailable, sorting on the CPU"); - const char* tileValue = std::getenv("SPLATKIT_METAL_TILE_RASTER"); - if (r->gpuSort_ && tileValue != nullptr && std::strcmp(tileValue, "1") == 0) { - r->computeRaster_ = r->tileRaster_.create(r->device_, r->library_); - LOGI("Experimental tile raster: %s", - r->computeRaster_ - ? "bounded hybrid enabled (512 candidates / local large-footprint fallback)" - : "unavailable; using hardware"); - } r->inFlight_ = dispatch_semaphore_create(kFramesInFlight); r->description_ = std::string(r->device_.name.UTF8String) + ", Metal"; LOGI("%s", r->description_.c_str()); @@ -142,19 +137,19 @@ DeviceCapabilities MetalSplatRenderer::deviceCapabilities() const { DeviceCapabilities caps; - // MetalLOD caps a hierarchy at 2.2M nodes; the engine's residency window is [100k, 32M]. - caps.limits.maxLodCapacitySplats = 2'200'000; + // MetalLOD caps a selected cut at 4M nodes; the engine's residency window is [100k, 32M]. + caps.limits.maxLodCapacitySplats = MetalLOD::kMaxBudget; caps.limits.minResidencyCapacitySplats = 100'000; caps.limits.maxResidencyCapacitySplats = 32'000'000; - // Hybrid screen tiles are the experimental tile raster, reported as it stands. - caps.supportsComputeTiles = computeRaster_; + // Hybrid screen tiles need GPU ordering; their pipelines are built on first request. + caps.supportsComputeTiles = gpuSort_; caps.supportsHiZOcclusion = false; caps.supportsSubgroups = gpuSort_; // Metal exposes no texture-dimension query; Apple family 7, the minimum this renderer // accepts, supports 16384x16384. caps.maxTextureDimension = device_ != nil ? 16384u : 0u; RenderPolicySupport& policy = caps.policy; - policy.fallback.raster = computeRaster_ ? RasterStrategy::hybrid : RasterStrategy::hardware; + policy.fallback.raster = RasterStrategy::hardware; policy.fallback.tileSize = 16; policy.fallback.lodErrorPixels = 1.0f; policy.fallback.alphaThreshold = 1.0f / 255.0f; @@ -163,9 +158,17 @@ depthBits_ == MetalRadixSort::KeyBits::Low16 ? SortKeyBits::low16 : SortKeyBits::full32; policy.fallback.enableFrustumCulling = true; policy.fallback.enableEarlyTermination = true; - // Only the key width is a public per-instance control today. The sub-pixel radius is - // meaningful only under the internal tight-culling experiment; alpha, tiles and - // occlusion stay fixed shader/rasterization choices. + // The key width and hybrid tiles are public per-instance controls. The sub-pixel radius is + // meaningful only under the internal tight-culling experiment; alpha, tile size and + // occlusion stay fixed shader/rasterization choices. Pure compute tiles are not built. + // The LOD error threshold applies live to a hierarchy world and at its next upload. + policy.lodErrorPixels = gpuSort_; + policy.minLodErrorPixels = 0.1f; + policy.maxLodErrorPixels = 16.0f; + policy.lodSplatLimit = gpuSort_; + policy.raster = gpuSort_; + policy.rasterMask = 1u << static_cast(RasterStrategy::hardware) | + 1u << static_cast(RasterStrategy::hybrid); policy.sortDepth = gpuSort_; policy.subpixelThreshold = visibility_.tightCulling(); return caps; @@ -175,22 +178,59 @@ const MetalRadixSort::KeyBits bits = policy.sortDepth == SortKeyBits::low16 ? MetalRadixSort::KeyBits::Low16 : MetalRadixSort::KeyBits::Full32; - if (bits == depthBits_ && policy.subpixelThreshold == minPixelRadius_) return true; + const bool tiles = policy.raster == RasterStrategy::hybrid; + const bool visibilityChanged = bits != depthBits_ || policy.subpixelThreshold != minPixelRadius_; + const bool lodChanged = + policy.lodErrorPixels != lodErrorPixels_ || policy.lodSplatLimit != lodSplatLimit_; + if (!visibilityChanged && tiles == computeRaster_ && !lodChanged) return true; if (!gpuSort_) { // No GPU ordering pipelines exist; nothing the policy can reach. if (reason != nullptr) *reason = "Metal GPU ordering is unavailable"; return false; } - // Rebuilding pipelines while frames may still encode against them is unsafe. + // Rebuilding pipelines or the target while frames may still encode against them is unsafe. waitIdle(); - if (!visibility_.reconfigure(policy.subpixelThreshold, bits)) { + // Each step that fails reverts the earlier ones, so a failure keeps the previous policy. + if (tiles && !tileRasterReady_) { + tileRasterReady_ = tileRaster_.create(device_, library_); + if (!tileRasterReady_) { + if (reason != nullptr) *reason = "Metal tile raster pipelines are unavailable"; + return false; + } + } + const auto setTiles = [this](bool enabled) { + computeRaster_ = enabled; + // The compute compositor writes into the target, which needs shader-write usage. + return createTarget(); + }; + const bool tilesChanged = tiles != computeRaster_; + if (tilesChanged && !setTiles(tiles)) { + setTiles(!tiles); + if (reason != nullptr) *reason = "Metal render target rebuild failed"; + return false; + } + if (visibilityChanged && !visibility_.reconfigure(policy.subpixelThreshold, bits)) { + if (tilesChanged) setTiles(!tiles); if (reason != nullptr) *reason = "Metal visibility pipeline rebuild failed"; return false; } minPixelRadius_ = policy.subpixelThreshold; depthBits_ = bits; - LOGI("Metal policy: %u-bit sort keys, %.3fpx sub-pixel radius", static_cast(bits), - minPixelRadius_); + if (tilesChanged && !tiles) tileRaster_.releaseScratch(); + // Plain configuration read by the next selection encode; it cannot fail, so it goes last. + lodErrorPixels_ = policy.lodErrorPixels; + lodSplatLimit_ = policy.lodSplatLimit; + lodPixels_ = lodErrorPixels_; + // Frames are idle here, so every readback so far describes the previous policy. + lodAdaptedReadback_ = lodReadbacks_.load(); + if (lod_) { + lod_->setPixelLimit(lodPixels_); + lod_->setSplatLimit(lodSplatLimit_); + } + LOGI("Metal policy: %u-bit sort keys, %.3fpx sub-pixel radius, %.2fpx LOD error, %u LOD " + "splat limit, %s raster", + static_cast(bits), minPixelRadius_, lodErrorPixels_, lodSplatLimit_, + computeRaster_ ? "hybrid" : "hardware"); return true; } @@ -361,18 +401,9 @@ if (!gpuSort_) return false; waitIdle(); auto lod = std::make_unique(); - float qualityPixels = 1.0f; - if (const char* text = std::getenv("SPLATKIT_METAL_LOD_QUALITY_PIXELS")) { - char* end = nullptr; - const float value = std::strtof(text, &end); - if (end == text || *end != '\0' || !std::isfinite(value) || value < 0) { - LOGE("invalid experimental LOD quality threshold"); - return false; - } - qualityPixels = value; - } - if (!lod->create(device_, library_) || !lod->upload(queue_, tree, budget, qualityPixels)) + if (!lod->create(device_, library_) || !lod->upload(queue_, tree, budget, lodErrorPixels_)) return false; + lod->setSplatLimit(lodSplatLimit_); // Compact projections and radix scratch scale with the cut, not all resident nodes. MetalVisibility visibility; if (!visibility.create(device_, library_, true, minPixelRadius_, depthBits_) || @@ -388,6 +419,8 @@ visibility_ = std::move(visibility); lod_ = std::move(lod); lodReadback_ = readback; + lodPixels_ = lodErrorPixels_; + lodAdaptedReadback_ = lodReadbacks_.load(); lastLodLimitedCount_.store(0); lastLodEvaluatedCount_.store(0); world_ = std::move(world); @@ -420,6 +453,30 @@ // The frame. +// Selection counts arrive a frame or two late, so the threshold climbs fast while refinements +// are denied and relaxes slowly, holding inside a 10% band below the limit. +void MetalSplatRenderer::adaptLodThreshold() { + constexpr float kRaise = 1.1f; + constexpr float kRelax = 1.03f; + constexpr float kMaxPixels = 64.0f; + const uint32_t readbacks = lodReadbacks_.load(); + if (readbacks == lodAdaptedReadback_) return; + lodAdaptedReadback_ = readbacks; + if (lastLodLimitedCount_.load() > 0) { + lodPixels_ = std::min(lodPixels_ * kRaise, std::max(kMaxPixels, lodErrorPixels_)); + } else if (lastSelectedCount_.load() < lod_->limit() / 10 * 9) { + lodPixels_ = std::max(lodPixels_ / kRelax, lodErrorPixels_); + } + lod_->setPixelLimit(lodPixels_); +} + +uint32_t MetalSplatRenderer::takePresentTimes(std::vector* times) { + times->clear(); + const std::lock_guard lock(presents_->mutex); + times->swap(presents_->times); + return std::exchange(presents_->dropped, 0); +} + bool MetalSplatRenderer::draw(const Frame& frame) { if (gpuFailed_.load()) return false; if (!ready() || splatPipelines_[0] == nil) return false; @@ -473,6 +530,7 @@ const int degree = std::clamp(std::min(frame.shDegree, world_->info().shDegree), 0, kMaxShDegree); if (lod_) { + adaptLodThreshold(); auto selection = [queue_ commandBuffer]; selection.label = @"GPU LOD selection"; lod_->encode(selection, uniforms_[slot]); @@ -489,6 +547,8 @@ std::atomic* limited = &lastLodLimitedCount_; std::atomic* evaluated = &lastLodEvaluatedCount_; const bool logLod = frame_ % 120 == 0; + std::atomic* readbacks = &lodReadbacks_; + const float pixels = lodPixels_; std::atomic* failed = &gpuFailed_; [selection addCompletedHandler:^(id done) { if (done.status == MTLCommandBufferStatusError) { @@ -500,9 +560,11 @@ const auto* counters = static_cast(selectedCount.contents); limited->store(counters[4]); evaluated->store(counters[5]); + readbacks->fetch_add(1); if (logLod) - LOGI("LOD SSE: %u selected, %u evaluated interiors, %u quality-limited refinements", - counters[0], counters[5], counters[4]); + LOGI("LOD SSE: %u selected, %u evaluated interiors, %u quality-limited refinements, " + "%.2f px", + counters[0], counters[5], counters[4], pixels); milliseconds->store((done.GPUEndTime - done.GPUStartTime) * 1000.0); }]; [selection commit]; @@ -635,6 +697,20 @@ } } + // presentedTime is when the frame reached the display, zero when it was never shown. + // The simulator's Metal has no presentation handler, so it reports no present timing. +#if !TARGET_OS_SIMULATOR + std::shared_ptr presents = presents_; + [drawable addPresentedHandler:^(id shown) { + const CFTimeInterval time = shown.presentedTime; + const std::lock_guard lock(presents->mutex); + if (time <= 0) { + ++presents->dropped; + } else if (presents->times.size() < 1024) { + presents->times.push_back(static_cast(time * 1e9)); + } + }]; +#endif [cmd presentDrawable:drawable]; dispatch_semaphore_t inFlight = inFlight_; std::atomic* gpuMillis = &lastGpuMillis_; diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.h b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.h index 388b560..cbf6bfa 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.h +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.h @@ -20,6 +20,8 @@ class MetalTileRaster { // [compute tiles, hardware tiles, invalid input, nonempty compute tiles]. // Read after slot completion. Compute tiles include background-only tiles. id diagnostics(uint32_t slot) const { return diagnostics_[slot]; } + // Frees the candidate scratch; the next encode allocates it again. + void releaseScratch(); private: id device_ = nil; diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.mm b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.mm index 40eaca5..d21dbcb 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.mm +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/MetalTileRaster.mm @@ -21,6 +21,13 @@ raster_.staticThreadgroupMemoryLength <= device.maxThreadgroupMemoryLength; } +void MetalTileRaster::releaseScratch() { + bins_ = nil; + counts_ = nil; + fallback_ = nil; + rectangles_ = nil; +} + bool MetalTileRaster::encode(id cmd, id uniforms, id projected, id order, id count, uint32_t capacity, id target, uint32_t slot) { diff --git a/packages/splatkit-ios/Sources/SplatKitCore/rendering/shaders/SplatLOD.metal b/packages/splatkit-ios/Sources/SplatKitCore/rendering/shaders/SplatLOD.metal index 6055205..a599157 100644 --- a/packages/splatkit-ios/Sources/SplatKitCore/rendering/shaders/SplatLOD.metal +++ b/packages/splatkit-ios/Sources/SplatKitCore/rendering/shaders/SplatLOD.metal @@ -72,8 +72,8 @@ kernel void evaluateSplatLOD(uint t [[thread_position_in_grid]], if (lane == 0 && t < state.active) groups[t / 32] = uint4(sum, drops, 0, 0); } -// Two-level parallel scan: 256 SIMD-group totals per block, then at most 269 -// block totals at the 2.2M safety limit. No global atomic contention. +// Two-level parallel scan: 256 SIMD-group totals per block, then at most 489 +// block totals at the 4M safety limit. No global atomic contention. kernel void scanSplatLODGroups(uint t [[thread_position_in_grid]], uint tid [[thread_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], diff --git a/packages/splatkit-ios/distribution/CHANGELOG.md b/packages/splatkit-ios/distribution/CHANGELOG.md index f3d941c..f8498c5 100644 --- a/packages/splatkit-ios/distribution/CHANGELOG.md +++ b/packages/splatkit-ios/distribution/CHANGELOG.md @@ -3,6 +3,29 @@ Notable changes to the SplatKit iOS SDK and the shared C++ engine it ships. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); alphas may break APIs. +## Unreleased + +### Added + +- `renderPolicy.raster` selects hybrid screen tiles per view, and `SKRenderPolicySupport.rasterMask` lists the strategies Metal builds: hardware and hybrid. + Hybrid is an experimental opt-in for scenes where many large translucent splats overlap each pixel; hardware stays the default and is faster on distant or sparse scenes. + The tile pipelines are built on the first request, and switching back to hardware frees the tile scratch. +- `renderPolicy.lodErrorPixels` applies live on Metal: a lower threshold refines a hierarchy world further, up to the budget it loaded with. +- Shared engine: `RenderPolicySupport::rasterMask`; a raster strategy outside it falls back with a warning. +- `renderPolicy.lodSplatLimit` caps the splats a hierarchy frame selects, live, below the loaded capacity. + When a cut would exceed the limit or the loaded capacity, Metal raises the error threshold frame by frame until it fits, so detail thins evenly instead of stopping wherever traversal ran out of room. +- Stats measure frames the display showed: Metal reports each drawable's presented time, `fps` counts shown frames, and `presentTiming`, `frameMillisP95`, `lowFps` (1% low over 5 seconds) and `droppedFrames` join `SKSplatStats` and `SplatStats`. + The periodic log line appends the same fields. + +### Changed + +- A hierarchy world's LOD capacity reaches 4M selected splats, up from 2.2M; `maxLodCapacitySplats` reports it. +- Walk mode refuses steps onto a floor more than 0.35 m higher, looking 0.25 m ahead, so it climbs stairs and steps over door tracks but no longer climbs counters, chairs or tables whose top the hip probe passes over, and slides along them when walked into at an angle. + +### Removed + +- The `SPLATKIT_METAL_TILE_RASTER` and `SPLATKIT_METAL_LOD_QUALITY_PIXELS` environment variables; set `renderPolicy.raster` and `renderPolicy.lodErrorPixels` instead. + ## [0.1.0-alpha.3] - 2026-09-16 ### Added diff --git a/packages/splatkit-ios/tests/MetalRasterTest.mm b/packages/splatkit-ios/tests/MetalRasterTest.mm index c87fa9f..02e163f 100644 --- a/packages/splatkit-ios/tests/MetalRasterTest.mm +++ b/packages/splatkit-ios/tests/MetalRasterTest.mm @@ -1,7 +1,6 @@ #include #include #include -#include #include #include "rendering/MetalSplatRenderer.h" @@ -13,22 +12,13 @@ namespace splatkit { namespace { -class TileMode { - public: - explicit TileMode(bool enabled) { - if (const char* value = std::getenv("SPLATKIT_METAL_TILE_RASTER")) previous_ = value; - setenv("SPLATKIT_METAL_TILE_RASTER", enabled ? "1" : "0", 1); - } - ~TileMode() { - if (previous_) - setenv("SPLATKIT_METAL_TILE_RASTER", previous_->c_str(), 1); - else - unsetenv("SPLATKIT_METAL_TILE_RASTER"); - } - - private: - std::optional previous_; -}; +// Hybrid screen tiles are per-view policy; hardware is every renderer's default. +void useHybridTiles(MetalSplatRenderer& renderer) { + RenderPolicy policy = renderer.deviceCapabilities().policy.fallback; + policy.raster = RasterStrategy::hybrid; + std::string reason; + ASSERT_TRUE(renderer.applyRenderPolicy(policy, &reason)) << reason; +} class MetalRasterTest : public testing::Test { protected: @@ -40,7 +30,6 @@ void SetUp() override { }; TEST_F(MetalRasterTest, WorldReadinessRequiresGpuCompletionAndResetsOnReplacement) { - TileMode option(false); auto renderer = MetalSplatRenderer::create(); ASSERT_NE(renderer, nullptr); auto layer = [CAMetalLayer layer]; @@ -80,12 +69,59 @@ void SetUp() override { EXPECT_FALSE(renderer->hasCompletedWorldFrame()); } +TEST_F(MetalRasterTest, HybridTilesAreAPerViewOptInThatTurnsBackOff) { + auto renderer = MetalSplatRenderer::create(); + ASSERT_NE(renderer, nullptr); + const DeviceCapabilities caps = renderer->deviceCapabilities(); + EXPECT_TRUE(caps.supportsComputeTiles); + EXPECT_EQ(caps.policy.fallback.raster, RasterStrategy::hardware); + RenderPolicy computeOnly = caps.policy.fallback; + computeOnly.raster = RasterStrategy::computeTile; + const RenderPolicyResolution resolution = resolveRenderPolicy(computeOnly, caps.policy); + EXPECT_EQ(resolution.effective.raster, RasterStrategy::hardware); + ASSERT_EQ(resolution.warnings.size(), 1u); + + auto layer = [CAMetalLayer layer]; + layer.drawableSize = CGSizeMake(64, 64); + renderer->setLayer(layer); + renderer->setDrawableSize(64, 64); + splat::SplatCloud cloud; + cloud.positions = {0, 0, -2}; + cloud.colors = {1, 0, 0}; + cloud.alphas = {1}; + cloud.covariances = {0.04f, 0, 0, 0.04f, 0, 0.04f}; + ASSERT_TRUE(renderer->uploadWorld(cloud, 0)); + SplatRenderer::Frame frame; + frame.orderSource = SplatRenderer::OrderSource::gpu; + const SplatRenderer::Range range{0, 1}; + frame.ranges = ⦥ + frame.rangeCount = 1; + frame.proj = splat::Mat4::perspective(1, 1, 0.1f, 100); + auto completed = dispatch_semaphore_create(0); + auto tilesAfterFrame = [&] { + renderer->captureNextFrame( + [&](std::vector, uint32_t, uint32_t) { dispatch_semaphore_signal(completed); }); + EXPECT_TRUE(renderer->draw(frame)); + EXPECT_EQ( + dispatch_semaphore_wait(completed, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)), 0); + const auto tiles = renderer->lastScreenTileStats(); + return tiles.compute + tiles.hardware; + }; + EXPECT_EQ(tilesAfterFrame(), 0u); + useHybridTiles(*renderer); + EXPECT_EQ(tilesAfterFrame(), 16u); + RenderPolicy hardware = caps.policy.fallback; + std::string reason; + ASSERT_TRUE(renderer->applyRenderPolicy(hardware, &reason)) << reason; + EXPECT_EQ(tilesAfterFrame(), 0u); +} + TEST_F(MetalRasterTest, HybridCompletesOverflowTilesWithTheFullHardwareImage) { std::vector images[2]; for (int mode = 0; mode < 2; ++mode) { - TileMode option(mode != 0); auto renderer = MetalSplatRenderer::create(); ASSERT_NE(renderer, nullptr); + if (mode != 0) useHybridTiles(*renderer); auto layer = [CAMetalLayer layer]; layer.drawableSize = CGSizeMake(64, 64); renderer->setLayer(layer); @@ -125,9 +161,9 @@ void SetUp() override { TEST_F(MetalRasterTest, LargeFootprintAndCrossTileBoundaryMatchHardwareImage) { std::vector images[2]; for (int mode = 0; mode < 2; ++mode) { - TileMode option(mode != 0); auto renderer = MetalSplatRenderer::create(); ASSERT_NE(renderer, nullptr); + if (mode != 0) useHybridTiles(*renderer); auto layer = [CAMetalLayer layer]; layer.drawableSize = CGSizeMake(128, 128); renderer->setLayer(layer); @@ -193,9 +229,9 @@ void SetUp() override { } std::vector images[2]; for (int mode = 0; mode < 2; ++mode) { - TileMode option(mode != 0); auto renderer = MetalSplatRenderer::create(); ASSERT_NE(renderer, nullptr); + if (mode != 0) useHybridTiles(*renderer); auto layer = [CAMetalLayer layer]; layer.drawableSize = CGSizeMake(64, 64); renderer->setLayer(layer); @@ -230,6 +266,70 @@ void SetUp() override { EXPECT_GT(images[1][(32 * 64 + 56) * 4 + 1], 80); } +TEST_F(MetalRasterTest, LodErrorPolicyRefinesALoadedHierarchyLive) { + splat::LodTree tree; + tree.leafCount = 64; + tree.nodes.positions = {0, 0, -8}; + tree.nodes.colors = {1, 1, 1}; + tree.nodes.alphas = {1}; + tree.nodes.covariances = {0.04f, 0, 0, 0.04f, 0, 0.04f}; + tree.layout.push_back({{0, 0, -8}, 0.4f, 1, 64}); + for (uint32_t i = 0; i < 64; ++i) { + const float x = (static_cast(i % 8) - 3.5f) * 0.1f; + const float y = (static_cast(i / 8) - 3.5f) * 0.1f; + tree.nodes.positions.insert(tree.nodes.positions.end(), {x, y, -8}); + tree.nodes.colors.insert(tree.nodes.colors.end(), {1, 1, 1}); + tree.nodes.alphas.push_back(1); + tree.nodes.covariances.insert(tree.nodes.covariances.end(), {0.001f, 0, 0, 0.001f, 0, 0.001f}); + tree.layout.push_back({{x, y, -8}, 0, 0, 0}); + } + auto renderer = MetalSplatRenderer::create(); + ASSERT_NE(renderer, nullptr); + const DeviceCapabilities caps = renderer->deviceCapabilities(); + EXPECT_TRUE(caps.policy.lodErrorPixels); + EXPECT_EQ(caps.limits.maxLodCapacitySplats, 4'000'000u); + auto layer = [CAMetalLayer layer]; + layer.drawableSize = CGSizeMake(64, 64); + renderer->setLayer(layer); + renderer->setDrawableSize(64, 64); + ASSERT_TRUE(renderer->uploadLodWorld(tree, 0, 64)); + SplatRenderer::Frame frame; + frame.orderSource = SplatRenderer::OrderSource::gpu; + frame.proj = splat::Mat4::perspective(1, 1, 0.1f, 100); + auto completed = dispatch_semaphore_create(0); + auto selectedAt = [&](float pixels, uint32_t limit = 0) { + RenderPolicy policy = caps.policy.fallback; + policy.lodErrorPixels = pixels; + policy.lodSplatLimit = limit; + std::string reason; + EXPECT_TRUE(renderer->applyRenderPolicy(policy, &reason)) << reason; + // Selection readback trails submission, so settle over a few frames. + for (int i = 0; i < 4; ++i) { + renderer->captureNextFrame( + [&](std::vector, uint32_t, uint32_t) { dispatch_semaphore_signal(completed); }); + EXPECT_TRUE(renderer->draw(frame)); + EXPECT_EQ( + dispatch_semaphore_wait(completed, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)), + 0); + } + return renderer->lastSelectedCount(); + }; + const uint32_t coarse = selectedAt(16); + const uint32_t fine = selectedAt(0.1f); + printf("[ LOD ] selected %u at 16 px, %u at 0.1 px\n", coarse, fine); + EXPECT_EQ(coarse, 1u); + EXPECT_EQ(fine, 64u); + EXPECT_EQ(selectedAt(16), 1u); + + // A limit below the refined cut denies the split and raises the threshold instead; lifting + // the limit restores the policy's threshold at once. + EXPECT_TRUE(caps.policy.lodSplatLimit); + EXPECT_EQ(selectedAt(0.1f, 32), 1u); + EXPECT_GT(renderer->lodPixels(), 0.1f); + EXPECT_EQ(selectedAt(0.1f), 64u); + EXPECT_FLOAT_EQ(renderer->lodPixels(), 0.1f); +} + TEST_F(MetalRasterTest, GpuOrderedSplatContributesToThePresentedPixels) { auto renderer = MetalSplatRenderer::create(); ASSERT_NE(renderer, nullptr);