From fa9367189068e73e4d704ea81094e1d50a26fa22 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 26 May 2026 08:27:44 -0700 Subject: [PATCH 1/7] android: CNSDK display-processor plug-in (Android arm of #263 + runtime PR #268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to displayxr-runtime PR #268, which splits the Android POC along the post-#263 plug-in boundary. Runtime side keeps the cross- cutting infrastructure (is_self_submitting vtable flag, vk_native Android plumbing, audit fixes); CNSDK + drv-side audit fixes live here. Adds `src/drv_leia_android/` as a sibling of the Windows drv_leia/, sharing the same plug-in entry contract (xrtPluginNegotiate) but selected at CMake configure time by `if(WIN32)` / `elseif(ANDROID)`. Source files leia_cnsdk.{cpp,h} — CNSDK C-ABI wrapper: core init (worker thread), interlacer lifecycle, atlas weave, face-tracking snapshot. Audit fixes B2/B4/B5/B10/B11/B12. leia_display_processor_cnsdk.{cpp,h} — xrt_display_processor vtable. is_self_submitting=true, atlas-mode only (no per- tile blit), DXR_HW_DBG + DXR_ATRACE blocks gated on XRT_DEBUG_ANDROID_VERBOSE. Audit B7 / B8 / B14. leia_plugin_android.c — entry point + iface. Only create_dp_vk is non-NULL (no D3D/Metal/GL on Android). id = "leia-cnsdk", probe always succeeds (POC pattern, async device init). Build wiring CMakeLists.txt (top-level) — relax `if(NOT WIN32) FATAL_ERROR` into `if(NOT WIN32 AND NOT ANDROID)`; branch add_subdirectory between src/drv_leia (Windows) and src/drv_leia_android (Android). Skip the NSIS installer on Android (APK is the install). src/drv_leia_android/CMakeLists.txt — new. find_package(CNSDK CONFIG REQUIRED) on CNSDK_ROOT; builds libdxrp050_leia_cnsdk.so matching the runtime's plug-in filename convention; symbol- hidden except xrtPluginNegotiate. Docs docs/cnsdk-android-calibration.md — moved from displayxr-runtime docs/cnsdk-axis-calibration branch per PR #271 plan, with file-path refs rewritten to point at src/drv_leia_android/. Status: compiles in the standalone repo; not yet wired into the runtime APK's Gradle (manual jniLibs/ drop per README until the multi-module gradle setup lands). First hardware install gated on Lume Pad arrival. Co-Authored-By: Claude Opus 4.7 (1M context) --- CMakeLists.txt | 43 +- docs/cnsdk-android-calibration.md | 113 ++++ src/drv_leia_android/CMakeLists.txt | 93 ++++ src/drv_leia_android/README.md | 60 +++ src/drv_leia_android/leia_cnsdk.cpp | 503 ++++++++++++++++++ src/drv_leia_android/leia_cnsdk.h | 193 +++++++ .../leia_display_processor_cnsdk.cpp | 456 ++++++++++++++++ .../leia_display_processor_cnsdk.h | 43 ++ src/drv_leia_android/leia_plugin_android.c | 229 ++++++++ 9 files changed, 1718 insertions(+), 15 deletions(-) create mode 100644 docs/cnsdk-android-calibration.md create mode 100644 src/drv_leia_android/CMakeLists.txt create mode 100644 src/drv_leia_android/README.md create mode 100644 src/drv_leia_android/leia_cnsdk.cpp create mode 100644 src/drv_leia_android/leia_cnsdk.h create mode 100644 src/drv_leia_android/leia_display_processor_cnsdk.cpp create mode 100644 src/drv_leia_android/leia_display_processor_cnsdk.h create mode 100644 src/drv_leia_android/leia_plugin_android.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 08d79a8..9d587b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,8 +13,12 @@ set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -if(NOT WIN32) - message(FATAL_ERROR "The Leia SR plug-in is Windows-only.") +if(NOT WIN32 AND NOT ANDROID) + message(FATAL_ERROR + "The Leia plug-in supports two targets:\n" + " - Windows / Leia SR (DisplayXR-LeiaSR.dll, drv_leia)\n" + " - Android / Leia CNSDK (libdxrp050_leia_cnsdk.so, drv_leia_android)\n" + "Configured platform is neither WIN32 nor ANDROID.") endif() if(NOT DEFINED BUILD_NUM) @@ -64,17 +68,26 @@ else() FetchContent_MakeAvailable(displayxr_runtime) endif() -# --- SR SDK ------------------------------------------------------------- -if(NOT DEFINED ENV{LEIASR_SDKROOT}) - message(FATAL_ERROR - "LEIASR_SDKROOT env var not set. Point at an extracted LeiaSR-SDK-*-win64 directory. " - "scripts/build-windows.bat fetches it from the sr-sdk-v* release on this repo.") -endif() -set(SR_PATH "$ENV{LEIASR_SDKROOT}") -list(APPEND CMAKE_PREFIX_PATH "${SR_PATH}") -find_package(simulatedreality CONFIG REQUIRED GLOBAL) -find_package(srDirectX CONFIG REQUIRED GLOBAL) -message(STATUS "Leia SR SDK at ${SR_PATH}") +if(WIN32) + # --- SR SDK (Windows / Leia SR plug-in) ---------------------------- + if(NOT DEFINED ENV{LEIASR_SDKROOT}) + message(FATAL_ERROR + "LEIASR_SDKROOT env var not set. Point at an extracted LeiaSR-SDK-*-win64 directory. " + "scripts/build-windows.bat fetches it from the sr-sdk-v* release on this repo.") + endif() + set(SR_PATH "$ENV{LEIASR_SDKROOT}") + list(APPEND CMAKE_PREFIX_PATH "${SR_PATH}") + find_package(simulatedreality CONFIG REQUIRED GLOBAL) + find_package(srDirectX CONFIG REQUIRED GLOBAL) + message(STATUS "Leia SR SDK at ${SR_PATH}") -add_subdirectory(src/drv_leia) -add_subdirectory(installer) + add_subdirectory(src/drv_leia) + add_subdirectory(installer) +elseif(ANDROID) + # --- CNSDK (Android / Leia CNSDK plug-in) -------------------------- + # CNSDK setup (find_package(CNSDK CONFIG)) lives inside + # src/drv_leia_android/CMakeLists.txt — it consumes CNSDK_ROOT, + # which the build environment supplies. No NSIS installer on + # Android; the .so ships in the runtime APK's jniLibs//. + add_subdirectory(src/drv_leia_android) +endif() diff --git a/docs/cnsdk-android-calibration.md b/docs/cnsdk-android-calibration.md new file mode 100644 index 0000000..a4612e6 --- /dev/null +++ b/docs/cnsdk-android-calibration.md @@ -0,0 +1,113 @@ +# CNSDK Android Calibration Procedure + +The Android POC's CNSDK display processor makes three assumptions about +CNSDK conventions that the SDK docs leave ambiguous. Once a Lume Pad is +attached, validate each in this order — they're listed cheapest-to-test +first. Audit references in brackets. + +## 1. Face position axis convention [B15] + +**What's assumed:** in `src/drv_leia_android/leia_cnsdk.cpp::leia_cnsdk_get_primary_face`, +CNSDK's `leia_core_get_primary_face` returns a 3-float position in +millimeters with axes x=right / y=up / z=toward viewer, origin at the +camera. The wrapper converts mm→m and subtracts the cached +`cameraCenterX/Y/Z` to land in `xrt_eye_position`'s display-relative +frame. + +**What CNSDK actually documents** (cnsdk/include/leia/headTracking/common/types.h): +- `leia_headtracking_detected_face.posePosition` is "Head location in + mm. The origin point is the location of the camera." ✓ matches. +- `leia_headtracking_face.point.pos` is "3D coordinate with camera + transform and Kalman filter applied." Axis convention not stated. +- `posePoseAngle` is "in radians. **The rotation is a left handed + coordinate system.**" Strong hint that positions may also follow a + left-handed system. + +**Test:** stand directly in front of the display, motionless. Note the +last reported eye position in logcat (look for `xrLocateViews` HUD +output or any debug printf you add). Then move your head 10 cm +**right**. Re-read. + +| Result | What it means | Fix | +|---|---|---| +| x increased by ~0.10 | Right-handed, x=right. Convention matches. ✓ | — | +| x decreased by ~0.10 | Left-handed, x=right OR right-handed with x=left | Flip sign of `out_x` in `leia_cnsdk_get_primary_face` | +| y changed by ~0.10 | Axes are rotated; CNSDK uses landscape vs portrait orientation differently | Swap x/y, possibly negate one | + +Repeat for **down** (y) and **toward display** (z). + +If the cube appears to track the wrong direction, this is the first +thing to check. + +## 2. Tile-to-eye mapping [B17] + +**What's assumed:** the SBS atlas's tile (col 0, row 0) is the **left +eye** view and (col 1, row 0) is the **right eye**. The DP's +`blit_atlas_to_per_view` blits tile 0 → `view_img[0]`, tile 1 → +`view_img[1]`. `leia_cnsdk_weave` then calls +`leia_interlacer_vulkan_set_view_for_texture_array(interlacer, 0, view_iv[0])` +and `(interlacer, 1, view_iv[1])` — passing left as view-0, right as +view-1. + +**What CNSDK actually does:** undocumented. CNSDK's interlacer treats +view 0 and view 1 as left and right per common convention, but this +isn't explicit in the headers. + +**Test:** display the [Khronos OpenXR cube test pattern](https://github.com/KhronosGroup/OpenXR-SDK-Source/tree/main/src/tests/hello_xr) +or any stereo-pair where left and right eyes have visually distinct +content (e.g., a number "1" rendered to left view, "2" to right). Look +at the Lume Pad with one eye covered, then the other. + +| What you see | Meaning | Fix | +|---|---|---| +| "1" with left eye, "2" with right | Convention matches. ✓ | — | +| "2" with left eye, "1" with right | Tile-to-eye mapping is swapped | In `leia_cnsdk_weave`, swap the args to `set_view_for_texture_array` (pass right as 0, left as 1). Or in the DP, swap the destination index in `blit_atlas_to_per_view`. | +| Both eyes see "1" and "2" overlaid as a ghost | Interlacing pattern broken — likely a calibration issue, not view assignment | Check `leia_core_get_device_config` is returning the right `displayDots` — out of scope of this doc | + +## 3. UV vertical flip [B18] + +**What's assumed:** `leia_cnsdk.cpp::leia_cnsdk_weave` calls +`leia_interlacer_set_flip_input_uv_vertical(interlacer, true)` because +Vulkan's NDC is Y-down (origin at top-left) and CNSDK is presumed to +default to OpenGL convention (Y-up, origin bottom-left). + +**Test:** display any content with clear top/bottom asymmetry — text +is ideal. A rendered string "HELLO" or just a single capital "A" works. + +| What you see | Meaning | Fix | +|---|---|---| +| Right-side-up "HELLO" | UV flip convention is correct ✓ | — | +| Upside-down "OLLEH" / "HELLO" vertically mirrored | CNSDK is already flipping; we're double-flipping | Change `leia_interlacer_set_flip_input_uv_vertical(interlacer, true)` to `false` in `leia_cnsdk.cpp::leia_cnsdk_weave` | +| Letters look correct but offset / cropped | Different bug — not a flip issue. Check view_width / view_height accounting | Out of scope of this doc | + +## Combined check + +Once all three are pinned down, the cube test app (when it lands — +B13d) should: +1. Render a spinning cube at the depth corresponding to the wearer's + head distance. +2. Move convincingly with head motion (left-right strafing). +3. Have correct horizontal disparity (look closer with one eye than the + other). +4. Show right-side-up text/numbers on debug overlays. + +If 1+3 are off, it's #1 (face axes). If 2 is off, it's #1 or #2 (eye +mapping). If 4 is off, it's #3 (UV flip). + +## Where the convention assumptions live + +| Assumption | File | Function | Audit ref | +|---|---|---|---| +| Face position axes / units | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_get_primary_face` | B15 | +| Tile-to-eye mapping | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_weave` (`set_view_for_texture_array`) | B17 | +| UV vertical flip | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_weave` (`set_flip_input_uv_vertical`) | B18 | + +All three are 1–3 line changes once you know the correct values. + +## Why these aren't fixed pre-emptively + +Each is a 50/50 coin flip in either direction. Picking wrong is worse +than picking right (off-by-180° on axes is more confusing than +matching the spec assumption). Lume Pad on-device test is the +definitive answer; this doc lets you arrive at hardware ready to flip +the right knob. diff --git a/src/drv_leia_android/CMakeLists.txt b/src/drv_leia_android/CMakeLists.txt new file mode 100644 index 0000000..ca28d3f --- /dev/null +++ b/src/drv_leia_android/CMakeLists.txt @@ -0,0 +1,93 @@ +# Copyright 2026, Leia Inc / DisplayXR +# SPDX-License-Identifier: BSL-1.0 +# +# DisplayXR Leia CNSDK plug-in (Android). +# +# Builds the CNSDK-backed display-processor as a shared library named +# `libdxrp050_leia_cnsdk.so`, matching the runtime's Android plug-in +# discovery convention (libdxrp_.so, where NNN is the +# probe-order and matches the iface's `id` field). See +# `docs/specs/runtime/plugin-discovery.md` §3.2 in the runtime tree. +# +# The .so is meant to be dropped into the runtime APK's +# `jniLibs//` so the runtime's target_plugin_loader.c (PR #309) +# discovers it via dladdr-of-self + dirname enumeration at +# `xrCreateInstance` time. + +if(NOT ANDROID) + message(FATAL_ERROR "drv_leia_android is Android-only.") +endif() + +# --- CNSDK ---------------------------------------------------------------- +# +# CNSDK ships as an AAR + extracted SDK tree (headers + .so + cmake +# config). The expected layout matches what was previously consumed by +# the runtime POC's openxr_android/build.gradle CNSDK_AAR rule: +# +# ${CNSDK_ROOT}/include/leia/sdk/*.h +# ${CNSDK_ROOT}/share/cmake/CNSDK/CNSDKConfig.cmake +# ${CNSDK_ROOT}/lib//libleiaSDK-faceTrackingInApp.so +# +# CI / dev set CNSDK_ROOT to the extracted release tree. Pinned to +# CNSDK 0.7.28 for the POC. + +if(NOT DEFINED CNSDK_ROOT) + if(DEFINED ENV{CNSDK_ROOT}) + set(CNSDK_ROOT "$ENV{CNSDK_ROOT}") + else() + message(FATAL_ERROR + "CNSDK_ROOT not set. Extract CNSDK 0.7.28 and set " + "CNSDK_ROOT (CMake var or env var) to the extracted dir. " + "Source: https://github.com/LeiaInc/leiainc.github.io/tree/master/CNSDK/cnsdk-android-0.7.28.zip") + endif() +endif() +list(APPEND CMAKE_PREFIX_PATH "${CNSDK_ROOT}") +find_package(CNSDK CONFIG REQUIRED) +message(STATUS "Leia CNSDK at ${CNSDK_ROOT}") + +# --- Plug-in target ------------------------------------------------------- + +set(DRV_LEIA_ANDROID_SOURCES + leia_cnsdk.cpp + leia_cnsdk.h + leia_display_processor_cnsdk.cpp + leia_display_processor_cnsdk.h + leia_plugin_android.c +) + +add_library(dxrp050_leia_cnsdk SHARED ${DRV_LEIA_ANDROID_SOURCES}) + +target_include_directories(dxrp050_leia_cnsdk PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(dxrp050_leia_cnsdk + PRIVATE + # Runtime aux surface (consumed via FetchContent or local + # checkout per top-level CMakeLists.txt). + xrt-interfaces + aux_util + aux_os + aux_math + aux_vk + # CNSDK Vulkan interlacer + face-tracker. + CNSDK::leiaSDK + # Android system libs. + log + android + vulkan +) + +# Debug-log + ATrace gating: when XRT_DEBUG_ANDROID_VERBOSE is defined, +# DXR_HW_DBG / DXR_ATRACE blocks compile in. Release builds drop them +# all. The runtime APK's Gradle Debug variant sets this; Release does +# not. For standalone CMake builds, pass -DXRT_DEBUG_ANDROID_VERBOSE=1 +# on the cmake command line. + +set_target_properties(dxrp050_leia_cnsdk PROPERTIES + OUTPUT_NAME "dxrp050_leia_cnsdk" + PREFIX "lib" # libdxrp050_leia_cnsdk.so (matches discovery convention) + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) diff --git a/src/drv_leia_android/README.md b/src/drv_leia_android/README.md new file mode 100644 index 0000000..2bc7246 --- /dev/null +++ b/src/drv_leia_android/README.md @@ -0,0 +1,60 @@ +# drv_leia_android — DisplayXR Leia CNSDK Plug-in + +The Android variant of the Leia display-processor plug-in. Wraps the +**Leia Computational Display SDK (CNSDK)** as an `xrt_display_processor` +implementation, shipping as `libdxrp050_leia_cnsdk.so`. + +The runtime's `target_plugin_loader.c` Android branch (PR #309 / commit +`c96c93ce8`) discovers this `.so` at `xrCreateInstance` time via +`dladdr`-of-self + dirname enumeration of files matching the +`libdxrp_.so` convention. Bundle this `.so` into the runtime +APK's `jniLibs//` (single-vendor mode); multi-APK discovery is v2 +(tracked at runtime #310). + +## Files + +| File | Purpose | +|---|---| +| `leia_cnsdk.{cpp,h}` | C++ wrapper around the CNSDK C ABI: core init, interlacer lifecycle, face-tracking worker thread, atlas weave entry point, device-config caching. | +| `leia_display_processor_cnsdk.{cpp,h}` | `xrt_display_processor` vtable wired to the wrapper. Advertises `is_self_submitting=true` so the compositor flushes its pre-DP cmd buffer + skips its own post-DP submit. Atlas mode only (per-tile blit removed — CNSDK splits the SBS atlas internally via `set_interlace_view_texture_atlas`). | +| `leia_plugin_android.c` | `xrtPluginNegotiate` entry point + `xrt_plugin_iface` vtable. `create_dp_vk` is the only non-NULL factory slot. | + +## Build + +```bash +# CNSDK 0.7.28 extracted somewhere on disk: +export CNSDK_ROOT=/path/to/cnsdk + +# Configure for arm64-v8a Android via the NDK toolchain: +cmake -S . -B build-android \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-29 \ + -G Ninja + +cmake --build build-android --target dxrp050_leia_cnsdk +``` + +Output: `build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so`. + +Copy that into your runtime APK's +`src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a/` before +running `./gradlew :src:xrt:targets:openxr_android:assembleInProcessDebug`. + +(End-to-end multi-module Gradle integration is a follow-up — see the +top-level repo README's "Android — POC build flow" section.) + +## CNSDK convention assumptions + +Three plug-in-side conventions are assumed in this POC (face axis +signs/units, tile-to-eye mapping, UV vertical flip). All three may +need single-line flips after first hardware bring-up — see +[`docs/cnsdk-android-calibration.md`](../../docs/cnsdk-android-calibration.md) +for the symptom-→-fix table. + +## Status + +POC — not yet validated on a Lume Pad. Compiles clean on the host. +First hardware install is gated on Lume Pad arrival; bring-up plan +lives in the runtime repo at +`docs/getting-started/android-bringup-checklist.md`. diff --git a/src/drv_leia_android/leia_cnsdk.cpp b/src/drv_leia_android/leia_cnsdk.cpp new file mode 100644 index 0000000..122f943 --- /dev/null +++ b/src/drv_leia_android/leia_cnsdk.cpp @@ -0,0 +1,503 @@ +// Copyright 2025, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief CNSDK wrapper implementation — isolates CNSDK headers + * from the rest of the compositor. + * @author David Fattal + * @ingroup drv_leia + */ + +#include "leia_cnsdk.h" + +#include "util/u_logging.h" + +#include +#include +#include +#include + +#ifdef XRT_OS_ANDROID +#include "android/android_globals.h" +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#include +#endif +#endif + +#include +#include +#include + + +// Hardware-bring-up debug logging. Gated by XRT_DEBUG_ANDROID_VERBOSE +// which is passed via cppFlags from the Android Debug build variant +// (src/xrt/targets/openxr_android/build.gradle::debug). Compiles to +// nothing in release. Tag "HW_DBG_CNSDK:" is greppable in logcat. +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#define DXR_HW_DBG(...) U_LOG_I("HW_DBG_CNSDK: " __VA_ARGS__) +#define DXR_HW_DBG_ONCE(...) do { \ + static bool _logged = false; \ + if (!_logged) { U_LOG_I("HW_DBG_CNSDK[once]: " __VA_ARGS__); _logged = true; } \ + } while (0) + +// ATrace RAII scope — captures show up in Perfetto / Studio Profiler. +// Same gate as DXR_HW_DBG; release builds compile to nothing. +struct AtraceScopeCnsdk { + AtraceScopeCnsdk(const char *name) { ATrace_beginSection(name); } + ~AtraceScopeCnsdk() { ATrace_endSection(); } +}; +#define DXR_ATRACE(name) AtraceScopeCnsdk _atrace_##__LINE__(name) +#else +#define DXR_HW_DBG(...) ((void)0) +#define DXR_HW_DBG_ONCE(...) ((void)0) +#define DXR_ATRACE(name) ((void)0) +#endif + + +/* + * + * Internal struct. + * + */ + +struct leia_cnsdk +{ + struct leia_core *core{nullptr}; + struct leia_interlacer *interlacer{nullptr}; + + // Face-tracking startup is offloaded to a worker thread because + // leia_core_enable_face_tracking is heavy (CNSDK docs explicitly warn + // against the main thread). Worker pattern: + // - Spawn in leia_cnsdk_create. + // - Worker polls leia_core_is_initialized until ready, then snapshots + // the camera center from leia_device_config (needed to convert + // CNSDK's camera-relative face positions into display-relative), + // calls enable + start face tracking, then sets + // face_tracking_started and exits. + // - Destroy sets shutting_down to ask the worker to bail if it's + // still in the polling phase, then joins. + // + // camera_center_{x,y,z}_m: cached at worker init. The `_m` suffix + // reminds the reader they're already mm→m converted before storage. + // These are read by leia_cnsdk_get_primary_face on the render thread + // only after face_tracking_started.load(acquire) returns true — the + // happens-before ordering of the atomic gives the read visibility on + // the worker's writes. + std::atomic face_tracking_started{false}; + std::atomic shutting_down{false}; + std::thread worker; + float camera_center_x_m{0.0f}; + float camera_center_y_m{0.0f}; + float camera_center_z_m{0.0f}; + + // Cached display metrics. Populated by the worker thread alongside + // the camera-center snapshot; the atomic flag gives the render + // thread happens-before visibility on the float/int writes. Once + // set, leia_cnsdk_get_display_metrics returns the cached values + // instead of calling get_device_config / release_device_config per + // frame — eliminates a per-frame allocation churn AND the + // concurrent-device-config-access concern (audit B9). + std::atomic display_metrics_cached{false}; + float display_width_m_cached{0.0f}; + float display_height_m_cached{0.0f}; + uint32_t display_pixel_w_cached{0}; + uint32_t display_pixel_h_cached{0}; + + // One-shot flag: once leia_interlacer_vulkan_initialize fails, give + // up rather than retrying every frame. Read + written only by the + // render thread (no concurrent access; no atomic needed). + bool interlacer_init_failed{false}; +}; + + +/* + * + * Private helpers. + * + */ + +namespace { + +void +face_tracking_worker(struct leia_cnsdk *cnsdk) +{ + using namespace std::chrono_literals; + + DXR_HW_DBG("worker: entered, waiting for leia_core_is_initialized"); + + // Phase 1: wait for the async core init to complete. Poll every 50 ms; + // honor shutdown promptly. + int poll_count = 0; + while (!cnsdk->shutting_down.load(std::memory_order_acquire)) { + if (cnsdk->core != nullptr && leia_core_is_initialized(cnsdk->core)) { + break; + } + if ((++poll_count % 20) == 0) { + DXR_HW_DBG("worker: still polling for core init (~%d s elapsed)", + poll_count / 20); + } + std::this_thread::sleep_for(50ms); + } + if (cnsdk->shutting_down.load(std::memory_order_acquire)) { + DXR_HW_DBG("worker: shutdown requested before core ready, exiting"); + return; + } + DXR_HW_DBG("worker: core initialized after %d polls", poll_count); + + // Phase 2a: snapshot all device-config values we need on the render + // thread (camera center for face-position translation; display + // metrics for Kooima projection). CNSDK doesn't annotate device + // config thread safety, so we keep it on this one worker thread and + // expose only cached values to the render thread via atomics. mm→m + // conversion happens at storage time so render-thread reads are + // branch-free. + struct leia_device_config *cfg = leia_core_get_device_config(cnsdk->core); + if (cfg != NULL) { + cnsdk->camera_center_x_m = cfg->cameraCenterX / 1000.0f; + cnsdk->camera_center_y_m = cfg->cameraCenterY / 1000.0f; + cnsdk->camera_center_z_m = cfg->cameraCenterZ / 1000.0f; + cnsdk->display_width_m_cached = (float)cfg->displaySizeInMm[0] / 1000.0f; + cnsdk->display_height_m_cached = (float)cfg->displaySizeInMm[1] / 1000.0f; + cnsdk->display_pixel_w_cached = (uint32_t)cfg->panelResolution[0]; + cnsdk->display_pixel_h_cached = (uint32_t)cfg->panelResolution[1]; + leia_core_release_device_config(cnsdk->core, cfg); + cnsdk->display_metrics_cached.store(true, std::memory_order_release); + DXR_HW_DBG("worker: cached metrics: %ux%u px, %.3fx%.3f m; cam=(%.3f, %.3f, %.3f) m", + cnsdk->display_pixel_w_cached, cnsdk->display_pixel_h_cached, + cnsdk->display_width_m_cached, cnsdk->display_height_m_cached, + cnsdk->camera_center_x_m, cnsdk->camera_center_y_m, + cnsdk->camera_center_z_m); + } else { + U_LOG_W("leia_core_get_device_config failed in worker; camera center + metrics stay default"); + } + + // Phase 2b: heavy enable + start. Single call, can't be interrupted — + // destroy will block on the join until this returns. + if (!leia_core_enable_face_tracking(cnsdk->core, true)) { + U_LOG_W("leia_core_enable_face_tracking failed (worker)"); + return; + } + leia_core_start_face_tracking(cnsdk->core, true); + + cnsdk->face_tracking_started.store(true, std::memory_order_release); + U_LOG_W("CNSDK face tracking started (worker)"); +} + +} // namespace + + +/* + * + * Public API. + * + */ + +extern "C" xrt_result_t +leia_cnsdk_create(struct leia_cnsdk **out_cnsdk) +{ + DXR_HW_DBG("leia_cnsdk_create: entering"); + leia_platform_on_library_load(); + + struct leia_core_init_configuration *config = leia_core_init_configuration_alloc(CNSDK_VERSION); + +#ifdef XRT_OS_ANDROID + leia_core_init_configuration_set_platform_android_java_vm(config, (JavaVM *)android_globals_get_vm()); + leia_core_init_configuration_set_platform_android_handle( + config, LEIA_CORE_ANDROID_HANDLE_ACTIVITY, (jobject)android_globals_get_activity()); +#endif + + leia_core_init_configuration_set_platform_log_level(config, kLeiaLogLevelTrace); + leia_core_init_configuration_set_enable_validation(config, true); + + struct leia_core *core = leia_core_init_async(config); + leia_core_init_configuration_free(config); + + if (core == NULL) { + U_LOG_E("leia_core_init_async failed"); + *out_cnsdk = NULL; + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + + leia_core_set_backlight(core, true); + + auto *cnsdk = new struct leia_cnsdk(); + cnsdk->core = core; + cnsdk->worker = std::thread(face_tracking_worker, cnsdk); + + DXR_HW_DBG("leia_cnsdk_create: core=%p, worker thread spawned", (void *)core); + *out_cnsdk = cnsdk; + return XRT_SUCCESS; +} + +extern "C" void +leia_cnsdk_destroy(struct leia_cnsdk **cnsdk_ptr) +{ + if (cnsdk_ptr == NULL || *cnsdk_ptr == NULL) { + return; + } + + struct leia_cnsdk *cnsdk = *cnsdk_ptr; + DXR_HW_DBG("leia_cnsdk_destroy: entering, core=%p", (void *)cnsdk->core); + + // Signal the worker, then join with a watchdog: if it doesn't finish + // within kWorkerJoinTimeoutMs, detach instead so destroy can return. + // The worker might be mid-leia_core_enable_face_tracking with no + // interruption hook — without the timeout, destroy can hang + // indefinitely on a CNSDK deadlock (audit B10). + // + // Detaching leaks the std::thread but is the only option short of + // CNSDK exposing a cancel API. + cnsdk->shutting_down.store(true, std::memory_order_release); + if (cnsdk->worker.joinable()) { + constexpr auto kWorkerJoinTimeoutMs = std::chrono::milliseconds(2000); + // std::thread::join doesn't take a timeout, so use a side thread + // that does the join and a condition variable to wait on with a + // deadline. Cheap on the happy path (the worker is usually + // already finished by destroy time, so join returns instantly). + std::atomic joined{false}; + std::thread joiner([&]() { + cnsdk->worker.join(); + joined.store(true, std::memory_order_release); + }); + const auto deadline = std::chrono::steady_clock::now() + kWorkerJoinTimeoutMs; + while (!joined.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + if (joined.load(std::memory_order_acquire)) { + joiner.join(); + } else { + U_LOG_W("CNSDK worker did not exit within %lld ms; detaching", + (long long)kWorkerJoinTimeoutMs.count()); + cnsdk->worker.detach(); + joiner.detach(); + } + } + + if (cnsdk->interlacer != NULL) { + // CNSDK 0.7.28 renamed the per-interlacer release; the core owns the + // interlacer lifetime, so the shutdown call needs both handles. + leia_interlacer_shutdown(cnsdk->core, cnsdk->interlacer); + cnsdk->interlacer = NULL; + } + + if (cnsdk->core != NULL) { + // CNSDK 0.7.28 renamed leia_core_release → leia_core_shutdown. + leia_core_shutdown(cnsdk->core); + cnsdk->core = NULL; + } + + leia_platform_on_library_unload(); + + delete cnsdk; + *cnsdk_ptr = NULL; +} + +extern "C" bool +leia_cnsdk_is_initialized(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return false; + } + return leia_core_is_initialized(cnsdk->core); +} + +extern "C" void +leia_cnsdk_on_pause(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return; + } + if (!leia_core_is_initialized(cnsdk->core)) { + DXR_HW_DBG("on_pause: skipped (core not initialized yet)"); + return; + } + DXR_HW_DBG("on_pause: forwarding to leia_core_on_pause"); + leia_core_on_pause(cnsdk->core); +} + +extern "C" void +leia_cnsdk_on_resume(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return; + } + if (!leia_core_is_initialized(cnsdk->core)) { + DXR_HW_DBG("on_resume: skipped (core not initialized yet)"); + return; + } + DXR_HW_DBG("on_resume: forwarding to leia_core_on_resume"); + leia_core_on_resume(cnsdk->core); +} + +extern "C" bool +leia_cnsdk_get_display_metrics(struct leia_cnsdk *cnsdk, + float *out_width_m, + float *out_height_m, + uint32_t *out_pixel_w, + uint32_t *out_pixel_h) +{ + // Worker thread snapshots all four values from the device config + // once, then sets the atomic. Render thread polls the atomic and + // reads the cached float/int fields. No per-frame get/release. + if (cnsdk == NULL || + !cnsdk->display_metrics_cached.load(std::memory_order_acquire)) { + return false; + } + + if (out_width_m != NULL) { + *out_width_m = cnsdk->display_width_m_cached; + } + if (out_height_m != NULL) { + *out_height_m = cnsdk->display_height_m_cached; + } + if (out_pixel_w != NULL) { + *out_pixel_w = cnsdk->display_pixel_w_cached; + } + if (out_pixel_h != NULL) { + *out_pixel_h = cnsdk->display_pixel_h_cached; + } + return true; +} + +extern "C" bool +leia_cnsdk_ensure_face_tracking_started(struct leia_cnsdk *cnsdk) +{ + // Worker thread handles enable + start; this is now a non-blocking + // status check. + if (cnsdk == NULL) { + return false; + } + return cnsdk->face_tracking_started.load(std::memory_order_acquire); +} + +extern "C" bool +leia_cnsdk_ensure_interlacer(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkFormat targetFmt) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return false; + } + if (cnsdk->interlacer != NULL) { + return true; + } + // One-shot give-up: once leia_interlacer_vulkan_initialize fails + // (typically permanently — wrong VkDevice format, no GPU memory, + // CNSDK lib mismatch), don't keep retrying every frame. + if (cnsdk->interlacer_init_failed) { + return false; + } + if (!leia_core_is_initialized(cnsdk->core)) { + return false; + } + + struct leia_interlacer_init_configuration *ic = leia_interlacer_init_configuration_alloc(); + // Atlas mode: CNSDK accepts the SBS atlas VkImage+View directly per + // frame via set_interlace_view_texture_atlas, and splits internally. + // No per-view image management on our side; the DP shrinks + // substantially. See feature/android-cnsdk-ci for the prior art + // (CNSDK 0.10.56 used a different API but same architectural idea). + leia_interlacer_init_configuration_set_use_atlas_for_views(ic, true); + // Views format = atlas format. Atlas is rendered to UNORM by + // comp_vk_native_renderer.c, so use UNORM here (audit B2). + cnsdk->interlacer = leia_interlacer_vulkan_initialize( + cnsdk->core, ic, device, physDev, VK_FORMAT_B8G8R8A8_UNORM, + targetFmt, VK_FORMAT_D32_SFLOAT, 3); + leia_interlacer_init_configuration_free(ic); + + if (cnsdk->interlacer == NULL) { + U_LOG_W("leia_interlacer_vulkan_initialize returned NULL; giving up (no retries)"); + cnsdk->interlacer_init_failed = true; + return false; + } + + // Tell CNSDK the atlas is laid out 2x1 SBS horizontal. This is the + // default but we set it explicitly so future layout changes + // (multi-view modes) only have to touch one place. + leia_interlacer_set_num_tiles(cnsdk->interlacer, 2, 1); + DXR_HW_DBG("ensure_interlacer: created interlacer=%p (atlas mode, 2x1, targetFmt=%d)", + (void *)cnsdk->interlacer, (int)targetFmt); + return true; +} + +extern "C" bool +leia_cnsdk_get_primary_face(struct leia_cnsdk *cnsdk, + float *out_x, + float *out_y, + float *out_z) +{ + if (cnsdk == NULL || cnsdk->core == NULL || + !cnsdk->face_tracking_started.load(std::memory_order_acquire)) { + return false; + } + + float position[3] = {0, 0, 0}; + struct leia_float_slice slice = {position, 3}; + if (!leia_core_get_primary_face(cnsdk->core, slice)) { + return false; + } + + // CNSDK returns millimeters relative to the camera. xrt_eye_position + // wants meters relative to the display center, so divide by 1000 then + // subtract the cached camera center (also already in meters). + const float pos_x_m = position[0] / 1000.0f - cnsdk->camera_center_x_m; + const float pos_y_m = position[1] / 1000.0f - cnsdk->camera_center_y_m; + const float pos_z_m = position[2] / 1000.0f - cnsdk->camera_center_z_m; + +#ifdef XRT_DEBUG_ANDROID_VERBOSE + // Throttle to once per ~second at 60 Hz so logcat stays readable. + static int dbg_face_counter = 0; + if ((dbg_face_counter++ % 60) == 0) { + DXR_HW_DBG("face: raw_mm=(%.1f, %.1f, %.1f) → out_m=(%.4f, %.4f, %.4f)", + position[0], position[1], position[2], pos_x_m, pos_y_m, pos_z_m); + } +#endif + + if (out_x != NULL) { *out_x = pos_x_m; } + if (out_y != NULL) { *out_y = pos_y_m; } + if (out_z != NULL) { *out_z = pos_z_m; } + return true; +} + +extern "C" void +leia_cnsdk_weave(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkImage atlas_image, + VkImageView atlas_view, + uint32_t atlas_width, + uint32_t atlas_height, + VkFormat targetFmt, + uint32_t w, + uint32_t h, + VkFramebuffer fb, + VkImage targetImage) +{ + (void)device; (void)physDev; (void)targetFmt; + + if (cnsdk == NULL || cnsdk->interlacer == NULL) { + return; + } + + DXR_ATRACE("dxr_cnsdk:weave"); + leia_interlacer_set_flip_input_uv_vertical(cnsdk->interlacer, true); + + // Atlas mode: hand CNSDK the SBS atlas VkImage+View each frame; it + // splits internally per the 2x1 layout set in ensure_interlacer. + // (Previously this function blitted tiles into per-view images and + // passed those — see git log for the per-tile-blit history.) + leia_interlacer_vulkan_set_interlace_view_texture_atlas( + cnsdk->interlacer, atlas_image, atlas_view); + leia_interlacer_set_source_views_size( + cnsdk->interlacer, (int32_t)atlas_width, (int32_t)atlas_height, + /*isHorizontalViews=*/true); + + leia_interlacer_set_shader_debug_mode(cnsdk->interlacer, LEIA_SHADER_DEBUG_MODE_NONE); + DXR_HW_DBG_ONCE("weave: first do_post_process atlas=%ux%u target=%ux%u", + atlas_width, atlas_height, w, h); + leia_interlacer_vulkan_do_post_process( + cnsdk->interlacer, w, h, false, fb, targetImage, NULL, + NULL, NULL, 0); +} diff --git a/src/drv_leia_android/leia_cnsdk.h b/src/drv_leia_android/leia_cnsdk.h new file mode 100644 index 0000000..20b70bf --- /dev/null +++ b/src/drv_leia_android/leia_cnsdk.h @@ -0,0 +1,193 @@ +// Copyright 2025, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Opaque wrapper around CNSDK (Android) interlacing API. + * + * Encapsulates leia_core and leia_interlacer so the compositor + * does not include CNSDK headers directly. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#pragma once + +#include "xrt/xrt_results.h" +#include "xrt/xrt_vulkan_includes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct leia_cnsdk; + +/*! + * Create and asynchronously initialise a CNSDK core + backlight. + * + * @param[out] out_cnsdk Receives the opaque handle (NULL on failure). + * @return XRT_SUCCESS on success. + */ +xrt_result_t +leia_cnsdk_create(struct leia_cnsdk **out_cnsdk); + +/*! + * Destroy a CNSDK handle and release all resources. + * + * @param cnsdk_ptr Pointer to handle; set to NULL on return. + */ +void +leia_cnsdk_destroy(struct leia_cnsdk **cnsdk_ptr); + +/*! + * Check whether the asynchronous core init has completed. + * + * @return true once the core is fully initialised. + */ +bool +leia_cnsdk_is_initialized(struct leia_cnsdk *cnsdk); + +/*! + * Notify CNSDK that the host Activity has paused (backgrounded). + * + * Wraps leia_core_on_pause. Safe to call any time after + * @ref leia_cnsdk_create — CNSDK no-ops if the core isn't initialized + * yet. Stops face-tracking cameras + dims the backlight; idempotent + * across repeated calls. + * + * Intended caller: OpenXR session state machine in oxr_session.c on + * entering XR_SESSION_STATE_STOPPING. + */ +void +leia_cnsdk_on_pause(struct leia_cnsdk *cnsdk); + +/*! + * Notify CNSDK that the host Activity has resumed (foregrounded). + * + * Wraps leia_core_on_resume. Counterpart of @ref leia_cnsdk_on_pause. + * + * Intended caller: OpenXR session state machine on entering + * XR_SESSION_STATE_READY after a pause. + */ +void +leia_cnsdk_on_resume(struct leia_cnsdk *cnsdk); + +/*! + * Fetch native display metrics from CNSDK's device config. + * + * The four values are snapshotted once by the face-tracking worker + * thread right after @ref leia_core_is_initialized first returns true + * and stored on the wrapper struct. Subsequent calls return the + * cached values without re-acquiring CNSDK's device config every + * frame. Returns false until that snapshot has happened; caller is + * expected to poll across frames. + * + * @param[out] out_width_m Display physical width in meters. + * @param[out] out_height_m Display physical height in meters. + * @param[out] out_pixel_w Panel pixel width. + * @param[out] out_pixel_h Panel pixel height. + * @return true if all outputs were populated. + */ +bool +leia_cnsdk_get_display_metrics(struct leia_cnsdk *cnsdk, + float *out_width_m, + float *out_height_m, + uint32_t *out_pixel_w, + uint32_t *out_pixel_h); + +/*! + * Non-blocking check for whether CNSDK face tracking is running. + * + * Enable + start happens asynchronously on a worker thread spawned by + * @ref leia_cnsdk_create, so callers can poll this every frame from the + * render thread without stalling. Returns true once the worker has + * finished enabling + starting; false until then (or permanently if + * the enable call failed). + * + * @return true once face tracking is started. + */ +bool +leia_cnsdk_ensure_face_tracking_started(struct leia_cnsdk *cnsdk); + +/*! + * Idempotent: lazily create the CNSDK Vulkan interlacer in atlas mode + * once @ref leia_core_is_initialized returns true. Safe to call every + * frame. Atlas mode means CNSDK accepts the SBS atlas VkImage+View + * directly via @ref leia_cnsdk_weave and does the L/R split internally; + * the DP doesn't have to manage per-view images or per-tile blits. + * + * @return true if the interlacer exists and is ready to weave. + */ +bool +leia_cnsdk_ensure_interlacer(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkFormat targetFmt); + +/*! + * Fetch the latest predicted primary face position from CNSDK. + * + * Returns false until face tracking is running and CNSDK has a face + * lock. Position is returned in meters relative to the **display + * center** (matching `xrt_eye_position`'s convention), even though + * CNSDK natively returns millimeters relative to the camera. The + * wrapper does the unit conversion + camera-center translation using + * the cached `cameraCenterX/Y/Z` from `leia_device_config` populated + * once the core is initialized. + * + * @param[out] out_x Face position X (meters, display-relative). + * @param[out] out_y Face position Y (meters, display-relative). + * @param[out] out_z Face position Z (meters, +toward viewer). + * @return true if a valid face was returned. + */ +bool +leia_cnsdk_get_primary_face(struct leia_cnsdk *cnsdk, + float *out_x, + float *out_y, + float *out_z); + +/*! + * Perform CNSDK Vulkan interlacing on an SBS atlas. + * + * Atlas mode: pass the runtime's pre-composited SBS atlas image+view + * directly. CNSDK does the L/R split internally via + * @ref leia_interlacer_vulkan_set_interlace_view_texture_atlas. The DP + * does no per-view image management and no per-frame blits, so there's + * no GPU stall between us and CNSDK — its `do_post_process` records + * and submits its own cmd buffer when it's ready. + * + * Caller must have first invoked @ref leia_cnsdk_ensure_interlacer; if + * the interlacer isn't ready yet this function is a no-op (no submit, + * no GPU side effects), making it safe to call every frame during the + * async core-init window. + * + * @param cnsdk Opaque CNSDK handle. + * @param device Vulkan logical device. + * @param physDev Vulkan physical device. + * @param atlas_image SBS atlas VkImage. + * @param atlas_view Matching VkImageView covering the full atlas. + * @param atlas_width Atlas width in pixels (= view_w * tile_columns). + * @param atlas_height Atlas height in pixels (= view_h * tile_rows). + * @param targetFmt Format of the target / swapchain image. + * @param w Target width in pixels. + * @param h Target height in pixels. + * @param fb Target framebuffer. + * @param targetImage Target VkImage (for CNSDK-side layout transitions). + */ +void +leia_cnsdk_weave(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkImage atlas_image, + VkImageView atlas_view, + uint32_t atlas_width, + uint32_t atlas_height, + VkFormat targetFmt, + uint32_t w, + uint32_t h, + VkFramebuffer fb, + VkImage targetImage); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.cpp b/src/drv_leia_android/leia_display_processor_cnsdk.cpp new file mode 100644 index 0000000..12e95f0 --- /dev/null +++ b/src/drv_leia_android/leia_display_processor_cnsdk.cpp @@ -0,0 +1,456 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Leia CNSDK display processor (Android), atlas-mode variant. + * + * Wraps leia_cnsdk as an xrt_display_processor. Uses CNSDK's atlas mode: + * the compositor's pre-composited SBS atlas VkImage+View is passed + * directly to CNSDK each frame via + * leia_interlacer_vulkan_set_interlace_view_texture_atlas, and CNSDK + * splits L/R internally. No per-view image management, no per-tile + * blit, no blit cmd buffer / semaphore / fence on our side. + * + * Display metrics + face-tracked eye positions come from CNSDK once the + * async core init completes (lazy on first query). Falls back to + * hardcoded Lume Pad 2 metrics and IPD-only eyes while the core is + * still booting. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#include "leia_display_processor_cnsdk.h" +#include "leia_cnsdk.h" + +#include "xrt/xrt_display_metrics.h" +#include "vk/vk_helpers.h" +#include "util/u_logging.h" + +#include + +#if defined(XRT_OS_ANDROID) && defined(XRT_DEBUG_ANDROID_VERBOSE) +#include +#endif + + +// Hardware-bring-up debug logging. Gated by XRT_DEBUG_ANDROID_VERBOSE +// (cppFlag from the Android Debug variant). Compiles to nothing in +// release. Tag "HW_DBG_DP:" greppable in logcat, separate from the +// CNSDK wrapper's HW_DBG_CNSDK tag. +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#define DXR_HW_DBG(...) U_LOG_I("HW_DBG_DP: " __VA_ARGS__) +#define DXR_HW_DBG_ONCE(...) do { \ + static bool _logged = false; \ + if (!_logged) { U_LOG_I("HW_DBG_DP[once]: " __VA_ARGS__); _logged = true; } \ + } while (0) +struct AtraceScopeDp { + AtraceScopeDp(const char *name) { ATrace_beginSection(name); } + ~AtraceScopeDp() { ATrace_endSection(); } +}; +#define DXR_ATRACE(name) AtraceScopeDp _atrace_##__LINE__(name) +#else +#define DXR_HW_DBG(...) ((void)0) +#define DXR_HW_DBG_ONCE(...) ((void)0) +#define DXR_ATRACE(name) ((void)0) +#endif + + +namespace { + +// Lume Pad 2-class defaults. Used until CNSDK reports the real device +// metrics through leia_core via the leia_cnsdk wrapper's cached values. +constexpr float kDefaultDisplayWidthM = 0.1934f; // ~12.4" diagonal, 16:10 +constexpr float kDefaultDisplayHeightM = 0.1209f; +constexpr uint32_t kDefaultDisplayPixelW = 2560; +constexpr uint32_t kDefaultDisplayPixelH = 1600; + +// Hardcoded IPD-only eye positions. Origin = display center; z is +// toward the user. 65 mm IPD, ~50 cm viewing distance. +constexpr float kIpdHalfM = 0.0325f; +constexpr float kEyeViewerDistM = 0.5f; + +struct leia_dp_cnsdk +{ + struct xrt_display_processor base; + struct leia_cnsdk *cnsdk; //!< Owned. + struct vk_bundle *vk; //!< Borrowed from compositor. + VkCommandPool cmd_pool; //!< Borrowed from compositor; used by mono passthrough. +}; + +inline leia_dp_cnsdk * +as_impl(struct xrt_display_processor *xdp) +{ + return reinterpret_cast(xdp); +} + +// Mono / 2D-mode passthrough: blit the single-tile atlas directly to +// the swapchain target. No CNSDK weave (would be wrong for 1x1). Same +// barrier dance as the atlas-side host-stall path — drain via +// vkQueueWaitIdle so xrEndFrame sees the target image ready. +bool +mono_passthrough_blit(leia_dp_cnsdk *impl, + VkImage atlas_image, + uint32_t atlas_width, + uint32_t atlas_height, + VkImage target_image, + uint32_t target_width, + uint32_t target_height) +{ + DXR_ATRACE("dxr_dp:mono_passthrough_blit"); + struct vk_bundle *vk = impl->vk; + + VkCommandBufferAllocateInfo ai = {}; + ai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + ai.commandPool = impl->cmd_pool; + ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + ai.commandBufferCount = 1; + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (vk->vkAllocateCommandBuffers(vk->device, &ai, &cmd) != VK_SUCCESS) { + return false; + } + + VkCommandBufferBeginInfo bi = {}; + bi.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + vk->vkBeginCommandBuffer(cmd, &bi); + + // Atlas: SHADER_READ_ONLY_OPTIMAL → TRANSFER_SRC_OPTIMAL. + VkImageMemoryBarrier atlas_to_src = {}; + atlas_to_src.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + atlas_to_src.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; + atlas_to_src.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + atlas_to_src.oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + atlas_to_src.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + atlas_to_src.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + atlas_to_src.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + atlas_to_src.image = atlas_image; + atlas_to_src.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + // Target: assume COLOR_ATTACHMENT_OPTIMAL (the compositor's pre-DP + // barrier set this in the window-target path; for texture-mode the + // shared image's layout is caller-dependent — we use UNDEFINED→ + // TRANSFER_DST which is always safe for clearing). + VkImageMemoryBarrier target_to_dst = {}; + target_to_dst.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + target_to_dst.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + target_to_dst.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + target_to_dst.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + target_to_dst.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + target_to_dst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + target_to_dst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + target_to_dst.image = target_image; + target_to_dst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + VkImageMemoryBarrier pre[2] = {atlas_to_src, target_to_dst}; + vk->vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 2, pre); + + // Stretch-blit the atlas to fill the target. Linear filter so a + // resolution mismatch doesn't go blocky. + VkImageBlit blit = {}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[0] = {0, 0, 0}; + blit.srcOffsets[1] = {(int32_t)atlas_width, (int32_t)atlas_height, 1}; + blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.dstOffsets[0] = {0, 0, 0}; + blit.dstOffsets[1] = {(int32_t)target_width, (int32_t)target_height, 1}; + vk->vkCmdBlitImage(cmd, + atlas_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + target_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, &blit, VK_FILTER_LINEAR); + + // Restore atlas to SHADER_READ for the compositor's invariant; + // leave target in COLOR_ATTACHMENT_OPTIMAL (xrEndFrame's contract). + VkImageMemoryBarrier atlas_back = atlas_to_src; + atlas_back.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + atlas_back.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + atlas_back.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + atlas_back.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkImageMemoryBarrier target_back = target_to_dst; + target_back.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + target_back.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + target_back.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + target_back.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + VkImageMemoryBarrier post[2] = {atlas_back, target_back}; + vk->vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, 0, nullptr, 0, nullptr, 2, post); + + vk->vkEndCommandBuffer(cmd); + + VkSubmitInfo si = {}; + si.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + si.commandBufferCount = 1; + si.pCommandBuffers = &cmd; + VkResult res = vk->vkQueueSubmit(vk->main_queue->queue, 1, &si, VK_NULL_HANDLE); + if (res == VK_SUCCESS) { + vk->vkQueueWaitIdle(vk->main_queue->queue); + } + vk->vkFreeCommandBuffers(vk->device, impl->cmd_pool, 1, &cmd); + DXR_HW_DBG_ONCE("mono_passthrough_blit: first frame (atlas %ux%u → target %ux%u)", + atlas_width, atlas_height, target_width, target_height); + return res == VK_SUCCESS; +} + +void +process_atlas_weave(struct xrt_display_processor *xdp, + VkCommandBuffer cmd_buffer, + VkImage_XDP atlas_image, + VkImageView atlas_view, + uint32_t view_width, + uint32_t view_height, + uint32_t tile_columns, + uint32_t tile_rows, + VkFormat_XDP view_format, + VkFramebuffer target_fb, + VkImage_XDP target_image, + uint32_t target_width, + uint32_t target_height, + VkFormat_XDP target_format, + int32_t canvas_offset_x, + int32_t canvas_offset_y, + uint32_t canvas_width, + uint32_t canvas_height) +{ + DXR_ATRACE("dxr_dp:process_atlas_weave"); + (void)cmd_buffer; // self-submitting: compositor passes VK_NULL_HANDLE + (void)view_format; + (void)canvas_offset_x; (void)canvas_offset_y; + (void)canvas_width; (void)canvas_height; + + leia_dp_cnsdk *impl = as_impl(xdp); + + if (tile_columns == 1 && tile_rows == 1) { + // Mono / 2D mode: no interlacing needed. Atlas IS the final + // image; blit it directly to the target so we still produce + // something visible. CNSDK doesn't weave here (would be wrong + // for 1x1) — only used in the runtime's 2D fallback or for + // non-3D-display vendors. Implementation moved below for + // readability. + if (impl->vk == nullptr || impl->cmd_pool == VK_NULL_HANDLE) { + return; + } + const uint32_t atlas_w_mono = view_width; + const uint32_t atlas_h_mono = view_height; + mono_passthrough_blit(impl, (VkImage)(uintptr_t)atlas_image, + atlas_w_mono, atlas_h_mono, + (VkImage)(uintptr_t)target_image, + target_width, target_height); + return; + } + if (tile_columns != 2 || tile_rows != 1) { + static bool warned = false; + if (!warned) { + U_LOG_W("CNSDK DP expects 2x1 SBS atlas or 1x1 mono, got %ux%u; skipping", + tile_columns, tile_rows); + warned = true; + } + return; + } + + if (impl->vk == nullptr) { + return; + } + + // Gate the weave on CNSDK interlacer readiness. Async core init may + // take several frames; until then the interlacer doesn't exist and + // leia_cnsdk_weave is a no-op. The compositor's pre-DP submit has + // already happened, so target is in COLOR_ATTACHMENT_OPTIMAL with + // undefined content for those frames — acceptable for POC. + if (!leia_cnsdk_ensure_interlacer(impl->cnsdk, + impl->vk->device, + impl->vk->physical_device, + (VkFormat)target_format)) { + return; + } + DXR_HW_DBG_ONCE("process_atlas_weave: first frame with ready interlacer"); + + const uint32_t atlas_w = view_width * tile_columns; + const uint32_t atlas_h = view_height * tile_rows; + +#ifdef XRT_DEBUG_ANDROID_VERBOSE + static int dp_dbg_frame = 0; + if ((dp_dbg_frame++ % 60) == 0) { + DXR_HW_DBG("process_atlas_weave[frame=%d]: atlas=%ux%u target=%ux%u fmt=%d", + dp_dbg_frame, atlas_w, atlas_h, target_width, target_height, (int)target_format); + } +#endif + + leia_cnsdk_weave(impl->cnsdk, + impl->vk->device, + impl->vk->physical_device, + (VkImage)(uintptr_t)atlas_image, + atlas_view, + atlas_w, + atlas_h, + (VkFormat)target_format, + target_width, + target_height, + target_fb, + (VkImage)(uintptr_t)target_image); +} + +bool +is_self_submitting_true(struct xrt_display_processor *xdp) +{ + (void)xdp; + return true; +} + +// Try to fetch CNSDK's predicted face position and derive L/R eyes from +// it (face X ± IPD/2). Falls back to a hardcoded IPD-only stub if face +// tracking isn't running yet (CNSDK core still async-initializing, or +// no face lock). +bool +get_predicted_eye_positions_ipd(struct xrt_display_processor *xdp, + struct xrt_eye_positions *out_eye_pos) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + bool tracked = false; + float fx = 0.0f, fy = 0.0f, fz = kEyeViewerDistM; + if (impl->cnsdk != nullptr) { + if (leia_cnsdk_ensure_face_tracking_started(impl->cnsdk) && + leia_cnsdk_get_primary_face(impl->cnsdk, &fx, &fy, &fz)) { + tracked = true; + } + } + + out_eye_pos->eyes[0].x = fx - kIpdHalfM; + out_eye_pos->eyes[0].y = fy; + out_eye_pos->eyes[0].z = fz; + out_eye_pos->eyes[1].x = fx + kIpdHalfM; + out_eye_pos->eyes[1].y = fy; + out_eye_pos->eyes[1].z = fz; + out_eye_pos->count = 2; + out_eye_pos->valid = true; + out_eye_pos->is_tracking = tracked; + return true; +} + +bool +get_display_dimensions_default(struct xrt_display_processor *xdp, + float *out_width_m, + float *out_height_m) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + if (impl->cnsdk != nullptr && + leia_cnsdk_get_display_metrics(impl->cnsdk, out_width_m, out_height_m, + nullptr, nullptr)) { + return true; + } + + *out_width_m = kDefaultDisplayWidthM; + *out_height_m = kDefaultDisplayHeightM; + return true; +} + +bool +get_display_pixel_info_default(struct xrt_display_processor *xdp, + uint32_t *out_pixel_width, + uint32_t *out_pixel_height, + int32_t *out_screen_left, + int32_t *out_screen_top) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + *out_screen_left = 0; + *out_screen_top = 0; + + if (impl->cnsdk != nullptr && + leia_cnsdk_get_display_metrics(impl->cnsdk, nullptr, nullptr, + out_pixel_width, out_pixel_height)) { + return true; + } + + *out_pixel_width = kDefaultDisplayPixelW; + *out_pixel_height = kDefaultDisplayPixelH; + return true; +} + +void +on_pause_cnsdk(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + if (impl->cnsdk != nullptr) { + leia_cnsdk_on_pause(impl->cnsdk); + } +} + +void +on_resume_cnsdk(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + if (impl->cnsdk != nullptr) { + leia_cnsdk_on_resume(impl->cnsdk); + } +} + +void +destroy_impl(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + // Drain all in-flight GPU work (especially CNSDK's interlacer + // submits) before destroying any handles CNSDK might still be + // reading. Same defensive idiom as audit B6 — applies in atlas mode + // because CNSDK still owns its own queue submit even though we + // don't have a per-view image / blit cmd buffer to wait on. + if (impl->vk != nullptr) { + impl->vk->vkDeviceWaitIdle(impl->vk->device); + } + + if (impl->cnsdk != nullptr) { + leia_cnsdk_destroy(&impl->cnsdk); + } + free(impl); +} + +} // namespace + +extern "C" xrt_result_t +leia_dp_factory_cnsdk(void *vk_bundle, + void *vk_cmd_pool, + void *window_handle, + int32_t target_format, + struct xrt_display_processor **out_xdp) +{ + (void)window_handle; (void)target_format; + + struct leia_cnsdk *cnsdk = nullptr; + xrt_result_t ret = leia_cnsdk_create(&cnsdk); + if (ret != XRT_SUCCESS || cnsdk == nullptr) { + U_LOG_W("leia_cnsdk_create failed (%d), falling back to no-DP path", (int)ret); + return ret != XRT_SUCCESS ? ret : XRT_ERROR_DEVICE_CREATION_FAILED; + } + + leia_dp_cnsdk *impl = static_cast(calloc(1, sizeof(*impl))); + if (impl == nullptr) { + leia_cnsdk_destroy(&cnsdk); + return XRT_ERROR_ALLOCATION; + } + + impl->cnsdk = cnsdk; + impl->vk = static_cast(vk_bundle); + impl->cmd_pool = (VkCommandPool)(uintptr_t)vk_cmd_pool; + + impl->base.process_atlas = process_atlas_weave; + impl->base.on_pause = on_pause_cnsdk; + impl->base.on_resume = on_resume_cnsdk; + impl->base.is_self_submitting = is_self_submitting_true; + impl->base.get_predicted_eye_positions = get_predicted_eye_positions_ipd; + impl->base.get_display_dimensions = get_display_dimensions_default; + impl->base.get_display_pixel_info = get_display_pixel_info_default; + impl->base.destroy = destroy_impl; + + *out_xdp = &impl->base; + + U_LOG_W("Leia CNSDK DP created (atlas mode)"); + DXR_HW_DBG("factory: impl=%p vk=%p cnsdk=%p", (void *)impl, (void *)impl->vk, (void *)impl->cnsdk); + return XRT_SUCCESS; +} diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.h b/src/drv_leia_android/leia_display_processor_cnsdk.h new file mode 100644 index 0000000..53e7362 --- /dev/null +++ b/src/drv_leia_android/leia_display_processor_cnsdk.h @@ -0,0 +1,43 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Leia CNSDK display processor (Android): wraps the CNSDK + * Vulkan interlacer as an @ref xrt_display_processor. + * + * Advertises `is_self_submitting = true`. The compositor flushes its + * pre-DP cmd buffer before calling process_atlas, which blits the SBS + * atlas tiles into per-view VkImages and then calls leia_cnsdk_weave — + * CNSDK records and submits its own command buffer internally. + * + * Display metrics + face-tracked eye positions come from CNSDK once the + * async core init completes (lazy on first query). Falls back to hardcoded + * Lume Pad 2 metrics and IPD-only eyes while the core is still booting. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#pragma once + +#include "xrt/xrt_display_processor.h" +#include "xrt/xrt_results.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * Factory matching @ref xrt_dp_factory_vk_fn_t. Creates and owns a + * leia_cnsdk handle for the lifetime of the display processor. + */ +xrt_result_t +leia_dp_factory_cnsdk(void *vk_bundle, + void *vk_cmd_pool, + void *window_handle, + int32_t target_format, + struct xrt_display_processor **out_xdp); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia_android/leia_plugin_android.c b/src/drv_leia_android/leia_plugin_android.c new file mode 100644 index 0000000..92585f5 --- /dev/null +++ b/src/drv_leia_android/leia_plugin_android.c @@ -0,0 +1,229 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Android variant of the Leia plug-in entry point. + * + * Mirrors @ref leia_plugin.c (Windows/SR) but wraps CNSDK instead of + * the SR SDK. Builds into `libdxrp050_leia_cnsdk.so`, shipped in the + * runtime APK's `jniLibs//` and discovered by the runtime's + * `target_plugin_loader.c` Android branch (PR #309 / commit c96c93ce8). + * + * Differences from the Windows plug-in: + * - No `probe()` registry/EDID check — the CNSDK device is the only + * thing this plug-in supports, so probe always succeeds. + * - Only `create_dp_vk` is non-NULL (Android has no D3D/Metal/GL + * compositor path today, and CNSDK is Vulkan-only). + * - `get_display_info` returns Lume Pad 2 defaults; CNSDK-derived + * metrics surface lazily via the DP's `get_display_dimensions` / + * `get_display_pixel_info` calls once the async core init completes. + * + * @author leaiss + * @ingroup drv_leia_android + */ + +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_results.h" +#include "xrt/xrt_device.h" + +#include "util/u_device.h" +#include "util/u_logging.h" + +#include "leia_display_processor_cnsdk.h" + +#include +#include + + +/* + * + * Minimal HMD device (no SR SDK probe — Lume Pad 2 hardcoded defaults). + * + */ + +struct leia_android_hmd +{ + struct xrt_device base; +}; + +static void +leia_android_hmd_destroy(struct xrt_device *xdev) +{ + struct leia_android_hmd *hmd = (struct leia_android_hmd *)xdev; + u_device_free(&hmd->base); +} + +static xrt_result_t +leia_android_hmd_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + int64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + (void)xdev; + (void)name; + (void)at_timestamp_ns; + out_relation->pose.orientation = (struct xrt_quat){0.0f, 0.0f, 0.0f, 1.0f}; + out_relation->pose.position = (struct xrt_vec3){0.0f, 0.0f, 0.0f}; + out_relation->relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT); + return XRT_SUCCESS; +} + +static struct xrt_device * +leia_android_hmd_create(void) +{ + enum u_device_alloc_flags flags = + (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); + struct leia_android_hmd *hmd = U_DEVICE_ALLOCATE(struct leia_android_hmd, flags, 1, 0); + + hmd->base.update_inputs = u_device_noop_update_inputs; + hmd->base.get_tracked_pose = leia_android_hmd_get_tracked_pose; + hmd->base.get_view_poses = u_device_get_view_poses; + hmd->base.get_visibility_mask = u_device_get_visibility_mask; + hmd->base.destroy = leia_android_hmd_destroy; + hmd->base.name = XRT_DEVICE_GENERIC_HMD; + hmd->base.device_type = XRT_DEVICE_TYPE_HMD; + + // Lume Pad 2 default panel + viewing distance. CNSDK fills these + // in lazily on first DP query once the async core init completes. + struct u_device_simple_info info = { + .display.w_pixels = 2560, + .display.h_pixels = 1600, + .display.w_meters = 0.235f, + .display.h_meters = 0.147f, + .lens_horizontal_separation_meters = 0.063f, + .lens_vertical_position_meters = 0.0735f, + .fov = {85.0f * (float)M_PI / 180.0f, + 85.0f * (float)M_PI / 180.0f}, + }; + u_device_setup_split_side_by_side(&hmd->base, &info); + + snprintf(hmd->base.str, XRT_DEVICE_NAME_LEN, "Leia CNSDK (Android)"); + snprintf(hmd->base.serial, XRT_DEVICE_NAME_LEN, "lume-pad-cnsdk"); + + return &hmd->base; +} + + +/* + * + * Vtable callbacks. + * + */ + +static xrt_result_t +leia_plugin_android_probe(struct xrt_plugin_instance **out_inst) +{ + // CNSDK device probe runs async on a worker thread; the plug-in + // can't synchronously confirm hardware presence at xrCreateInstance + // time. POC pattern: always claim the slot on Android. The + // downstream leia_dp_factory_cnsdk path bails cleanly if CNSDK + // fails to initialize later. + *out_inst = NULL; + return XRT_SUCCESS; +} + +static xrt_result_t +leia_plugin_android_create_device(struct xrt_plugin_instance *inst, struct xrt_device **out_dev) +{ + (void)inst; + struct xrt_device *xdev = leia_android_hmd_create(); + if (xdev == NULL) { + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + *out_dev = xdev; + return XRT_SUCCESS; +} + +static void +leia_plugin_android_destroy(struct xrt_plugin_instance *inst) +{ + (void)inst; +} + +static bool +leia_plugin_android_get_display_info(struct xrt_plugin_instance *inst, + struct xrt_device *xdev, + struct xrt_plugin_display_info *out_info) +{ + (void)inst; + (void)xdev; + + // Lume Pad 2 defaults. CNSDK device-config is async — the DP's + // get_display_dimensions / get_display_pixel_info read it lazily, + // so these baseline values keep the runtime usable while the + // CNSDK core is still booting. + out_info->display_pixel_width = 2560; + out_info->display_pixel_height = 1600; + out_info->recommended_view_scale_x = 1.0f; + out_info->recommended_view_scale_y = 1.0f; + out_info->display_width_m = 0.235f; + out_info->display_height_m = 0.147f; + out_info->nominal_viewer_x_m = 0.0f; + out_info->nominal_viewer_y_m = 0.0f; + out_info->nominal_viewer_z_m = 0.50f; + + // CNSDK exposes face position only — no MANUAL pose-stream API. + out_info->supported_eye_tracking_modes = 1u; // MANAGED_BIT + out_info->default_eye_tracking_mode = 0u; // MANAGED + + return true; +} + + +/* + * + * Vtable. + * + */ + +static struct xrt_plugin_iface g_leia_android_iface = { + .struct_size = sizeof(struct xrt_plugin_iface), + .reserved_0 = 0, + + .id = "leia-cnsdk", + .display_name = "DisplayXR Leia CNSDK (Android)", + .vendor = "Leia Inc.", + .version = NULL, + + .probe = leia_plugin_android_probe, + .create_device = leia_plugin_android_create_device, + + .create_dp_vk = leia_dp_factory_cnsdk, + .create_dp_d3d11 = NULL, + .create_dp_d3d12 = NULL, + .create_dp_gl = NULL, + .create_dp_metal = NULL, + + .destroy = leia_plugin_android_destroy, + + .get_display_info = leia_plugin_android_get_display_info, + + .set_pose_source = NULL, // CNSDK pose comes from face tracking, not an external source +}; + + +/* + * + * Entry point. + * + */ + +XRT_PLUGIN_EXPORT xrt_result_t +xrtPluginNegotiate(uint32_t runtime_api_version, + const struct xrt_plugin_host_iface *host, + struct xrt_plugin_iface **out_iface, + uint32_t *out_plugin_api_version) +{ + (void)host; + + *out_plugin_api_version = XRT_PLUGIN_API_VERSION_CURRENT; + + if (runtime_api_version != XRT_PLUGIN_API_VERSION_CURRENT) { + *out_iface = NULL; + return XRT_ERROR_PROBER_NOT_SUPPORTED; + } + + *out_iface = &g_leia_android_iface; + return XRT_SUCCESS; +} From 8f449025f70304c83171eef6aae95aac17ef6f97 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 26 May 2026 09:05:51 -0700 Subject: [PATCH 2/7] android: flip CMAKE_FIND_ROOT_PATH_MODE_PACKAGE to BOTH in drv_leia_android The Android NDK toolchain (android.toolchain.cmake) defaults CMAKE_FIND_ROOT_PATH_MODE_PACKAGE to ONLY, which restricts find_package() to NDK sysroot dirs and silently ignores anything we add to CMAKE_PREFIX_PATH outside that. Result: find_package(CNSDK CONFIG REQUIRED) found nothing even with CNSDK_ROOT set + appended to CMAKE_PREFIX_PATH, breaking the standalone NDK build with "Could not find a package configuration file provided by 'CNSDK'". Flip the mode to BOTH at the top of src/drv_leia_android/CMakeLists.txt so the CNSDK install at CNSDK_ROOT becomes searchable. Mirrors what the runtime APK's gradle does via cmake.arguments "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH". Also widen .gitignore to cover build-*/ so out-of-tree builds like build-android/ don't end up staged. Verified end-to-end: cmake -B build-android -G Ninja \ -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \ -DCMAKE_MAKE_PROGRAM=$NDK_CMAKE/bin/ninja.exe \ -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-29 \ -DCNSDK_ROOT=$RUNTIME/cnsdk \ -DDXR_RUNTIME_SOURCE_DIR=$RUNTIME \ -DEigen3_DIR=$RUNTIME/.../intermediates/eigen/eigen-3.4.0/cmake cmake --build build-android --target dxrp050_leia_cnsdk Output: libdxrp050_leia_cnsdk.so (1.8 MB, xrtPluginNegotiate exported) Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + src/drv_leia_android/CMakeLists.txt | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index f897eca..9fc0548 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # Build artifacts build/ +build-*/ _package/ out/ *.user diff --git a/src/drv_leia_android/CMakeLists.txt b/src/drv_leia_android/CMakeLists.txt index ca28d3f..5d19fa2 100644 --- a/src/drv_leia_android/CMakeLists.txt +++ b/src/drv_leia_android/CMakeLists.txt @@ -41,6 +41,12 @@ if(NOT DEFINED CNSDK_ROOT) "Source: https://github.com/LeiaInc/leiainc.github.io/tree/master/CNSDK/cnsdk-android-0.7.28.zip") endif() endif() +# Android NDK toolchain defaults CMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY, +# which restricts find_package to NDK sysroot dirs and ignores +# CMAKE_PREFIX_PATH outside that. Flip it to BOTH so the CNSDK install +# at ${CNSDK_ROOT} is searchable (matches the runtime APK's gradle +# cmake.arguments setup). +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) list(APPEND CMAKE_PREFIX_PATH "${CNSDK_ROOT}") find_package(CNSDK CONFIG REQUIRED) message(STATUS "Leia CNSDK at ${CNSDK_ROOT}") From 2f9b68878c4de0c323bac64615f2de76466473d3 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 2 Jun 2026 08:47:22 -0700 Subject: [PATCH 3/7] android(abi): set DP struct_size header for ABI major v2 (ADR-020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Android CNSDK display-processor factory wired six optional vtable slots (on_pause, on_resume, is_self_submitting, get_predicted_eye_positions, get_display_dimensions, get_display_pixel_info) but never set the struct_size header introduced by ABI v2. The impl is calloc'd, so it stayed 0 — under the v2 runtime, XRT_DP_HAS_SLOT bounds every optional slot against struct_size and reads them all as absent. Most damaging: is_self_submitting=true would be ignored, regressing the self-submitting atlas path into a per-frame double submit. Mirrors the drv_leia (Windows) migration in a6c0d7d. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/drv_leia_android/leia_display_processor_cnsdk.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.cpp b/src/drv_leia_android/leia_display_processor_cnsdk.cpp index 12e95f0..2f9a80e 100644 --- a/src/drv_leia_android/leia_display_processor_cnsdk.cpp +++ b/src/drv_leia_android/leia_display_processor_cnsdk.cpp @@ -439,6 +439,13 @@ leia_dp_factory_cnsdk(void *vk_bundle, impl->vk = static_cast(vk_bundle); impl->cmd_pool = (VkCommandPool)(uintptr_t)vk_cmd_pool; + /* ABI major v2 (ADR-020): the 8-byte struct_size header tells the + * runtime how far this vtable extends. The runtime gates every optional + * slot below on XRT_DP_HAS_SLOT(xdp, field), bounded against this value — + * leave it 0 (calloc default) and is_self_submitting / on_pause / on_resume / + * eye-positions / display-dims all read as absent, silently regressing the + * self-submitting atlas path into a double-submit. */ + impl->base.struct_size = static_cast(sizeof(struct xrt_display_processor)); impl->base.process_atlas = process_atlas_weave; impl->base.on_pause = on_pause_cnsdk; impl->base.on_resume = on_resume_cnsdk; From 28cd04f0eb283874b93304403005ce912cd26231 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 2 Jun 2026 09:46:17 -0700 Subject: [PATCH 4/7] =?UTF-8?q?android:=20review=20fixes=20=E2=80=94=20exp?= =?UTF-8?q?licit=20probe=5Fdisplays=3DNULL,=20atlas-mode=20doc,=20mono=20N?= =?UTF-8?q?ULL-guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-ABI-v2 review pass: - leia_plugin_android.c: make the probe_displays omission deliberate with an explicit `.probe_displays = NULL` + rationale. struct_size spans the full v2 iface, so the runtime's gate sees the slot, finds NULL, and uses the synthesized primary claim — correct for a single-panel Android device with no monitor enumeration (Windows arm implements it over EDID; Android has none). - leia_display_processor_cnsdk.h: fix stale doc that described the obsolete per-tile-blit path; current code is atlas mode (SBS VkImage handed straight to leia_cnsdk_weave, no per-view blit). - leia_display_processor_cnsdk.cpp: NULL-guard vk / vk->main_queue in the mono 1x1 passthrough fallback before dereferencing main_queue->queue, mirroring the atlas-weave path's existing guard. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/drv_leia_android/leia_display_processor_cnsdk.cpp | 8 ++++++++ src/drv_leia_android/leia_display_processor_cnsdk.h | 6 +++--- src/drv_leia_android/leia_plugin_android.c | 10 ++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.cpp b/src/drv_leia_android/leia_display_processor_cnsdk.cpp index 2f9a80e..88cee9b 100644 --- a/src/drv_leia_android/leia_display_processor_cnsdk.cpp +++ b/src/drv_leia_android/leia_display_processor_cnsdk.cpp @@ -99,6 +99,14 @@ mono_passthrough_blit(leia_dp_cnsdk *impl, { DXR_ATRACE("dxr_dp:mono_passthrough_blit"); struct vk_bundle *vk = impl->vk; + if (vk == nullptr || vk->main_queue == nullptr) { + // Defensive: this mono 1x1 fallback submits on the main queue + // directly. The atlas-weave path guards vk==nullptr already; + // mirror it here so a partially-initialized bundle can't deref + // vk->main_queue->queue below. + U_LOG_W("mono_passthrough_blit: vk bundle/main_queue not ready, skipping"); + return false; + } VkCommandBufferAllocateInfo ai = {}; ai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.h b/src/drv_leia_android/leia_display_processor_cnsdk.h index 53e7362..2a6c3f8 100644 --- a/src/drv_leia_android/leia_display_processor_cnsdk.h +++ b/src/drv_leia_android/leia_display_processor_cnsdk.h @@ -6,9 +6,9 @@ * Vulkan interlacer as an @ref xrt_display_processor. * * Advertises `is_self_submitting = true`. The compositor flushes its - * pre-DP cmd buffer before calling process_atlas, which blits the SBS - * atlas tiles into per-view VkImages and then calls leia_cnsdk_weave — - * CNSDK records and submits its own command buffer internally. + * pre-DP cmd buffer before calling process_atlas, which hands the SBS + * atlas VkImage directly to leia_cnsdk_weave (atlas mode — no per-view + * blit) — CNSDK records and submits its own command buffer internally. * * Display metrics + face-tracked eye positions come from CNSDK once the * async core init completes (lazy on first query). Falls back to hardcoded diff --git a/src/drv_leia_android/leia_plugin_android.c b/src/drv_leia_android/leia_plugin_android.c index 92585f5..4d86820 100644 --- a/src/drv_leia_android/leia_plugin_android.c +++ b/src/drv_leia_android/leia_plugin_android.c @@ -200,6 +200,16 @@ static struct xrt_plugin_iface g_leia_android_iface = { .get_display_info = leia_plugin_android_get_display_info, .set_pose_source = NULL, // CNSDK pose comes from face tracking, not an external source + + // probe_displays (v1.9.0 / ADR-015 / #69): deliberately omitted on Android. + // It exists for per-monitor claim discovery on desktop multi-display setups + // (the Windows arm in drv_leia/leia_plugin.c implements it over EDID + // enumeration). Android is a single fixed-panel device with no monitor + // enumeration, so we leave it NULL — struct_size still spans the full v2 + // iface, so the runtime's struct_size gate sees the slot, finds it NULL, and + // falls back to query_source_claims()'s synthesized primary claim off a + // successful binary probe(). Validated end-to-end via android-smoketest.sh. + .probe_displays = NULL, }; From ba1070beb3aaea645a262359f5015d357752c433 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 26 May 2026 12:37:42 -0700 Subject: [PATCH 5/7] android: tighten plug-in symbol exports with --exclude-libs,ALL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plug-in discovery contract §4 ("single-symbol export discipline"): plug-ins must expose only xrtPluginNegotiate via the dynamic symbol table. The existing C_/CXX_VISIBILITY_PRESET=hidden setting only hides symbols compiled from THIS target's own sources — symbols pulled in from linked static libs (aux_util, aux_vk, aux_android jni glue, etc.) keep whatever visibility they had when compiled, which defaults to "default" everywhere. Result before this commit: $ llvm-nm -D --defined-only --extern-only libdxrp050_leia_cnsdk.so | wc -l 1233 (one intentional + 1232 leaked from static libs) After: $ llvm-nm -D --defined-only --extern-only libdxrp050_leia_cnsdk.so | wc -l 1 $ llvm-nm -D --defined-only --extern-only libdxrp050_leia_cnsdk.so 0000000000036970 T xrtPluginNegotiate LINKER:--exclude-libs,ALL applies a "hidden" visibility filter to every archive linked into the .so, so static-lib symbols stay private even when compiled with default visibility. The XRT_PLUGIN_EXPORT macro on xrtPluginNegotiate marks it as explicitly visible so the runtime loader's dlsym still resolves it. Gated on (ANDROID OR UNIX) since --exclude-libs,ALL is a GNU ld / lld flag. Windows uses /EXPORT def-file discipline elsewhere. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/drv_leia_android/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/drv_leia_android/CMakeLists.txt b/src/drv_leia_android/CMakeLists.txt index 5d19fa2..da98354 100644 --- a/src/drv_leia_android/CMakeLists.txt +++ b/src/drv_leia_android/CMakeLists.txt @@ -97,3 +97,15 @@ set_target_properties(dxrp050_leia_cnsdk PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON ) + +# Single-symbol export discipline (plug-in discovery contract §4): +# only xrtPluginNegotiate should be in the dynamic symbol table. +# C_/CXX_VISIBILITY_PRESET=hidden only affects this target's own +# sources, not symbols pulled in from linked static libs (aux_util, +# aux_vk, jni glue, etc.). --exclude-libs,ALL applies a "hidden" +# visibility filter to ALL static libs, so their symbols stay +# private to this .so even if compiled with default visibility. +# XRT_PLUGIN_EXPORT on xrtPluginNegotiate keeps it explicitly visible. +if(ANDROID OR UNIX) + target_link_options(dxrp050_leia_cnsdk PRIVATE "LINKER:--exclude-libs,ALL") +endif() From 70e45dbc0dd41fe7e7d7c14cc13f7631fce9a1a0 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 26 May 2026 12:40:07 -0700 Subject: [PATCH 6/7] =?UTF-8?q?android:=20scripts/build-android.sh=20?= =?UTF-8?q?=E2=80=94=20single-cmd=20plug-in=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps the multi-arg cmake invocation needed to produce libdxrp050_leia_cnsdk.so for Android arm64-v8a. Without this script, a bring-up dev has to remember: - NDK toolchain file path - Ninja binary path inside the SDK's bundled cmake (not on PATH) - CNSDK_ROOT (extracted release tree, not the source checkout) - DXR_RUNTIME_SOURCE_DIR (sibling runtime checkout) - Eigen3_DIR (gradle-fetched stub config under the runtime tree) - ABI + platform settings All of these drift across machines. The script auto-resolves each from sensible defaults relative to the script's location, with explicit env- var overrides documented at the top. Hard failure with actionable error messages if any required dep is missing (NDK version not installed, CNSDK not extracted, runtime checkout not next to the plug-in, etc.). Mirrors scripts/build-windows.bat's shape — same one-line usage pattern, same env-var convention. Adds a [target] arg supporting: - dxrp050_leia_cnsdk (default) — build the plug-in .so - clean — wipe build-android/ Verified end-to-end: $ scripts/build-android.sh clean Wiping build-android/ $ scripts/build-android.sh dxrp050_leia_cnsdk ... === Built: build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so === -rw-r--r-- 1.7M libdxrp050_leia_cnsdk.so Drop into the runtime APK's jniLibs//: cp build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so ... Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/build-android.sh | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/build-android.sh diff --git a/scripts/build-android.sh b/scripts/build-android.sh new file mode 100644 index 0000000..d86242b --- /dev/null +++ b/scripts/build-android.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# ============================================================ +# DisplayXR Leia CNSDK Plug-in — Android Build Helper +# ============================================================ +# Wraps the multi-arg cmake invocation needed to produce +# libdxrp050_leia_cnsdk.so for Android arm64-v8a. Without this +# script, a bring-up dev has to remember: +# - NDK toolchain file path +# - Ninja binary path inside the SDK's bundled cmake +# - CNSDK_ROOT (extracted release tree, not source checkout) +# - DXR_RUNTIME_SOURCE_DIR (sibling runtime checkout) +# - Eigen3_DIR (gradle-fetched stub config) +# - ABI + platform settings +# All of those drift across machines; this script auto-resolves +# them with sensible defaults and explicit env-var overrides. +# +# Usage: scripts/build-android.sh [target] +# target = dxrp050_leia_cnsdk (default) — only the plug-in .so +# = clean — wipe build-android/ +# +# Required: +# ANDROID_SDK_ROOT or ANDROID_HOME — Android SDK install dir +# (must contain ndk// and +# cmake//bin/ninja.exe). +# +# Optional env (auto-detected with defaults): +# ANDROID_NDK_VERSION — NDK to use (default: 26.3.11579264) +# CNSDK_ROOT — extracted CNSDK 0.7.28 release tree +# (default: ../openxr-3d-display/cnsdk) +# DXR_RUNTIME_SOURCE_DIR — local runtime checkout +# (default: ../openxr-3d-display) +# EIGEN3_DIR — Eigen3Config.cmake location. If unset, +# the script points at the gradle-fetched +# Eigen under the runtime checkout's +# openxr_android/build/intermediates/. +# That dir only exists AFTER a successful +# runtime APK build; if you're building +# the plug-in standalone (no runtime APK +# built yet), set EIGEN3_DIR explicitly. +# +# Output: build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so +# ============================================================ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "${SCRIPT_DIR}/.." && pwd)" +TARGET="${1:-dxrp050_leia_cnsdk}" + +# Resolve Android SDK +: "${ANDROID_SDK_ROOT:=${ANDROID_HOME:-}}" +if [ -z "${ANDROID_SDK_ROOT}" ] || [ ! -d "${ANDROID_SDK_ROOT}" ]; then + echo "ERROR: ANDROID_SDK_ROOT not set or doesn't exist." + echo " Set ANDROID_SDK_ROOT (or ANDROID_HOME) to your Android SDK install dir." + exit 1 +fi +echo "ANDROID_SDK_ROOT: ${ANDROID_SDK_ROOT}" + +# Resolve NDK +: "${ANDROID_NDK_VERSION:=26.3.11579264}" +NDK_DIR="${ANDROID_SDK_ROOT}/ndk/${ANDROID_NDK_VERSION}" +if [ ! -d "${NDK_DIR}" ]; then + echo "ERROR: NDK ${ANDROID_NDK_VERSION} not installed at ${NDK_DIR}." + echo " Install via Android Studio SDK Manager or 'sdkmanager \"ndk;${ANDROID_NDK_VERSION}\"'." + exit 1 +fi +TOOLCHAIN_FILE="${NDK_DIR}/build/cmake/android.toolchain.cmake" +echo "NDK: ${NDK_DIR}" + +# Resolve Ninja (bundled with Android SDK's cmake) +NINJA="$(find "${ANDROID_SDK_ROOT}/cmake" -name ninja.exe -o -name ninja 2>/dev/null | head -1)" +if [ -z "${NINJA}" ]; then + echo "ERROR: ninja not found under ${ANDROID_SDK_ROOT}/cmake." + echo " Install via Android Studio SDK Manager (Tools > Settings > SDK > SDK Tools > CMake)." + exit 1 +fi +echo "Ninja: ${NINJA}" + +# Resolve runtime source dir +: "${DXR_RUNTIME_SOURCE_DIR:=${REPO}/../openxr-3d-display}" +if [ ! -f "${DXR_RUNTIME_SOURCE_DIR}/CMakeLists.txt" ]; then + echo "ERROR: DXR_RUNTIME_SOURCE_DIR=${DXR_RUNTIME_SOURCE_DIR} doesn't look like a runtime checkout." + echo " Point it at a local clone of DisplayXR/displayxr-runtime." + exit 1 +fi +echo "Runtime: ${DXR_RUNTIME_SOURCE_DIR}" + +# Resolve CNSDK +: "${CNSDK_ROOT:=${DXR_RUNTIME_SOURCE_DIR}/cnsdk}" +if [ ! -f "${CNSDK_ROOT}/share/cmake/CNSDK/CNSDKConfig.cmake" ]; then + echo "ERROR: CNSDK_ROOT=${CNSDK_ROOT} missing share/cmake/CNSDK/CNSDKConfig.cmake." + echo " Extract CNSDK 0.7.28 release zip and set CNSDK_ROOT to the extracted dir." + echo " Source: https://github.com/LeiaInc/leiainc.github.io/tree/master/CNSDK/cnsdk-android-0.7.28.zip" + exit 1 +fi +echo "CNSDK: ${CNSDK_ROOT}" + +# Resolve Eigen3 dir +: "${EIGEN3_DIR:=${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/build/intermediates/eigen/eigen-3.4.0/cmake}" +if [ ! -f "${EIGEN3_DIR}/Eigen3Config.cmake" ]; then + echo "ERROR: EIGEN3_DIR=${EIGEN3_DIR} missing Eigen3Config.cmake." + echo " Either (a) build the runtime APK first to materialize the gradle-fetched Eigen at the default path, or" + echo " (b) set EIGEN3_DIR explicitly to a dir containing Eigen3Config.cmake." + exit 1 +fi +echo "Eigen3: ${EIGEN3_DIR}" + +# Handle clean target +if [ "${TARGET}" = "clean" ]; then + echo "Wiping build-android/" + rm -rf "${REPO}/build-android" + exit 0 +fi + +# Configure +cd "${REPO}" +if [ ! -f build-android/CMakeCache.txt ]; then + echo + echo "=== Configuring ===" + cmake -S . -B build-android -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="${TOOLCHAIN_FILE}" \ + -DCMAKE_MAKE_PROGRAM="${NINJA}" \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-29 \ + -DCNSDK_ROOT="${CNSDK_ROOT}" \ + -DDXR_RUNTIME_SOURCE_DIR="${DXR_RUNTIME_SOURCE_DIR}" \ + -DEigen3_DIR="${EIGEN3_DIR}" +fi + +# Build +echo +echo "=== Building ${TARGET} ===" +cmake --build build-android --target "${TARGET}" + +# Report +echo +SO=build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so +if [ -f "${SO}" ]; then + echo "=== Built: $(pwd)/${SO} ===" + ls -l "${SO}" + echo + echo "Drop into the runtime APK's jniLibs//:" + echo " cp ${SO} ${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a/" +fi From cfef49d9e0ec4ef9c277a5c8c44bb6be473ea314 Mon Sep 17 00:00:00 2001 From: leaiss Date: Tue, 26 May 2026 13:41:11 -0700 Subject: [PATCH 7/7] android: link CNSDK in-app variant + bundle transitive .so deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to make the plug-in self-sufficient on a stock-Android device (the emulator validation pass surfaced both): 1. src/drv_leia_android/CMakeLists.txt — link CNSDK::leiaSDK-faceTrackingInApp (in-process face tracking) instead of CNSDK::leiaSDK (which CNSDKConfig.cmake aliases to the faceTrackingInService variant, requiring a separate Leia face-tracking service running). CNSDK 0.7.28's Targets.cmake only exposes the InService variant — define our own IMPORTED target around the InApp .so file (which DOES exist on disk at ${CNSDK_ROOT}/lib//libleiaSDK-faceTrackingInApp.so). Reuse the existing InService target's INTERFACE_INCLUDE_DIRECTORIES so includes resolve. 2. scripts/build-android.sh — new `install-runtime-jnilibs` target. Builds the plug-in .so and ALSO copies the CNSDK transitive .so deps (extracted from the InApp AAR + SNPE third-party AAR) into the runtime APK's jniLibs// alongside the plug-in. Without this, dlopen of libdxrp050_leia_cnsdk.so fails at runtime with "library libleiaSDK-faceTrackingInApp.so not found" (and then libblink.so, libSNPE.so, ...). Bundling the full transitive set into the runtime APK = 16 .so files total, ~80 MB added. Usage: scripts/build-android.sh install-runtime-jnilibs cd ./gradlew :src:xrt:targets:openxr_android:assembleInProcessDebug --rerun-tasks De-duplicates libc++_shared.so (CNSDK ships its own copy; the runtime build already ships one — version skew between the two causes dlopen failures). Known limitation surfaced during emulator validation: even with all transitive .so files bundled into the runtime APK, the Android linker fails dlopen of the plug-in's DT_NEEDED libs because the dlopen happens in the TEST APP's classloader namespace (clns-7), which only sees the test app's own /data/app//lib//. The fix is in target_plugin_loader.c (runtime side): use android_dlopen_ext with a custom namespace whose library_search_paths includes the RUNTIME APK's lib dir. That's a runtime-side change tracked separately. On Lume Pad the namespace issue probably doesn't apply because Leia's preinstalled OS likely places CNSDK at a system path. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/build-android.sh | 89 +++++++++++++++++++++++++++-- src/drv_leia_android/CMakeLists.txt | 35 +++++++++++- 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/scripts/build-android.sh b/scripts/build-android.sh index d86242b..383e8fc 100644 --- a/scripts/build-android.sh +++ b/scripts/build-android.sh @@ -17,6 +17,12 @@ # Usage: scripts/build-android.sh [target] # target = dxrp050_leia_cnsdk (default) — only the plug-in .so # = clean — wipe build-android/ +# = install-runtime-jnilibs — build .so + copy plug-in +# .so + CNSDK transitive +# .so files into the +# runtime APK's jniLibs/ +# (the runtime APK then +# needs `assembleInProcessDebug --rerun-tasks`) # # Required: # ANDROID_SDK_ROOT or ANDROID_HOME — Android SDK install dir @@ -112,6 +118,16 @@ if [ "${TARGET}" = "clean" ]; then exit 0 fi +# install-runtime-jnilibs: build the plug-in .so AND copy it (plus +# the CNSDK transitive .so deps) into the runtime APK's jniLibs/. +# Internally just delegates the build step to the default target, +# then does the copies. +INSTALL_JNILIBS=false +if [ "${TARGET}" = "install-runtime-jnilibs" ]; then + INSTALL_JNILIBS=true + TARGET=dxrp050_leia_cnsdk +fi + # Configure cd "${REPO}" if [ ! -f build-android/CMakeCache.txt ]; then @@ -135,10 +151,75 @@ cmake --build build-android --target "${TARGET}" # Report echo SO=build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so -if [ -f "${SO}" ]; then - echo "=== Built: $(pwd)/${SO} ===" - ls -l "${SO}" +if [ ! -f "${SO}" ]; then + exit 0 +fi +echo "=== Built: $(pwd)/${SO} ===" +ls -l "${SO}" + +# Install into runtime APK's jniLibs/ when requested. +if [ "${INSTALL_JNILIBS}" = "true" ]; then + JNI_DIR="${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a" + mkdir -p "${JNI_DIR}" + echo + echo "=== Installing into ${JNI_DIR} ===" + + # Plug-in .so itself. + cp -f "${SO}" "${JNI_DIR}/" + echo " + $(basename "${SO}")" + + # CNSDK transitive .so deps. The plug-in's leia_cnsdk_get_* + # entry points need libleiaSDK-faceTrackingInApp.so (loaded + # via DT_NEEDED), which in turn needs libblink.so and + # liblicense_utils.so. These ship in the sdk-faceTrackingInApp + # AAR — extract and copy. + AAR_NAME="sdk-faceTrackingInApp" + AAR_VERSION_FILE="${CNSDK_ROOT}/VERSION.txt" + if [ -f "${AAR_VERSION_FILE}" ]; then + CNSDK_VERSION="$(cat "${AAR_VERSION_FILE}" | tr -d ' \r\n')" + else + CNSDK_VERSION="0.7.28" + fi + AAR="${CNSDK_ROOT}/android/${AAR_NAME}-${CNSDK_VERSION}.aar" + if [ ! -f "${AAR}" ]; then + echo "WARNING: ${AAR} not found — runtime APK won't have CNSDK transitive .so deps and the plug-in will fail to dlopen on device." + else + # AAR is a zip; extract just the arm64-v8a .so files. + TMP_AAR=$(mktemp -d) + unzip -q -j "${AAR}" 'jni/arm64-v8a/*.so' -d "${TMP_AAR}" + for so in "${TMP_AAR}"/*.so; do + cp -f "${so}" "${JNI_DIR}/" + echo " + $(basename "${so}")" + done + rm -rf "${TMP_AAR}" + fi + + # SNPE — Qualcomm's Snapdragon Neural Processing Engine, used by + # CNSDK's libblink.so (face-tracking inference). Ships as a separate + # AAR under CNSDK's third_party tree. Bundle these unless they're + # known to come from the platform (e.g. system /vendor/ on Lume Pad). + SNPE_AAR="${CNSDK_ROOT}/android/third_party/snpe-release.aar" + if [ -f "${SNPE_AAR}" ]; then + TMP_SNPE=$(mktemp -d) + unzip -q -j "${SNPE_AAR}" 'jni/arm64-v8a/*.so' -d "${TMP_SNPE}" + for so in "${TMP_SNPE}"/*.so; do + # libc++_shared.so already shipped by the runtime build — + # double-include can cause version-skew dlopen failures. + if [ "$(basename "${so}")" = "libc++_shared.so" ]; then + continue + fi + cp -f "${so}" "${JNI_DIR}/" + echo " + $(basename "${so}")" + done + rm -rf "${TMP_SNPE}" + fi + + echo + echo "Now build the runtime APK with the jniLibs picked up:" + echo " cd ${DXR_RUNTIME_SOURCE_DIR} && ./gradlew :src:xrt:targets:openxr_android:assembleInProcessDebug --rerun-tasks" +else echo - echo "Drop into the runtime APK's jniLibs//:" + echo "Drop into the runtime APK's jniLibs// (or re-run this script with" + echo "the 'install-runtime-jnilibs' target to do this + CNSDK transitive .so deps in one step):" echo " cp ${SO} ${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a/" fi diff --git a/src/drv_leia_android/CMakeLists.txt b/src/drv_leia_android/CMakeLists.txt index da98354..f7dfde9 100644 --- a/src/drv_leia_android/CMakeLists.txt +++ b/src/drv_leia_android/CMakeLists.txt @@ -51,6 +51,35 @@ list(APPEND CMAKE_PREFIX_PATH "${CNSDK_ROOT}") find_package(CNSDK CONFIG REQUIRED) message(STATUS "Leia CNSDK at ${CNSDK_ROOT}") +# CNSDK 0.7.28's CMake config only exports targets for the +# faceTrackingInService variant (out-of-process face tracking via +# a separate Leia system service). For in-process operation on a +# stock device + emulator testing we want the faceTrackingInApp +# variant, which CNSDK ships as a sibling .so but doesn't expose a +# target for. Define our own IMPORTED target around it. +if(NOT TARGET CNSDK::leiaSDK-faceTrackingInApp) + set(_inapp_lib "${CNSDK_ROOT}/lib/${CMAKE_ANDROID_ARCH_ABI}/libleiaSDK-faceTrackingInApp.so") + if(NOT EXISTS "${_inapp_lib}") + message(FATAL_ERROR "CNSDK in-app face-tracking lib not found: ${_inapp_lib}") + endif() + # Reuse CNSDK::leiaSDK-faceTrackingInService-shared's include dirs + # (they're identical between variants — same headers, different + # backing impl) so the plug-in's #include etc. + # resolve. + get_target_property(_cnsdk_includes + CNSDK::leiaSDK-faceTrackingInService-shared + INTERFACE_INCLUDE_DIRECTORIES) + add_library(CNSDK::leiaSDK-faceTrackingInApp-shared SHARED IMPORTED) + set_target_properties(CNSDK::leiaSDK-faceTrackingInApp-shared PROPERTIES + IMPORTED_LOCATION "${_inapp_lib}" + INTERFACE_INCLUDE_DIRECTORIES "${_cnsdk_includes}" + ) + add_library(CNSDK::leiaSDK-faceTrackingInApp INTERFACE IMPORTED) + set_target_properties(CNSDK::leiaSDK-faceTrackingInApp PROPERTIES + INTERFACE_LINK_LIBRARIES "CNSDK::leiaSDK-faceTrackingInApp-shared" + ) +endif() + # --- Plug-in target ------------------------------------------------------- set(DRV_LEIA_ANDROID_SOURCES @@ -76,8 +105,10 @@ target_link_libraries(dxrp050_leia_cnsdk aux_os aux_math aux_vk - # CNSDK Vulkan interlacer + face-tracker. - CNSDK::leiaSDK + # CNSDK Vulkan interlacer + in-process face-tracker. + # (The InApp variant is what we ship; see the IMPORTED-target + # definition above for why we don't just use CNSDK::leiaSDK.) + CNSDK::leiaSDK-faceTrackingInApp # Android system libs. log android