diff --git a/.gitignore b/.gitignore index f897eca..9fc0548 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # Build artifacts build/ +build-*/ _package/ out/ *.user diff --git a/CMakeLists.txt b/CMakeLists.txt index 08d79a8..9d587b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,8 +13,12 @@ set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -if(NOT WIN32) - message(FATAL_ERROR "The Leia SR plug-in is Windows-only.") +if(NOT WIN32 AND NOT ANDROID) + message(FATAL_ERROR + "The Leia plug-in supports two targets:\n" + " - Windows / Leia SR (DisplayXR-LeiaSR.dll, drv_leia)\n" + " - Android / Leia CNSDK (libdxrp050_leia_cnsdk.so, drv_leia_android)\n" + "Configured platform is neither WIN32 nor ANDROID.") endif() if(NOT DEFINED BUILD_NUM) @@ -64,17 +68,26 @@ else() FetchContent_MakeAvailable(displayxr_runtime) endif() -# --- SR SDK ------------------------------------------------------------- -if(NOT DEFINED ENV{LEIASR_SDKROOT}) - message(FATAL_ERROR - "LEIASR_SDKROOT env var not set. Point at an extracted LeiaSR-SDK-*-win64 directory. " - "scripts/build-windows.bat fetches it from the sr-sdk-v* release on this repo.") -endif() -set(SR_PATH "$ENV{LEIASR_SDKROOT}") -list(APPEND CMAKE_PREFIX_PATH "${SR_PATH}") -find_package(simulatedreality CONFIG REQUIRED GLOBAL) -find_package(srDirectX CONFIG REQUIRED GLOBAL) -message(STATUS "Leia SR SDK at ${SR_PATH}") +if(WIN32) + # --- SR SDK (Windows / Leia SR plug-in) ---------------------------- + if(NOT DEFINED ENV{LEIASR_SDKROOT}) + message(FATAL_ERROR + "LEIASR_SDKROOT env var not set. Point at an extracted LeiaSR-SDK-*-win64 directory. " + "scripts/build-windows.bat fetches it from the sr-sdk-v* release on this repo.") + endif() + set(SR_PATH "$ENV{LEIASR_SDKROOT}") + list(APPEND CMAKE_PREFIX_PATH "${SR_PATH}") + find_package(simulatedreality CONFIG REQUIRED GLOBAL) + find_package(srDirectX CONFIG REQUIRED GLOBAL) + message(STATUS "Leia SR SDK at ${SR_PATH}") -add_subdirectory(src/drv_leia) -add_subdirectory(installer) + add_subdirectory(src/drv_leia) + add_subdirectory(installer) +elseif(ANDROID) + # --- CNSDK (Android / Leia CNSDK plug-in) -------------------------- + # CNSDK setup (find_package(CNSDK CONFIG)) lives inside + # src/drv_leia_android/CMakeLists.txt — it consumes CNSDK_ROOT, + # which the build environment supplies. No NSIS installer on + # Android; the .so ships in the runtime APK's jniLibs//. + add_subdirectory(src/drv_leia_android) +endif() diff --git a/docs/cnsdk-android-calibration.md b/docs/cnsdk-android-calibration.md new file mode 100644 index 0000000..a4612e6 --- /dev/null +++ b/docs/cnsdk-android-calibration.md @@ -0,0 +1,113 @@ +# CNSDK Android Calibration Procedure + +The Android POC's CNSDK display processor makes three assumptions about +CNSDK conventions that the SDK docs leave ambiguous. Once a Lume Pad is +attached, validate each in this order — they're listed cheapest-to-test +first. Audit references in brackets. + +## 1. Face position axis convention [B15] + +**What's assumed:** in `src/drv_leia_android/leia_cnsdk.cpp::leia_cnsdk_get_primary_face`, +CNSDK's `leia_core_get_primary_face` returns a 3-float position in +millimeters with axes x=right / y=up / z=toward viewer, origin at the +camera. The wrapper converts mm→m and subtracts the cached +`cameraCenterX/Y/Z` to land in `xrt_eye_position`'s display-relative +frame. + +**What CNSDK actually documents** (cnsdk/include/leia/headTracking/common/types.h): +- `leia_headtracking_detected_face.posePosition` is "Head location in + mm. The origin point is the location of the camera." ✓ matches. +- `leia_headtracking_face.point.pos` is "3D coordinate with camera + transform and Kalman filter applied." Axis convention not stated. +- `posePoseAngle` is "in radians. **The rotation is a left handed + coordinate system.**" Strong hint that positions may also follow a + left-handed system. + +**Test:** stand directly in front of the display, motionless. Note the +last reported eye position in logcat (look for `xrLocateViews` HUD +output or any debug printf you add). Then move your head 10 cm +**right**. Re-read. + +| Result | What it means | Fix | +|---|---|---| +| x increased by ~0.10 | Right-handed, x=right. Convention matches. ✓ | — | +| x decreased by ~0.10 | Left-handed, x=right OR right-handed with x=left | Flip sign of `out_x` in `leia_cnsdk_get_primary_face` | +| y changed by ~0.10 | Axes are rotated; CNSDK uses landscape vs portrait orientation differently | Swap x/y, possibly negate one | + +Repeat for **down** (y) and **toward display** (z). + +If the cube appears to track the wrong direction, this is the first +thing to check. + +## 2. Tile-to-eye mapping [B17] + +**What's assumed:** the SBS atlas's tile (col 0, row 0) is the **left +eye** view and (col 1, row 0) is the **right eye**. The DP's +`blit_atlas_to_per_view` blits tile 0 → `view_img[0]`, tile 1 → +`view_img[1]`. `leia_cnsdk_weave` then calls +`leia_interlacer_vulkan_set_view_for_texture_array(interlacer, 0, view_iv[0])` +and `(interlacer, 1, view_iv[1])` — passing left as view-0, right as +view-1. + +**What CNSDK actually does:** undocumented. CNSDK's interlacer treats +view 0 and view 1 as left and right per common convention, but this +isn't explicit in the headers. + +**Test:** display the [Khronos OpenXR cube test pattern](https://github.com/KhronosGroup/OpenXR-SDK-Source/tree/main/src/tests/hello_xr) +or any stereo-pair where left and right eyes have visually distinct +content (e.g., a number "1" rendered to left view, "2" to right). Look +at the Lume Pad with one eye covered, then the other. + +| What you see | Meaning | Fix | +|---|---|---| +| "1" with left eye, "2" with right | Convention matches. ✓ | — | +| "2" with left eye, "1" with right | Tile-to-eye mapping is swapped | In `leia_cnsdk_weave`, swap the args to `set_view_for_texture_array` (pass right as 0, left as 1). Or in the DP, swap the destination index in `blit_atlas_to_per_view`. | +| Both eyes see "1" and "2" overlaid as a ghost | Interlacing pattern broken — likely a calibration issue, not view assignment | Check `leia_core_get_device_config` is returning the right `displayDots` — out of scope of this doc | + +## 3. UV vertical flip [B18] + +**What's assumed:** `leia_cnsdk.cpp::leia_cnsdk_weave` calls +`leia_interlacer_set_flip_input_uv_vertical(interlacer, true)` because +Vulkan's NDC is Y-down (origin at top-left) and CNSDK is presumed to +default to OpenGL convention (Y-up, origin bottom-left). + +**Test:** display any content with clear top/bottom asymmetry — text +is ideal. A rendered string "HELLO" or just a single capital "A" works. + +| What you see | Meaning | Fix | +|---|---|---| +| Right-side-up "HELLO" | UV flip convention is correct ✓ | — | +| Upside-down "OLLEH" / "HELLO" vertically mirrored | CNSDK is already flipping; we're double-flipping | Change `leia_interlacer_set_flip_input_uv_vertical(interlacer, true)` to `false` in `leia_cnsdk.cpp::leia_cnsdk_weave` | +| Letters look correct but offset / cropped | Different bug — not a flip issue. Check view_width / view_height accounting | Out of scope of this doc | + +## Combined check + +Once all three are pinned down, the cube test app (when it lands — +B13d) should: +1. Render a spinning cube at the depth corresponding to the wearer's + head distance. +2. Move convincingly with head motion (left-right strafing). +3. Have correct horizontal disparity (look closer with one eye than the + other). +4. Show right-side-up text/numbers on debug overlays. + +If 1+3 are off, it's #1 (face axes). If 2 is off, it's #1 or #2 (eye +mapping). If 4 is off, it's #3 (UV flip). + +## Where the convention assumptions live + +| Assumption | File | Function | Audit ref | +|---|---|---|---| +| Face position axes / units | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_get_primary_face` | B15 | +| Tile-to-eye mapping | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_weave` (`set_view_for_texture_array`) | B17 | +| UV vertical flip | `src/drv_leia_android/leia_cnsdk.cpp` | `leia_cnsdk_weave` (`set_flip_input_uv_vertical`) | B18 | + +All three are 1–3 line changes once you know the correct values. + +## Why these aren't fixed pre-emptively + +Each is a 50/50 coin flip in either direction. Picking wrong is worse +than picking right (off-by-180° on axes is more confusing than +matching the spec assumption). Lume Pad on-device test is the +definitive answer; this doc lets you arrive at hardware ready to flip +the right knob. diff --git a/scripts/build-android.sh b/scripts/build-android.sh new file mode 100644 index 0000000..383e8fc --- /dev/null +++ b/scripts/build-android.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# ============================================================ +# DisplayXR Leia CNSDK Plug-in — Android Build Helper +# ============================================================ +# Wraps the multi-arg cmake invocation needed to produce +# libdxrp050_leia_cnsdk.so for Android arm64-v8a. Without this +# script, a bring-up dev has to remember: +# - NDK toolchain file path +# - Ninja binary path inside the SDK's bundled cmake +# - CNSDK_ROOT (extracted release tree, not source checkout) +# - DXR_RUNTIME_SOURCE_DIR (sibling runtime checkout) +# - Eigen3_DIR (gradle-fetched stub config) +# - ABI + platform settings +# All of those drift across machines; this script auto-resolves +# them with sensible defaults and explicit env-var overrides. +# +# Usage: scripts/build-android.sh [target] +# target = dxrp050_leia_cnsdk (default) — only the plug-in .so +# = clean — wipe build-android/ +# = install-runtime-jnilibs — build .so + copy plug-in +# .so + CNSDK transitive +# .so files into the +# runtime APK's jniLibs/ +# (the runtime APK then +# needs `assembleInProcessDebug --rerun-tasks`) +# +# Required: +# ANDROID_SDK_ROOT or ANDROID_HOME — Android SDK install dir +# (must contain ndk// and +# cmake//bin/ninja.exe). +# +# Optional env (auto-detected with defaults): +# ANDROID_NDK_VERSION — NDK to use (default: 26.3.11579264) +# CNSDK_ROOT — extracted CNSDK 0.7.28 release tree +# (default: ../openxr-3d-display/cnsdk) +# DXR_RUNTIME_SOURCE_DIR — local runtime checkout +# (default: ../openxr-3d-display) +# EIGEN3_DIR — Eigen3Config.cmake location. If unset, +# the script points at the gradle-fetched +# Eigen under the runtime checkout's +# openxr_android/build/intermediates/. +# That dir only exists AFTER a successful +# runtime APK build; if you're building +# the plug-in standalone (no runtime APK +# built yet), set EIGEN3_DIR explicitly. +# +# Output: build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so +# ============================================================ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "${SCRIPT_DIR}/.." && pwd)" +TARGET="${1:-dxrp050_leia_cnsdk}" + +# Resolve Android SDK +: "${ANDROID_SDK_ROOT:=${ANDROID_HOME:-}}" +if [ -z "${ANDROID_SDK_ROOT}" ] || [ ! -d "${ANDROID_SDK_ROOT}" ]; then + echo "ERROR: ANDROID_SDK_ROOT not set or doesn't exist." + echo " Set ANDROID_SDK_ROOT (or ANDROID_HOME) to your Android SDK install dir." + exit 1 +fi +echo "ANDROID_SDK_ROOT: ${ANDROID_SDK_ROOT}" + +# Resolve NDK +: "${ANDROID_NDK_VERSION:=26.3.11579264}" +NDK_DIR="${ANDROID_SDK_ROOT}/ndk/${ANDROID_NDK_VERSION}" +if [ ! -d "${NDK_DIR}" ]; then + echo "ERROR: NDK ${ANDROID_NDK_VERSION} not installed at ${NDK_DIR}." + echo " Install via Android Studio SDK Manager or 'sdkmanager \"ndk;${ANDROID_NDK_VERSION}\"'." + exit 1 +fi +TOOLCHAIN_FILE="${NDK_DIR}/build/cmake/android.toolchain.cmake" +echo "NDK: ${NDK_DIR}" + +# Resolve Ninja (bundled with Android SDK's cmake) +NINJA="$(find "${ANDROID_SDK_ROOT}/cmake" -name ninja.exe -o -name ninja 2>/dev/null | head -1)" +if [ -z "${NINJA}" ]; then + echo "ERROR: ninja not found under ${ANDROID_SDK_ROOT}/cmake." + echo " Install via Android Studio SDK Manager (Tools > Settings > SDK > SDK Tools > CMake)." + exit 1 +fi +echo "Ninja: ${NINJA}" + +# Resolve runtime source dir +: "${DXR_RUNTIME_SOURCE_DIR:=${REPO}/../openxr-3d-display}" +if [ ! -f "${DXR_RUNTIME_SOURCE_DIR}/CMakeLists.txt" ]; then + echo "ERROR: DXR_RUNTIME_SOURCE_DIR=${DXR_RUNTIME_SOURCE_DIR} doesn't look like a runtime checkout." + echo " Point it at a local clone of DisplayXR/displayxr-runtime." + exit 1 +fi +echo "Runtime: ${DXR_RUNTIME_SOURCE_DIR}" + +# Resolve CNSDK +: "${CNSDK_ROOT:=${DXR_RUNTIME_SOURCE_DIR}/cnsdk}" +if [ ! -f "${CNSDK_ROOT}/share/cmake/CNSDK/CNSDKConfig.cmake" ]; then + echo "ERROR: CNSDK_ROOT=${CNSDK_ROOT} missing share/cmake/CNSDK/CNSDKConfig.cmake." + echo " Extract CNSDK 0.7.28 release zip and set CNSDK_ROOT to the extracted dir." + echo " Source: https://github.com/LeiaInc/leiainc.github.io/tree/master/CNSDK/cnsdk-android-0.7.28.zip" + exit 1 +fi +echo "CNSDK: ${CNSDK_ROOT}" + +# Resolve Eigen3 dir +: "${EIGEN3_DIR:=${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/build/intermediates/eigen/eigen-3.4.0/cmake}" +if [ ! -f "${EIGEN3_DIR}/Eigen3Config.cmake" ]; then + echo "ERROR: EIGEN3_DIR=${EIGEN3_DIR} missing Eigen3Config.cmake." + echo " Either (a) build the runtime APK first to materialize the gradle-fetched Eigen at the default path, or" + echo " (b) set EIGEN3_DIR explicitly to a dir containing Eigen3Config.cmake." + exit 1 +fi +echo "Eigen3: ${EIGEN3_DIR}" + +# Handle clean target +if [ "${TARGET}" = "clean" ]; then + echo "Wiping build-android/" + rm -rf "${REPO}/build-android" + exit 0 +fi + +# install-runtime-jnilibs: build the plug-in .so AND copy it (plus +# the CNSDK transitive .so deps) into the runtime APK's jniLibs/. +# Internally just delegates the build step to the default target, +# then does the copies. +INSTALL_JNILIBS=false +if [ "${TARGET}" = "install-runtime-jnilibs" ]; then + INSTALL_JNILIBS=true + TARGET=dxrp050_leia_cnsdk +fi + +# Configure +cd "${REPO}" +if [ ! -f build-android/CMakeCache.txt ]; then + echo + echo "=== Configuring ===" + cmake -S . -B build-android -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="${TOOLCHAIN_FILE}" \ + -DCMAKE_MAKE_PROGRAM="${NINJA}" \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-29 \ + -DCNSDK_ROOT="${CNSDK_ROOT}" \ + -DDXR_RUNTIME_SOURCE_DIR="${DXR_RUNTIME_SOURCE_DIR}" \ + -DEigen3_DIR="${EIGEN3_DIR}" +fi + +# Build +echo +echo "=== Building ${TARGET} ===" +cmake --build build-android --target "${TARGET}" + +# Report +echo +SO=build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so +if [ ! -f "${SO}" ]; then + exit 0 +fi +echo "=== Built: $(pwd)/${SO} ===" +ls -l "${SO}" + +# Install into runtime APK's jniLibs/ when requested. +if [ "${INSTALL_JNILIBS}" = "true" ]; then + JNI_DIR="${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a" + mkdir -p "${JNI_DIR}" + echo + echo "=== Installing into ${JNI_DIR} ===" + + # Plug-in .so itself. + cp -f "${SO}" "${JNI_DIR}/" + echo " + $(basename "${SO}")" + + # CNSDK transitive .so deps. The plug-in's leia_cnsdk_get_* + # entry points need libleiaSDK-faceTrackingInApp.so (loaded + # via DT_NEEDED), which in turn needs libblink.so and + # liblicense_utils.so. These ship in the sdk-faceTrackingInApp + # AAR — extract and copy. + AAR_NAME="sdk-faceTrackingInApp" + AAR_VERSION_FILE="${CNSDK_ROOT}/VERSION.txt" + if [ -f "${AAR_VERSION_FILE}" ]; then + CNSDK_VERSION="$(cat "${AAR_VERSION_FILE}" | tr -d ' \r\n')" + else + CNSDK_VERSION="0.7.28" + fi + AAR="${CNSDK_ROOT}/android/${AAR_NAME}-${CNSDK_VERSION}.aar" + if [ ! -f "${AAR}" ]; then + echo "WARNING: ${AAR} not found — runtime APK won't have CNSDK transitive .so deps and the plug-in will fail to dlopen on device." + else + # AAR is a zip; extract just the arm64-v8a .so files. + TMP_AAR=$(mktemp -d) + unzip -q -j "${AAR}" 'jni/arm64-v8a/*.so' -d "${TMP_AAR}" + for so in "${TMP_AAR}"/*.so; do + cp -f "${so}" "${JNI_DIR}/" + echo " + $(basename "${so}")" + done + rm -rf "${TMP_AAR}" + fi + + # SNPE — Qualcomm's Snapdragon Neural Processing Engine, used by + # CNSDK's libblink.so (face-tracking inference). Ships as a separate + # AAR under CNSDK's third_party tree. Bundle these unless they're + # known to come from the platform (e.g. system /vendor/ on Lume Pad). + SNPE_AAR="${CNSDK_ROOT}/android/third_party/snpe-release.aar" + if [ -f "${SNPE_AAR}" ]; then + TMP_SNPE=$(mktemp -d) + unzip -q -j "${SNPE_AAR}" 'jni/arm64-v8a/*.so' -d "${TMP_SNPE}" + for so in "${TMP_SNPE}"/*.so; do + # libc++_shared.so already shipped by the runtime build — + # double-include can cause version-skew dlopen failures. + if [ "$(basename "${so}")" = "libc++_shared.so" ]; then + continue + fi + cp -f "${so}" "${JNI_DIR}/" + echo " + $(basename "${so}")" + done + rm -rf "${TMP_SNPE}" + fi + + echo + echo "Now build the runtime APK with the jniLibs picked up:" + echo " cd ${DXR_RUNTIME_SOURCE_DIR} && ./gradlew :src:xrt:targets:openxr_android:assembleInProcessDebug --rerun-tasks" +else + echo + echo "Drop into the runtime APK's jniLibs// (or re-run this script with" + echo "the 'install-runtime-jnilibs' target to do this + CNSDK transitive .so deps in one step):" + echo " cp ${SO} ${DXR_RUNTIME_SOURCE_DIR}/src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a/" +fi diff --git a/src/drv_leia_android/CMakeLists.txt b/src/drv_leia_android/CMakeLists.txt new file mode 100644 index 0000000..f7dfde9 --- /dev/null +++ b/src/drv_leia_android/CMakeLists.txt @@ -0,0 +1,142 @@ +# Copyright 2026, Leia Inc / DisplayXR +# SPDX-License-Identifier: BSL-1.0 +# +# DisplayXR Leia CNSDK plug-in (Android). +# +# Builds the CNSDK-backed display-processor as a shared library named +# `libdxrp050_leia_cnsdk.so`, matching the runtime's Android plug-in +# discovery convention (libdxrp_.so, where NNN is the +# probe-order and matches the iface's `id` field). See +# `docs/specs/runtime/plugin-discovery.md` §3.2 in the runtime tree. +# +# The .so is meant to be dropped into the runtime APK's +# `jniLibs//` so the runtime's target_plugin_loader.c (PR #309) +# discovers it via dladdr-of-self + dirname enumeration at +# `xrCreateInstance` time. + +if(NOT ANDROID) + message(FATAL_ERROR "drv_leia_android is Android-only.") +endif() + +# --- CNSDK ---------------------------------------------------------------- +# +# CNSDK ships as an AAR + extracted SDK tree (headers + .so + cmake +# config). The expected layout matches what was previously consumed by +# the runtime POC's openxr_android/build.gradle CNSDK_AAR rule: +# +# ${CNSDK_ROOT}/include/leia/sdk/*.h +# ${CNSDK_ROOT}/share/cmake/CNSDK/CNSDKConfig.cmake +# ${CNSDK_ROOT}/lib//libleiaSDK-faceTrackingInApp.so +# +# CI / dev set CNSDK_ROOT to the extracted release tree. Pinned to +# CNSDK 0.7.28 for the POC. + +if(NOT DEFINED CNSDK_ROOT) + if(DEFINED ENV{CNSDK_ROOT}) + set(CNSDK_ROOT "$ENV{CNSDK_ROOT}") + else() + message(FATAL_ERROR + "CNSDK_ROOT not set. Extract CNSDK 0.7.28 and set " + "CNSDK_ROOT (CMake var or env var) to the extracted dir. " + "Source: https://github.com/LeiaInc/leiainc.github.io/tree/master/CNSDK/cnsdk-android-0.7.28.zip") + endif() +endif() +# Android NDK toolchain defaults CMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY, +# which restricts find_package to NDK sysroot dirs and ignores +# CMAKE_PREFIX_PATH outside that. Flip it to BOTH so the CNSDK install +# at ${CNSDK_ROOT} is searchable (matches the runtime APK's gradle +# cmake.arguments setup). +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH) +list(APPEND CMAKE_PREFIX_PATH "${CNSDK_ROOT}") +find_package(CNSDK CONFIG REQUIRED) +message(STATUS "Leia CNSDK at ${CNSDK_ROOT}") + +# CNSDK 0.7.28's CMake config only exports targets for the +# faceTrackingInService variant (out-of-process face tracking via +# a separate Leia system service). For in-process operation on a +# stock device + emulator testing we want the faceTrackingInApp +# variant, which CNSDK ships as a sibling .so but doesn't expose a +# target for. Define our own IMPORTED target around it. +if(NOT TARGET CNSDK::leiaSDK-faceTrackingInApp) + set(_inapp_lib "${CNSDK_ROOT}/lib/${CMAKE_ANDROID_ARCH_ABI}/libleiaSDK-faceTrackingInApp.so") + if(NOT EXISTS "${_inapp_lib}") + message(FATAL_ERROR "CNSDK in-app face-tracking lib not found: ${_inapp_lib}") + endif() + # Reuse CNSDK::leiaSDK-faceTrackingInService-shared's include dirs + # (they're identical between variants — same headers, different + # backing impl) so the plug-in's #include etc. + # resolve. + get_target_property(_cnsdk_includes + CNSDK::leiaSDK-faceTrackingInService-shared + INTERFACE_INCLUDE_DIRECTORIES) + add_library(CNSDK::leiaSDK-faceTrackingInApp-shared SHARED IMPORTED) + set_target_properties(CNSDK::leiaSDK-faceTrackingInApp-shared PROPERTIES + IMPORTED_LOCATION "${_inapp_lib}" + INTERFACE_INCLUDE_DIRECTORIES "${_cnsdk_includes}" + ) + add_library(CNSDK::leiaSDK-faceTrackingInApp INTERFACE IMPORTED) + set_target_properties(CNSDK::leiaSDK-faceTrackingInApp PROPERTIES + INTERFACE_LINK_LIBRARIES "CNSDK::leiaSDK-faceTrackingInApp-shared" + ) +endif() + +# --- Plug-in target ------------------------------------------------------- + +set(DRV_LEIA_ANDROID_SOURCES + leia_cnsdk.cpp + leia_cnsdk.h + leia_display_processor_cnsdk.cpp + leia_display_processor_cnsdk.h + leia_plugin_android.c +) + +add_library(dxrp050_leia_cnsdk SHARED ${DRV_LEIA_ANDROID_SOURCES}) + +target_include_directories(dxrp050_leia_cnsdk PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(dxrp050_leia_cnsdk + PRIVATE + # Runtime aux surface (consumed via FetchContent or local + # checkout per top-level CMakeLists.txt). + xrt-interfaces + aux_util + aux_os + aux_math + aux_vk + # CNSDK Vulkan interlacer + in-process face-tracker. + # (The InApp variant is what we ship; see the IMPORTED-target + # definition above for why we don't just use CNSDK::leiaSDK.) + CNSDK::leiaSDK-faceTrackingInApp + # Android system libs. + log + android + vulkan +) + +# Debug-log + ATrace gating: when XRT_DEBUG_ANDROID_VERBOSE is defined, +# DXR_HW_DBG / DXR_ATRACE blocks compile in. Release builds drop them +# all. The runtime APK's Gradle Debug variant sets this; Release does +# not. For standalone CMake builds, pass -DXRT_DEBUG_ANDROID_VERBOSE=1 +# on the cmake command line. + +set_target_properties(dxrp050_leia_cnsdk PROPERTIES + OUTPUT_NAME "dxrp050_leia_cnsdk" + PREFIX "lib" # libdxrp050_leia_cnsdk.so (matches discovery convention) + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON +) + +# Single-symbol export discipline (plug-in discovery contract §4): +# only xrtPluginNegotiate should be in the dynamic symbol table. +# C_/CXX_VISIBILITY_PRESET=hidden only affects this target's own +# sources, not symbols pulled in from linked static libs (aux_util, +# aux_vk, jni glue, etc.). --exclude-libs,ALL applies a "hidden" +# visibility filter to ALL static libs, so their symbols stay +# private to this .so even if compiled with default visibility. +# XRT_PLUGIN_EXPORT on xrtPluginNegotiate keeps it explicitly visible. +if(ANDROID OR UNIX) + target_link_options(dxrp050_leia_cnsdk PRIVATE "LINKER:--exclude-libs,ALL") +endif() diff --git a/src/drv_leia_android/README.md b/src/drv_leia_android/README.md new file mode 100644 index 0000000..2bc7246 --- /dev/null +++ b/src/drv_leia_android/README.md @@ -0,0 +1,60 @@ +# drv_leia_android — DisplayXR Leia CNSDK Plug-in + +The Android variant of the Leia display-processor plug-in. Wraps the +**Leia Computational Display SDK (CNSDK)** as an `xrt_display_processor` +implementation, shipping as `libdxrp050_leia_cnsdk.so`. + +The runtime's `target_plugin_loader.c` Android branch (PR #309 / commit +`c96c93ce8`) discovers this `.so` at `xrCreateInstance` time via +`dladdr`-of-self + dirname enumeration of files matching the +`libdxrp_.so` convention. Bundle this `.so` into the runtime +APK's `jniLibs//` (single-vendor mode); multi-APK discovery is v2 +(tracked at runtime #310). + +## Files + +| File | Purpose | +|---|---| +| `leia_cnsdk.{cpp,h}` | C++ wrapper around the CNSDK C ABI: core init, interlacer lifecycle, face-tracking worker thread, atlas weave entry point, device-config caching. | +| `leia_display_processor_cnsdk.{cpp,h}` | `xrt_display_processor` vtable wired to the wrapper. Advertises `is_self_submitting=true` so the compositor flushes its pre-DP cmd buffer + skips its own post-DP submit. Atlas mode only (per-tile blit removed — CNSDK splits the SBS atlas internally via `set_interlace_view_texture_atlas`). | +| `leia_plugin_android.c` | `xrtPluginNegotiate` entry point + `xrt_plugin_iface` vtable. `create_dp_vk` is the only non-NULL factory slot. | + +## Build + +```bash +# CNSDK 0.7.28 extracted somewhere on disk: +export CNSDK_ROOT=/path/to/cnsdk + +# Configure for arm64-v8a Android via the NDK toolchain: +cmake -S . -B build-android \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-29 \ + -G Ninja + +cmake --build build-android --target dxrp050_leia_cnsdk +``` + +Output: `build-android/src/drv_leia_android/libdxrp050_leia_cnsdk.so`. + +Copy that into your runtime APK's +`src/xrt/targets/openxr_android/src/main/jniLibs/arm64-v8a/` before +running `./gradlew :src:xrt:targets:openxr_android:assembleInProcessDebug`. + +(End-to-end multi-module Gradle integration is a follow-up — see the +top-level repo README's "Android — POC build flow" section.) + +## CNSDK convention assumptions + +Three plug-in-side conventions are assumed in this POC (face axis +signs/units, tile-to-eye mapping, UV vertical flip). All three may +need single-line flips after first hardware bring-up — see +[`docs/cnsdk-android-calibration.md`](../../docs/cnsdk-android-calibration.md) +for the symptom-→-fix table. + +## Status + +POC — not yet validated on a Lume Pad. Compiles clean on the host. +First hardware install is gated on Lume Pad arrival; bring-up plan +lives in the runtime repo at +`docs/getting-started/android-bringup-checklist.md`. diff --git a/src/drv_leia_android/leia_cnsdk.cpp b/src/drv_leia_android/leia_cnsdk.cpp new file mode 100644 index 0000000..122f943 --- /dev/null +++ b/src/drv_leia_android/leia_cnsdk.cpp @@ -0,0 +1,503 @@ +// Copyright 2025, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief CNSDK wrapper implementation — isolates CNSDK headers + * from the rest of the compositor. + * @author David Fattal + * @ingroup drv_leia + */ + +#include "leia_cnsdk.h" + +#include "util/u_logging.h" + +#include +#include +#include +#include + +#ifdef XRT_OS_ANDROID +#include "android/android_globals.h" +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#include +#endif +#endif + +#include +#include +#include + + +// Hardware-bring-up debug logging. Gated by XRT_DEBUG_ANDROID_VERBOSE +// which is passed via cppFlags from the Android Debug build variant +// (src/xrt/targets/openxr_android/build.gradle::debug). Compiles to +// nothing in release. Tag "HW_DBG_CNSDK:" is greppable in logcat. +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#define DXR_HW_DBG(...) U_LOG_I("HW_DBG_CNSDK: " __VA_ARGS__) +#define DXR_HW_DBG_ONCE(...) do { \ + static bool _logged = false; \ + if (!_logged) { U_LOG_I("HW_DBG_CNSDK[once]: " __VA_ARGS__); _logged = true; } \ + } while (0) + +// ATrace RAII scope — captures show up in Perfetto / Studio Profiler. +// Same gate as DXR_HW_DBG; release builds compile to nothing. +struct AtraceScopeCnsdk { + AtraceScopeCnsdk(const char *name) { ATrace_beginSection(name); } + ~AtraceScopeCnsdk() { ATrace_endSection(); } +}; +#define DXR_ATRACE(name) AtraceScopeCnsdk _atrace_##__LINE__(name) +#else +#define DXR_HW_DBG(...) ((void)0) +#define DXR_HW_DBG_ONCE(...) ((void)0) +#define DXR_ATRACE(name) ((void)0) +#endif + + +/* + * + * Internal struct. + * + */ + +struct leia_cnsdk +{ + struct leia_core *core{nullptr}; + struct leia_interlacer *interlacer{nullptr}; + + // Face-tracking startup is offloaded to a worker thread because + // leia_core_enable_face_tracking is heavy (CNSDK docs explicitly warn + // against the main thread). Worker pattern: + // - Spawn in leia_cnsdk_create. + // - Worker polls leia_core_is_initialized until ready, then snapshots + // the camera center from leia_device_config (needed to convert + // CNSDK's camera-relative face positions into display-relative), + // calls enable + start face tracking, then sets + // face_tracking_started and exits. + // - Destroy sets shutting_down to ask the worker to bail if it's + // still in the polling phase, then joins. + // + // camera_center_{x,y,z}_m: cached at worker init. The `_m` suffix + // reminds the reader they're already mm→m converted before storage. + // These are read by leia_cnsdk_get_primary_face on the render thread + // only after face_tracking_started.load(acquire) returns true — the + // happens-before ordering of the atomic gives the read visibility on + // the worker's writes. + std::atomic face_tracking_started{false}; + std::atomic shutting_down{false}; + std::thread worker; + float camera_center_x_m{0.0f}; + float camera_center_y_m{0.0f}; + float camera_center_z_m{0.0f}; + + // Cached display metrics. Populated by the worker thread alongside + // the camera-center snapshot; the atomic flag gives the render + // thread happens-before visibility on the float/int writes. Once + // set, leia_cnsdk_get_display_metrics returns the cached values + // instead of calling get_device_config / release_device_config per + // frame — eliminates a per-frame allocation churn AND the + // concurrent-device-config-access concern (audit B9). + std::atomic display_metrics_cached{false}; + float display_width_m_cached{0.0f}; + float display_height_m_cached{0.0f}; + uint32_t display_pixel_w_cached{0}; + uint32_t display_pixel_h_cached{0}; + + // One-shot flag: once leia_interlacer_vulkan_initialize fails, give + // up rather than retrying every frame. Read + written only by the + // render thread (no concurrent access; no atomic needed). + bool interlacer_init_failed{false}; +}; + + +/* + * + * Private helpers. + * + */ + +namespace { + +void +face_tracking_worker(struct leia_cnsdk *cnsdk) +{ + using namespace std::chrono_literals; + + DXR_HW_DBG("worker: entered, waiting for leia_core_is_initialized"); + + // Phase 1: wait for the async core init to complete. Poll every 50 ms; + // honor shutdown promptly. + int poll_count = 0; + while (!cnsdk->shutting_down.load(std::memory_order_acquire)) { + if (cnsdk->core != nullptr && leia_core_is_initialized(cnsdk->core)) { + break; + } + if ((++poll_count % 20) == 0) { + DXR_HW_DBG("worker: still polling for core init (~%d s elapsed)", + poll_count / 20); + } + std::this_thread::sleep_for(50ms); + } + if (cnsdk->shutting_down.load(std::memory_order_acquire)) { + DXR_HW_DBG("worker: shutdown requested before core ready, exiting"); + return; + } + DXR_HW_DBG("worker: core initialized after %d polls", poll_count); + + // Phase 2a: snapshot all device-config values we need on the render + // thread (camera center for face-position translation; display + // metrics for Kooima projection). CNSDK doesn't annotate device + // config thread safety, so we keep it on this one worker thread and + // expose only cached values to the render thread via atomics. mm→m + // conversion happens at storage time so render-thread reads are + // branch-free. + struct leia_device_config *cfg = leia_core_get_device_config(cnsdk->core); + if (cfg != NULL) { + cnsdk->camera_center_x_m = cfg->cameraCenterX / 1000.0f; + cnsdk->camera_center_y_m = cfg->cameraCenterY / 1000.0f; + cnsdk->camera_center_z_m = cfg->cameraCenterZ / 1000.0f; + cnsdk->display_width_m_cached = (float)cfg->displaySizeInMm[0] / 1000.0f; + cnsdk->display_height_m_cached = (float)cfg->displaySizeInMm[1] / 1000.0f; + cnsdk->display_pixel_w_cached = (uint32_t)cfg->panelResolution[0]; + cnsdk->display_pixel_h_cached = (uint32_t)cfg->panelResolution[1]; + leia_core_release_device_config(cnsdk->core, cfg); + cnsdk->display_metrics_cached.store(true, std::memory_order_release); + DXR_HW_DBG("worker: cached metrics: %ux%u px, %.3fx%.3f m; cam=(%.3f, %.3f, %.3f) m", + cnsdk->display_pixel_w_cached, cnsdk->display_pixel_h_cached, + cnsdk->display_width_m_cached, cnsdk->display_height_m_cached, + cnsdk->camera_center_x_m, cnsdk->camera_center_y_m, + cnsdk->camera_center_z_m); + } else { + U_LOG_W("leia_core_get_device_config failed in worker; camera center + metrics stay default"); + } + + // Phase 2b: heavy enable + start. Single call, can't be interrupted — + // destroy will block on the join until this returns. + if (!leia_core_enable_face_tracking(cnsdk->core, true)) { + U_LOG_W("leia_core_enable_face_tracking failed (worker)"); + return; + } + leia_core_start_face_tracking(cnsdk->core, true); + + cnsdk->face_tracking_started.store(true, std::memory_order_release); + U_LOG_W("CNSDK face tracking started (worker)"); +} + +} // namespace + + +/* + * + * Public API. + * + */ + +extern "C" xrt_result_t +leia_cnsdk_create(struct leia_cnsdk **out_cnsdk) +{ + DXR_HW_DBG("leia_cnsdk_create: entering"); + leia_platform_on_library_load(); + + struct leia_core_init_configuration *config = leia_core_init_configuration_alloc(CNSDK_VERSION); + +#ifdef XRT_OS_ANDROID + leia_core_init_configuration_set_platform_android_java_vm(config, (JavaVM *)android_globals_get_vm()); + leia_core_init_configuration_set_platform_android_handle( + config, LEIA_CORE_ANDROID_HANDLE_ACTIVITY, (jobject)android_globals_get_activity()); +#endif + + leia_core_init_configuration_set_platform_log_level(config, kLeiaLogLevelTrace); + leia_core_init_configuration_set_enable_validation(config, true); + + struct leia_core *core = leia_core_init_async(config); + leia_core_init_configuration_free(config); + + if (core == NULL) { + U_LOG_E("leia_core_init_async failed"); + *out_cnsdk = NULL; + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + + leia_core_set_backlight(core, true); + + auto *cnsdk = new struct leia_cnsdk(); + cnsdk->core = core; + cnsdk->worker = std::thread(face_tracking_worker, cnsdk); + + DXR_HW_DBG("leia_cnsdk_create: core=%p, worker thread spawned", (void *)core); + *out_cnsdk = cnsdk; + return XRT_SUCCESS; +} + +extern "C" void +leia_cnsdk_destroy(struct leia_cnsdk **cnsdk_ptr) +{ + if (cnsdk_ptr == NULL || *cnsdk_ptr == NULL) { + return; + } + + struct leia_cnsdk *cnsdk = *cnsdk_ptr; + DXR_HW_DBG("leia_cnsdk_destroy: entering, core=%p", (void *)cnsdk->core); + + // Signal the worker, then join with a watchdog: if it doesn't finish + // within kWorkerJoinTimeoutMs, detach instead so destroy can return. + // The worker might be mid-leia_core_enable_face_tracking with no + // interruption hook — without the timeout, destroy can hang + // indefinitely on a CNSDK deadlock (audit B10). + // + // Detaching leaks the std::thread but is the only option short of + // CNSDK exposing a cancel API. + cnsdk->shutting_down.store(true, std::memory_order_release); + if (cnsdk->worker.joinable()) { + constexpr auto kWorkerJoinTimeoutMs = std::chrono::milliseconds(2000); + // std::thread::join doesn't take a timeout, so use a side thread + // that does the join and a condition variable to wait on with a + // deadline. Cheap on the happy path (the worker is usually + // already finished by destroy time, so join returns instantly). + std::atomic joined{false}; + std::thread joiner([&]() { + cnsdk->worker.join(); + joined.store(true, std::memory_order_release); + }); + const auto deadline = std::chrono::steady_clock::now() + kWorkerJoinTimeoutMs; + while (!joined.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + if (joined.load(std::memory_order_acquire)) { + joiner.join(); + } else { + U_LOG_W("CNSDK worker did not exit within %lld ms; detaching", + (long long)kWorkerJoinTimeoutMs.count()); + cnsdk->worker.detach(); + joiner.detach(); + } + } + + if (cnsdk->interlacer != NULL) { + // CNSDK 0.7.28 renamed the per-interlacer release; the core owns the + // interlacer lifetime, so the shutdown call needs both handles. + leia_interlacer_shutdown(cnsdk->core, cnsdk->interlacer); + cnsdk->interlacer = NULL; + } + + if (cnsdk->core != NULL) { + // CNSDK 0.7.28 renamed leia_core_release → leia_core_shutdown. + leia_core_shutdown(cnsdk->core); + cnsdk->core = NULL; + } + + leia_platform_on_library_unload(); + + delete cnsdk; + *cnsdk_ptr = NULL; +} + +extern "C" bool +leia_cnsdk_is_initialized(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return false; + } + return leia_core_is_initialized(cnsdk->core); +} + +extern "C" void +leia_cnsdk_on_pause(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return; + } + if (!leia_core_is_initialized(cnsdk->core)) { + DXR_HW_DBG("on_pause: skipped (core not initialized yet)"); + return; + } + DXR_HW_DBG("on_pause: forwarding to leia_core_on_pause"); + leia_core_on_pause(cnsdk->core); +} + +extern "C" void +leia_cnsdk_on_resume(struct leia_cnsdk *cnsdk) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return; + } + if (!leia_core_is_initialized(cnsdk->core)) { + DXR_HW_DBG("on_resume: skipped (core not initialized yet)"); + return; + } + DXR_HW_DBG("on_resume: forwarding to leia_core_on_resume"); + leia_core_on_resume(cnsdk->core); +} + +extern "C" bool +leia_cnsdk_get_display_metrics(struct leia_cnsdk *cnsdk, + float *out_width_m, + float *out_height_m, + uint32_t *out_pixel_w, + uint32_t *out_pixel_h) +{ + // Worker thread snapshots all four values from the device config + // once, then sets the atomic. Render thread polls the atomic and + // reads the cached float/int fields. No per-frame get/release. + if (cnsdk == NULL || + !cnsdk->display_metrics_cached.load(std::memory_order_acquire)) { + return false; + } + + if (out_width_m != NULL) { + *out_width_m = cnsdk->display_width_m_cached; + } + if (out_height_m != NULL) { + *out_height_m = cnsdk->display_height_m_cached; + } + if (out_pixel_w != NULL) { + *out_pixel_w = cnsdk->display_pixel_w_cached; + } + if (out_pixel_h != NULL) { + *out_pixel_h = cnsdk->display_pixel_h_cached; + } + return true; +} + +extern "C" bool +leia_cnsdk_ensure_face_tracking_started(struct leia_cnsdk *cnsdk) +{ + // Worker thread handles enable + start; this is now a non-blocking + // status check. + if (cnsdk == NULL) { + return false; + } + return cnsdk->face_tracking_started.load(std::memory_order_acquire); +} + +extern "C" bool +leia_cnsdk_ensure_interlacer(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkFormat targetFmt) +{ + if (cnsdk == NULL || cnsdk->core == NULL) { + return false; + } + if (cnsdk->interlacer != NULL) { + return true; + } + // One-shot give-up: once leia_interlacer_vulkan_initialize fails + // (typically permanently — wrong VkDevice format, no GPU memory, + // CNSDK lib mismatch), don't keep retrying every frame. + if (cnsdk->interlacer_init_failed) { + return false; + } + if (!leia_core_is_initialized(cnsdk->core)) { + return false; + } + + struct leia_interlacer_init_configuration *ic = leia_interlacer_init_configuration_alloc(); + // Atlas mode: CNSDK accepts the SBS atlas VkImage+View directly per + // frame via set_interlace_view_texture_atlas, and splits internally. + // No per-view image management on our side; the DP shrinks + // substantially. See feature/android-cnsdk-ci for the prior art + // (CNSDK 0.10.56 used a different API but same architectural idea). + leia_interlacer_init_configuration_set_use_atlas_for_views(ic, true); + // Views format = atlas format. Atlas is rendered to UNORM by + // comp_vk_native_renderer.c, so use UNORM here (audit B2). + cnsdk->interlacer = leia_interlacer_vulkan_initialize( + cnsdk->core, ic, device, physDev, VK_FORMAT_B8G8R8A8_UNORM, + targetFmt, VK_FORMAT_D32_SFLOAT, 3); + leia_interlacer_init_configuration_free(ic); + + if (cnsdk->interlacer == NULL) { + U_LOG_W("leia_interlacer_vulkan_initialize returned NULL; giving up (no retries)"); + cnsdk->interlacer_init_failed = true; + return false; + } + + // Tell CNSDK the atlas is laid out 2x1 SBS horizontal. This is the + // default but we set it explicitly so future layout changes + // (multi-view modes) only have to touch one place. + leia_interlacer_set_num_tiles(cnsdk->interlacer, 2, 1); + DXR_HW_DBG("ensure_interlacer: created interlacer=%p (atlas mode, 2x1, targetFmt=%d)", + (void *)cnsdk->interlacer, (int)targetFmt); + return true; +} + +extern "C" bool +leia_cnsdk_get_primary_face(struct leia_cnsdk *cnsdk, + float *out_x, + float *out_y, + float *out_z) +{ + if (cnsdk == NULL || cnsdk->core == NULL || + !cnsdk->face_tracking_started.load(std::memory_order_acquire)) { + return false; + } + + float position[3] = {0, 0, 0}; + struct leia_float_slice slice = {position, 3}; + if (!leia_core_get_primary_face(cnsdk->core, slice)) { + return false; + } + + // CNSDK returns millimeters relative to the camera. xrt_eye_position + // wants meters relative to the display center, so divide by 1000 then + // subtract the cached camera center (also already in meters). + const float pos_x_m = position[0] / 1000.0f - cnsdk->camera_center_x_m; + const float pos_y_m = position[1] / 1000.0f - cnsdk->camera_center_y_m; + const float pos_z_m = position[2] / 1000.0f - cnsdk->camera_center_z_m; + +#ifdef XRT_DEBUG_ANDROID_VERBOSE + // Throttle to once per ~second at 60 Hz so logcat stays readable. + static int dbg_face_counter = 0; + if ((dbg_face_counter++ % 60) == 0) { + DXR_HW_DBG("face: raw_mm=(%.1f, %.1f, %.1f) → out_m=(%.4f, %.4f, %.4f)", + position[0], position[1], position[2], pos_x_m, pos_y_m, pos_z_m); + } +#endif + + if (out_x != NULL) { *out_x = pos_x_m; } + if (out_y != NULL) { *out_y = pos_y_m; } + if (out_z != NULL) { *out_z = pos_z_m; } + return true; +} + +extern "C" void +leia_cnsdk_weave(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkImage atlas_image, + VkImageView atlas_view, + uint32_t atlas_width, + uint32_t atlas_height, + VkFormat targetFmt, + uint32_t w, + uint32_t h, + VkFramebuffer fb, + VkImage targetImage) +{ + (void)device; (void)physDev; (void)targetFmt; + + if (cnsdk == NULL || cnsdk->interlacer == NULL) { + return; + } + + DXR_ATRACE("dxr_cnsdk:weave"); + leia_interlacer_set_flip_input_uv_vertical(cnsdk->interlacer, true); + + // Atlas mode: hand CNSDK the SBS atlas VkImage+View each frame; it + // splits internally per the 2x1 layout set in ensure_interlacer. + // (Previously this function blitted tiles into per-view images and + // passed those — see git log for the per-tile-blit history.) + leia_interlacer_vulkan_set_interlace_view_texture_atlas( + cnsdk->interlacer, atlas_image, atlas_view); + leia_interlacer_set_source_views_size( + cnsdk->interlacer, (int32_t)atlas_width, (int32_t)atlas_height, + /*isHorizontalViews=*/true); + + leia_interlacer_set_shader_debug_mode(cnsdk->interlacer, LEIA_SHADER_DEBUG_MODE_NONE); + DXR_HW_DBG_ONCE("weave: first do_post_process atlas=%ux%u target=%ux%u", + atlas_width, atlas_height, w, h); + leia_interlacer_vulkan_do_post_process( + cnsdk->interlacer, w, h, false, fb, targetImage, NULL, + NULL, NULL, 0); +} diff --git a/src/drv_leia_android/leia_cnsdk.h b/src/drv_leia_android/leia_cnsdk.h new file mode 100644 index 0000000..20b70bf --- /dev/null +++ b/src/drv_leia_android/leia_cnsdk.h @@ -0,0 +1,193 @@ +// Copyright 2025, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Opaque wrapper around CNSDK (Android) interlacing API. + * + * Encapsulates leia_core and leia_interlacer so the compositor + * does not include CNSDK headers directly. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#pragma once + +#include "xrt/xrt_results.h" +#include "xrt/xrt_vulkan_includes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct leia_cnsdk; + +/*! + * Create and asynchronously initialise a CNSDK core + backlight. + * + * @param[out] out_cnsdk Receives the opaque handle (NULL on failure). + * @return XRT_SUCCESS on success. + */ +xrt_result_t +leia_cnsdk_create(struct leia_cnsdk **out_cnsdk); + +/*! + * Destroy a CNSDK handle and release all resources. + * + * @param cnsdk_ptr Pointer to handle; set to NULL on return. + */ +void +leia_cnsdk_destroy(struct leia_cnsdk **cnsdk_ptr); + +/*! + * Check whether the asynchronous core init has completed. + * + * @return true once the core is fully initialised. + */ +bool +leia_cnsdk_is_initialized(struct leia_cnsdk *cnsdk); + +/*! + * Notify CNSDK that the host Activity has paused (backgrounded). + * + * Wraps leia_core_on_pause. Safe to call any time after + * @ref leia_cnsdk_create — CNSDK no-ops if the core isn't initialized + * yet. Stops face-tracking cameras + dims the backlight; idempotent + * across repeated calls. + * + * Intended caller: OpenXR session state machine in oxr_session.c on + * entering XR_SESSION_STATE_STOPPING. + */ +void +leia_cnsdk_on_pause(struct leia_cnsdk *cnsdk); + +/*! + * Notify CNSDK that the host Activity has resumed (foregrounded). + * + * Wraps leia_core_on_resume. Counterpart of @ref leia_cnsdk_on_pause. + * + * Intended caller: OpenXR session state machine on entering + * XR_SESSION_STATE_READY after a pause. + */ +void +leia_cnsdk_on_resume(struct leia_cnsdk *cnsdk); + +/*! + * Fetch native display metrics from CNSDK's device config. + * + * The four values are snapshotted once by the face-tracking worker + * thread right after @ref leia_core_is_initialized first returns true + * and stored on the wrapper struct. Subsequent calls return the + * cached values without re-acquiring CNSDK's device config every + * frame. Returns false until that snapshot has happened; caller is + * expected to poll across frames. + * + * @param[out] out_width_m Display physical width in meters. + * @param[out] out_height_m Display physical height in meters. + * @param[out] out_pixel_w Panel pixel width. + * @param[out] out_pixel_h Panel pixel height. + * @return true if all outputs were populated. + */ +bool +leia_cnsdk_get_display_metrics(struct leia_cnsdk *cnsdk, + float *out_width_m, + float *out_height_m, + uint32_t *out_pixel_w, + uint32_t *out_pixel_h); + +/*! + * Non-blocking check for whether CNSDK face tracking is running. + * + * Enable + start happens asynchronously on a worker thread spawned by + * @ref leia_cnsdk_create, so callers can poll this every frame from the + * render thread without stalling. Returns true once the worker has + * finished enabling + starting; false until then (or permanently if + * the enable call failed). + * + * @return true once face tracking is started. + */ +bool +leia_cnsdk_ensure_face_tracking_started(struct leia_cnsdk *cnsdk); + +/*! + * Idempotent: lazily create the CNSDK Vulkan interlacer in atlas mode + * once @ref leia_core_is_initialized returns true. Safe to call every + * frame. Atlas mode means CNSDK accepts the SBS atlas VkImage+View + * directly via @ref leia_cnsdk_weave and does the L/R split internally; + * the DP doesn't have to manage per-view images or per-tile blits. + * + * @return true if the interlacer exists and is ready to weave. + */ +bool +leia_cnsdk_ensure_interlacer(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkFormat targetFmt); + +/*! + * Fetch the latest predicted primary face position from CNSDK. + * + * Returns false until face tracking is running and CNSDK has a face + * lock. Position is returned in meters relative to the **display + * center** (matching `xrt_eye_position`'s convention), even though + * CNSDK natively returns millimeters relative to the camera. The + * wrapper does the unit conversion + camera-center translation using + * the cached `cameraCenterX/Y/Z` from `leia_device_config` populated + * once the core is initialized. + * + * @param[out] out_x Face position X (meters, display-relative). + * @param[out] out_y Face position Y (meters, display-relative). + * @param[out] out_z Face position Z (meters, +toward viewer). + * @return true if a valid face was returned. + */ +bool +leia_cnsdk_get_primary_face(struct leia_cnsdk *cnsdk, + float *out_x, + float *out_y, + float *out_z); + +/*! + * Perform CNSDK Vulkan interlacing on an SBS atlas. + * + * Atlas mode: pass the runtime's pre-composited SBS atlas image+view + * directly. CNSDK does the L/R split internally via + * @ref leia_interlacer_vulkan_set_interlace_view_texture_atlas. The DP + * does no per-view image management and no per-frame blits, so there's + * no GPU stall between us and CNSDK — its `do_post_process` records + * and submits its own cmd buffer when it's ready. + * + * Caller must have first invoked @ref leia_cnsdk_ensure_interlacer; if + * the interlacer isn't ready yet this function is a no-op (no submit, + * no GPU side effects), making it safe to call every frame during the + * async core-init window. + * + * @param cnsdk Opaque CNSDK handle. + * @param device Vulkan logical device. + * @param physDev Vulkan physical device. + * @param atlas_image SBS atlas VkImage. + * @param atlas_view Matching VkImageView covering the full atlas. + * @param atlas_width Atlas width in pixels (= view_w * tile_columns). + * @param atlas_height Atlas height in pixels (= view_h * tile_rows). + * @param targetFmt Format of the target / swapchain image. + * @param w Target width in pixels. + * @param h Target height in pixels. + * @param fb Target framebuffer. + * @param targetImage Target VkImage (for CNSDK-side layout transitions). + */ +void +leia_cnsdk_weave(struct leia_cnsdk *cnsdk, + VkDevice device, + VkPhysicalDevice physDev, + VkImage atlas_image, + VkImageView atlas_view, + uint32_t atlas_width, + uint32_t atlas_height, + VkFormat targetFmt, + uint32_t w, + uint32_t h, + VkFramebuffer fb, + VkImage targetImage); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.cpp b/src/drv_leia_android/leia_display_processor_cnsdk.cpp new file mode 100644 index 0000000..88cee9b --- /dev/null +++ b/src/drv_leia_android/leia_display_processor_cnsdk.cpp @@ -0,0 +1,471 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Leia CNSDK display processor (Android), atlas-mode variant. + * + * Wraps leia_cnsdk as an xrt_display_processor. Uses CNSDK's atlas mode: + * the compositor's pre-composited SBS atlas VkImage+View is passed + * directly to CNSDK each frame via + * leia_interlacer_vulkan_set_interlace_view_texture_atlas, and CNSDK + * splits L/R internally. No per-view image management, no per-tile + * blit, no blit cmd buffer / semaphore / fence on our side. + * + * Display metrics + face-tracked eye positions come from CNSDK once the + * async core init completes (lazy on first query). Falls back to + * hardcoded Lume Pad 2 metrics and IPD-only eyes while the core is + * still booting. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#include "leia_display_processor_cnsdk.h" +#include "leia_cnsdk.h" + +#include "xrt/xrt_display_metrics.h" +#include "vk/vk_helpers.h" +#include "util/u_logging.h" + +#include + +#if defined(XRT_OS_ANDROID) && defined(XRT_DEBUG_ANDROID_VERBOSE) +#include +#endif + + +// Hardware-bring-up debug logging. Gated by XRT_DEBUG_ANDROID_VERBOSE +// (cppFlag from the Android Debug variant). Compiles to nothing in +// release. Tag "HW_DBG_DP:" greppable in logcat, separate from the +// CNSDK wrapper's HW_DBG_CNSDK tag. +#ifdef XRT_DEBUG_ANDROID_VERBOSE +#define DXR_HW_DBG(...) U_LOG_I("HW_DBG_DP: " __VA_ARGS__) +#define DXR_HW_DBG_ONCE(...) do { \ + static bool _logged = false; \ + if (!_logged) { U_LOG_I("HW_DBG_DP[once]: " __VA_ARGS__); _logged = true; } \ + } while (0) +struct AtraceScopeDp { + AtraceScopeDp(const char *name) { ATrace_beginSection(name); } + ~AtraceScopeDp() { ATrace_endSection(); } +}; +#define DXR_ATRACE(name) AtraceScopeDp _atrace_##__LINE__(name) +#else +#define DXR_HW_DBG(...) ((void)0) +#define DXR_HW_DBG_ONCE(...) ((void)0) +#define DXR_ATRACE(name) ((void)0) +#endif + + +namespace { + +// Lume Pad 2-class defaults. Used until CNSDK reports the real device +// metrics through leia_core via the leia_cnsdk wrapper's cached values. +constexpr float kDefaultDisplayWidthM = 0.1934f; // ~12.4" diagonal, 16:10 +constexpr float kDefaultDisplayHeightM = 0.1209f; +constexpr uint32_t kDefaultDisplayPixelW = 2560; +constexpr uint32_t kDefaultDisplayPixelH = 1600; + +// Hardcoded IPD-only eye positions. Origin = display center; z is +// toward the user. 65 mm IPD, ~50 cm viewing distance. +constexpr float kIpdHalfM = 0.0325f; +constexpr float kEyeViewerDistM = 0.5f; + +struct leia_dp_cnsdk +{ + struct xrt_display_processor base; + struct leia_cnsdk *cnsdk; //!< Owned. + struct vk_bundle *vk; //!< Borrowed from compositor. + VkCommandPool cmd_pool; //!< Borrowed from compositor; used by mono passthrough. +}; + +inline leia_dp_cnsdk * +as_impl(struct xrt_display_processor *xdp) +{ + return reinterpret_cast(xdp); +} + +// Mono / 2D-mode passthrough: blit the single-tile atlas directly to +// the swapchain target. No CNSDK weave (would be wrong for 1x1). Same +// barrier dance as the atlas-side host-stall path — drain via +// vkQueueWaitIdle so xrEndFrame sees the target image ready. +bool +mono_passthrough_blit(leia_dp_cnsdk *impl, + VkImage atlas_image, + uint32_t atlas_width, + uint32_t atlas_height, + VkImage target_image, + uint32_t target_width, + uint32_t target_height) +{ + DXR_ATRACE("dxr_dp:mono_passthrough_blit"); + struct vk_bundle *vk = impl->vk; + if (vk == nullptr || vk->main_queue == nullptr) { + // Defensive: this mono 1x1 fallback submits on the main queue + // directly. The atlas-weave path guards vk==nullptr already; + // mirror it here so a partially-initialized bundle can't deref + // vk->main_queue->queue below. + U_LOG_W("mono_passthrough_blit: vk bundle/main_queue not ready, skipping"); + return false; + } + + VkCommandBufferAllocateInfo ai = {}; + ai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + ai.commandPool = impl->cmd_pool; + ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + ai.commandBufferCount = 1; + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (vk->vkAllocateCommandBuffers(vk->device, &ai, &cmd) != VK_SUCCESS) { + return false; + } + + VkCommandBufferBeginInfo bi = {}; + bi.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + vk->vkBeginCommandBuffer(cmd, &bi); + + // Atlas: SHADER_READ_ONLY_OPTIMAL → TRANSFER_SRC_OPTIMAL. + VkImageMemoryBarrier atlas_to_src = {}; + atlas_to_src.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + atlas_to_src.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; + atlas_to_src.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + atlas_to_src.oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + atlas_to_src.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + atlas_to_src.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + atlas_to_src.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + atlas_to_src.image = atlas_image; + atlas_to_src.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + // Target: assume COLOR_ATTACHMENT_OPTIMAL (the compositor's pre-DP + // barrier set this in the window-target path; for texture-mode the + // shared image's layout is caller-dependent — we use UNDEFINED→ + // TRANSFER_DST which is always safe for clearing). + VkImageMemoryBarrier target_to_dst = {}; + target_to_dst.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + target_to_dst.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + target_to_dst.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + target_to_dst.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + target_to_dst.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + target_to_dst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + target_to_dst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + target_to_dst.image = target_image; + target_to_dst.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + VkImageMemoryBarrier pre[2] = {atlas_to_src, target_to_dst}; + vk->vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 2, pre); + + // Stretch-blit the atlas to fill the target. Linear filter so a + // resolution mismatch doesn't go blocky. + VkImageBlit blit = {}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[0] = {0, 0, 0}; + blit.srcOffsets[1] = {(int32_t)atlas_width, (int32_t)atlas_height, 1}; + blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.dstOffsets[0] = {0, 0, 0}; + blit.dstOffsets[1] = {(int32_t)target_width, (int32_t)target_height, 1}; + vk->vkCmdBlitImage(cmd, + atlas_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + target_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, &blit, VK_FILTER_LINEAR); + + // Restore atlas to SHADER_READ for the compositor's invariant; + // leave target in COLOR_ATTACHMENT_OPTIMAL (xrEndFrame's contract). + VkImageMemoryBarrier atlas_back = atlas_to_src; + atlas_back.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + atlas_back.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + atlas_back.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + atlas_back.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkImageMemoryBarrier target_back = target_to_dst; + target_back.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + target_back.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + target_back.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + target_back.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + VkImageMemoryBarrier post[2] = {atlas_back, target_back}; + vk->vkCmdPipelineBarrier(cmd, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, 0, nullptr, 0, nullptr, 2, post); + + vk->vkEndCommandBuffer(cmd); + + VkSubmitInfo si = {}; + si.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + si.commandBufferCount = 1; + si.pCommandBuffers = &cmd; + VkResult res = vk->vkQueueSubmit(vk->main_queue->queue, 1, &si, VK_NULL_HANDLE); + if (res == VK_SUCCESS) { + vk->vkQueueWaitIdle(vk->main_queue->queue); + } + vk->vkFreeCommandBuffers(vk->device, impl->cmd_pool, 1, &cmd); + DXR_HW_DBG_ONCE("mono_passthrough_blit: first frame (atlas %ux%u → target %ux%u)", + atlas_width, atlas_height, target_width, target_height); + return res == VK_SUCCESS; +} + +void +process_atlas_weave(struct xrt_display_processor *xdp, + VkCommandBuffer cmd_buffer, + VkImage_XDP atlas_image, + VkImageView atlas_view, + uint32_t view_width, + uint32_t view_height, + uint32_t tile_columns, + uint32_t tile_rows, + VkFormat_XDP view_format, + VkFramebuffer target_fb, + VkImage_XDP target_image, + uint32_t target_width, + uint32_t target_height, + VkFormat_XDP target_format, + int32_t canvas_offset_x, + int32_t canvas_offset_y, + uint32_t canvas_width, + uint32_t canvas_height) +{ + DXR_ATRACE("dxr_dp:process_atlas_weave"); + (void)cmd_buffer; // self-submitting: compositor passes VK_NULL_HANDLE + (void)view_format; + (void)canvas_offset_x; (void)canvas_offset_y; + (void)canvas_width; (void)canvas_height; + + leia_dp_cnsdk *impl = as_impl(xdp); + + if (tile_columns == 1 && tile_rows == 1) { + // Mono / 2D mode: no interlacing needed. Atlas IS the final + // image; blit it directly to the target so we still produce + // something visible. CNSDK doesn't weave here (would be wrong + // for 1x1) — only used in the runtime's 2D fallback or for + // non-3D-display vendors. Implementation moved below for + // readability. + if (impl->vk == nullptr || impl->cmd_pool == VK_NULL_HANDLE) { + return; + } + const uint32_t atlas_w_mono = view_width; + const uint32_t atlas_h_mono = view_height; + mono_passthrough_blit(impl, (VkImage)(uintptr_t)atlas_image, + atlas_w_mono, atlas_h_mono, + (VkImage)(uintptr_t)target_image, + target_width, target_height); + return; + } + if (tile_columns != 2 || tile_rows != 1) { + static bool warned = false; + if (!warned) { + U_LOG_W("CNSDK DP expects 2x1 SBS atlas or 1x1 mono, got %ux%u; skipping", + tile_columns, tile_rows); + warned = true; + } + return; + } + + if (impl->vk == nullptr) { + return; + } + + // Gate the weave on CNSDK interlacer readiness. Async core init may + // take several frames; until then the interlacer doesn't exist and + // leia_cnsdk_weave is a no-op. The compositor's pre-DP submit has + // already happened, so target is in COLOR_ATTACHMENT_OPTIMAL with + // undefined content for those frames — acceptable for POC. + if (!leia_cnsdk_ensure_interlacer(impl->cnsdk, + impl->vk->device, + impl->vk->physical_device, + (VkFormat)target_format)) { + return; + } + DXR_HW_DBG_ONCE("process_atlas_weave: first frame with ready interlacer"); + + const uint32_t atlas_w = view_width * tile_columns; + const uint32_t atlas_h = view_height * tile_rows; + +#ifdef XRT_DEBUG_ANDROID_VERBOSE + static int dp_dbg_frame = 0; + if ((dp_dbg_frame++ % 60) == 0) { + DXR_HW_DBG("process_atlas_weave[frame=%d]: atlas=%ux%u target=%ux%u fmt=%d", + dp_dbg_frame, atlas_w, atlas_h, target_width, target_height, (int)target_format); + } +#endif + + leia_cnsdk_weave(impl->cnsdk, + impl->vk->device, + impl->vk->physical_device, + (VkImage)(uintptr_t)atlas_image, + atlas_view, + atlas_w, + atlas_h, + (VkFormat)target_format, + target_width, + target_height, + target_fb, + (VkImage)(uintptr_t)target_image); +} + +bool +is_self_submitting_true(struct xrt_display_processor *xdp) +{ + (void)xdp; + return true; +} + +// Try to fetch CNSDK's predicted face position and derive L/R eyes from +// it (face X ± IPD/2). Falls back to a hardcoded IPD-only stub if face +// tracking isn't running yet (CNSDK core still async-initializing, or +// no face lock). +bool +get_predicted_eye_positions_ipd(struct xrt_display_processor *xdp, + struct xrt_eye_positions *out_eye_pos) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + bool tracked = false; + float fx = 0.0f, fy = 0.0f, fz = kEyeViewerDistM; + if (impl->cnsdk != nullptr) { + if (leia_cnsdk_ensure_face_tracking_started(impl->cnsdk) && + leia_cnsdk_get_primary_face(impl->cnsdk, &fx, &fy, &fz)) { + tracked = true; + } + } + + out_eye_pos->eyes[0].x = fx - kIpdHalfM; + out_eye_pos->eyes[0].y = fy; + out_eye_pos->eyes[0].z = fz; + out_eye_pos->eyes[1].x = fx + kIpdHalfM; + out_eye_pos->eyes[1].y = fy; + out_eye_pos->eyes[1].z = fz; + out_eye_pos->count = 2; + out_eye_pos->valid = true; + out_eye_pos->is_tracking = tracked; + return true; +} + +bool +get_display_dimensions_default(struct xrt_display_processor *xdp, + float *out_width_m, + float *out_height_m) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + if (impl->cnsdk != nullptr && + leia_cnsdk_get_display_metrics(impl->cnsdk, out_width_m, out_height_m, + nullptr, nullptr)) { + return true; + } + + *out_width_m = kDefaultDisplayWidthM; + *out_height_m = kDefaultDisplayHeightM; + return true; +} + +bool +get_display_pixel_info_default(struct xrt_display_processor *xdp, + uint32_t *out_pixel_width, + uint32_t *out_pixel_height, + int32_t *out_screen_left, + int32_t *out_screen_top) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + *out_screen_left = 0; + *out_screen_top = 0; + + if (impl->cnsdk != nullptr && + leia_cnsdk_get_display_metrics(impl->cnsdk, nullptr, nullptr, + out_pixel_width, out_pixel_height)) { + return true; + } + + *out_pixel_width = kDefaultDisplayPixelW; + *out_pixel_height = kDefaultDisplayPixelH; + return true; +} + +void +on_pause_cnsdk(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + if (impl->cnsdk != nullptr) { + leia_cnsdk_on_pause(impl->cnsdk); + } +} + +void +on_resume_cnsdk(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + if (impl->cnsdk != nullptr) { + leia_cnsdk_on_resume(impl->cnsdk); + } +} + +void +destroy_impl(struct xrt_display_processor *xdp) +{ + leia_dp_cnsdk *impl = as_impl(xdp); + + // Drain all in-flight GPU work (especially CNSDK's interlacer + // submits) before destroying any handles CNSDK might still be + // reading. Same defensive idiom as audit B6 — applies in atlas mode + // because CNSDK still owns its own queue submit even though we + // don't have a per-view image / blit cmd buffer to wait on. + if (impl->vk != nullptr) { + impl->vk->vkDeviceWaitIdle(impl->vk->device); + } + + if (impl->cnsdk != nullptr) { + leia_cnsdk_destroy(&impl->cnsdk); + } + free(impl); +} + +} // namespace + +extern "C" xrt_result_t +leia_dp_factory_cnsdk(void *vk_bundle, + void *vk_cmd_pool, + void *window_handle, + int32_t target_format, + struct xrt_display_processor **out_xdp) +{ + (void)window_handle; (void)target_format; + + struct leia_cnsdk *cnsdk = nullptr; + xrt_result_t ret = leia_cnsdk_create(&cnsdk); + if (ret != XRT_SUCCESS || cnsdk == nullptr) { + U_LOG_W("leia_cnsdk_create failed (%d), falling back to no-DP path", (int)ret); + return ret != XRT_SUCCESS ? ret : XRT_ERROR_DEVICE_CREATION_FAILED; + } + + leia_dp_cnsdk *impl = static_cast(calloc(1, sizeof(*impl))); + if (impl == nullptr) { + leia_cnsdk_destroy(&cnsdk); + return XRT_ERROR_ALLOCATION; + } + + impl->cnsdk = cnsdk; + impl->vk = static_cast(vk_bundle); + impl->cmd_pool = (VkCommandPool)(uintptr_t)vk_cmd_pool; + + /* ABI major v2 (ADR-020): the 8-byte struct_size header tells the + * runtime how far this vtable extends. The runtime gates every optional + * slot below on XRT_DP_HAS_SLOT(xdp, field), bounded against this value — + * leave it 0 (calloc default) and is_self_submitting / on_pause / on_resume / + * eye-positions / display-dims all read as absent, silently regressing the + * self-submitting atlas path into a double-submit. */ + impl->base.struct_size = static_cast(sizeof(struct xrt_display_processor)); + impl->base.process_atlas = process_atlas_weave; + impl->base.on_pause = on_pause_cnsdk; + impl->base.on_resume = on_resume_cnsdk; + impl->base.is_self_submitting = is_self_submitting_true; + impl->base.get_predicted_eye_positions = get_predicted_eye_positions_ipd; + impl->base.get_display_dimensions = get_display_dimensions_default; + impl->base.get_display_pixel_info = get_display_pixel_info_default; + impl->base.destroy = destroy_impl; + + *out_xdp = &impl->base; + + U_LOG_W("Leia CNSDK DP created (atlas mode)"); + DXR_HW_DBG("factory: impl=%p vk=%p cnsdk=%p", (void *)impl, (void *)impl->vk, (void *)impl->cnsdk); + return XRT_SUCCESS; +} diff --git a/src/drv_leia_android/leia_display_processor_cnsdk.h b/src/drv_leia_android/leia_display_processor_cnsdk.h new file mode 100644 index 0000000..2a6c3f8 --- /dev/null +++ b/src/drv_leia_android/leia_display_processor_cnsdk.h @@ -0,0 +1,43 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Leia CNSDK display processor (Android): wraps the CNSDK + * Vulkan interlacer as an @ref xrt_display_processor. + * + * Advertises `is_self_submitting = true`. The compositor flushes its + * pre-DP cmd buffer before calling process_atlas, which hands the SBS + * atlas VkImage directly to leia_cnsdk_weave (atlas mode — no per-view + * blit) — CNSDK records and submits its own command buffer internally. + * + * Display metrics + face-tracked eye positions come from CNSDK once the + * async core init completes (lazy on first query). Falls back to hardcoded + * Lume Pad 2 metrics and IPD-only eyes while the core is still booting. + * + * @author David Fattal + * @ingroup drv_leia + */ + +#pragma once + +#include "xrt/xrt_display_processor.h" +#include "xrt/xrt_results.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * Factory matching @ref xrt_dp_factory_vk_fn_t. Creates and owns a + * leia_cnsdk handle for the lifetime of the display processor. + */ +xrt_result_t +leia_dp_factory_cnsdk(void *vk_bundle, + void *vk_cmd_pool, + void *window_handle, + int32_t target_format, + struct xrt_display_processor **out_xdp); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia_android/leia_plugin_android.c b/src/drv_leia_android/leia_plugin_android.c new file mode 100644 index 0000000..4d86820 --- /dev/null +++ b/src/drv_leia_android/leia_plugin_android.c @@ -0,0 +1,239 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Android variant of the Leia plug-in entry point. + * + * Mirrors @ref leia_plugin.c (Windows/SR) but wraps CNSDK instead of + * the SR SDK. Builds into `libdxrp050_leia_cnsdk.so`, shipped in the + * runtime APK's `jniLibs//` and discovered by the runtime's + * `target_plugin_loader.c` Android branch (PR #309 / commit c96c93ce8). + * + * Differences from the Windows plug-in: + * - No `probe()` registry/EDID check — the CNSDK device is the only + * thing this plug-in supports, so probe always succeeds. + * - Only `create_dp_vk` is non-NULL (Android has no D3D/Metal/GL + * compositor path today, and CNSDK is Vulkan-only). + * - `get_display_info` returns Lume Pad 2 defaults; CNSDK-derived + * metrics surface lazily via the DP's `get_display_dimensions` / + * `get_display_pixel_info` calls once the async core init completes. + * + * @author leaiss + * @ingroup drv_leia_android + */ + +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_results.h" +#include "xrt/xrt_device.h" + +#include "util/u_device.h" +#include "util/u_logging.h" + +#include "leia_display_processor_cnsdk.h" + +#include +#include + + +/* + * + * Minimal HMD device (no SR SDK probe — Lume Pad 2 hardcoded defaults). + * + */ + +struct leia_android_hmd +{ + struct xrt_device base; +}; + +static void +leia_android_hmd_destroy(struct xrt_device *xdev) +{ + struct leia_android_hmd *hmd = (struct leia_android_hmd *)xdev; + u_device_free(&hmd->base); +} + +static xrt_result_t +leia_android_hmd_get_tracked_pose(struct xrt_device *xdev, + enum xrt_input_name name, + int64_t at_timestamp_ns, + struct xrt_space_relation *out_relation) +{ + (void)xdev; + (void)name; + (void)at_timestamp_ns; + out_relation->pose.orientation = (struct xrt_quat){0.0f, 0.0f, 0.0f, 1.0f}; + out_relation->pose.position = (struct xrt_vec3){0.0f, 0.0f, 0.0f}; + out_relation->relation_flags = (enum xrt_space_relation_flags)( + XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT); + return XRT_SUCCESS; +} + +static struct xrt_device * +leia_android_hmd_create(void) +{ + enum u_device_alloc_flags flags = + (enum u_device_alloc_flags)(U_DEVICE_ALLOC_HMD | U_DEVICE_ALLOC_TRACKING_NONE); + struct leia_android_hmd *hmd = U_DEVICE_ALLOCATE(struct leia_android_hmd, flags, 1, 0); + + hmd->base.update_inputs = u_device_noop_update_inputs; + hmd->base.get_tracked_pose = leia_android_hmd_get_tracked_pose; + hmd->base.get_view_poses = u_device_get_view_poses; + hmd->base.get_visibility_mask = u_device_get_visibility_mask; + hmd->base.destroy = leia_android_hmd_destroy; + hmd->base.name = XRT_DEVICE_GENERIC_HMD; + hmd->base.device_type = XRT_DEVICE_TYPE_HMD; + + // Lume Pad 2 default panel + viewing distance. CNSDK fills these + // in lazily on first DP query once the async core init completes. + struct u_device_simple_info info = { + .display.w_pixels = 2560, + .display.h_pixels = 1600, + .display.w_meters = 0.235f, + .display.h_meters = 0.147f, + .lens_horizontal_separation_meters = 0.063f, + .lens_vertical_position_meters = 0.0735f, + .fov = {85.0f * (float)M_PI / 180.0f, + 85.0f * (float)M_PI / 180.0f}, + }; + u_device_setup_split_side_by_side(&hmd->base, &info); + + snprintf(hmd->base.str, XRT_DEVICE_NAME_LEN, "Leia CNSDK (Android)"); + snprintf(hmd->base.serial, XRT_DEVICE_NAME_LEN, "lume-pad-cnsdk"); + + return &hmd->base; +} + + +/* + * + * Vtable callbacks. + * + */ + +static xrt_result_t +leia_plugin_android_probe(struct xrt_plugin_instance **out_inst) +{ + // CNSDK device probe runs async on a worker thread; the plug-in + // can't synchronously confirm hardware presence at xrCreateInstance + // time. POC pattern: always claim the slot on Android. The + // downstream leia_dp_factory_cnsdk path bails cleanly if CNSDK + // fails to initialize later. + *out_inst = NULL; + return XRT_SUCCESS; +} + +static xrt_result_t +leia_plugin_android_create_device(struct xrt_plugin_instance *inst, struct xrt_device **out_dev) +{ + (void)inst; + struct xrt_device *xdev = leia_android_hmd_create(); + if (xdev == NULL) { + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + *out_dev = xdev; + return XRT_SUCCESS; +} + +static void +leia_plugin_android_destroy(struct xrt_plugin_instance *inst) +{ + (void)inst; +} + +static bool +leia_plugin_android_get_display_info(struct xrt_plugin_instance *inst, + struct xrt_device *xdev, + struct xrt_plugin_display_info *out_info) +{ + (void)inst; + (void)xdev; + + // Lume Pad 2 defaults. CNSDK device-config is async — the DP's + // get_display_dimensions / get_display_pixel_info read it lazily, + // so these baseline values keep the runtime usable while the + // CNSDK core is still booting. + out_info->display_pixel_width = 2560; + out_info->display_pixel_height = 1600; + out_info->recommended_view_scale_x = 1.0f; + out_info->recommended_view_scale_y = 1.0f; + out_info->display_width_m = 0.235f; + out_info->display_height_m = 0.147f; + out_info->nominal_viewer_x_m = 0.0f; + out_info->nominal_viewer_y_m = 0.0f; + out_info->nominal_viewer_z_m = 0.50f; + + // CNSDK exposes face position only — no MANUAL pose-stream API. + out_info->supported_eye_tracking_modes = 1u; // MANAGED_BIT + out_info->default_eye_tracking_mode = 0u; // MANAGED + + return true; +} + + +/* + * + * Vtable. + * + */ + +static struct xrt_plugin_iface g_leia_android_iface = { + .struct_size = sizeof(struct xrt_plugin_iface), + .reserved_0 = 0, + + .id = "leia-cnsdk", + .display_name = "DisplayXR Leia CNSDK (Android)", + .vendor = "Leia Inc.", + .version = NULL, + + .probe = leia_plugin_android_probe, + .create_device = leia_plugin_android_create_device, + + .create_dp_vk = leia_dp_factory_cnsdk, + .create_dp_d3d11 = NULL, + .create_dp_d3d12 = NULL, + .create_dp_gl = NULL, + .create_dp_metal = NULL, + + .destroy = leia_plugin_android_destroy, + + .get_display_info = leia_plugin_android_get_display_info, + + .set_pose_source = NULL, // CNSDK pose comes from face tracking, not an external source + + // probe_displays (v1.9.0 / ADR-015 / #69): deliberately omitted on Android. + // It exists for per-monitor claim discovery on desktop multi-display setups + // (the Windows arm in drv_leia/leia_plugin.c implements it over EDID + // enumeration). Android is a single fixed-panel device with no monitor + // enumeration, so we leave it NULL — struct_size still spans the full v2 + // iface, so the runtime's struct_size gate sees the slot, finds it NULL, and + // falls back to query_source_claims()'s synthesized primary claim off a + // successful binary probe(). Validated end-to-end via android-smoketest.sh. + .probe_displays = NULL, +}; + + +/* + * + * Entry point. + * + */ + +XRT_PLUGIN_EXPORT xrt_result_t +xrtPluginNegotiate(uint32_t runtime_api_version, + const struct xrt_plugin_host_iface *host, + struct xrt_plugin_iface **out_iface, + uint32_t *out_plugin_api_version) +{ + (void)host; + + *out_plugin_api_version = XRT_PLUGIN_API_VERSION_CURRENT; + + if (runtime_api_version != XRT_PLUGIN_API_VERSION_CURRENT) { + *out_iface = NULL; + return XRT_ERROR_PROBER_NOT_SUPPORTED; + } + + *out_iface = &g_leia_android_iface; + return XRT_SUCCESS; +}