diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 8ba4cbeff..c3ced68a7 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -466,8 +466,8 @@ jobs: fi status_url="${KITMAKER_API_ENDPOINT%/}/v0/status/${KITMAKER_RELEASE_UUID}" - # Limit total polling time to at most ~10 minutes (20 * 30s) - max_attempts=20 + # Limit total polling time to under an hour with exponential backoff starting at 30s + max_attempts=15 sleep_seconds=30 for ((attempt=1; attempt<=max_attempts; attempt++)); do @@ -495,6 +495,7 @@ jobs: fi sleep "${sleep_seconds}" + sleep_seconds=$(( sleep_seconds * 125 / 100 )) done echo "Timed out waiting for Kitmaker release ${KITMAKER_RELEASE_UUID} to complete" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml new file mode 100644 index 000000000..3ebc1cb72 --- /dev/null +++ b/.github/workflows/release-tag.yml @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Manually-triggered release workflow. +# +# Run from a release/X.Y.x branch via the Actions UI. The workflow: +# 1. Validates the branch is release/X.Y.x and that VERSION matches. +# 2. Computes the patch number the same way CMake does (commits since +# the last change to VERSION). +# 3. Verifies no v* tag already exists at HEAD and the computed tag name +# isn't already taken on another commit. +# 4. Verifies the most recent Build Ubuntu run on this commit was green. +# 5. Pushes vX.Y.PATCH (annotated) using a PAT so build-ubuntu.yml fires +# on the tag and runs the release publish chain. +# 6. Creates a draft GitHub release with auto-generated notes. +# +# Access is gated by environment: release (admin reviewers required). + +name: Release Tag + +on: + workflow_dispatch: + inputs: + skip_ci_check: + description: "Skip the 'latest Build Ubuntu run on this SHA was green' check. Use only if you know what you're doing." + type: boolean + default: false + +concurrency: + group: release-tag-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + release-tag: + runs-on: ubuntu-latest + environment: release + permissions: + contents: read + actions: read + + steps: + - name: Validate ref is a release branch + run: | + set -euo pipefail + if [[ ! "${GITHUB_REF}" =~ ^refs/heads/release/([0-9]+)\.([0-9]+)\.x$ ]]; then + echo "::error::This workflow may only run from a release/X.Y.x branch (got: ${GITHUB_REF})." + exit 1 + fi + echo "BRANCH_MAJOR=${BASH_REMATCH[1]}" >> "${GITHUB_ENV}" + echo "BRANCH_MINOR=${BASH_REMATCH[2]}" >> "${GITHUB_ENV}" + + - name: Checkout (full history) + uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.RELEASE_SYNC_TOKEN }} + + - name: Validate VERSION matches branch + run: | + set -euo pipefail + version_content="$(tr -d '[:space:]' < VERSION)" + if [[ ! "${version_content}" =~ ^([0-9]+)\.([0-9]+)\.x$ ]]; then + echo "::error::VERSION must be MAJOR.MINOR.x format (got: '${version_content}')." + exit 1 + fi + if [[ "${BASH_REMATCH[1]}" != "${BRANCH_MAJOR}" || "${BASH_REMATCH[2]}" != "${BRANCH_MINOR}" ]]; then + echo "::error::VERSION (${version_content}) does not match branch release/${BRANCH_MAJOR}.${BRANCH_MINOR}.x." + exit 1 + fi + + - name: Compute patch number and tag + id: compute + run: | + set -euo pipefail + version_commit="$(git rev-list -n 1 HEAD -- VERSION)" + if [[ -z "${version_commit}" ]]; then + echo "::error::Could not locate the commit that last modified VERSION." + exit 1 + fi + patch="$(git rev-list --count "${version_commit}..HEAD")" + if ! [[ "${patch}" =~ ^[0-9]+$ ]]; then + echo "::error::Computed patch is not a number: '${patch}'." + exit 1 + fi + tag="v${BRANCH_MAJOR}.${BRANCH_MINOR}.${patch}" + sha="$(git rev-parse HEAD)" + echo "Computed release tag: ${tag} at ${sha}" + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "patch=${patch}" >> "${GITHUB_OUTPUT}" + echo "sha=${sha}" >> "${GITHUB_OUTPUT}" + + - name: Check no existing tag at HEAD (idempotent re-run is allowed) + id: head-tag-check + run: | + set -euo pipefail + tag="${{ steps.compute.outputs.tag }}" + sha="${{ steps.compute.outputs.sha }}" + existing_at_head="$(git tag --points-at "${sha}" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || true)" + if [[ -z "${existing_at_head}" ]]; then + echo "head_already_tagged=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + # Allow idempotent re-run when the existing tag matches what we'd compute. + if echo "${existing_at_head}" | grep -qx "${tag}"; then + echo "::notice::HEAD is already tagged ${tag}; skipping tag push, will still ensure draft release exists." + echo "head_already_tagged=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + echo "::error::HEAD already has a release tag (${existing_at_head}) different from the computed tag (${tag}). Refusing to tag the same commit twice." + exit 1 + + - name: Check computed tag is not taken on a different commit + if: steps.head-tag-check.outputs.head_already_tagged == 'false' + run: | + set -euo pipefail + tag="${{ steps.compute.outputs.tag }}" + if git rev-parse --verify --quiet "refs/tags/${tag}" >/dev/null; then + existing_sha="$(git rev-list -n 1 "refs/tags/${tag}")" + echo "::error::Tag ${tag} already exists on a different commit (${existing_sha}). Resolve manually." + exit 1 + fi + + - name: Verify latest Build Ubuntu run on this SHA was green + if: ${{ !inputs.skip_ci_check }} + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ steps.compute.outputs.sha }} + run: | + set -euo pipefail + # Find the most recent completed Build Ubuntu run for this commit. + run_json="$(gh run list \ + --workflow build-ubuntu.yml \ + --commit "${SHA}" \ + --status completed \ + --limit 1 \ + --json conclusion,databaseId,headSha,event)" + count="$(jq 'length' <<< "${run_json}")" + if [[ "${count}" -eq 0 ]]; then + echo "::error::No completed Build Ubuntu run found for ${SHA}. Push the branch and let CI finish before tagging, or set skip_ci_check=true." + exit 1 + fi + conclusion="$(jq -r '.[0].conclusion' <<< "${run_json}")" + run_id="$(jq -r '.[0].databaseId' <<< "${run_json}")" + if [[ "${conclusion}" != "success" ]]; then + echo "::error::Latest Build Ubuntu run on ${SHA} concluded '${conclusion}' (run id ${run_id}). Refusing to tag." + exit 1 + fi + echo "Build Ubuntu run ${run_id} on ${SHA} was successful." + + - name: Configure git identity + if: steps.head-tag-check.outputs.head_already_tagged == 'false' + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com" + + - name: Create and push annotated tag + if: steps.head-tag-check.outputs.head_already_tagged == 'false' + run: | + set -euo pipefail + tag="${{ steps.compute.outputs.tag }}" + git tag -a "${tag}" -m "Release ${tag}" + # Pushed via RELEASE_SYNC_TOKEN (set on the actions/checkout step) so the + # tag push triggers build-ubuntu.yml; GITHUB_TOKEN-pushed tags would not. + git push origin "${tag}" + echo "Pushed ${tag}; this will trigger build-ubuntu.yml on the tag ref." + + - name: Create or refresh draft GitHub release + env: + GH_TOKEN: ${{ secrets.RELEASE_SYNC_TOKEN }} + run: | + set -euo pipefail + tag="${{ steps.compute.outputs.tag }}" + # Find the previous tag on the same X.Y line for note generation. + prev_tag="$(git tag --list "v${BRANCH_MAJOR}.${BRANCH_MINOR}.*" --sort=-v:refname \ + | grep -vx "${tag}" | head -1 || true)" + notes_args=() + if [[ -n "${prev_tag}" ]]; then + notes_args+=(--notes-start-tag "${prev_tag}") + fi + if gh release view "${tag}" >/dev/null 2>&1; then + echo "::notice::Release ${tag} already exists; leaving as-is." + else + gh release create "${tag}" \ + --draft \ + --title "${tag}" \ + --generate-notes "${notes_args[@]}" + echo "Draft release ${tag} created." + fi + + - name: Summary + run: | + { + echo "### Release Tag Summary" + echo "" + echo "- Branch: \`${GITHUB_REF#refs/heads/}\`" + echo "- Commit: \`${{ steps.compute.outputs.sha }}\`" + echo "- Computed tag: \`${{ steps.compute.outputs.tag }}\`" + echo "- Already tagged at HEAD: \`${{ steps.head-tag-check.outputs.head_already_tagged }}\`" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.nspect-allowlist.toml b/.nspect-allowlist.toml new file mode 100644 index 000000000..e2d6631fc --- /dev/null +++ b/.nspect-allowlist.toml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = "1.0.0" + +[oss.excluded] +[[oss.excluded.directories]] + +paths = ['docs'] +comment = 'NVIDIA specific documentation' diff --git a/README.md b/README.md index 51ed693ed..314712759 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ SPDX-License-Identifier: Apache-2.0 **The unified framework for high-fidelity ego-centric and robotics data collection.** [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/) -[![Isaac Lab](https://img.shields.io/badge/Isaac%20Lab-3.0.0-orange.svg)](https://isaac-sim.github.io/IsaacLab/) -[![numpy](https://img.shields.io/badge/numpy-1.22%2B-lightgrey.svg)](https://numpy.org/) +[![Isaac Lab](https://img.shields.io/badge/Isaac%20Lab-3.0.0-orange.svg)](https://isaac-sim.github.io/IsaacLab/develop) +[![numpy](https://img.shields.io/badge/numpy-1.23%2B-lightgrey.svg)](https://numpy.org/) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) @@ -60,6 +60,6 @@ Our [documentation page](https://nvidia.github.io/IsaacTeleop) provides everythi ### Install & Run Isaac Lab -Isaac Tepeop Core is design to work side by side with [NVIDIA Isaac Lab](https://github.com/isaac-sim/IsaacLab) starting with Isaac Lab 3.0 EA release. +Isaac Teleop is design to work side by side with [NVIDIA Isaac Lab](https://github.com/isaac-sim/IsaacLab) starting with Isaac Lab [3.0 Beta release](https://github.com/isaac-sim/IsaacLab/releases/tag/v3.0.0-beta). -To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.github.io/IsaacLab/main/source/setup/installation/index.html) guide for more details. Then follow the [CloudXR teleoperation in Isaac Lab](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html) to get started with Teleoperation in Sim. +To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.github.io/IsaacLab/develop/source/setup/installation/index.html) guide for more details. Then follow the [CloudXR teleoperation in Isaac Lab](https://isaac-sim.github.io/IsaacLab/develop/source/how-to/cloudxr_teleoperation.html) to get started with Teleoperation in Sim. diff --git a/VERSION b/VERSION index 1bb9786c7..d230cb102 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.x +1.1.x diff --git a/deps/cloudxr/.env.default b/deps/cloudxr/.env.default index 429f7ff6b..0d747ef48 100644 --- a/deps/cloudxr/.env.default +++ b/deps/cloudxr/.env.default @@ -5,8 +5,8 @@ ########################################################### # CloudXR Docker Images and Configs ########################################################### -CXR_RUNTIME_SDK_VERSION=6.1.0-rc2 -CXR_WEB_SDK_VERSION=6.1.0-rc6 +CXR_RUNTIME_SDK_VERSION=6.1.0 +CXR_WEB_SDK_VERSION=6.1.0 CXR_HOST_VOLUME_PATH=$HOME/.cloudxr ########################################################### diff --git a/deps/cloudxr/Dockerfile.runtime-ngc b/deps/cloudxr/Dockerfile.runtime-ngc new file mode 100644 index 000000000..eeb31f7c5 --- /dev/null +++ b/deps/cloudxr/Dockerfile.runtime-ngc @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Minimal CloudXR runtime image. +# The SDK must be downloaded first (see scripts/download_cloudxr_runtime_sdk.sh). +# +# Build context: deps/cloudxr/ (contains CloudXR--Linux--sdk.tar.gz) +# +# Environment overrides: +# NV_DEVICE_PROFILE Device profile (default: auto-webrtc). +# Values: auto-native, auto-webrtc, +# apple-vision-pro, ipad-pro, quest3 + +# ------------------------------------------------------------------- +# Stage 0: Extract the SDK tarball. +# ------------------------------------------------------------------- +FROM scratch AS sdk +ARG TARGETARCH +ARG CXR_RUNTIME_SDK_VERSION +ADD CloudXR-${CXR_RUNTIME_SDK_VERSION}-Linux-${TARGETARCH}-sdk.tar.gz /sdk/ + +# ------------------------------------------------------------------- +# Stage 1: Runtime image. +# ------------------------------------------------------------------- +FROM ubuntu:24.04 + +ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=all +ENV LD_LIBRARY_PATH=/opt/cloudxr + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libatomic1 \ + libbsd0 \ + libegl1 \ + libgl1 \ + libglx0 \ + libvulkan1 \ + libx11-6 \ + libxext6 \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +# Vulkan ICD + EGL vendor so the loader can find the NVIDIA driver. +RUN mkdir -p /etc/vulkan/icd.d /usr/share/glvnd/egl_vendor.d +COPY runtime/data/10_nvidia.json /usr/share/glvnd/egl_vendor.d/ +COPY runtime/data/nvidia_icd.json /etc/vulkan/icd.d/ +ENV VK_DRIVER_FILES=/etc/vulkan/icd.d/nvidia_icd.json + +RUN mkdir -p /openxr/run && chmod 777 /openxr /openxr/run && \ + touch /var/run/utmp + +WORKDIR /opt/cloudxr + +COPY runtime/main.py . +COPY --from=sdk /sdk/*.so /sdk/*.so.* ./ +COPY --from=sdk /sdk/openxr_cloudxr.json . + +# CloudXR / OpenXR runtime configuration. +ENV XR_RUNTIME_JSON=/openxr/openxr_cloudxr.json +ENV NV_CXR_RUNTIME_DIR=/openxr/run +ENV XRT_NO_STDIN=true +ENV NV_CXR_FILE_LOGGING=false +ENV NV_DEVICE_PROFILE=auto-webrtc + +COPY --chmod=0755 runtime/eula.sh /eula.sh +COPY --chmod=0755 runtime/entrypoint-ngc.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh", "python3", "/opt/cloudxr/main.py"] diff --git a/deps/cloudxr/Dockerfile.test b/deps/cloudxr/Dockerfile.test index 203382df4..fb706dd6f 100644 --- a/deps/cloudxr/Dockerfile.test +++ b/deps/cloudxr/Dockerfile.test @@ -31,7 +31,8 @@ COPY examples/oxr/python/ /app/tests/ WORKDIR /app/tests RUN uv venv --python $PYTHON_VERSION /app/venv && \ . /app/venv/bin/activate && \ - uv pip install --find-links=/app/install/wheels isaacteleop numpy + uv pip install --find-links=/app/install/wheels --no-index --no-deps --no-cache-dir 'isaacteleop' && \ + uv pip install isaacteleop # Set environment variables ENV PATH="/app/venv/bin:$PATH" diff --git a/deps/cloudxr/docker-compose.yaml b/deps/cloudxr/docker-compose.yaml index dd615843d..454b68be4 100644 --- a/deps/cloudxr/docker-compose.yaml +++ b/deps/cloudxr/docker-compose.yaml @@ -1,10 +1,33 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# CloudXR compose override for web/proxy services and runtime mode. -# Used together with docker-compose.runtime.yaml. - services: + # CloudXR Runtime + cloudxr-runtime: + build: + context: . + dockerfile: Dockerfile.runtime-ngc + args: + CXR_RUNTIME_SDK_VERSION: ${CXR_RUNTIME_SDK_VERSION} + container_name: cloudxr-runtime-${CXR_RUNTIME_SDK_VERSION} + user: "${CXR_UID:-1000}:${CXR_GID:-1000}" + network_mode: host + healthcheck: + test: ["CMD", "test", "-f", "/openxr/run/runtime_started"] + interval: 1s + timeout: 1s + retries: 10 + start_period: 5s + environment: + - ACCEPT_EULA=${ACCEPT_CLOUDXR_EULA:-Y} + - NV_CXR_ENABLE_PUSH_DEVICES=${NV_CXR_ENABLE_PUSH_DEVICES} + - NV_CXR_ENABLE_TENSOR_DATA=${NV_CXR_ENABLE_TENSOR_DATA} + - NV_DEVICE_PROFILE=${NV_DEVICE_PROFILE:-auto-webrtc} + - NV_GPU_INDEX=${NV_GPU_INDEX:-0} + volumes: + - openxr-volume:/openxr/ + runtime: nvidia + # WebSocket SSL Proxy Service wss-proxy: build: @@ -40,3 +63,11 @@ services: volumes: - ./webxr_client:/app/webxr_client restart: unless-stopped + +volumes: + openxr-volume: + driver: local + driver_opts: + type: none + o: bind + device: ${CXR_HOST_VOLUME_PATH:-/tmp/cloudxr} diff --git a/deps/cloudxr/runtime/entrypoint-ngc.sh b/deps/cloudxr/runtime/entrypoint-ngc.sh new file mode 100755 index 000000000..89dbeaac3 --- /dev/null +++ b/deps/cloudxr/runtime/entrypoint-ngc.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -e + +mkdir -p /openxr/run +cp -f /opt/cloudxr/libopenxr_cloudxr.so /openxr/ +cp -f /opt/cloudxr/openxr_cloudxr.json /openxr/ +rm -f /openxr/run/ipc_cloudxr /openxr/run/runtime_started +exec /eula.sh "$@" diff --git a/deps/cloudxr/runtime/main.py b/deps/cloudxr/runtime/main.py new file mode 100644 index 000000000..0e4b0d4e4 --- /dev/null +++ b/deps/cloudxr/runtime/main.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import ctypes +import signal + +lib = ctypes.CDLL("libcloudxr.so") +svc = ctypes.c_void_p() + + +def stop(sig, frame): + lib.nv_cxr_service_stop(svc) + + +lib.nv_cxr_service_create(ctypes.byref(svc)) +signal.signal(signal.SIGINT, stop) +signal.signal(signal.SIGTERM, stop) +lib.nv_cxr_service_start(svc) +lib.nv_cxr_service_join(svc) +lib.nv_cxr_service_destroy(svc) diff --git a/deps/cloudxr/webxr_client/helpers/DeviceProfiles.ts b/deps/cloudxr/webxr_client/helpers/DeviceProfiles.ts index 0a99dc0a8..5a85a44ca 100644 --- a/deps/cloudxr/webxr_client/helpers/DeviceProfiles.ts +++ b/deps/cloudxr/webxr_client/helpers/DeviceProfiles.ts @@ -47,6 +47,8 @@ export interface DeviceProfile { cloudxr?: { perEyeWidth?: number; perEyeHeight?: number; + reprojectionGridCols?: number; + reprojectionGridRows?: number; deviceFrameRate?: number; maxStreamingBitrateKbps?: number; codec?: 'av1' | 'h264' | 'h265'; @@ -100,6 +102,9 @@ const QUEST3S_PROFILE: DeviceProfile = { id: 'quest3s', label: 'Quest 3S', description: 'Same as Quest 3 for now.', + cloudxr: { + ...QUEST3_PROFILE.cloudxr!, + }, }; // Quest 2: same as Quest 3 but default codec H.265 (no hardware AV1 support). @@ -108,7 +113,12 @@ const QUEST2_PROFILE: DeviceProfile = { id: 'quest2', label: 'Quest 2', description: 'Same as Quest 3 except using H.265.', - cloudxr: { ...QUEST3_PROFILE.cloudxr!, codec: 'h265' }, + cloudxr: { + ...QUEST3_PROFILE.cloudxr!, + reprojectionGridCols: 64, + reprojectionGridRows: 64, + codec: 'h265', + }, }; // Pico 4 Ultra defaults are conservative until device-specific validation is complete. @@ -130,6 +140,8 @@ const PICO4ULTRA_PROFILE: DeviceProfile = { cloudxr: { perEyeWidth: 2048, perEyeHeight: 1792, + reprojectionGridCols: 64, + reprojectionGridRows: 64, deviceFrameRate: 90, maxStreamingBitrateKbps: 100000, codec: 'av1', diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index 9cab2cc26..8726bd697 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -215,6 +215,8 @@ export default function CloudXRComponent({ signalingResourcePath: connectionConfig.resourcePath, perEyeWidth: config.perEyeWidth, perEyeHeight: config.perEyeHeight, + reprojectionGridCols: config.reprojectionGridCols, + reprojectionGridRows: config.reprojectionGridRows, codec: config.codec, gl: gl, referenceSpace: referenceSpace, diff --git a/deps/cloudxr/webxr_client/helpers/utils.ts b/deps/cloudxr/webxr_client/helpers/utils.ts index 06b4e9fb6..8345d3cc3 100644 --- a/deps/cloudxr/webxr_client/helpers/utils.ts +++ b/deps/cloudxr/webxr_client/helpers/utils.ts @@ -134,6 +134,12 @@ export interface CloudXRConfig { /** Height of each eye in pixels (must be multiple of 16) */ perEyeHeight: number; + /** Depth reprojection mesh grid vertex columns (not cell columns). Undefined uses factor mode. */ + reprojectionGridCols?: number; + + /** Depth reprojection mesh grid vertex rows (not cell rows). Undefined uses factor mode. */ + reprojectionGridRows?: number; + /** Target frame rate for the XR device in frames per second (FPS) */ deviceFrameRate: number; @@ -210,6 +216,24 @@ export function getResolutionFromInputs( }; } +/** + * Reads reprojection grid values from two number inputs. + * Blank means "use default factor mode", returned as undefined. + */ +export function getGridFromInputs( + reprojectionGridColsInput: HTMLInputElement, + reprojectionGridRowsInput: HTMLInputElement +): { reprojectionGridCols: number | undefined; reprojectionGridRows: number | undefined } { + const colsRaw = reprojectionGridColsInput.value.trim(); + const rowsRaw = reprojectionGridRowsInput.value.trim(); + const cols = colsRaw === '' ? undefined : parseInt(reprojectionGridColsInput.value, 10); + const rows = rowsRaw === '' ? undefined : parseInt(reprojectionGridRowsInput.value, 10); + return { + reprojectionGridCols: cols === undefined || Number.isFinite(cols) ? cols : NaN, + reprojectionGridRows: rows === undefined || Number.isFinite(rows) ? rows : NaN, + }; +} + /** * Determines connection configuration based on protocol and user inputs * Supports both direct WSS connections and proxy routing for HTTPS diff --git a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx index 0b70e0311..d0eb20aa3 100644 --- a/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXR2DUI.tsx @@ -44,13 +44,17 @@ import { import { CloudXRConfig, enableLocalStorage, + getGridFromInputs, getResolutionFromInputs, setSelectValueIfAvailable, setupCertificateAcceptanceLink, } from '@helpers/utils'; import { + getGridValidationError, + getGridValidationMessageForConnect, getResolutionValidationError, getResolutionValidationMessageForConnect, + validateDepthReprojectionGrid, validatePerEyeResolution, } from '@nvidia/cloudxr'; @@ -83,10 +87,18 @@ export class CloudXR2DUI { private perEyeWidthInput!: HTMLInputElement; /** Input field for per-eye height configuration */ private perEyeHeightInput!: HTMLInputElement; + /** Input field for reprojection mesh grid X (columns) */ + private reprojectionGridColsInput!: HTMLInputElement; + /** Input field for reprojection mesh grid Y (rows) */ + private reprojectionGridRowsInput!: HTMLInputElement; /** Inline resolution validation under width input */ private resolutionWidthValidationMessage: HTMLElement | null = null; /** Inline resolution validation under height input */ private resolutionHeightValidationMessage: HTMLElement | null = null; + /** Inline grid validation under reprojection grid columns input */ + private reprojectionGridColsValidationMessage: HTMLElement | null = null; + /** Inline grid validation under reprojection grid rows input */ + private reprojectionGridRowsValidationMessage: HTMLElement | null = null; private validationMessageBox!: HTMLElement; private validationMessageText!: HTMLElement; /** Dropdown to enable pose smoothing */ @@ -201,12 +213,20 @@ export class CloudXR2DUI { this.codecSelect = this.getElement('codec'); this.perEyeWidthInput = this.getElement('perEyeWidth'); this.perEyeHeightInput = this.getElement('perEyeHeight'); + this.reprojectionGridColsInput = this.getElement('reprojectionGridCols'); + this.reprojectionGridRowsInput = this.getElement('reprojectionGridRows'); this.resolutionWidthValidationMessage = document.getElementById( 'resolutionWidthValidationMessage' ); this.resolutionHeightValidationMessage = document.getElementById( 'resolutionHeightValidationMessage' ); + this.reprojectionGridColsValidationMessage = document.getElementById( + 'reprojectionGridColsValidationMessage' + ); + this.reprojectionGridRowsValidationMessage = document.getElementById( + 'reprojectionGridRowsValidationMessage' + ); this.enablePoseSmoothingSelect = this.getElement('enablePoseSmoothing'); this.posePredictionFactorInput = this.getElement('posePredictionFactor'); this.posePredictionFactorValue = this.getElement('posePredictionFactorValue'); @@ -264,6 +284,8 @@ export class CloudXR2DUI { useSecureConnection: useSecure, perEyeWidth: 2048, perEyeHeight: 1792, + reprojectionGridCols: 0, + reprojectionGridRows: 0, deviceFrameRate: 90, maxStreamingBitrateMbps: 150, codec: 'av1', @@ -292,6 +314,8 @@ export class CloudXR2DUI { enableLocalStorage(this.portInput, 'port'); enableLocalStorage(this.perEyeWidthInput, 'perEyeWidth'); enableLocalStorage(this.perEyeHeightInput, 'perEyeHeight'); + enableLocalStorage(this.reprojectionGridColsInput, 'reprojectionGridCols'); + enableLocalStorage(this.reprojectionGridRowsInput, 'reprojectionGridRows'); enableLocalStorage(this.proxyUrlInput, 'proxyUrl'); enableLocalStorage(this.deviceFrameRateSelect, 'deviceFrameRate'); enableLocalStorage(this.maxStreamingBitrateMbpsSelect, 'maxStreamingBitrateMbps'); @@ -370,6 +394,16 @@ export class CloudXR2DUI { addListener(this.perEyeHeightInput, 'blur', updateResValidation); addListener(this.perEyeHeightInput, 'keyup', updateResValidation); this.updateResolutionValidationMessage(); + const updateGridValidation = () => this.updateGridValidationMessage(); + addListener(this.reprojectionGridColsInput, 'input', onProfileLinkedChange); + addListener(this.reprojectionGridColsInput, 'change', onProfileLinkedChange); + addListener(this.reprojectionGridColsInput, 'blur', updateGridValidation); + addListener(this.reprojectionGridColsInput, 'keyup', updateGridValidation); + addListener(this.reprojectionGridRowsInput, 'input', onProfileLinkedChange); + addListener(this.reprojectionGridRowsInput, 'change', onProfileLinkedChange); + addListener(this.reprojectionGridRowsInput, 'blur', updateGridValidation); + addListener(this.reprojectionGridRowsInput, 'keyup', updateGridValidation); + this.updateGridValidationMessage(); addListener(this.deviceFrameRateSelect, 'change', onProfileLinkedChange); addListener(this.maxStreamingBitrateMbpsSelect, 'change', onProfileLinkedChange); addListener(this.codecSelect, 'change', onProfileLinkedChange); @@ -440,13 +474,50 @@ export class CloudXR2DUI { this.updateConnectButtonState(); } + /** Update inline grid validation under each input. */ + private updateGridValidationMessage(): void { + const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( + this.reprojectionGridColsInput, + this.reprojectionGridRowsInput + ); + const { reprojectionGridColsError, reprojectionGridRowsError } = validateDepthReprojectionGrid( + reprojectionGridCols, + reprojectionGridRows + ); + if (this.reprojectionGridColsValidationMessage) { + const showGridCols = reprojectionGridColsError ?? ''; + this.reprojectionGridColsValidationMessage.textContent = showGridCols; + this.reprojectionGridColsValidationMessage.className = showGridCols + ? 'config-text resolution-validation-error' + : 'config-text'; + } + if (this.reprojectionGridRowsValidationMessage) { + const showGridRows = reprojectionGridRowsError ?? ''; + this.reprojectionGridRowsValidationMessage.textContent = showGridRows; + this.reprojectionGridRowsValidationMessage.className = showGridRows + ? 'config-text resolution-validation-error' + : 'config-text'; + } + this.updateConnectButtonState(); + } + /** Disable Connect button and show validation error when resolution invalid; enable when valid. */ public updateConnectButtonState(): void { const { w, h } = getResolutionFromInputs(this.perEyeWidthInput, this.perEyeHeightInput); + const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( + this.reprojectionGridColsInput, + this.reprojectionGridRowsInput + ); const resolutionError = getResolutionValidationError(w, h); + const gridError = getGridValidationError(reprojectionGridCols, reprojectionGridRows); const connectMessage = getResolutionValidationMessageForConnect(w, h); - if (connectMessage) { - this.validationMessageText.textContent = connectMessage; + const gridConnectMessage = getGridValidationMessageForConnect( + reprojectionGridCols, + reprojectionGridRows + ); + const combinedConnectMessage = [connectMessage, gridConnectMessage].filter(Boolean).join(' '); + if (combinedConnectMessage) { + this.validationMessageText.textContent = combinedConnectMessage; this.validationMessageBox.className = 'validation-message-box show'; } else { this.validationMessageText.textContent = ''; @@ -454,7 +525,7 @@ export class CloudXR2DUI { } // Only update button when idle (don't override "CONNECT (starting...)" or "CONNECT (XR session active)") if (this.startButton && this.startButton.innerHTML === 'CONNECT') { - const shouldEnable = !resolutionError; + const shouldEnable = !resolutionError && !gridError; this.setStartButtonState(!shouldEnable, 'CONNECT'); } } @@ -478,12 +549,18 @@ export class CloudXR2DUI { this.perEyeWidthInput, this.perEyeHeightInput ); + const { reprojectionGridCols, reprojectionGridRows } = getGridFromInputs( + this.reprojectionGridColsInput, + this.reprojectionGridRowsInput + ); const newConfiguration: AppConfig = { serverIP: this.serverIpInput.value || this.getDefaultConfiguration().serverIP, port: portValue || defaultPort, useSecureConnection: useSecure, perEyeWidth, perEyeHeight, + reprojectionGridCols, + reprojectionGridRows, deviceFrameRate: parseInt(this.deviceFrameRateSelect.value) || this.getDefaultConfiguration().deviceFrameRate, @@ -560,6 +637,10 @@ export class CloudXR2DUI { if (cloudxr.perEyeHeight !== undefined) { this.perEyeHeightInput.value = String(cloudxr.perEyeHeight); } + this.reprojectionGridColsInput.value = + cloudxr.reprojectionGridCols !== undefined ? String(cloudxr.reprojectionGridCols) : ''; + this.reprojectionGridRowsInput.value = + cloudxr.reprojectionGridRows !== undefined ? String(cloudxr.reprojectionGridRows) : ''; if (cloudxr.deviceFrameRate !== undefined) { setSelectValueIfAvailable(this.deviceFrameRateSelect, String(cloudxr.deviceFrameRate)); } @@ -600,6 +681,8 @@ export class CloudXR2DUI { try { localStorage.setItem('perEyeWidth', this.perEyeWidthInput.value); localStorage.setItem('perEyeHeight', this.perEyeHeightInput.value); + localStorage.setItem('reprojectionGridCols', this.reprojectionGridColsInput.value); + localStorage.setItem('reprojectionGridRows', this.reprojectionGridRowsInput.value); localStorage.setItem('deviceFrameRate', this.deviceFrameRateSelect.value); localStorage.setItem('maxStreamingBitrateMbps', this.maxStreamingBitrateMbpsSelect.value); localStorage.setItem('codec', this.codecSelect.value); @@ -666,7 +749,11 @@ export class CloudXR2DUI { this.handleConnectClick = async () => { const cfg = this.getConfiguration(); const resolutionError = getResolutionValidationError(cfg.perEyeWidth, cfg.perEyeHeight); - if (resolutionError) { + const gridError = getGridValidationError( + cfg.reprojectionGridCols, + cfg.reprojectionGridRows + ); + if (resolutionError || gridError) { this.updateConnectButtonState(); return; } diff --git a/deps/cloudxr/webxr_client/src/index.html b/deps/cloudxr/webxr_client/src/index.html index 2a1c80b9b..5e754a790 100644 --- a/deps/cloudxr/webxr_client/src/index.html +++ b/deps/cloudxr/webxr_client/src/index.html @@ -653,6 +653,17 @@

Debug Settings

Configure the per-eye resolution. Width must be a multiple of 16 (min 128); height must be a multiple of 64 (min 128).
+ + +
+ + +
+
+ Grid rules: leave either field blank to use factor mode, or set both to integers >= 2 for explicit mesh resolution. +
diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css index 368f677fa..ede035085 100644 --- a/docs/source/_static/css/custom.css +++ b/docs/source/_static/css/custom.css @@ -252,6 +252,26 @@ img.license-badge { margin-bottom: 1rem; } +/* Next steps panels: same image size and aspect ratio in both cards; no margins */ +#next-steps .sd-card-header, +#next-steps .sd-card-header.docutils, +.sd-card-header:has(img[alt="Isaac Lab"]), +.sd-card-header:has(img[alt="Isaac ROS"]) { + margin: 0 !important; + padding: 0 !important; + border: none !important; +} +#next-steps .sd-card-header img, +.sd-card-header:has(img[alt="Isaac Lab"]) img, +.sd-card-header:has(img[alt="Isaac ROS"]) img { + width: 100% !important; + height: 160px !important; + object-fit: cover !important; + display: block !important; + margin: 0 !important; + padding: 0 !important; +} + /* Local contents box (e.g. "Steps"): title and list must be readable (nav.contents from theme) */ nav.contents .topic-title, nav.contents.local .topic-title, @@ -265,3 +285,14 @@ nav.contents.local .simple a, nav.contents ul a { color: var(--pst-color-text-base) !important; } + +/* Disable click-to-zoom on figure images (add class "no-image-zoom" to the figure directive) */ +figure a.no-image-zoom:has(> img) { + pointer-events: none; + cursor: default; +} + +/* Lighter text for trademark notice (theme muted color; works in light and dark mode) */ +.trademark-notice { + color: var(--pst-color-text-muted) !important; +} diff --git a/docs/source/_static/hardware-req-01.svg b/docs/source/_static/hardware-req-01.svg new file mode 100644 index 000000000..04ea6e154 --- /dev/null +++ b/docs/source/_static/hardware-req-01.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/hardware-req-02.svg b/docs/source/_static/hardware-req-02.svg new file mode 100644 index 000000000..baebe3b92 --- /dev/null +++ b/docs/source/_static/hardware-req-02.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/hardware-req-03.svg b/docs/source/_static/hardware-req-03.svg new file mode 100644 index 000000000..7b3321020 --- /dev/null +++ b/docs/source/_static/hardware-req-03.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/hardware-req-04.svg b/docs/source/_static/hardware-req-04.svg new file mode 100644 index 000000000..f526be945 --- /dev/null +++ b/docs/source/_static/hardware-req-04.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/isaac-teleop-architecture.svg b/docs/source/_static/isaac-teleop-architecture.svg index 86178e1cf..08baab6a9 100644 --- a/docs/source/_static/isaac-teleop-architecture.svg +++ b/docs/source/_static/isaac-teleop-architecture.svg @@ -1 +1 @@ - + diff --git a/docs/source/_static/isaaclab.jpg b/docs/source/_static/isaaclab.jpg new file mode 100644 index 000000000..16072e704 Binary files /dev/null and b/docs/source/_static/isaaclab.jpg differ diff --git a/docs/source/_static/isaacros.png b/docs/source/_static/isaacros.png new file mode 100644 index 000000000..128c6a6bb Binary files /dev/null and b/docs/source/_static/isaacros.png differ diff --git a/docs/source/getting_started/build_from_source.rst b/docs/source/getting_started/build_from_source.rst index cf5d5c5db..334cb5f57 100644 --- a/docs/source/getting_started/build_from_source.rst +++ b/docs/source/getting_started/build_from_source.rst @@ -32,7 +32,7 @@ the list of dependencies. On **Ubuntu**, install build tools and clang-format: .. code-block:: bash sudo apt-get update - sudo apt-get install -y build-essential cmake libx11-dev clang-format-14 ccache + sudo apt-get install -y build-essential cmake libx11-dev clang-format-14 ccache patchelf Our build system uses `uv`_ for Python version and dependency management. Install `uv`_ if not already installed: @@ -91,7 +91,7 @@ From the project root: .. code-block:: bash cmake -B build - cmake --build build + cmake --build build --parallel cmake --install build This will: diff --git a/docs/source/getting_started/quick_start.rst b/docs/source/getting_started/quick_start.rst index cdc032c65..cb4cdb357 100644 --- a/docs/source/getting_started/quick_start.rst +++ b/docs/source/getting_started/quick_start.rst @@ -33,7 +33,7 @@ In a new terminal, install the package from PyPI (or from a local wheel if you b .. code-block:: bash # From PyPI - uv pip install isaacteleop[cloudxr,retargeters]~=1.0 --extra-index-url https://pypi.nvidia.com + pip install isaacteleop[cloudxr,retargeters]~=1.0.0 --extra-index-url https://pypi.nvidia.com Instead of installing the package from PyPI, you can build from source and install the local wheel. See :doc:`build_from_source` for more details. @@ -302,8 +302,46 @@ The example runs for 20 seconds and then exits. To try other examples, see Next steps ---------- -- `Teleoperation and Imitation Learning with Isaac Lab Mimic`_ — learn how to use CloudXR to teleoperate a simulated - robot in Isaac Lab +.. grid:: 2 + :gutter: 3 + + .. grid-item-card:: + + .. image:: ../_static/isaaclab.jpg + :alt: Isaac Lab + + ^^^^^^^^^^^^^ + + **Teleoperation in Isaac Lab** + + Follow instructions in `Teleoperation and Imitation Learning with Isaac Lab Mimic`_ to know + more about how to collect demonstrations with Isaac Lab and how to augment them with Isaac + Lab Mimic and train imitation learning policies. + + If you are new to Isaac Lab, follow instructions in `Isaac Lab Quick Start`_ to get started. + + .. grid-item-card:: + + .. image:: ../_static/isaacros.png + :alt: Isaac ROS + + ^^^^^^^^^^^^^ + + **Teleoperation with Isaac ROS** + + Check out the :code-dir:`examples/teleop_ros2/` directory for an example on how to make a + ROS 2 message publisher using Isaac Teleop. + + We are also working on a Unitree G1-based end-to-end teleoperation, data collection, and + imitation learning solution for ROS2 in an upcoming `Isaac ROS`_ release. Stay tuned! + + .. rst-class:: trademark-notice + + *ROS is a trademark of Open Robotics.* + +More Information +---------------- + - :doc:`teleop_session` — learn how ``TeleopSession`` works and how to build custom retargeting pipelines - :doc:`build_from_source` — build the C++ core, Python bindings, and plugins @@ -315,6 +353,7 @@ Next steps .. _`CloudXR documentation`: https://docs.nvidia.com/cloudxr-sdk/latest/index.html .. _`CloudXR.js documentation`: https://docs.nvidia.com/cloudxr-sdk/latest/usr_guide/cloudxr_js/index.html .. _`Isaac XR Teleop Sample Client for Apple Vision Pro`: https://github.com/isaac-sim/isaac-xr-teleop-sample-client-apple -.. _`Isaac Lab Quick Start`: https://isaac-sim.github.io/IsaacLab/main/source/setup/quickstart.html -.. _`Teleoperation and Imitation Learning with Isaac Lab Mimic`: https://isaac-sim.github.io/IsaacLab/main/source/overview/imitation-learning/teleop_imitation.html#teleoperation-imitation-learning +.. _`Isaac Lab Quick Start`: https://isaac-sim.github.io/IsaacLab/develop/source/setup/quickstart.html +.. _`Teleoperation and Imitation Learning with Isaac Lab Mimic`: https://isaac-sim.github.io/IsaacLab/develop/source/overview/imitation-learning/teleop_imitation.html#teleoperation-imitation-learning .. _`CloudXR network setup`: https://docs.nvidia.com/cloudxr-sdk/latest/requirement/network_setup.html#ports-and-firewalls +.. _`Isaac ROS`: https://nvidia-isaac-ros.github.io diff --git a/docs/source/index.rst b/docs/source/index.rst index 110375480..46f91dfaf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -34,6 +34,7 @@ Table of Contents :caption: Overview :titlesonly: + Isaac Teleop overview/architecture overview/ecosystem diff --git a/docs/source/overview/ecosystem.rst b/docs/source/overview/ecosystem.rst index fe474c533..9a3b0161f 100644 --- a/docs/source/overview/ecosystem.rst +++ b/docs/source/overview/ecosystem.rst @@ -54,18 +54,19 @@ see :ref:`device-interface-device-plugin` for details. - Hand tracking (26 joints), spatial controllers - `Isaac XR Teleop Sample Client`_ (visionOS app) - Build from source; see :ref:`Connect Apple Vision Pro ` - * - Meta Quest 3 + * - Meta Quest 2/3/3S - Motion controllers (triggers, thumbsticks, squeeze), hand tracking - `Isaac Teleop Web Client`_ (browser) - See :ref:`Connect Quest and Pico ` * - Pico 4 Ultra - Motion controllers, hand tracking - `Isaac Teleop Web Client`_ (browser) - - Requires Pico OS 15.4.4U+ + - Requires Pico OS 15.4.4U or newer * - `Pico Motion Tracker`_ - Full body tracking - `Isaac Teleop Web Client`_ (browser) - - Requires Pico OS 15.4.4U+ + - | Requires Pico OS 15.4.4U or newer + | Requires Pico Browser 4.0.40 or newer (Enterprise enabled) In addition to the fully integrated XR headsets, Isaac Teleop also supports standalone input devices. Those devices are typically directly connected to the workstation where the Isaac Teleop @@ -95,30 +96,46 @@ The following input devices and device categories are planned for support in the .. list-table:: Planned Input Devices :header-rows: 1 - :widths: 20 25 25 15 + :widths: 20 25 25 25 * - Device - Input Modes - Client / Connection - - ETA - * - Keyboard and Mouse - - Keyboard and mouse input - - CLI tool - - Planned - * - Master Manipulators - - Gello, Haply, JoyLo etc. - - CLI tool - - Planned - * - Exoskeletons - - TBA - - TBA - - TBA + - Status + * - JoyLo + - Master Manipulators + - CLI tool with USB connection + - Planning, see `#272 `_ + * - Gello + - Master Manipulators + - CLI tool with USB connection + - Planning, see `#273 `_ + * - Haply + - Master Manipulators + - CLI tool with USB connection + - Planning, see `#274 `_ + * - SO-101 + - Master Manipulators + - CLI tool with USB connection + - Planning, see `#275 `_ + * - 3D Space Mouse + - HID input + - CLI tool with USB connection + - Planning, see `#276 `_ + * - Gamepad + - HID input + - CLI tool with USB/Bluetooth connection + - Planning, see `#277 `_ + * - Keyboard + - HID input + - CLI tool with USB/Bluetooth connection + - Planning, see `#278 `_ Targeted Robotics Embodiments ----------------------------- - Retarget the standardized device outputs to different embodiments. -- `Reference retargeter implementations `_, +- `Reference retargeter implementations `_, including popular embodiments such as Unitree G1. - `Retargeter tuning UI `_ to facilitate live retargeter tuning. diff --git a/docs/source/references/license.rst b/docs/source/references/license.rst index 613a5d75e..247bdcb0e 100644 --- a/docs/source/references/license.rst +++ b/docs/source/references/license.rst @@ -15,6 +15,7 @@ Known exceptions `MIT license `_ or `Boost Software License 1.0 `_. - CloudXR SDK is NVIDIA's proprietary software, licensed under the `NVIDIA CloudXR License `_. +- The documentation is built using `NVIDIA Sphinx Theme `_. License headers --------------- diff --git a/docs/source/references/requirements.rst b/docs/source/references/requirements.rst index ebf2922ca..3dc511219 100644 --- a/docs/source/references/requirements.rst +++ b/docs/source/references/requirements.rst @@ -1,28 +1,112 @@ System Requirements =================== +Isaac Teleop can be used for teleoperation of robots, data collection, and human-centric data +collection. The hardware & software requirements are different for each use case. -Hardware requirements ----------------------- +Teleoperation to Robots with Input Devices +------------------------------------------ -**Minimum** +.. figure:: ../_static/hardware-req-02.svg + :width: 75% + :alt: Hardware requirements for teleoperation with extra input devices + :class: no-image-zoom -For real robot teleop & data collection: +When using extra input devices (such as Manus Gloves, Logitech Rudder Pedals, OAK-D Camera, etc.), +Isaac Teleop needs to be **run on a laptop or workstation**, that is connected to the robot via +network. The minimum requirements for the laptop/workstation are: -- **CPU**: x86 or ARM (Jetson Thor) -- **GPU**: NVIDIA GPU required +.. list-table:: + :widths: 30 70 + :header-rows: 1 -**Simulation Ready** + * - Component + - Requirement + * - CPU / GPU + - x86_64 [#jetson-nano]_, NVIDIA GPU required + * - OS + - Ubuntu 22.04 or 24.04 + * - Python + - 3.10 / 3.11 / 3.12 + * - CUDA + - 12.8 or newer + * - NVIDIA Driver + - 580.95.05 or newer -For running simulation with Isaac Sim and Isaac Lab with RTX rendering: +Teleoperation with Isaac Sim and Isaac Lab +------------------------------------------- -- **CPU**: AMD Ryzen Threadripper 7960x (Recommended) -- **GPU**: 1x RTX 6000 Pro (Blackwell) or 2x RTX 6000 (Ada) +.. figure:: ../_static/hardware-req-03.svg + :width: 75% + :alt: Hardware requirements for teleoperation with Isaac Sim and Isaac Lab + :class: no-image-zoom -Software Requirements ---------------------- +For running simulation with Isaac Sim and Isaac Lab with RTX rendering, the CloudXR server and +Isaac Teleop sessions need to be **run on the same workstation with Isaac Sim and Isaac Lab**. The +workstation's system requirements are driven by Isaac Sim and Isaac Lab [#isaacsim-req]_. -- **OS**: Ubuntu 22.04 or 24.04 -- **Python**: 3.10 or newer (version configured in root `CMakeLists.txt`) -- **CUDA**: 12.8 (Recommended) -- **NVIDIA Driver**: 580.95.05 (Recommended) +The recommended workstation configuration for Sim-based Teleop is: + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Component + - Requirement + * - CPU + - AMD Ryzen Threadripper 7960x + * - GPU + - 1x RTX 6000 Pro (Blackwell) or 2x RTX 6000 (Ada) + * - OS + - Ubuntu 22.04 [#isaaclab-req]_ + * - Python + - 3.12 [#isaaclab-req]_ + * - CUDA + - 12.8 or newer + * - NVIDIA Driver + - 580.95.05 or newer + +If you are only using XR headsets for teleoperation, you can host the workstation in the cloud. +See `Isaac Lab Cloud Deployment `_ +for more details. + +Human-centric Data Collection +------------------------------ + +Isaac Teleop can also be used for human-centric data collection without a robot or simulator in the +loop. In this case, Isaac Teleop needs to be **run on a laptop or workstation**, that is connected +to device peripherals. The minimum requirements for the laptop/workstation are: + +.. figure:: ../_static/hardware-req-04.svg + :width: 75% + :alt: Hardware requirements for human-centric data collection + :class: no-image-zoom + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Component + - Requirement + * - CPU / GPU + - x86_64 [#jetson-nano]_, NVIDIA GPU required + * - OS + - Ubuntu 22.04 or 24.04 + * - Python + - 3.10 / 3.11 / 3.12 + * - CUDA + - 12.8 or newer + * - NVIDIA Driver + - 580.95.05 or newer + +.. rubric:: Footnotes + +.. [#jetson-nano] Jetson Nano support is being considered. See + `#271 `_ + for more details. Please up-vote and/or comment on the issue if you are interested in this feature. + +.. [#isaacsim-req] Please refer to `Isaac Sim System Requirements `_ + for more details. + +.. [#isaaclab-req] Please refer to `Isaac Lab System Requirements `_ + for more details. diff --git a/examples/oxr/python/pyproject.toml b/examples/oxr/python/pyproject.toml index 8282ece4a..0933ea356 100644 --- a/examples/oxr/python/pyproject.toml +++ b/examples/oxr/python/pyproject.toml @@ -8,5 +8,5 @@ description = "OpenXR Tracking Python Examples" requires-python = ">=3.10,<3.13" dependencies = [ "isaacteleop", - "numpy>=1.22.2", + "numpy>=1.23.0", ] diff --git a/examples/retargeting/python/pyproject.toml b/examples/retargeting/python/pyproject.toml index eaa5a041a..8037a98dc 100644 --- a/examples/retargeting/python/pyproject.toml +++ b/examples/retargeting/python/pyproject.toml @@ -8,5 +8,5 @@ description = "Retargeting Engine Python Examples" requires-python = ">=3.10,<3.13" dependencies = [ "isaacteleop[ui,retargeters-lite]", - "numpy>=1.22.2", + "numpy>=1.23.0", ] diff --git a/examples/teleop_session_manager/python/pyproject.toml b/examples/teleop_session_manager/python/pyproject.toml index 88cd48476..8fd4ce79b 100644 --- a/examples/teleop_session_manager/python/pyproject.toml +++ b/examples/teleop_session_manager/python/pyproject.toml @@ -8,5 +8,5 @@ description = "TeleopUtils Python Examples" requires-python = ">=3.10,<3.13" dependencies = [ "isaacteleop[ui]", - "numpy>=1.22.2", + "numpy>=1.23.0", ] diff --git a/scripts/download_cloudxr_runtime_sdk.sh b/scripts/download_cloudxr_runtime_sdk.sh index 31e835d34..5e0dca933 100755 --- a/scripts/download_cloudxr_runtime_sdk.sh +++ b/scripts/download_cloudxr_runtime_sdk.sh @@ -1,15 +1,16 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Downloads the CloudXR Runtime SDK if not already present using NGC. # The SDK tarball (CloudXR--Linux--sdk.tar.gz) is placed in deps/cloudxr/ # for use by Dockerfile.runtime. # -# Two ways to obtain the SDK: -# 1) NGC (default): requires ngc CLI; downloads from public or private NGC. -# 2) Local tarball: place CloudXR--Linux--sdk.tar.gz in deps/cloudxr/. +# Three ways to obtain the SDK (tried in order): +# 1) Local tarball: place CloudXR--Linux--sdk.tar.gz in deps/cloudxr/. +# 2) Public NGC: downloads via wget from the public NGC resource API (no NGC CLI needed). +# 3) Private NGC: requires ngc CLI; downloads from private NGC org. set -Eeuo pipefail @@ -78,18 +79,16 @@ install_from_local_tarball() { } # ----------------------------------------------------------------------------- -# NGC: resource path and download/install logic -# Resource: nvidia/cloudxr-runtime:${CXR_RUNTIME_SDK_VERSION} +# Public NGC: download via wget from the public NGC resource API +# Resource: nvidia/cloudxr-runtime-for-isaac-teleop/${CXR_RUNTIME_SDK_VERSION} # ----------------------------------------------------------------------------- install_from_public_ngc() { - local SDK_RESOURCE="nvidia/cloudxr-runtime:${CXR_RUNTIME_SDK_VERSION}" - local SDK_DOWNLOAD_DIR="$GIT_ROOT/deps/cloudxr/.sdk-download" + local NGC_URL="https://api.ngc.nvidia.com/v2/resources/org/nvidia/cloudxr-runtime-for-isaac-teleop/${CXR_RUNTIME_SDK_VERSION}/files?redirect=true&path=${SDK_FILE}" - if ! command -v ngc &> /dev/null; then - echo -e "${RED}Error: NGC CLI not found. Please install it first.${NC}" + if ! command -v wget &> /dev/null; then + echo -e "${RED}Error: wget not found. Please install it first.${NC}" echo -e "To use a local SDK instead, place $SDK_FILE in deps/cloudxr/" - echo -e "Visit: https://ngc.nvidia.com/setup/installers/cli" - exit 1 + return 1 fi echo -e "${GREEN}========================================${NC}" @@ -97,37 +96,17 @@ install_from_public_ngc() { echo -e "${GREEN}========================================${NC}" echo "" - mkdir -p "$SDK_DOWNLOAD_DIR" + mkdir -p "$CXR_DEPLOYMENT_DIR" - echo -e "${YELLOW}[1/2] Downloading CloudXR Runtime SDK from NGC...${NC}" - cd "$SDK_DOWNLOAD_DIR" - if ! ngc registry resource download-version \ - --team no-team \ - --file "$SDK_FILE" \ - "$SDK_RESOURCE"; then + echo -e "${YELLOW}Downloading CloudXR Runtime SDK from NGC...${NC}" + if ! wget --content-disposition \ + --output-document "$CXR_DEPLOYMENT_DIR/$SDK_FILE" \ + "$NGC_URL"; then echo -e "${RED}Error: Failed to download CloudXR Runtime SDK from NGC${NC}" + rm -f "$CXR_DEPLOYMENT_DIR/$SDK_FILE" return 1 fi - local DOWNLOADED_DIR="$(basename "$(find . -mindepth 1 -maxdepth 1 -type d -name 'cloudxr-runtime_v*' -print -quit)")" - if [ -z "$DOWNLOADED_DIR" ]; then - echo -e "${RED}Error: Failed to find downloaded SDK directory${NC}" - return 1 - fi - - echo -e "${GREEN}✓ CloudXR Runtime SDK downloaded${NC}" - echo "" - - echo -e "${YELLOW}[2/2] Installing SDK to deps/cloudxr/...${NC}" - cd "$SDK_DOWNLOAD_DIR/$DOWNLOADED_DIR" - - if [ ! -f "$SDK_FILE" ]; then - echo -e "${RED}Error: $SDK_FILE not found in downloaded SDK${NC}" - return 1 - fi - - mv "$SDK_FILE" "$CXR_DEPLOYMENT_DIR/" - echo -e "${GREEN}✓ CloudXR Runtime SDK installed successfully${NC}" echo "" } diff --git a/scripts/download_cloudxr_sdk.sh b/scripts/download_cloudxr_sdk.sh index 66a622175..468471c89 100755 --- a/scripts/download_cloudxr_sdk.sh +++ b/scripts/download_cloudxr_sdk.sh @@ -1,16 +1,17 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Downloads the CloudXR Web Client EA SDK if not already present using NGC. +# Downloads the CloudXR Web Client SDK if not already present using NGC. # The SDK is extracted to deps/cloudxr/cloudxr-web-sdk-${CXR_WEB_SDK_VERSION}/ for use by Dockerfile.web-app. # -# Two ways to obtain the SDK: -# 1) NGC (default): requires ngc CLI; downloads nvidia/cloudxr-js-early-access:${CXR_WEB_SDK_VERSION}. -# 2) Local tarball: place cloudxr-web-sdk-${CXR_WEB_SDK_VERSION}.tar.gz in deps/cloudxr/. +# Three ways to obtain the SDK (tried in order): +# 1) Local tarball: place cloudxr-web-sdk-${CXR_WEB_SDK_VERSION}.tar.gz in deps/cloudxr/. # The tarball must extract to the same layout as the NGC release: root must contain # isaac/ and nvidia-cloudxr-${CXR_WEB_SDK_VERSION}.tgz (optionally inside a single top-level directory). +# 2) Public NGC: downloads via wget from the public NGC resource API (no NGC CLI needed). +# 3) Private NGC: requires ngc CLI; downloads from private NGC org. set -Eeuo pipefail @@ -88,56 +89,34 @@ install_from_local_tarball() { } # ----------------------------------------------------------------------------- -# NGC: resource path and download/install logic -# Resource: nvidia/cloudxr-js-early-access:${CXR_WEB_SDK_VERSION} +# Public NGC: download via wget from the public NGC resource API +# Resource: nvidia/cloudxr-js/${CXR_WEB_SDK_VERSION} # ----------------------------------------------------------------------------- install_from_public_ngc() { - local SDK_RESOURCE="nvidia/cloudxr-js-early-access:${CXR_WEB_SDK_VERSION}" - local SDK_DOWNLOAD_DIR="$GIT_ROOT/deps/cloudxr/.sdk-download" + local NGC_URL="https://api.ngc.nvidia.com/v2/resources/org/nvidia/cloudxr-js/${CXR_WEB_SDK_VERSION}/files?redirect=true&path=${SDK_FILE}" - if ! command -v ngc &> /dev/null; then - echo -e "${RED}Error: NGC CLI not found. Please install it first.${NC}" + if ! command -v wget &> /dev/null; then + echo -e "${RED}Error: wget not found. Please install it first.${NC}" echo -e "To use a local SDK instead, place $SDK_FILE in deps/cloudxr/" - echo -e "Visit: https://ngc.nvidia.com/setup/installers/cli" - exit 1 - fi - - echo -e "${GREEN}========================================${NC}" - echo -e "${GREEN}Downloading CloudXR Web Client EA SDK${NC}" - echo -e "${GREEN}========================================${NC}" - echo "" - - mkdir -p "$SDK_DOWNLOAD_DIR" - - echo -e "${YELLOW}[1/3] Downloading CloudXR Web SDK from NGC...${NC}" - cd "$SDK_DOWNLOAD_DIR" - if ! ngc registry resource download-version \ - --team no-team \ - --file "$SDK_FILE" \ - "$SDK_RESOURCE"; then - echo -e "${RED}Error: Failed to download CloudXR Web SDK from NGC${NC}" return 1 fi - local DOWNLOADED_DIR="$(basename "$(find . -mindepth 1 -maxdepth 1 -type d -name 'cloudxr-js-early-access_v*' -print -quit)")" - if [ -z "$DOWNLOADED_DIR" ]; then - echo -e "${RED}Error: Failed to find downloaded SDK directory${NC}" - return 1 - fi - - echo -e "${GREEN}✓ CloudXR Web SDK downloaded${NC}" + echo -e "${GREEN}==================================${NC}" + echo -e "${GREEN}Downloading CloudXR Web Client SDK${NC}" + echo -e "${GREEN}==================================${NC}" echo "" - echo -e "${YELLOW}[2/2] Installing SDK to deps/cloudxr/...${NC}" - cd "$SDK_DOWNLOAD_DIR/$DOWNLOADED_DIR" + mkdir -p "$CXR_DEPLOYMENT_DIR" - if [ ! -f "$SDK_FILE" ]; then - echo -e "${RED}Error: $SDK_FILE not found in downloaded SDK${NC}" + echo -e "${YELLOW}Downloading CloudXR Web SDK from NGC...${NC}" + if ! wget --content-disposition \ + --output-document "$CXR_DEPLOYMENT_DIR/$SDK_FILE" \ + "$NGC_URL"; then + echo -e "${RED}Error: Failed to download CloudXR Web SDK from NGC${NC}" + rm -f "$CXR_DEPLOYMENT_DIR/$SDK_FILE" return 1 fi - mv "$SDK_FILE" "$CXR_DEPLOYMENT_DIR/" - echo -e "${GREEN}✓ CloudXR Web SDK installed successfully${NC}" echo "" } diff --git a/scripts/run_cloudxr_via_docker.sh b/scripts/run_cloudxr_via_docker.sh index 0b7b88669..1a4ff40f9 100755 --- a/scripts/run_cloudxr_via_docker.sh +++ b/scripts/run_cloudxr_via_docker.sh @@ -15,6 +15,9 @@ source scripts/setup_cloudxr_env.sh # Check CloudXR EULA acceptance ./scripts/check_cloudxr_eula.sh || exit 1 +# Download CloudXR Runtime SDK if not already present +./scripts/download_cloudxr_runtime_sdk.sh || exit 1 + # Download CloudXR Web SDK if not already present ./scripts/download_cloudxr_sdk.sh || exit 1 @@ -31,6 +34,5 @@ fi $COMPOSE_CMD \ --env-file "$ENV_DEFAULT" \ --env-file "$ENV_LOCAL" \ - -f deps/cloudxr/docker-compose.runtime.yaml \ -f deps/cloudxr/docker-compose.yaml \ up --build diff --git a/src/core/cloudxr/python/CMakeLists.txt b/src/core/cloudxr/python/CMakeLists.txt index e4384e353..66075ab32 100644 --- a/src/core/cloudxr/python/CMakeLists.txt +++ b/src/core/cloudxr/python/CMakeLists.txt @@ -78,6 +78,24 @@ else() COMMENT "Extracting CloudXR SDK tarball into python_package/.../cloudxr/native/" ) add_dependencies(cloudxr_native_bundle cloudxr_native_dir) + + # libcloudxr.so in the SDK tarball carries a NEEDED entry for libssl.so.3 even + # though it never calls any OpenSSL functions. When Python (which bundles its own + # OpenSSL) loads libcloudxr.so, the loader pulls in a second, incompatible + # libssl.so.3 and NVST crashes. Strip the spurious dependency at build time. + # A future SDK built with -Wl,--as-needed will make this step unnecessary. + find_program(PATCHELF_EXECUTABLE patchelf) + if(PATCHELF_EXECUTABLE) + add_custom_target(cloudxr_patchelf ALL + COMMAND ${PATCHELF_EXECUTABLE} --remove-needed libssl.so.3 + "${CLOUDXR_NATIVE_DIR}/libcloudxr.so" + COMMENT "Stripping spurious libssl.so.3 dependency from libcloudxr.so" + ) + add_dependencies(cloudxr_patchelf cloudxr_native_bundle) + else() + message(WARNING "patchelf not found; libcloudxr.so will retain its libssl.so.3 " + "dependency. Install patchelf to avoid OpenSSL symbol conflicts.") + endif() endif() # Copy CloudXR Python module to package structure (always; pure Python is cross-platform) @@ -94,7 +112,11 @@ add_custom_target(cloudxr_python ALL COMMENT "Copying cloudxr Python files to package structure" ) if(CLOUDXR_NATIVE_AVAILABLE) - add_dependencies(cloudxr_python cloudxr_native_bundle) + if(PATCHELF_EXECUTABLE) + add_dependencies(cloudxr_python cloudxr_patchelf) + else() + add_dependencies(cloudxr_python cloudxr_native_bundle) + endif() endif() install( diff --git a/src/core/cloudxr/python/runtime.py b/src/core/cloudxr/python/runtime.py index 4eb09316d..4cd0de8f5 100644 --- a/src/core/cloudxr/python/runtime.py +++ b/src/core/cloudxr/python/runtime.py @@ -190,7 +190,8 @@ def run() -> None: os.close(devnull_fd) lib_path = os.path.join(sdk_path, "libcloudxr.so") - lib = ctypes.CDLL(lib_path) + deepbind = getattr(os, "RTLD_DEEPBIND", 0) + lib = ctypes.CDLL(lib_path, mode=deepbind) svc = ctypes.c_void_p() # Signal handler must only call stop() after create() has run; avoid calling with null svc. state = {"service_created": False, "interrupted": False} diff --git a/src/core/deviceio/cpp/full_body_tracker_pico.cpp b/src/core/deviceio/cpp/full_body_tracker_pico.cpp index d1e88ef61..6e0a45ac5 100644 --- a/src/core/deviceio/cpp/full_body_tracker_pico.cpp +++ b/src/core/deviceio/cpp/full_body_tracker_pico.cpp @@ -75,7 +75,9 @@ FullBodyTrackerPicoImpl::FullBodyTrackerPicoImpl(const OpenXRSessionHandles& han } if (!body_tracking_props.supportsBodyTracking) { - throw std::runtime_error("Body tracking not supported by this system"); + std::cerr << "[FullBodyTrackerPico] Body tracking not supported by this system, running in limp mode" + << std::endl; + return; } } else @@ -90,12 +92,6 @@ FullBodyTrackerPicoImpl::FullBodyTrackerPicoImpl(const OpenXRSessionHandles& han loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrLocateBodyJointsBD", reinterpret_cast(&pfn_locate_body_joints_)); - if (!pfn_create_body_tracker_ || !pfn_destroy_body_tracker_ || !pfn_locate_body_joints_) - { - throw std::runtime_error("Failed to get body tracking function pointers"); - } - - // Create body tracker for full body joints (24 joints) XrBodyTrackerCreateInfoBD create_info{ XR_TYPE_BODY_TRACKER_CREATE_INFO_BD }; create_info.next = nullptr; create_info.jointSet = XR_BODY_JOINT_SET_FULL_BODY_JOINTS_BD; @@ -111,11 +107,9 @@ FullBodyTrackerPicoImpl::FullBodyTrackerPicoImpl(const OpenXRSessionHandles& han FullBodyTrackerPicoImpl::~FullBodyTrackerPicoImpl() { - // pfn_destroy_body_tracker_ should never be null (verified in constructor) - assert(pfn_destroy_body_tracker_ != nullptr && "pfn_destroy_body_tracker must not be null"); - if (body_tracker_ != XR_NULL_HANDLE) { + assert(pfn_destroy_body_tracker_ != nullptr && "pfn_destroy_body_tracker must not be null"); pfn_destroy_body_tracker_(body_tracker_); body_tracker_ = XR_NULL_HANDLE; } @@ -125,6 +119,12 @@ bool FullBodyTrackerPicoImpl::update(XrTime time) { last_update_time_ = time; + if (body_tracker_ == XR_NULL_HANDLE) + { + tracked_.data.reset(); + return true; + } + XrBodyJointsLocateInfoBD locate_info{ XR_TYPE_BODY_JOINTS_LOCATE_INFO_BD }; locate_info.next = nullptr; locate_info.baseSpace = base_space_; diff --git a/src/core/python/requirements.txt b/src/core/python/requirements.txt index 4e7c4369d..3d6b003b9 100644 --- a/src/core/python/requirements.txt +++ b/src/core/python/requirements.txt @@ -1 +1 @@ -numpy>=1.22.2 +numpy>=1.23.0 diff --git a/src/core/retargeting_engine/python/utilities/controller_transform.py b/src/core/retargeting_engine/python/utilities/controller_transform.py index 5092bba26..d8a78de1f 100644 --- a/src/core/retargeting_engine/python/utilities/controller_transform.py +++ b/src/core/retargeting_engine/python/utilities/controller_transform.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ @@ -11,8 +11,6 @@ provided by a TransformSource node. """ -import copy - import numpy as np from ..interface.base_retargeter import BaseRetargeter @@ -21,9 +19,10 @@ from ..interface.tensor_group_type import OptionalType from ..tensor_types import ControllerInput, ControllerInputIndex, TransformMatrix from .transform_utils import ( + _copy_tensor_group_slots_from_dlpack_input, decompose_transform, - transform_position, transform_orientation, + transform_position, ) @@ -120,24 +119,13 @@ def _transform_controller( translation: np.ndarray, ) -> None: """Apply the transform to a single controller's data.""" - # Deep-copy all fields from input to output (avoid aliasing) - for i in range(len(inp)): - out[i] = copy.deepcopy(inp[i]) + _copy_tensor_group_slots_from_dlpack_input(inp, out) - # Transform pose fields in-place on the output buffers transform_position( - np.from_dlpack(out[ControllerInputIndex.GRIP_POSITION]), - rotation, - translation, - ) - transform_orientation( - np.from_dlpack(out[ControllerInputIndex.GRIP_ORIENTATION]), rotation + out[ControllerInputIndex.GRIP_POSITION], rotation, translation ) + transform_orientation(out[ControllerInputIndex.GRIP_ORIENTATION], rotation) transform_position( - np.from_dlpack(out[ControllerInputIndex.AIM_POSITION]), - rotation, - translation, - ) - transform_orientation( - np.from_dlpack(out[ControllerInputIndex.AIM_ORIENTATION]), rotation + out[ControllerInputIndex.AIM_POSITION], rotation, translation ) + transform_orientation(out[ControllerInputIndex.AIM_ORIENTATION], rotation) diff --git a/src/core/retargeting_engine/python/utilities/hand_transform.py b/src/core/retargeting_engine/python/utilities/hand_transform.py index af6c8710e..757ab0a5d 100644 --- a/src/core/retargeting_engine/python/utilities/hand_transform.py +++ b/src/core/retargeting_engine/python/utilities/hand_transform.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ @@ -15,8 +15,6 @@ the wrist while leaving other joints untransformed would break the hand skeleton. """ -import copy - import numpy as np from ..interface.base_retargeter import BaseRetargeter @@ -25,9 +23,10 @@ from ..interface.tensor_group_type import OptionalType from ..tensor_types import HandInput, HandInputIndex, TransformMatrix from .transform_utils import ( + _copy_tensor_group_slots_from_dlpack_input, decompose_transform, - transform_positions_batch, transform_orientations_batch, + transform_positions_batch, ) @@ -126,14 +125,9 @@ def _transform_hand( translation: np.ndarray, ) -> None: """Apply the transform to a single hand's joint data.""" - # Deep-copy all fields from input to output (avoid aliasing) - for i in range(len(inp)): - out[i] = copy.deepcopy(inp[i]) + _copy_tensor_group_slots_from_dlpack_input(inp, out) - # Transform pose fields in-place on the output buffers transform_positions_batch( - np.from_dlpack(out[HandInputIndex.JOINT_POSITIONS]), rotation, translation - ) - transform_orientations_batch( - np.from_dlpack(out[HandInputIndex.JOINT_ORIENTATIONS]), rotation + out[HandInputIndex.JOINT_POSITIONS], rotation, translation ) + transform_orientations_batch(out[HandInputIndex.JOINT_ORIENTATIONS], rotation) diff --git a/src/core/retargeting_engine/python/utilities/head_transform.py b/src/core/retargeting_engine/python/utilities/head_transform.py index 22f97598e..12ddb5553 100644 --- a/src/core/retargeting_engine/python/utilities/head_transform.py +++ b/src/core/retargeting_engine/python/utilities/head_transform.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ @@ -11,8 +11,6 @@ provided by a TransformSource node. """ -import copy - import numpy as np from ..interface.base_retargeter import BaseRetargeter @@ -20,9 +18,10 @@ from ..interface.tensor_group_type import OptionalType from ..tensor_types import HeadPose, HeadPoseIndex, TransformMatrix from .transform_utils import ( + _copy_tensor_group_slots_from_dlpack_input, decompose_transform, - transform_position, transform_orientation, + transform_position, ) @@ -103,12 +102,7 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N inp = inputs["head"] out = outputs["head"] - # Deep-copy all fields from input to output (avoid aliasing) - for i in range(len(inp)): - out[i] = copy.deepcopy(inp[i]) + _copy_tensor_group_slots_from_dlpack_input(inp, out) - # Transform pose fields in-place on the output buffers - transform_position( - np.from_dlpack(out[HeadPoseIndex.POSITION]), rotation, translation - ) - transform_orientation(np.from_dlpack(out[HeadPoseIndex.ORIENTATION]), rotation) + transform_position(out[HeadPoseIndex.POSITION], rotation, translation) + transform_orientation(out[HeadPoseIndex.ORIENTATION], rotation) diff --git a/src/core/retargeting_engine/python/utilities/transform_utils.py b/src/core/retargeting_engine/python/utilities/transform_utils.py index d373b7c6d..d157fc6a3 100644 --- a/src/core/retargeting_engine/python/utilities/transform_utils.py +++ b/src/core/retargeting_engine/python/utilities/transform_utils.py @@ -13,8 +13,13 @@ hands_source.py, controllers_source.py). """ +import copy +from typing import Tuple, Union + import numpy as np -from typing import Tuple + +from ..interface.tensor_group import OptionalTensorGroup, TensorGroup +from ..tensor_types import NDArrayType def validate_transform_matrix(matrix: np.ndarray) -> np.ndarray: @@ -44,6 +49,19 @@ def validate_transform_matrix(matrix: np.ndarray) -> np.ndarray: return matrix +def _copy_tensor_group_slots_from_dlpack_input( + inp: Union[OptionalTensorGroup, TensorGroup], + out: Union[OptionalTensorGroup, TensorGroup], +) -> None: + """Copy slots *inp* → *out*; ndarray slots via ``from_dlpack`` then ``.copy()``.""" + + for i, tensor_type in enumerate(inp.group_type.types): + if isinstance(tensor_type, NDArrayType): + out[i] = np.from_dlpack(inp[i]).copy() + else: + out[i] = copy.deepcopy(inp[i]) + + def decompose_transform(matrix: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: """ Decompose a 4x4 homogeneous transform into rotation and translation. diff --git a/src/core/retargeting_engine_tests/python/test_transform_numpy_versions.py b/src/core/retargeting_engine_tests/python/test_transform_numpy_versions.py new file mode 100644 index 000000000..b9d78e51e --- /dev/null +++ b/src/core/retargeting_engine_tests/python/test_transform_numpy_versions.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Run pose-transform smoke under isolated NumPy versions (1.23.x and 2.x). + +NumPy 1.23 adds the public ``numpy.from_dlpack`` API used by the transform path. +CTest sets ``PYTHONPATH`` to the built ``python_package`` tree; each test spawns +``uv run --no-project`` with a pinned NumPy. + +Python version comes from the **same interpreter as pytest** (CI matrix / +``ISAAC_TELEOP_PYTHON_VERSION``), not a hard-coded list. That keeps ``uv run`` +aligned with the wheel ABI under ``PYTHONPATH`` (native extensions match). + +The NumPy **1.23.5** pin is **skipped** on Python 3.12+ (no viable wheels / install +for that interpreter). NumPy **2.x** runs on every matrix Python. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +_SMOKE = Path(__file__).resolve().parent / "transform_numpy_version_smoke.py" + + +@pytest.mark.parametrize( + ("numpy_pin", "version_key"), + [ + ("numpy==1.23.5", "1.23"), + ("numpy>=2.0.0,<3", "2"), + ], +) +def test_head_transform_smoke_isolated_numpy(numpy_pin: str, version_key: str) -> None: + if shutil.which("uv") is None: + pytest.skip("uv not on PATH") + if "PYTHONPATH" not in os.environ: + pytest.skip( + "PYTHONPATH unset (run under CTest or point at build python_package/)" + ) + if not _SMOKE.is_file(): + pytest.fail(f"missing smoke script: {_SMOKE}") + + if version_key == "1.23" and sys.version_info >= (3, 12): + pytest.skip( + "numpy==1.23.5 is not supported on Python 3.12+ (no wheels; matrix uses 3.12/3.13)" + ) + + py = f"{sys.version_info.major}.{sys.version_info.minor}" + cmd = [ + "uv", + "run", + "--no-project", + "--python", + py, + "--with", + numpy_pin, + "python", + str(_SMOKE), + version_key, + ] + subprocess.run(cmd, check=True, env=os.environ.copy()) diff --git a/src/core/retargeting_engine_tests/python/test_transforms.py b/src/core/retargeting_engine_tests/python/test_transforms.py index 9d1380049..f122d69cb 100644 --- a/src/core/retargeting_engine_tests/python/test_transforms.py +++ b/src/core/retargeting_engine_tests/python/test_transforms.py @@ -7,6 +7,7 @@ Covers: - transform_utils: validate, decompose, position/orientation transforms - HeadTransform, HandTransform, ControllerTransform retargeter nodes +- Optional inputs: propagation when absent, and absent/present toggling across calls """ import pytest @@ -952,3 +953,204 @@ def test_absent_output_raises_on_read(self): with pytest.raises(ValueError, match="absent"): _ = result["head"][HeadPoseIndex.POSITION] + + +class TestTransformOptionalNoneToggle: + """ + Optional inputs alternate absent/present across successive calls. + + Ensures no stale state: after ``None``, a later present input still gets + the correct transform, and vice versa. + """ + + def test_head_transform_absent_present_cycle(self) -> None: + node = HeadTransform("head_toggle") + xform_90 = _make_transform_input(_rotation_z_90()) + xform_id = _make_transform_input(_identity_4x4()) + absent = OptionalTensorGroup(HeadPose()) + + r1 = _run_retargeter(node, {"head": absent, "transform": xform_id}) + assert r1["head"].is_none + + active = TensorGroup(HeadPose()) + active[HeadPoseIndex.POSITION] = np.array([1.0, 0.0, 0.0], dtype=np.float32) + active[HeadPoseIndex.ORIENTATION] = np.array( + [0.0, 0.0, 0.0, 1.0], dtype=np.float32 + ) + active[HeadPoseIndex.IS_VALID] = True + + r2 = _run_retargeter(node, {"head": active, "transform": xform_90}) + assert not r2["head"].is_none + npt.assert_array_almost_equal( + np.from_dlpack(r2["head"][HeadPoseIndex.POSITION]), + [0.0, 1.0, 0.0], + decimal=5, + ) + + r3 = _run_retargeter(node, {"head": absent, "transform": xform_id}) + assert r3["head"].is_none + + r4 = _run_retargeter(node, {"head": active, "transform": xform_id}) + assert not r4["head"].is_none + npt.assert_array_almost_equal( + np.from_dlpack(r4["head"][HeadPoseIndex.POSITION]), + [1.0, 0.0, 0.0], + decimal=5, + ) + + def test_controller_transform_left_optional_toggles(self) -> None: + node = ControllerTransform("ctrl_toggle") + xform_90 = _make_transform_input(_rotation_z_90()) + xform_id = _make_transform_input(_identity_4x4()) + left_absent = OptionalTensorGroup(ControllerInput()) + id_quat = np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float32) + + def _active_left_grip_1_0_0() -> TensorGroup: + tg = TensorGroup(ControllerInput()) + tg[ControllerInputIndex.GRIP_POSITION] = np.array( + [1.0, 0.0, 0.0], dtype=np.float32 + ) + tg[ControllerInputIndex.GRIP_ORIENTATION] = id_quat.copy() + tg[ControllerInputIndex.GRIP_IS_VALID] = True + tg[ControllerInputIndex.AIM_POSITION] = np.zeros(3, dtype=np.float32) + tg[ControllerInputIndex.AIM_ORIENTATION] = id_quat.copy() + tg[ControllerInputIndex.AIM_IS_VALID] = True + tg[ControllerInputIndex.PRIMARY_CLICK] = 0.0 + tg[ControllerInputIndex.SECONDARY_CLICK] = 0.0 + tg[ControllerInputIndex.THUMBSTICK_CLICK] = 0.0 + tg[ControllerInputIndex.THUMBSTICK_X] = 0.0 + tg[ControllerInputIndex.THUMBSTICK_Y] = 0.0 + tg[ControllerInputIndex.SQUEEZE_VALUE] = 0.0 + tg[ControllerInputIndex.TRIGGER_VALUE] = 0.5 + return tg + + right = TensorGroup(ControllerInput()) + right[ControllerInputIndex.GRIP_POSITION] = np.array( + [1.0, 2.0, 3.0], dtype=np.float32 + ) + right[ControllerInputIndex.GRIP_ORIENTATION] = id_quat.copy() + right[ControllerInputIndex.GRIP_IS_VALID] = True + right[ControllerInputIndex.AIM_POSITION] = np.zeros(3, dtype=np.float32) + right[ControllerInputIndex.AIM_ORIENTATION] = id_quat.copy() + right[ControllerInputIndex.AIM_IS_VALID] = True + right[ControllerInputIndex.PRIMARY_CLICK] = 0.0 + right[ControllerInputIndex.SECONDARY_CLICK] = 0.0 + right[ControllerInputIndex.THUMBSTICK_CLICK] = 0.0 + right[ControllerInputIndex.THUMBSTICK_X] = 0.0 + right[ControllerInputIndex.THUMBSTICK_Y] = 0.0 + right[ControllerInputIndex.SQUEEZE_VALUE] = 0.0 + right[ControllerInputIndex.TRIGGER_VALUE] = 0.5 + + r1 = _run_retargeter( + node, + { + "controller_left": left_absent, + "controller_right": right, + "transform": xform_id, + }, + ) + assert r1["controller_left"].is_none + assert not r1["controller_right"].is_none + + left = _active_left_grip_1_0_0() + r2 = _run_retargeter( + node, + { + "controller_left": left, + "controller_right": right, + "transform": xform_90, + }, + ) + assert not r2["controller_left"].is_none + npt.assert_array_almost_equal( + np.from_dlpack(r2["controller_left"][ControllerInputIndex.GRIP_POSITION]), + [0.0, 1.0, 0.0], + decimal=5, + ) + + r3 = _run_retargeter( + node, + { + "controller_left": left_absent, + "controller_right": right, + "transform": xform_id, + }, + ) + assert r3["controller_left"].is_none + + r4 = _run_retargeter( + node, + { + "controller_left": _active_left_grip_1_0_0(), + "controller_right": right, + "transform": xform_id, + }, + ) + npt.assert_array_almost_equal( + np.from_dlpack(r4["controller_left"][ControllerInputIndex.GRIP_POSITION]), + [1.0, 0.0, 0.0], + decimal=5, + ) + + def test_hand_transform_left_optional_toggles(self) -> None: + node = HandTransform("hand_toggle") + xform_90 = _make_transform_input(_rotation_z_90()) + xform_id = _make_transform_input(_identity_4x4()) + left_absent = OptionalTensorGroup(HandInput()) + + def _active_left_joint0_1_0_0() -> TensorGroup: + tg = TensorGroup(HandInput()) + positions = np.zeros((NUM_HAND_JOINTS, 3), dtype=np.float32) + positions[0] = [1.0, 0.0, 0.0] + orientations = np.zeros((NUM_HAND_JOINTS, 4), dtype=np.float32) + orientations[:, 3] = 1.0 + tg[HandInputIndex.JOINT_POSITIONS] = positions + tg[HandInputIndex.JOINT_ORIENTATIONS] = orientations + tg[HandInputIndex.JOINT_RADII] = ( + np.ones(NUM_HAND_JOINTS, dtype=np.float32) * 0.01 + ) + tg[HandInputIndex.JOINT_VALID] = np.ones(NUM_HAND_JOINTS, dtype=np.uint8) + return tg + + right = TensorGroup(HandInput()) + right[HandInputIndex.JOINT_POSITIONS] = np.zeros( + (NUM_HAND_JOINTS, 3), dtype=np.float32 + ) + right[HandInputIndex.JOINT_ORIENTATIONS] = np.tile( + np.array([0, 0, 0, 1], dtype=np.float32), (NUM_HAND_JOINTS, 1) + ) + right[HandInputIndex.JOINT_RADII] = np.ones(NUM_HAND_JOINTS, dtype=np.float32) + right[HandInputIndex.JOINT_VALID] = np.ones(NUM_HAND_JOINTS, dtype=np.uint8) + + r1 = _run_retargeter( + node, + {"hand_left": left_absent, "hand_right": right, "transform": xform_id}, + ) + assert r1["hand_left"].is_none + assert not r1["hand_right"].is_none + + left = _active_left_joint0_1_0_0() + r2 = _run_retargeter( + node, + {"hand_left": left, "hand_right": right, "transform": xform_90}, + ) + assert not r2["hand_left"].is_none + pos2 = np.from_dlpack(r2["hand_left"][HandInputIndex.JOINT_POSITIONS]) + npt.assert_array_almost_equal(pos2[0], [0.0, 1.0, 0.0], decimal=5) + + r3 = _run_retargeter( + node, + {"hand_left": left_absent, "hand_right": right, "transform": xform_id}, + ) + assert r3["hand_left"].is_none + + r4 = _run_retargeter( + node, + { + "hand_left": _active_left_joint0_1_0_0(), + "hand_right": right, + "transform": xform_id, + }, + ) + pos4 = np.from_dlpack(r4["hand_left"][HandInputIndex.JOINT_POSITIONS]) + npt.assert_array_almost_equal(pos4[0], [1.0, 0.0, 0.0], decimal=5) diff --git a/src/core/retargeting_engine_tests/python/transform_numpy_version_smoke.py b/src/core/retargeting_engine_tests/python/transform_numpy_version_smoke.py new file mode 100644 index 000000000..dd328a989 --- /dev/null +++ b/src/core/retargeting_engine_tests/python/transform_numpy_version_smoke.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Smoke entrypoint for ``test_transform_numpy_versions``. + +Run under ``uv run --no-project --with 'numpy==…'`` so the interpreter sees a +pinned NumPy while ``PYTHONPATH`` still points at the built ``isaacteleop`` +package. Invoked as: + + python transform_numpy_version_smoke.py <1.23|2> +""" + +from __future__ import annotations + +import sys + + +def main() -> None: + if len(sys.argv) != 2 or sys.argv[1] not in ("1.23", "2"): + raise SystemExit("usage: transform_numpy_version_smoke.py <1.23|2>") + + key = sys.argv[1] + + import numpy as np + + if key == "1.23": + if not np.__version__.startswith("1.23"): + raise AssertionError(f"expected NumPy 1.23.x, got {np.__version__}") + if not hasattr(np, "from_dlpack"): + raise AssertionError("expected numpy.from_dlpack (NumPy 1.23+)") + else: + major = int(np.__version__.split(".")[0]) + if major < 2: + raise AssertionError(f"expected NumPy 2.x+, got {np.__version__}") + + from isaacteleop.retargeting_engine.interface import TensorGroup + from isaacteleop.retargeting_engine.tensor_types import ( + HeadPose, + HeadPoseIndex, + TransformMatrix, + ) + from isaacteleop.retargeting_engine.utilities import HeadTransform + + head_in = TensorGroup(HeadPose()) + head_in[HeadPoseIndex.POSITION] = np.array([1.0, 2.0, 3.0], dtype=np.float32) + head_in[HeadPoseIndex.ORIENTATION] = np.array( + [0.0, 0.0, 0.0, 1.0], dtype=np.float32 + ) + head_in[HeadPoseIndex.IS_VALID] = True + + xform_in = TensorGroup(TransformMatrix()) + xform_in[0] = np.eye(4, dtype=np.float32) + + node = HeadTransform("numpy_smoke_head") + result = node({"head": head_in, "transform": xform_in}) + out = result["head"] + pos = out[HeadPoseIndex.POSITION] + if not np.allclose(pos, [1.0, 2.0, 3.0], rtol=0, atol=1e-4): + raise AssertionError(f"unexpected position {pos!r}") + + +if __name__ == "__main__": + main()