Bring in Sharpa retargeter from V2D

.github/actions/setup-v2d-src/action.yml

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Setup robotic_grounding source bundle
+description: |
+  Clone jiwenc-nv/v2d:retargeter at the SHA pinned in deps/v2d/version.txt
+  and copy `robotic_grounding/` into deps/v2d/src/robotic_grounding/. The
+  Teleop wheel build (src/core/python/CMakeLists.txt) then bundles that
+  subtree alongside isaacteleop, so `pip install isaacteleop[grounding]`
+  on the resulting wheel works without a separate robotic_grounding install.
+
+  Forks without V2D_RETARGETER_TOKEN no-op cleanly: the action exits
+  without populating deps/v2d/src/, the wheel build skips bundling, and
+  Sharpa retargeter tests skip via the `_HAS_PINOCCHIO` guard.
+
+inputs:
+  github-token:
+    description: PAT with read access to jiwenc-nv/v2d. Empty on forks; action no-ops then.
+    required: false
+    default: ''
+
+outputs:
+  bundled:
+    description: '"true" if deps/v2d/src/robotic_grounding/ was populated, else "false".'
+    value: ${{ steps.locate.outputs.bundled }}
+
+runs:
+  using: composite
+  steps:
+    - name: Skip on forks (no token)
+      if: inputs.github-token == ''
+      shell: bash
+      run: |
+        echo "::warning::V2D_RETARGETER_TOKEN is empty; skipping robotic_grounding source fetch."
+        echo "Sharpa retargeter tests will skip via the _HAS_PINOCCHIO guard."
+
+    - name: Read pinned V2D ref
+      id: pin
+      if: inputs.github-token != ''
+      shell: bash
+      run: |
+        if [ ! -f deps/v2d/version.txt ]; then
+          echo "::error::deps/v2d/version.txt is missing; cannot pin V2D commit."
+          exit 1
+        fi
+        SHA=$(grep -vE '^\s*(#|$)' deps/v2d/version.txt | head -1 | tr -d '[:space:]')
+        if [ -z "$SHA" ]; then
+          echo "::error::deps/v2d/version.txt contains no SHA."
+          exit 1
+        fi
+        if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
+          echo "::error::deps/v2d/version.txt must pin a full 40-char git commit SHA, not a branch or tag (got: ${SHA})."
+          exit 1
+        fi
+        echo "sha=${SHA}" >> "$GITHUB_OUTPUT"
+        echo "Pinned V2D ref: ${SHA}"
+
+    - name: Cache robotic_grounding source bundle
+      id: cache
+      if: inputs.github-token != ''
+      uses: actions/cache@v5
+      with:
+        path: deps/v2d/src
+        # Source bundle is pure files; cache key only depends on V2D SHA.
+        key: v2d-src-${{ steps.pin.outputs.sha }}
+
+    - name: Clone robotic_grounding source
+      if: inputs.github-token != '' && steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github-token }}
+        V2D_REF: ${{ steps.pin.outputs.sha }}
+      run: |
+        set -euo pipefail
+        rm -rf /tmp/v2d deps/v2d/src
+        mkdir -p deps/v2d/src
+
+        # Whole retargeter branch is ~25 MB so a normal clone is fine.
+        gh repo clone jiwenc-nv/v2d /tmp/v2d -- --branch retargeter
+        git -C /tmp/v2d checkout "${V2D_REF}"
+
+        cp -a \
+          /tmp/v2d/robotic_grounding/source/robotic_grounding/robotic_grounding \
+          deps/v2d/src/robotic_grounding
+        find deps/v2d/src/robotic_grounding -type d -name __pycache__ \
+          -exec rm -rf {} + 2>/dev/null || true
+
+        rm -rf /tmp/v2d
+
+    - name: Locate bundle
+      id: locate
+      shell: bash
+      run: |
+        if [ -f deps/v2d/src/robotic_grounding/__init__.py ]; then
+          echo "bundled=true" >> "$GITHUB_OUTPUT"
+          echo "robotic_grounding/ source present at deps/v2d/src/robotic_grounding/"
+        else
+          echo "bundled=false" >> "$GITHUB_OUTPUT"
+          echo "::warning::deps/v2d/src/robotic_grounding/ is missing; wheel build will skip [grounding]."
+        fi

.github/workflows/build-ubuntu.yml

@@ -60,6 +60,22 @@ jobs:
       with:
         ngc-api-key: ${{ secrets.NGC_TELEOP_CORE_GITHUB_SERVICE_KEY }}
 
+    # robotic_grounding source bundle for the [grounding] extra (Sharpa retargeter).
+    # Disabled on release branches (and PRs targeting them) to keep release
+    # artifacts free of V2D source: V2D is private/pre-release and shouldn't
+    # ship with anything stamped as a release. Also no-ops on forks that
+    # lack V2D_RETARGETER_TOKEN. In every disabled case the wheel build skips
+    # bundling robotic_grounding/, and Sharpa tests skip via the
+    # _HAS_PINOCCHIO guard. Must run BEFORE `Configure CMake` so the wheel
+    # staging step in src/core/python/CMakeLists.txt picks up
+    # deps/v2d/src/robotic_grounding/.
+    - name: Setup robotic_grounding source bundle
+      id: setup-v2d-src
+      if: ${{ !startsWith(github.ref_name, 'release/') && !startsWith(github.base_ref, 'release/') }}
+      uses: ./.github/actions/setup-v2d-src
+      with:
+        github-token: ${{ secrets.V2D_RETARGETER_TOKEN }}
+
     # Hunter cache - caches depthai dependencies (OpenSSL, CURL, etc.)
     - name: Cache Hunter packages
       uses: actions/cache@v5
@@ -91,7 +107,8 @@ jobs:
           -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DBUILD_PLUGIN_OAK_CAMERA=ON \
           -DBUILD_VIZ=ON \
-          -DENABLE_CLOUDXR_BUNDLE_CHECK=ON
+          -DENABLE_CLOUDXR_BUNDLE_CHECK=ON \
+          -DBUNDLE_ROBOTIC_GROUNDING=${{ steps.setup-v2d-src.outputs.bundled || 'false' }}
 
     - name: Build
       run: cmake --build build --parallel 4

.gitignore

@@ -50,3 +50,11 @@ dist/
 
 # SPDX report
 project.spdx
+
+# robotic_grounding source bundle (cloned locally / in CI from
+# jiwenc-nv/v2d:retargeter; never committed -- repopulated deterministically
+# from deps/v2d/version.txt). The Teleop wheel build copies this subtree
+# into the wheel staging dir so [grounding] users get robotic_grounding
+# without a separate install. V2D source must NOT be tracked here.
+deps/v2d/src/
+deps/v2d/wheels/

deps/v2d/version.txt

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Pinned commit on jiwenc-nv/v2d:retargeter that the [grounding] extra
+# (Sharpa retargeter) is built against. Edit this single line to upgrade
+# the bundled robotic_grounding; rerun scripts/setup_v2d_src.sh after.
+#
+# See docs/source/references/grounding_extra.rst for the end-to-end flow.
+
+27657fbbc85af946eaf58484aaf0bf6fead215e3

docs/source/index.rst

@@ -59,7 +59,7 @@ Table of Contents
    references/requirements
    references/build
    device/index
-   references/retargeting
+   references/retargeting/index
    references/mcap_record_replay
    references/oob_teleop_control
    references/license

docs/source/references/retargeting.rst → docs/source/references/retargeting/index.rst

@@ -308,4 +308,10 @@ If the built-in retargeters do not cover your use case, you can implement a cust
    ``IDeviceIOSource`` subclass for custom input devices.
 
 See the `retargeters README <https://github.com/NVIDIA/IsaacTeleop/blob/main/src/retargeters/README.md>`_
-and :doc:`Contributing Guide <../getting_started/contributing>` for details.
+and :doc:`Contributing Guide <../../getting_started/contributing>` for details.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Retargeter setup guides
+
+   sharpa

docs/source/references/retargeting/sharpa.rst

@@ -0,0 +1,216 @@
+.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+.. SPDX-License-Identifier: Apache-2.0
+
+Retargeter: Manus to Sharpa
+===========================
+
+``SharpaHandRetargeter`` maps live hand-tracking poses from a `Manus glove
+<https://www.manus-meta.com/>`_ (or any other source feeding the OpenXR
+hand-tracking layer) onto Sharpa hand joint angles, frame by frame, via
+optimization-based inverse kinematics. ``SharpaBiManualRetargeter`` is a
+thin combiner that interleaves left and right outputs into a single
+target-ordered vector for downstream control.
+
+At a glance
+-----------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 22 78
+
+   * - Stage
+     - What happens
+   * - Input
+     - 26-joint OpenXR ``HandInput`` for the configured side (xyzw quats),
+       sourced from a Manus glove plugin or any other OpenXR hand-tracking
+       provider.
+   * - Repack
+     - Drop OpenXR palm + non-thumb metacarpals to land on the canonical
+       MANO 21-joint layout, and convert quaternions to wxyz.
+   * - IK
+     - ``robotic_grounding.retarget.hand_kinematics.SharpaHandKinematics``
+       runs Pink IK on a Pinocchio model loaded from the Sharpa MJCF, with
+       a FreeFlyer root that re-anchors the wrist each frame.
+   * - Warm-start
+     - Previous-frame qpos is kept and reused (with the wrist re-pinned to
+       the new tracker reading) to keep the IK locally smooth. A frame
+       with any invalid input joint zeros the output and resets the
+       warm-start.
+   * - Output
+     - Sharpa finger DOFs (everything Pinocchio reports past the
+       FreeFlyer), optionally reordered by ``hand_joint_names``.
+
+The retargeter intentionally contains no IK math itself: joint orderings,
+frame mappings, rotation corrections, and Pink/Pinocchio configuration all
+live in ``robotic_grounding`` (V2D). This module is the OpenXR-shaped
+adapter on top of it.
+
+.. seealso::
+
+   :doc:`/device/manus` -- installing the Manus plugin so its tracking
+   shows up on the OpenXR hand layer that this retargeter consumes.
+
+   :doc:`index` -- the broader retargeting interface and pipeline-builder
+   pattern.
+
+Why the ``[grounding]`` extra exists
+------------------------------------
+
+The Sharpa kinematics model, MJCFs, meshes, and the IK setup that this
+retargeter calls all live in the ``robotic_grounding`` package (part of
+V2D). V2D is on track to be fully open sourced; until then its source
+isn't public, so the wheel has two build modes:
+
+* **Default** — wheel ships without ``robotic_grounding``. Sharpa
+  retargeter imports skip cleanly, so forks and OSS contributors can
+  build and use the rest of Teleop unaffected.
+* **With** ``-DBUNDLE_ROBOTIC_GROUNDING=TRUE`` — the build pulls
+  ``robotic_grounding`` from the pinned SHA in ``deps/v2d/version.txt``
+  and bundles it into the wheel. Installing ``isaacteleop[grounding]``
+  then resolves all imports, and the Sharpa MJCFs ship with the wheel.
+
+.. note::
+
+   ``-DBUNDLE_ROBOTIC_GROUNDING=TRUE`` is a temporary bridge. Once V2D is
+   fully open sourced, ``robotic_grounding`` will be a normal public
+   dependency of the ``[grounding]`` extra and the bundling flag (along
+   with ``scripts/setup_v2d_src.sh`` and the ``V2D_RETARGETER_TOKEN``
+   gating in CI) will go away.
+
+The next two sections cover that opt-in build, how to use the retargeter
+once the extra is in place, and how to verify the install.
+
+Build the ``[grounding]`` extra
+-------------------------------
+
+Prerequisites:
+
+* `gh CLI <https://cli.github.com/>`_ installed and ``gh auth login``\ 'd
+  with read access to ``jiwenc-nv/v2d``.
+* A configured Teleop build tree.
+
+.. code-block:: console
+
+   $ scripts/setup_v2d_src.sh
+   $ cmake -B build -DBUNDLE_ROBOTIC_GROUNDING=TRUE <other flags...>
+   $ cmake --build build --target python_wheel
+   $ uv pip install -e .[grounding]
+
+The first command populates ``deps/v2d/src/robotic_grounding/`` from the
+SHA pinned in ``deps/v2d/version.txt``. The CMake flag tells the wheel
+build to bundle that subtree alongside ``isaacteleop``.
+
+If the wheel was built without ``-DBUNDLE_ROBOTIC_GROUNDING=TRUE``, the
+import raises ``ModuleNotFoundError`` with a pointer back to
+``scripts/setup_v2d_src.sh``.
+
+Use it from Python
+------------------
+
+.. code-block:: python
+
+   from isaacteleop.retargeters import (
+       SharpaHandRetargeter,
+       SharpaHandRetargeterConfig,
+   )
+
+The Sharpa MJCFs and meshes ship inside the bundled ``robotic_grounding``
+package -- resolve them with ``importlib.resources``:
+
+.. code-block:: python
+
+   from importlib.resources import files
+
+   xml_dir = files("robotic_grounding") / "assets" / "xmls" / "sharpawave"
+   right_mjcf = str(xml_dir / "right_sharpawave_nomesh.xml")  # mesh-free, fast
+   # or "right_sharpawave.xml" if you also have the STL meshes
+
+   cfg = SharpaHandRetargeterConfig(hand_side="right", robot_asset_path=right_mjcf)
+   retargeter = SharpaHandRetargeter(cfg, name="sharpa_right")
+
+Key ``SharpaHandRetargeterConfig`` fields:
+
+* ``robot_asset_path`` — the Sharpa MJCF path (``..._nomesh.xml`` is the
+  mesh-free variant used in tests and on machines without the STLs).
+* ``hand_side`` — ``"left"`` or ``"right"``.
+* ``hand_joint_names`` — optional output ordering override; defaults to
+  whatever finger joints Pinocchio discovers in the MJCF, in model order.
+* ``source_to_robot_scale`` — MANO-to-robot length scale.
+* ``solver`` / ``max_iter`` / ``frequency`` /
+  ``frame_tasks_converged_threshold`` — Pink IK knobs forwarded to
+  ``SharpaHandKinematics``.
+
+For bimanual control, instantiate two ``SharpaHandRetargeter``\ s and
+wrap them with ``SharpaBiManualRetargeter`` so a single output vector is
+produced in your target joint order.
+
+Run the example
+---------------
+
+The repo ships a bimanual demo at
+``examples/retargeting/python/sharpa_hand_retargeter_demo.py``:
+
+.. code-block:: console
+
+   # Synthetic curl animation (no headset, no GUI required):
+   $ python examples/retargeting/python/sharpa_hand_retargeter_demo.py --synthetic
+
+   # Live bimanual from a connected Quest headset:
+   $ python examples/retargeting/python/sharpa_hand_retargeter_demo.py
+
+   # Custom MJCFs (e.g. the mesh-bearing variants):
+   $ python examples/retargeting/python/sharpa_hand_retargeter_demo.py \
+       --left-mjcf  /path/to/left_sharpawave.xml \
+       --right-mjcf /path/to/right_sharpawave.xml
+
+The synthetic mode is the smoke test: if it animates a curl trajectory
+and prints non-zero finger qpos each frame, the install is good.
+
+Validate
+--------
+
+Two checks; either by itself is sufficient.
+
+**End-to-end pytest** -- exercises the full Pinocchio + Pink IK pipeline
+through the Teleop wrapper (init, warm-start persistence, open vs. curled
+hand, absent-hand zeros, etc.):
+
+.. code-block:: console
+
+   $ ctest --test-dir build -R retargeting_test_sharpa_hand_retargeter --output-on-failure
+   ...
+   100% tests passed, 0 tests failed out of 1
+
+The ``Test command`` line printed by ``ctest -V`` should include
+``--extra grounding``. If it doesn't, the wheel build skipped bundling --
+re-check that ``cmake -B build`` was invoked with
+``-DBUNDLE_ROBOTIC_GROUNDING=TRUE`` after running ``setup_v2d_src.sh``.
+
+**Full retargeting suite** -- regression coverage in case the wrapper
+introduced a typing or import regression elsewhere:
+
+.. code-block:: console
+
+   $ ctest --test-dir build -R '^retargeting_' --output-on-failure
+   ...
+   100% tests passed, 0 tests failed out of 16
+
+CI
+--
+
+The workflow at ``.github/workflows/build-ubuntu.yml`` runs the same flow
+via the ``setup-v2d-src`` composite action, gated on the
+``V2D_RETARGETER_TOKEN`` repo secret (a PAT scoped read-only to
+``jiwenc-nv/v2d``). The action sets ``-DBUNDLE_ROBOTIC_GROUNDING`` from
+its own ``bundled`` output; on forks without the secret the action no-ops
+and the flag is ``false``.
+
+Public artifact safety: a Release-only step strips ``robotic_grounding/``
+out of every wheel before ``actions/upload-artifact`` runs, so V2D source
+never reaches the public artifact channel.
+
+Bumping the bundled ``robotic_grounding``
+-----------------------------------------
+
+Edit the SHA in ``deps/v2d/version.txt`` and rerun
+``scripts/setup_v2d_src.sh``.