ROS: verify topic data in integration tests

.github/workflows/build-ubuntu.yml

@@ -491,98 +491,42 @@ jobs:
           -t teleop_ros2_ref:${{ env.ROS_DISTRO }} .
 
     - name: Verify application modes
-      env:
-        ACCEPT_CLOUDXR_EULA: Y
-        CXR_BUILD_CONTEXT: ${{ github.workspace }}
-        CXR_RUNTIME_NETWORK_MODE: bridge
-        PYTHON_VERSION: ${{ matrix.python_version }}
-        ISAACTELEOP_PIP_DEBUG: "0"
-        ISAACTELEOP_PIP_FIND_LINKS: /workspace/install/wheels
-        ISAACTELEOP_PIP_SPEC: isaacteleop[cloudxr]
       run: |
-        source scripts/setup_cloudxr_env.sh
+        set -euo pipefail
 
         PROJECT_NAME="isaacteleop-test-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.arch }}-${{ env.ROS_DISTRO }}"
         LOG_FILE_BASE="/tmp/ros2_test_output_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}_${{ matrix.arch }}_${{ env.ROS_DISTRO }}"
+        REPLAY_MCAP="/tmp/teleop_ros2_test_input_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}_${{ matrix.arch }}_${{ env.ROS_DISTRO }}.mcap"
         STARTUP_MARKER="TeleopSession started successfully"
         READINESS_TIMEOUT_SEC=30
         READINESS_POLL_INTERVAL_SEC=1
 
         # Cleanup
-        trap 'docker compose -p "$PROJECT_NAME" \
-          --env-file "$ENV_DEFAULT" \
-          ${ENV_LOCAL:+--env-file "$ENV_LOCAL"} \
-          -f deps/cloudxr/docker-compose.runtime.yaml \
-          -f deps/cloudxr/docker-compose.test.yaml \
-          -f deps/cloudxr/docker-compose.override.yaml \
-          down -v' EXIT
-
-        # Start CloudXR runtime
-        cat <<EOF > deps/cloudxr/docker-compose.override.yaml
-        services:
-          cloudxr-runtime:
-            container_name: "cloudxr-runtime-$PROJECT_NAME"
-        EOF
-
-        docker compose -p "$PROJECT_NAME" \
-          --env-file "$ENV_DEFAULT" \
-          ${ENV_LOCAL:+--env-file "$ENV_LOCAL"} \
-          -f deps/cloudxr/docker-compose.runtime.yaml \
-          -f deps/cloudxr/docker-compose.test.yaml \
-          -f deps/cloudxr/docker-compose.override.yaml \
-          up --build -d cloudxr-runtime
-
-        # Wait for healthy
-        healthy=false
-        for i in {1..30}; do
-          if docker compose -p "$PROJECT_NAME" \
-            --env-file "$ENV_DEFAULT" \
-            ${ENV_LOCAL:+--env-file "$ENV_LOCAL"} \
-            -f deps/cloudxr/docker-compose.runtime.yaml \
-            -f deps/cloudxr/docker-compose.test.yaml \
-            -f deps/cloudxr/docker-compose.override.yaml \
-            ps cloudxr-runtime | grep -q "healthy"; then
-            healthy=true
-            break
-          fi
-          sleep 2
-        done
+        trap 'docker ps -aq --filter "name=teleop-ros2-$PROJECT_NAME" | xargs -r docker rm -f >/dev/null 2>&1 || true; rm -f "$REPLAY_MCAP"' EXIT
 
-        if [ "$healthy" = false ]; then
-          echo "Error: CloudXR runtime failed to become healthy"
-          docker compose -p "$PROJECT_NAME" \
-            --env-file "$ENV_DEFAULT" \
-            ${ENV_LOCAL:+--env-file "$ENV_LOCAL"} \
-            -f deps/cloudxr/docker-compose.runtime.yaml \
-            -f deps/cloudxr/docker-compose.test.yaml \
-            -f deps/cloudxr/docker-compose.override.yaml \
-            logs cloudxr-runtime
-          exit 1
-        fi
+        docker run --rm \
+          --user "$(id -u):$(id -g)" \
+          -v /tmp:/tmp \
+          --entrypoint bash teleop_ros2_ref:${{ env.ROS_DISTRO }} -c \
+          "source /usr/local/bin/teleop-env-setup && exec /opt/isaacteleop/install/examples/teleop_ros2/cpp/integration_tests/teleop_ros2_mcap_generator '$REPLAY_MCAP'"
 
-        # Run tests
         for mode in controller_teleop hand_teleop controller_raw full_body; do
           echo "Testing mode: $mode"
           LOG_FILE="${LOG_FILE_BASE}_${mode}.log"
           RUN_NAME="teleop-ros2-${PROJECT_NAME}-${mode//_/-}"
+          verifier_rc=0
           docker_rc=0
           started=false
 
           rm -f "$LOG_FILE"
 
-          # This test overrides the image entrypoint, so keep uv's runtime
-          # resolver pointed at the locally built arm64 nlopt wheel.
           docker run -d \
             --name "$RUN_NAME" \
-            --gpus all \
             --network host \
-            -v "$CXR_HOST_VOLUME_PATH:$CXR_HOST_VOLUME_PATH" \
             -v /tmp:/tmp \
-            -e NV_CXR_RUNTIME_DIR="$CXR_HOST_VOLUME_PATH/.cloudxr/run" \
-            -e XR_RUNTIME_JSON="$CXR_HOST_VOLUME_PATH/.cloudxr/openxr_cloudxr.json" \
             -e PYTHONUNBUFFERED=1 \
             --entrypoint bash teleop_ros2_ref:${{ env.ROS_DISTRO }} -c \
-            "source /usr/local/bin/teleop-env-setup && exec uv run --find-links=/opt/nlopt-wheels teleop_ros2_node.py --ros-args -p mode:=$mode -p use_mock_operators:=true" >/dev/null
+            "source /usr/local/bin/teleop-env-setup && exec uv run --no-sync teleop_ros2_node.py --ros-args -p mode:=$mode -p mcap_replay_path:=$REPLAY_MCAP" >/dev/null
 
           deadline=$((SECONDS + READINESS_TIMEOUT_SEC))
           while [ "$SECONDS" -lt "$deadline" ]; do
@@ -605,6 +549,15 @@ jobs:
           docker logs "$RUN_NAME" > "$LOG_FILE" 2>&1 || true
 
           if [ "$started" = true ]; then
+            if docker exec \
+              -e PYTHONUNBUFFERED=1 \
+              "$RUN_NAME" bash -c \
+              "source /usr/local/bin/teleop-env-setup && exec uv run --no-sync integration_tests/teleop_ros2_topic_verifier.py --mode $mode"; then
+              verifier_rc=0
+            else
+              verifier_rc=$?
+            fi
+            docker logs "$RUN_NAME" > "$LOG_FILE" 2>&1 || true
             docker stop --time 5 "$RUN_NAME" >/dev/null || true
             docker wait "$RUN_NAME" >/dev/null || true
           else
@@ -629,6 +582,10 @@ jobs:
           if [ "$started" != true ]; then
             exit 1
           fi
+          if [ "$verifier_rc" -ne 0 ]; then
+            echo "Error: topic verifier failed in mode $mode (exit code $verifier_rc)"
+            exit "$verifier_rc"
+          fi
         done
 
   # These are the actual dependency jobs for publish-wheel. publish-wheel

examples/teleop_ros2/AGENTS.md

@@ -12,3 +12,4 @@ SPDX-License-Identifier: Apache-2.0
 ## Python Node Layout
 
 - In `python/teleop_ros2_node.py`, preserve the existing grouped/sorted organization for global non-member helpers and `TeleopRos2Node` member functions: scan the surrounding order before inserting, and do not place helpers near call sites when the existing section is sorted.
+- In Python integration test verifier code, do not use bare `assert` for runtime validation; Python optimization can disable it, so raise explicit exceptions from validators.

examples/teleop_ros2/CMakeLists.txt

@@ -5,6 +5,9 @@ cmake_minimum_required(VERSION 3.20)
 project(teleop_ros2_examples)
 
 include(${CMAKE_SOURCE_DIR}/cmake/InstallPythonExample.cmake)
+
+add_subdirectory(cpp/integration_tests)
+
 install_python_example(DESTINATION examples/teleop_ros2/python
     EXTRA_UV_EXTRA_BUILD_DEPS "nlopt = [\"numpy\"]")
 

examples/teleop_ros2/README.md

@@ -154,6 +154,24 @@ The `mode` parameter selects the teleoperation scenario and which topics are pub
 
 Example: `--ros-args -p mode:=controller_raw`
 
+### MCAP Replay
+
+Set `mcap_replay_path` to run the same ROS 2 publisher from recorded DeviceIO
+tracker data instead of live OpenXR/DeviceIO inputs:
+
+```bash
+docker run --rm --net=host --ipc=host \
+  -v /tmp:/tmp \
+  --name teleop_ros2_ref \
+  teleop_ros2_ref --ros-args -p mode:=controller_raw \
+  -p mcap_replay_path:=/tmp/teleop_ros2_input.mcap
+```
+
+The installed integration test utility
+`examples/teleop_ros2/cpp/integration_tests/teleop_ros2_mcap_generator` creates a
+deterministic fixture with controller, hand, pedal, and full-body samples for CI
+coverage.
+
 ## Echo Topics
 
 ```bash

examples/teleop_ros2/cpp/integration_tests/CMakeLists.txt

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20)
+
+add_executable(teleop_ros2_mcap_generator
+    mcap_generator.cpp
+)
+
+target_link_libraries(teleop_ros2_mcap_generator
+    PRIVATE
+        deviceio::deviceio_session
+        mcap::mcap_core
+)
+
+install(TARGETS teleop_ros2_mcap_generator
+    RUNTIME DESTINATION examples/teleop_ros2/cpp/integration_tests
+)

examples/teleop_ros2/cpp/integration_tests/mcap_generator.cpp

@@ -0,0 +1,188 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include <mcap/recording_traits.hpp>
+#include <mcap/tracker_channels.hpp>
+#include <mcap/writer.hpp>
+#include <schema/controller_generated.h>
+#include <schema/full_body_generated.h>
+#include <schema/hand_generated.h>
+#include <schema/pedals_generated.h>
+#include <schema/timestamp_generated.h>
+
+#include <cstdint>
+#include <exception>
+#include <filesystem>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace
+{
+
+using ControllerChannels = core::McapTrackerChannels<core::ControllerSnapshotRecord, core::ControllerSnapshot>;
+using HandChannels = core::McapTrackerChannels<core::HandPoseRecord, core::HandPose>;
+using PedalChannels = core::McapTrackerChannels<core::Generic3AxisPedalOutputRecord, core::Generic3AxisPedalOutput>;
+using FullBodyChannels = core::McapTrackerChannels<core::FullBodyPosePicoRecord, core::FullBodyPosePico>;
+
+constexpr int kDefaultFrameCount = 1800;
+constexpr int64_t kFramePeriodNs = 16'666'667;
+
+std::vector<std::string> to_strings(auto channels)
+{
+    std::vector<std::string> result;
+    result.reserve(channels.size());
+    for (std::string_view channel : channels)
+    {
+        result.emplace_back(channel);
+    }
+    return result;
+}
+
+std::shared_ptr<core::ControllerSnapshotT> make_controller_sample(bool left, int frame)
+{
+    const float side = left ? -1.0f : 1.0f;
+    const float delta = 0.001f * static_cast<float>(frame);
+    auto sample = std::make_shared<core::ControllerSnapshotT>();
+    sample->grip_pose = std::make_shared<core::ControllerPose>(
+        core::Pose(core::Point(0.15f * side, 0.10f + delta, 1.10f), core::Quaternion(0.0f, 0.0f, 0.0f, 1.0f)), true);
+    sample->aim_pose = std::make_shared<core::ControllerPose>(
+        core::Pose(core::Point(0.20f * side, 0.15f + delta, 1.20f), core::Quaternion(0.0f, 0.0f, 0.0f, 1.0f)), true);
+    sample->inputs = std::make_shared<core::ControllerInputState>(
+        true, !left, false, left, 0.25f * side, left ? 0.40f : -0.40f, 0.55f, 0.70f);
+    return sample;
+}
+
+std::shared_ptr<core::HandPoseT> make_hand_sample(bool left, int frame)
+{
+    const float side = left ? -1.0f : 1.0f;
+    const float delta = 0.0005f * static_cast<float>(frame);
+    auto sample = std::make_shared<core::HandPoseT>();
+    sample->joints = std::make_unique<core::HandJoints>();
+    for (int joint = 0; joint < core::HandJoint_NUM_JOINTS; ++joint)
+    {
+        const float joint_f = static_cast<float>(joint);
+        const float x = 0.05f * side + 0.003f * side * joint_f;
+        const float y = 0.03f + 0.006f * joint_f + delta;
+        const float z = 1.00f + 0.004f * joint_f;
+        const core::Point position(x, y, z);
+        const core::Pose pose(position, core::Quaternion(0.0f, 0.0f, 0.0f, 1.0f));
+        sample->joints->mutable_poses()->Mutate(joint, core::HandJointPose(pose, true, 0.010f));
+    }
+    return sample;
+}
+
+std::shared_ptr<core::Generic3AxisPedalOutputT> make_pedal_sample(int frame)
+{
+    auto sample = std::make_shared<core::Generic3AxisPedalOutputT>();
+    sample->left_pedal = 0.20f;
+    sample->right_pedal = 0.80f;
+    sample->rudder = (frame % 2 == 0) ? 0.15f : -0.15f;
+    return sample;
+}
+
+std::shared_ptr<core::FullBodyPosePicoT> make_full_body_sample(int frame)
+{
+    const float delta = 0.0005f * static_cast<float>(frame);
+    auto sample = std::make_shared<core::FullBodyPosePicoT>();
+    sample->joints = std::make_unique<core::BodyJointsPico>();
+    for (int joint = 0; joint < core::BodyJointPico_NUM_JOINTS; ++joint)
+    {
+        const float joint_f = static_cast<float>(joint);
+        const float x = 0.01f * joint_f;
+        const float y = -0.02f + 0.002f * joint_f + delta;
+        const float z = 0.80f + 0.01f * joint_f;
+        const core::Point position(x, y, z);
+        const core::Pose pose(position, core::Quaternion(0.0f, 0.0f, 0.0f, 1.0f));
+        sample->joints->mutable_joints()->Mutate(joint, core::BodyJointPose(pose, true));
+    }
+    return sample;
+}
+
+std::unique_ptr<mcap::McapWriter> open_writer(const std::filesystem::path& path)
+{
+    if (path.has_parent_path())
+    {
+        std::filesystem::create_directories(path.parent_path());
+    }
+
+    auto writer = std::make_unique<mcap::McapWriter>();
+    mcap::McapWriterOptions options("teleop-ros2-integration-test");
+    options.compression = mcap::Compression::None;
+    const auto status = writer->open(path.string(), options);
+    if (!status.ok())
+    {
+        throw std::runtime_error("failed to open MCAP writer for " + path.string() + ": " + status.message);
+    }
+    return writer;
+}
+
+void write_fixture(const std::filesystem::path& output_path, int frame_count)
+{
+    auto writer = open_writer(output_path);
+    const auto controller_names = to_strings(core::ControllerRecordingTraits::recording_channels);
+    const auto hand_names = to_strings(core::HandRecordingTraits::recording_channels);
+    const auto pedal_names = to_strings(core::PedalRecordingTraits::recording_channels);
+    const auto full_body_names = to_strings(core::FullBodyPicoRecordingTraits::recording_channels);
+
+    ControllerChannels controller_channels(
+        *writer, "controllers", core::ControllerRecordingTraits::schema_name, controller_names);
+    HandChannels hand_channels(*writer, "hands", core::HandRecordingTraits::schema_name, hand_names);
+    PedalChannels pedal_channels(*writer, "pedals", core::PedalRecordingTraits::schema_name, pedal_names);
+    FullBodyChannels full_body_channels(
+        *writer, "full_body", core::FullBodyPicoRecordingTraits::schema_name, full_body_names);
+
+    for (int frame = 0; frame < frame_count; ++frame)
+    {
+        const int64_t time_ns = static_cast<int64_t>(frame + 1) * kFramePeriodNs;
+        const core::DeviceDataTimestamp timestamp(time_ns, time_ns, time_ns);
+        controller_channels.write(0, timestamp, make_controller_sample(true, frame));
+        controller_channels.write(1, timestamp, make_controller_sample(false, frame));
+        hand_channels.write(0, timestamp, make_hand_sample(true, frame));
+        hand_channels.write(1, timestamp, make_hand_sample(false, frame));
+        pedal_channels.write(0, timestamp, make_pedal_sample(frame));
+        full_body_channels.write(0, timestamp, make_full_body_sample(frame));
+    }
+
+    writer->close();
+}
+
+int parse_frame_count(const char* value)
+{
+    const int frame_count = std::stoi(value);
+    if (frame_count <= 0)
+    {
+        throw std::invalid_argument("frame count must be positive");
+    }
+    return frame_count;
+}
+
+} // namespace
+
+int main(int argc, char** argv)
+try
+{
+    if (argc < 2 || argc > 3)
+    {
+        std::cerr << "Usage: " << argv[0] << " <output.mcap> [frame_count]\n";
+        return 2;
+    }
+
+    const std::filesystem::path output_path(argv[1]);
+    const int frame_count = argc == 3 ? parse_frame_count(argv[2]) : kDefaultFrameCount;
+    write_fixture(output_path, frame_count);
+    std::cout << "Wrote " << frame_count << " teleop ROS 2 replay frames to " << output_path << "\n";
+    return 0;
+}
+catch (const std::exception& e)
+{
+    std::cerr << argv[0] << ": " << e.what() << "\n";
+    return 1;
+}
+catch (...)
+{
+    std::cerr << argv[0] << ": Unknown error occurred\n";
+    return 1;
+}