diff --git a/.gitignore b/.gitignore index da26fa7..dffeeb8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ __pycache__/ build/ dist/ *.egg-info/ - +outputs # IDE .idea/ .vscode/ @@ -38,3 +38,5 @@ datasets/* # Private external task files private_tasks/* !private_tasks/.gitkeep + +*.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 263ebed..d582623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -135,4 +135,8 @@ RUN printf "numpy==1.26.0\n" > /tmp/sim-constraints.txt \ RUN python -m pip install --upgrade pip==26.0.1 \ && python -m pip install --no-deps numpy==1.26.0 +# Fix Vulkan ICD for headless EGL rendering (no X server) +RUN echo '{"file_format_version":"1.0.0","ICD":{"library_path":"libEGL_nvidia.so.0","api_version":"1.4.312"}}' \ + > /etc/vulkan/icd.d/nvidia_icd.json + CMD ["/bin/bash"] diff --git a/Makefile b/Makefile index 0a17a9e..2c167d7 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,7 @@ launch-isaaclab-glowsai-4090: build-isaaclab docker run --rm -it \ --name $(CONTAINER_NAME)-glowsai-4090 \ --gpus '"device=0"' \ + --device /dev/dri/card0:/dev/dri/card0 \ --net=host \ --ipc=host \ --ulimit memlock=-1 \ @@ -114,7 +115,6 @@ launch-isaaclab-glowsai-4090: build-isaaclab -v /tmp/.X11-unix:/tmp/.X11-unix:rw \ -v /opt/VirtualGL:/opt/VirtualGL:ro \ -v /usr/share/vulkan/icd.d:/usr/share/vulkan/icd.d:ro \ - -v /etc/vulkan/icd.d:/etc/vulkan/icd.d:ro \ -e DISPLAY=:1 \ -e USE_VNC=1 \ -e OMNI_KIT_ACCEPT_EULA=Y \ @@ -136,12 +136,13 @@ launch-isaaclab-glowsai-4090: build-isaaclab exec /bin/bash \ ' -# ---- Launch: GlowsAI L40S (VirtualGL + VNC display :1) ----------------------- +# ---- Launch: GlowsAI L40S (VirtualGL + VNC display :2) ----------------------- launch-isaaclab-glowsai-l40s: build-isaaclab @set -e; \ docker run --rm -it \ --name $(CONTAINER_NAME)-glowsai-l40s \ --gpus '"device=0"' \ + --device /dev/dri/card0:/dev/dri/card0 \ --net=host \ --ipc=host \ --ulimit memlock=-1 \ @@ -153,7 +154,6 @@ launch-isaaclab-glowsai-l40s: build-isaaclab -v /tmp/.X11-unix:/tmp/.X11-unix:rw \ -v /opt/VirtualGL:/opt/VirtualGL:ro \ -v /usr/share/vulkan/icd.d:/usr/share/vulkan/icd.d:ro \ - -v /etc/vulkan/icd.d:/etc/vulkan/icd.d:ro \ -e DISPLAY=:1 \ -e USE_VNC=1 \ -e VGL_DISPLAY=egl0 \ diff --git a/README.md b/README.md index 8761bee..845f376 100644 --- a/README.md +++ b/README.md @@ -127,9 +127,14 @@ python scripts/datagen/generate.py \ --record \ --use_lerobot_recorder \ --lerobot_dataset_repo_id ${HF_USER}/ \ + --augment_pose_factor 10 \ + --augment_global_xy_jitter 0.01 \ + --augment_local_xy_jitter 0.01 \ --object_poses data//object_poses.json ``` +`--object_poses` still provides the base UMI scene setups, but `--augment_pose_factor` lets Step 3 replay more synthetic episodes from the same small set of demonstrations. For example, `16` source entries with `--augment_pose_factor 10` become `160` replay episodes. + Upload the recorded dataset to Hugging Face Hub: ```bash diff --git a/docs/getting_started.md b/docs/getting_started.md index 3d9297b..4ad6cef 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -239,6 +239,8 @@ hf download ${HF_USER}/ --local-dir data/ ### Run data generation +`object_poses.json` only supplies the per-episode object placements used to initialize the scene. If your UMI pipeline only gives you a small number of usable entries, expand them here with pose augmentation before training. + Available tasks: - `HCIS-CupStacking-SingleArm-v0` - `HCIS-CutleryArrangement-SingleArm-v0` @@ -253,9 +255,19 @@ python scripts/datagen/generate.py \ --record \ --use_lerobot_recorder \ --lerobot_dataset_repo_id ${HF_USER}/ \ + --augment_pose_factor 10 \ + --augment_global_xy_jitter 0.01 \ + --augment_local_xy_jitter 0.01 \ --object_poses data//object_poses.json ``` +Recommended starting point when you only have about `16` usable UMI entries: + +- `--augment_pose_factor 10` to turn `16` base entries into about `160` replay episodes. +- Keep `--augment_yaw_jitter_deg 0` at first for maximum stability. +- Add `--augment_mix_objects` only if you need more diversity after confirming the scripted policy still succeeds reliably. +- For `HCIS-CutleryArrangement-SingleArm-v0`, Step 3 also blends a subset of augmented scenes toward the eval initial-pose distribution while keeping the rest broader, so training does not collapse onto only the public eval start states. + ### Upload the generated dataset ```bash diff --git a/docs/synthetic_data_generation.md b/docs/synthetic_data_generation.md index b87fbab..a64f0bf 100644 --- a/docs/synthetic_data_generation.md +++ b/docs/synthetic_data_generation.md @@ -154,10 +154,13 @@ python scripts/datagen/generate.py \ --num_envs 1 \ --device cuda \ --enable_cameras \ - --num_demos 50 \ --record \ --use_lerobot_recorder \ - --lerobot_dataset_repo_id HF-USER/name + --lerobot_dataset_repo_id HF-USER/name \ + --augment_pose_factor 10 \ + --augment_global_xy_jitter 0.01 \ + --augment_local_xy_jitter 0.01 \ + --object_poses data//object_poses.json ``` Key flags: @@ -172,7 +175,12 @@ Key flags: | `--use_lerobot_recorder` | Swap the default `StreamingRecorderManager` for `LeRobotRecorderManager`, which writes the LeRobot dataset format on disk instead of HDF5. | | `--lerobot_dataset_repo_id HF-USER/name` | Passed straight into `LeRobotDatasetCfg(repo_id=..., fps=args_cli.lerobot_dataset_fps)`. Names the on-disk dataset and the eventual HF Hub repo. | | `--lerobot_dataset_fps` | Frame rate the dataset is written at. Default `30`. | -| `--num_demos 50` | Stop after **50 successful** episodes. With `--use_lerobot_recorder` the recorder runs in `EXPORT_SUCCEEDED_ONLY` mode, so failed rollouts do not count toward the target. | +| `--object_poses data/.../object_poses.json` | Base replay set from UMI. Every `status == "full"` entry becomes one source episode before augmentation. | +| `--augment_pose_factor 10` | Multiply the replay set in Step 3. Example: `16` source entries become `160` replay episodes. | +| `--augment_global_xy_jitter`, `--augment_local_xy_jitter` | Scene-level and per-object translation jitter in meters. | +| `--augment_yaw_jitter_deg` | Optional world-yaw jitter. Keep it at `0` first if you want the safest scripted rollouts. | +| `--augment_mix_objects` | Recombine object poses across episodes before jittering. Higher diversity, slightly higher risk. | +| `--cutlery_eval_pose_fraction`, `--cutlery_eval_pose_jitter` | For the cutlery task only: rewrite part of the augmented replay set so fork/knife starts resemble the eval initial-pose distribution, while keeping the remaining episodes broader. | | `--resume` | Append to an existing dataset (`EXPORT_SUCCEEDED_ONLY_RESUME`) instead of starting fresh. | | `--seed` | Optional. Defaults to `int(time.time())`. | @@ -182,7 +190,7 @@ What happens at runtime: 2. `_configure_env_cfg(...)` flips `env_cfg.recorders.dataset_export_mode` to `EXPORT_SUCCEEDED_ONLY` and rewires the `success` termination so the recorder controls episode endings. 3. `_replace_recorder_manager(...)` instantiates `LeRobotRecorderManager(env_cfg.recorders, LeRobotDatasetCfg(repo_id=..., fps=...), env)`. 4. The main loop calls `sm.pre_step → sm.get_action → env.step → sm.advance` until `sm.is_episode_done`. On episode end, `sm.check_success(env)` decides whether the recorder commits the episode. -5. Once `recorder_manager.exported_successful_episode_count >= num_demos`, the script exits cleanly and `recorder_manager.finalize()` writes the LeRobot dataset to disk. +5. Once every replay episode from `--object_poses` (after optional augmentation) has been attempted, the script exits cleanly and `recorder_manager.finalize()` writes the LeRobot dataset to disk. The dataset lands locally first — the `repo_id` only names the directory at this stage. Upload is a separate step. @@ -220,5 +228,5 @@ Paste your `repo_id` (e.g. `HF-USER/name`) into the Space to browse the dataset. - [ ] `packages/simulator/src/simulator/tasks/__init__.py` imports the new subpackage. - [ ] State machine subclasses `leisaac.datagen.state_machine.base.StateMachineBase` and implements `setup`, `pre_step`, `get_action`, `advance`, `reset`, `check_success`, `is_episode_done`. - [ ] `scripts/datagen/generate.py::TASK_REGISTRY` has `: (StateMachineClass, "")`. -- [ ] Run with `--record --use_lerobot_recorder --enable_cameras --num_demos N --lerobot_dataset_repo_id HF-USER/name`. +- [ ] Run with `--record --use_lerobot_recorder --enable_cameras --object_poses ... --lerobot_dataset_repo_id HF-USER/name`. - [ ] `hf auth login`, then `hf upload --repo-type dataset`. diff --git a/packages/simulator/src/simulator/datagen/state_machine/cutlery_arrangement.py b/packages/simulator/src/simulator/datagen/state_machine/cutlery_arrangement.py index d73dbd5..5242ee9 100644 --- a/packages/simulator/src/simulator/datagen/state_machine/cutlery_arrangement.py +++ b/packages/simulator/src/simulator/datagen/state_machine/cutlery_arrangement.py @@ -76,9 +76,9 @@ "panda_finger_joint2": 0.04, } -# Pick order: fork first (place on +y / left of plate), then knife (place on -y / right) +# Pick order: knife first, then fork. _PICK_ORDER = (_KNIFE_NAME, _FORK_NAME) -_PLACE_X_SIGNS = (+1.0, -1.0) # fork → +x of plate, knife → -x of plate +_PLACE_X_SIGNS = (+1.0, -1.0) # knife → +x of plate, fork → -x of plate _PHASE_DURATIONS_PER_OBJECT = (180, 130, 20, 160, 170, 15, 30) _PHASES_PER_OBJECT = len(_PHASE_DURATIONS_PER_OBJECT) @@ -140,8 +140,8 @@ def _find_body_index(robot, body_name: str) -> int: class CutleryArrangementStateMachine(StateMachineBase): """Scripted Franka policy for arranging cutlery around a plate. - Picks up the fork and places it on the +y (left) side of the plate, - then picks up the knife and places it on the -y (right) side. + Picks up the knife and places it on the +x (right) side of the plate, + then picks up the fork and places it on the -x (left) side. Each object goes through 7 phases: diff --git a/packages/simulator/src/simulator/tasks/cutlery_arrangement/cutlery_arrangement_env_cfg.py b/packages/simulator/src/simulator/tasks/cutlery_arrangement/cutlery_arrangement_env_cfg.py index e8698ab..120afc4 100644 --- a/packages/simulator/src/simulator/tasks/cutlery_arrangement/cutlery_arrangement_env_cfg.py +++ b/packages/simulator/src/simulator/tasks/cutlery_arrangement/cutlery_arrangement_env_cfg.py @@ -89,7 +89,7 @@ def cutlery_arranged( knife_cfg: SceneEntityCfg, max_dist_xy: float, ) -> torch.Tensor: - """Termination: fork on +y side of plate, knife on -y side, both within max_dist_xy.""" + """Termination: fork on -x side of plate, knife on +x side, both within max_dist_xy.""" plate: RigidObject = env.scene[plate_cfg.name] fork: RigidObject = env.scene[fork_cfg.name] knife: RigidObject = env.scene[knife_cfg.name] @@ -106,8 +106,8 @@ def cutlery_arranged( done = torch.logical_and(done, fork_dist_xy <= max_dist_xy) done = torch.logical_and(done, knife_dist_xy <= max_dist_xy) - fork_on_left = fork_pos[:, 0] > plate_pos[:, 0] - knife_on_right = knife_pos[:, 0] < plate_pos[:, 0] + fork_on_left = fork_pos[:, 0] < plate_pos[:, 0] + knife_on_right = knife_pos[:, 0] > plate_pos[:, 0] done = torch.logical_and(done, fork_on_left) done = torch.logical_and(done, knife_on_right) diff --git a/packages/simulator/src/simulator/utils/object_pose_augmentation.py b/packages/simulator/src/simulator/utils/object_pose_augmentation.py new file mode 100644 index 0000000..e7c9dba --- /dev/null +++ b/packages/simulator/src/simulator/utils/object_pose_augmentation.py @@ -0,0 +1,263 @@ +"""Pose augmentation utilities for synthetic data generation. + +These helpers operate on the world-frame episode poses returned by +``load_episode_poses``. The goal is to expand a small number of UMI-derived +scene setups into more simulator rollouts without depending on the raw UMI +trajectory, which is not used by the current training pipeline. +""" + +from __future__ import annotations + +import math +import random + +from simulator.utils.object_poses_loader import WorldPose + +EpisodeWorldPoses = dict[str, WorldPose] + +_CUTLERY_EVAL_BASE_XY: dict[str, tuple[float, float]] = { + "knife": (0.50, -0.10), + "fork": (0.55, -0.10), +} + + +class PoseAugmentationError(ValueError): + """Raised when pose augmentation inputs are malformed.""" + + +def augment_episode_world_poses( + episodes: list[EpisodeWorldPoses], + *, + factor: int, + seed: int, + global_xy_jitter: float = 0.0, + local_xy_jitter: float = 0.0, + yaw_jitter_deg: float = 0.0, + min_object_distance: float = 0.0, + mix_objects: bool = False, + max_attempts: int = 64, +) -> list[EpisodeWorldPoses]: + """Expand replay episodes with conservative world-frame perturbations. + + Args: + episodes: Base replay episodes from ``load_episode_poses``. + factor: Dataset multiplier. ``1`` returns a copy of the original list. + seed: RNG seed for deterministic augmentation. + global_xy_jitter: Uniform translation range applied to the full scene. + local_xy_jitter: Additional per-object uniform translation range. + yaw_jitter_deg: Uniform world-yaw jitter applied to every object. + min_object_distance: Minimum allowed XY distance between any two objects. + mix_objects: If True, build new episodes by sampling each object from the + full per-object pose bank instead of preserving original pairings. + max_attempts: Rejection-sampling attempts per synthetic episode. + """ + if factor < 1: + raise PoseAugmentationError(f"factor must be >= 1, got {factor}") + if max_attempts < 1: + raise PoseAugmentationError(f"max_attempts must be >= 1, got {max_attempts}") + if any(value < 0.0 for value in (global_xy_jitter, local_xy_jitter, yaw_jitter_deg, min_object_distance)): + raise PoseAugmentationError("jitter and min_object_distance values must be non-negative") + if not episodes: + return [] + + object_names = tuple(sorted(episodes[0].keys())) + if not object_names: + raise PoseAugmentationError("episodes must contain at least one object") + for ep_idx, episode in enumerate(episodes[1:], start=1): + names = tuple(sorted(episode.keys())) + if names != object_names: + raise PoseAugmentationError( + f"episode {ep_idx} object set {names} does not match episode 0 object set {object_names}" + ) + + rng = random.Random(seed) + pose_bank = { + name: [_clone_world_pose(episode[name]) for episode in episodes] + for name in object_names + } + out = [_clone_episode(episode) for episode in episodes] + + if factor == 1: + return out + + yaw_jitter_rad = math.radians(yaw_jitter_deg) + for episode in episodes: + for _ in range(factor - 1): + out.append( + _sample_augmented_episode( + base_episode=episode, + pose_bank=pose_bank, + object_names=object_names, + rng=rng, + global_xy_jitter=global_xy_jitter, + local_xy_jitter=local_xy_jitter, + yaw_jitter_rad=yaw_jitter_rad, + min_object_distance=min_object_distance, + mix_objects=mix_objects, + max_attempts=max_attempts, + ) + ) + return out + + +def inject_cutlery_eval_pose_distribution( + episodes: list[EpisodeWorldPoses], + *, + seed: int, + eval_like_fraction: float, + eval_xy_jitter: float, + replaceable_start_index: int = 0, + min_object_distance: float = 0.0, + max_attempts: int = 64, +) -> list[EpisodeWorldPoses]: + """Blend eval-like cutlery starts into an existing replay set. + + A subset of the replaceable episodes is rewritten so fork/knife spawn near + the public eval distribution, while the remaining episodes keep the broader + UMI-derived or augmented scene coverage. This helps datagen see some scenes + close to eval without collapsing the whole dataset onto that narrow range. + """ + if not episodes: + return [] + if not 0.0 <= eval_like_fraction <= 1.0: + raise PoseAugmentationError( + f"eval_like_fraction must be in [0, 1], got {eval_like_fraction}" + ) + if eval_xy_jitter < 0.0: + raise PoseAugmentationError(f"eval_xy_jitter must be non-negative, got {eval_xy_jitter}") + if max_attempts < 1: + raise PoseAugmentationError(f"max_attempts must be >= 1, got {max_attempts}") + if not 0 <= replaceable_start_index <= len(episodes): + raise PoseAugmentationError( + "replaceable_start_index must be within the episode list bounds" + ) + + object_names = set(episodes[0].keys()) + if not set(_CUTLERY_EVAL_BASE_XY).issubset(object_names): + return [_clone_episode(episode) for episode in episodes] + + out = [_clone_episode(episode) for episode in episodes] + replaceable_indices = list(range(replaceable_start_index, len(out))) + if not replaceable_indices or eval_like_fraction == 0.0: + return out + + rng = random.Random(seed) + target_count = max(1, round(len(replaceable_indices) * eval_like_fraction)) + for episode_index in rng.sample(replaceable_indices, k=min(target_count, len(replaceable_indices))): + source_episode = rng.choice(out[:replaceable_start_index] or out) + out[episode_index] = _sample_cutlery_eval_episode( + base_episode=source_episode, + rng=rng, + eval_xy_jitter=eval_xy_jitter, + min_object_distance=min_object_distance, + max_attempts=max_attempts, + ) + return out + + +def _sample_augmented_episode( + *, + base_episode: EpisodeWorldPoses, + pose_bank: dict[str, list[WorldPose]], + object_names: tuple[str, ...], + rng: random.Random, + global_xy_jitter: float, + local_xy_jitter: float, + yaw_jitter_rad: float, + min_object_distance: float, + mix_objects: bool, + max_attempts: int, +) -> EpisodeWorldPoses: + for _ in range(max_attempts): + dx_global = rng.uniform(-global_xy_jitter, global_xy_jitter) + dy_global = rng.uniform(-global_xy_jitter, global_xy_jitter) + yaw_delta = rng.uniform(-yaw_jitter_rad, yaw_jitter_rad) if yaw_jitter_rad > 0.0 else 0.0 + episode: EpisodeWorldPoses = {} + + for name in object_names: + source_pose = rng.choice(pose_bank[name]) if mix_objects else base_episode[name] + pos, quat = source_pose + dx_local = rng.uniform(-local_xy_jitter, local_xy_jitter) + dy_local = rng.uniform(-local_xy_jitter, local_xy_jitter) + episode[name] = ( + (pos[0] + dx_global + dx_local, pos[1] + dy_global + dy_local, pos[2]), + _rotate_world_yaw(quat, yaw_delta), + ) + + if _has_valid_separation(episode, min_object_distance): + return episode + + return _clone_episode(base_episode) + + +def _sample_cutlery_eval_episode( + *, + base_episode: EpisodeWorldPoses, + rng: random.Random, + eval_xy_jitter: float, + min_object_distance: float, + max_attempts: int, +) -> EpisodeWorldPoses: + for _ in range(max_attempts): + episode = _clone_episode(base_episode) + for name, (base_x, base_y) in _CUTLERY_EVAL_BASE_XY.items(): + pos, quat = episode[name] + episode[name] = ( + ( + base_x + rng.uniform(-eval_xy_jitter, eval_xy_jitter), + base_y + rng.uniform(-eval_xy_jitter, eval_xy_jitter), + pos[2], + ), + quat, + ) + if _has_valid_separation(episode, min_object_distance): + return episode + return _clone_episode(base_episode) + + +def _has_valid_separation(episode: EpisodeWorldPoses, min_object_distance: float) -> bool: + if min_object_distance <= 0.0 or len(episode) < 2: + return True + poses = list(episode.values()) + min_dist_sq = min_object_distance * min_object_distance + for idx, (pos_a, _) in enumerate(poses): + for pos_b, _ in poses[idx + 1 :]: + dx = pos_a[0] - pos_b[0] + dy = pos_a[1] - pos_b[1] + if dx * dx + dy * dy < min_dist_sq: + return False + return True + + +def _rotate_world_yaw(quat_wxyz: tuple[float, float, float, float], yaw_delta: float) -> tuple[float, float, float, float]: + if abs(yaw_delta) < 1e-12: + return tuple(float(v) for v in quat_wxyz) + half = yaw_delta * 0.5 + delta = (math.cos(half), 0.0, 0.0, math.sin(half)) + return _quat_mul_wxyz(delta, quat_wxyz) + + +def _quat_mul_wxyz( + q1: tuple[float, float, float, float], + q2: tuple[float, float, float, float], +) -> tuple[float, float, float, float]: + w1, x1, y1, z1 = q1 + w2, x2, y2, z2 = q2 + return ( + w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, + w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, + w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, + w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, + ) + + +def _clone_episode(episode: EpisodeWorldPoses) -> EpisodeWorldPoses: + return {name: _clone_world_pose(pose) for name, pose in episode.items()} + + +def _clone_world_pose(world_pose: WorldPose) -> WorldPose: + pos, quat = world_pose + return ( + (float(pos[0]), float(pos[1]), float(pos[2])), + (float(quat[0]), float(quat[1]), float(quat[2]), float(quat[3])), + ) diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..f81d6f2 --- /dev/null +++ b/run.sh @@ -0,0 +1,67 @@ +make launch-isaaclab-glowsai-l40s +# data generation +python scripts/datagen/generate.py \ + --task HCIS-CutleryArrangement-SingleArm-v0 \ + --num_envs 1 \ + --device cuda \ + --headless \ + --enable_cameras \ + --record \ + --use_lerobot_recorder \ + --lerobot_dataset_repo_id fanyi000/cultery_synth \ + --augment_pose_factor 10 \ + --augment_global_xy_jitter 0.01 \ + --augment_local_xy_jitter 0.05 \ + --object_poses data/AI-final-49/object_poses.json + +# use some dummy data +python scripts/datagen/generate.py \ + --task HCIS-CutleryArrangement-SingleArm-v0 \ + --num_envs 1 \ + --device cuda \ + --headless \ + --enable_cameras \ + --record \ + --use_lerobot_recorder \ + --lerobot_dataset_repo_id fanyi000/cultery_synth \ + --augment_pose_factor 10 \ + --augment_global_xy_jitter 0.01 \ + --augment_local_xy_jitter 0.05 \ + --object_poses data/AI-final-49/object_poses_combined.json + +# training +uv sync && source .venv/bin/activate +HF_HUB_DISABLE_XET=1 lerobot-train \ + --dataset.repo_id=${HF_USER}/cultery_synth \ + --policy.type=diffusion \ + --output_dir=outputs/diffusion_v2 \ + --job_name=cupstacking \ + --policy.device=cuda \ + --wandb.enable=true \ + --policy.repo_id=${HF_USER}/my_policy_diffusion + +# training exp1 +uv sync && source .venv/bin/activate +HF_HUB_DISABLE_XET=1 lerobot-train \ + --dataset.repo_id=${HF_USER}/cultery_synth \ + --policy.type=act \ + --output_dir=outputs/act_v1 \ + --job_name=cupstacking \ + --policy.device=cuda \ + --wandb.enable=true \ + --policy.repo_id=${HF_USER}/my_policy_act + +# evaluation +hf download fanyi000/my_policy_diffusion --local-dir outputs/diffusion_v3 +make launch-isaaclab-glowsai-l40s +python scripts/rollout.py \ + --task=eval/cutlery_arrangement_eval.py \ + --policy_type=lerobot-diffusion \ + --policy_checkpoint_path=outputs/diffusion_v3 \ + --policy_action_horizon=8 \ + --device=cuda \ + --headless \ + --enable_cameras \ + --eval_rounds=30 \ + --episode_length_s=60 \ + 2>&1 | tee /workspace/aicapstone/eval_results.txt \ No newline at end of file diff --git a/scripts/analyze_data_quality.py b/scripts/analyze_data_quality.py new file mode 100644 index 0000000..6359d0a --- /dev/null +++ b/scripts/analyze_data_quality.py @@ -0,0 +1,261 @@ +"""Data quality analysis for Section 4.1 of the report.""" + +import json +import math +import sys +from pathlib import Path + +try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import numpy as np +except ImportError: + print("ERROR: matplotlib / numpy not found.") + sys.exit(1) + +# ── constants ──────────────────────────────────────────────────────────────── +ANCHOR_X, ANCHOR_Y = 0.40, 0.10 +PLATE_POS = (0.50, -0.40) +EVAL_JITTER = 0.05 +ROBOT_BASE = (0.35, -0.74) +R_MAX_H = 0.845 # Franka horizontal reach at grasp height +R_MIN_H = 0.20 + +DATA_DIR = Path(__file__).parent.parent / "data" / "AI-final-49" +OUT_DIR = Path(__file__).parent.parent / "data" / "figures" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +# ── helpers ────────────────────────────────────────────────────────────────── +def to_world(tvec): + return ANCHOR_X + tvec[0], ANCHOR_Y + tvec[1] + +def in_workspace(wx, wy): + d = math.sqrt((wx - ROBOT_BASE[0])**2 + (wy - ROBOT_BASE[1])**2) + return R_MIN_H <= d <= R_MAX_H + +def load_positions(path): + with open(path) as f: + data = json.load(f) + result = [] + for ep in data: + if ep.get("status") != "full": + continue + pos = {} + for obj in ep["objects"]: + if obj["object_name"] == "plate": + continue + pos[obj["object_name"]] = to_world(obj["tvec"]) + source = "synthetic" if ep["video_name"].startswith("synthetic") else "real" + result.append({"positions": pos, "source": source}) + return result + +def workspace_arc(cx, cy, r, n=360): + thetas = np.linspace(0, 2 * math.pi, n) + return cx + r * np.cos(thetas), cy + r * np.sin(thetas) + +# ── Figure 1: Data pipeline ────────────────────────────────────────────────── +def fig_pipeline(): + with open(DATA_DIR / "object_poses_combined.json") as f: + combined = json.load(f) + real_n = sum(1 for e in combined if not e["video_name"].startswith("synthetic")) + syn_n = sum(1 for e in combined if e["video_name"].startswith("synthetic")) + base_n = real_n + syn_n + + stages = ["UMI\nRecorded\n(raw)", "UMI\nIn Workspace\n(filtered)", + "Base Episodes\n(real + synthetic)", "Datagen\nEpisodes\n(×10 augment)"] + counts = [49, real_n, base_n, base_n * 10] + colors = ["#d9534f", "#f0ad4e", "#5bc0de", "#5cb85c"] + + fig, ax = plt.subplots(figsize=(9, 4.5)) + bars = ax.bar(stages, counts, color=colors, width=0.5, edgecolor="white", linewidth=1.2) + for bar, count in zip(bars, counts): + ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 10, + str(count), ha="center", va="bottom", fontweight="bold", fontsize=13) + ax.set_ylim(0, max(counts) * 1.15) + ax.set_ylabel("Number of Episodes", fontsize=11) + ax.set_title("Data Pipeline: UMI Recording → Datagen Episodes", fontsize=13, fontweight="bold") + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + plt.tight_layout() + out = OUT_DIR / "fig1_data_pipeline.png" + plt.savefig(out, dpi=150) + plt.close() + print(f"Saved: {out}") + + +# ── Figure 2: Spatial coverage ─────────────────────────────────────────────── +def fig_spatial_coverage(): + combined = load_positions(DATA_DIR / "object_poses_combined.json") + + fork_real_x, fork_real_y = [], [] + fork_syn_x, fork_syn_y = [], [] + knife_real_x, knife_real_y = [], [] + knife_syn_x, knife_syn_y = [], [] + + for ep in combined: + src, pos = ep["source"], ep["positions"] + if "fork" in pos: + (fork_real_x if src == "real" else fork_syn_x ).append(pos["fork"][0]) + (fork_real_y if src == "real" else fork_syn_y ).append(pos["fork"][1]) + if "knife" in pos: + (knife_real_x if src == "real" else knife_syn_x).append(pos["knife"][0]) + (knife_real_y if src == "real" else knife_syn_y).append(pos["knife"][1]) + + fig, ax = plt.subplots(figsize=(7, 7)) + + # workspace boundary (annulus) + ox, oy = workspace_arc(*ROBOT_BASE, R_MAX_H) + ix, iy = workspace_arc(*ROBOT_BASE, R_MIN_H) + ax.fill(ox, oy, color="#e3f2fd", alpha=0.5, zorder=0) + ax.plot(ox, oy, color="#1565C0", lw=1.5, linestyle="--", label="Franka workspace boundary") + ax.fill(ix, iy, color="white", alpha=1.0, zorder=1) + + # eval range boxes + for label, (dx, dy), col in [("Eval range — fork", (0.55, -0.10), "#2196F3"), + ("Eval range — knife", (0.50, -0.10), "#FF5722")]: + ax.add_patch(plt.Rectangle((dx - EVAL_JITTER, dy - EVAL_JITTER), + 2*EVAL_JITTER, 2*EVAL_JITTER, + lw=1.5, edgecolor=col, facecolor=col, alpha=0.15, + linestyle="--", label=label)) + ax.plot(dx, dy, "*", ms=11, color=col, zorder=5) + + # data points + ax.scatter(fork_syn_x, fork_syn_y, c="#90CAF9", s=30, alpha=0.6, marker="^") + ax.scatter(fork_real_x, fork_real_y, c="#1565C0", s=55, alpha=0.9, marker="^") + ax.scatter(knife_syn_x, knife_syn_y, c="#FFCC80", s=30, alpha=0.6, marker="s") + ax.scatter(knife_real_x, knife_real_y, c="#BF360C", s=55, alpha=0.9, marker="s") + ax.plot(*PLATE_POS, "o", ms=12, color="#555", zorder=5) + + # manual legend below the plot + legend_elements = [ + plt.Line2D([0],[0], color="#1565C0", lw=1.5, linestyle="--", label="Franka workspace boundary"), + plt.scatter([],[], c="#1565C0", s=55, marker="^", label=f"Fork — real UMI (n={len(fork_real_x)})"), + plt.scatter([],[], c="#90CAF9", s=30, marker="^", label=f"Fork — synthetic (n={len(fork_syn_x)})"), + plt.scatter([],[], c="#BF360C", s=55, marker="s", label=f"Knife — real UMI (n={len(knife_real_x)})"), + plt.scatter([],[], c="#FFCC80", s=30, marker="s", label=f"Knife — synthetic (n={len(knife_syn_x)})"), + plt.Rectangle((0,0),1,1, fc="#2196F3", alpha=0.3, label="Eval range — fork"), + plt.Rectangle((0,0),1,1, fc="#FF5722", alpha=0.3, label="Eval range — knife"), + plt.Line2D([0],[0], marker="o", ms=9, color="#555", lw=0, label="Plate (fixed)"), + ] + ax.legend(handles=legend_elements, fontsize=8, loc="lower left", + bbox_to_anchor=(0.0, -0.38), ncol=3, frameon=True) + + ax.set_xlabel("X (m)", fontsize=11) + ax.set_ylabel("Y (m)", fontsize=11) + ax.set_title("Object Starting Positions: Training Data vs Eval Range", fontsize=12, fontweight="bold") + ax.set_xlim(-0.10, 0.90) + ax.set_ylim(-0.80, 0.15) + ax.set_aspect("equal") + ax.grid(True, alpha=0.25) + plt.subplots_adjust(bottom=0.30) + out = OUT_DIR / "fig2_spatial_coverage.png" + plt.savefig(out, dpi=150, bbox_inches="tight") + plt.close() + print(f"Saved: {out}") + + +# ── Figure 3: UMI filtering ────────────────────────────────────────────────── +def fig_umi_filtering(): + with open(DATA_DIR / "object_poses.json") as f: + raw = json.load(f) + + valid_f, valid_k, invalid_f, invalid_k = [], [], [], [] + + for ep in raw: + if ep.get("status") != "full": + continue + pos = {o["object_name"]: to_world(o["tvec"]) + for o in ep["objects"] if o["object_name"] != "plate"} + ok = all(in_workspace(*p) for p in pos.values()) + for name, (wx, wy) in pos.items(): + bucket_v = valid_f if name == "fork" else valid_k + bucket_i = invalid_f if name == "fork" else invalid_k + (bucket_v if ok else bucket_i).append((wx, wy)) + + fig, ax = plt.subplots(figsize=(7, 7)) + + # workspace (annulus) + ox, oy = workspace_arc(*ROBOT_BASE, R_MAX_H) + ix, iy = workspace_arc(*ROBOT_BASE, R_MIN_H) + ax.fill(ox, oy, color="#e8f5e9", alpha=0.4, zorder=0) + ax.plot(ox, oy, color="#2e7d32", lw=2, linestyle="--", label="Franka workspace boundary") + ax.fill(ix, iy, color="white", alpha=1.0, zorder=1) + + if invalid_f: + ax.scatter(*zip(*invalid_f), c="#ef9a9a", s=50, marker="^", alpha=0.75, zorder=3) + if invalid_k: + ax.scatter(*zip(*invalid_k), c="#ef9a9a", s=50, marker="s", alpha=0.75, zorder=3) + if valid_f: + ax.scatter(*zip(*valid_f), c="#1b5e20", s=65, marker="^", alpha=0.9, zorder=4) + if valid_k: + ax.scatter(*zip(*valid_k), c="#1b5e20", s=65, marker="s", alpha=0.9, zorder=4) + + ax.plot(*PLATE_POS, "o", ms=12, color="#555", zorder=5) + ax.plot(*ROBOT_BASE, "kD", ms=10, zorder=6) + + nv = len(set(map(id, valid_f))) # approx per-episode count + legend_elements = [ + plt.Line2D([0],[0], color="#2e7d32", lw=2, linestyle="--", label="Franka workspace boundary"), + plt.scatter([],[], c="#1b5e20", s=65, marker="^", label=f"Fork — valid ({len(valid_f)})"), + plt.scatter([],[], c="#1b5e20", s=65, marker="s", label=f"Knife — valid ({len(valid_k)})"), + plt.scatter([],[], c="#ef9a9a", s=50, marker="^", label=f"Fork — discarded ({len(invalid_f)})"), + plt.scatter([],[], c="#ef9a9a", s=50, marker="s", label=f"Knife — discarded ({len(invalid_k)})"), + plt.Line2D([0],[0], marker="o", ms=9, color="#555", lw=0, label="Plate (fixed)"), + plt.Line2D([0],[0], marker="D", ms=9, color="k", lw=0, label="Robot base (0.35, −0.74)"), + ] + ax.legend(handles=legend_elements, fontsize=8, loc="lower left", + bbox_to_anchor=(0.0, -0.30), ncol=2, frameon=True) + + n_valid_ep = sum(1 for e in raw if e.get("status") == "full" + and all(in_workspace(*to_world(o["tvec"])) + for o in e["objects"] if o["object_name"] != "plate")) + n_total_ep = sum(1 for e in raw if e.get("status") == "full") + + ax.set_xlabel("X (m)", fontsize=11) + ax.set_ylabel("Y (m)", fontsize=11) + ax.set_title(f"UMI Data Filtering: {n_valid_ep} valid / {n_total_ep} episodes " + f"({n_valid_ep/n_total_ep*100:.0f}% usable)", + fontsize=12, fontweight="bold") + ax.set_xlim(-0.30, 1.00) + ax.set_ylim(-0.85, 0.35) + ax.set_aspect("equal") + ax.grid(True, alpha=0.25) + plt.subplots_adjust(bottom=0.25) + out = OUT_DIR / "fig3_umi_filtering.png" + plt.savefig(out, dpi=150, bbox_inches="tight") + plt.close() + print(f"Saved: {out}") + + +# ── summary ─────────────────────────────────────────────────────────────────── +def print_summary(): + with open(DATA_DIR / "object_poses.json") as f: + raw = json.load(f) + total = sum(1 for e in raw if e.get("status") == "full") + valid = sum(1 for e in raw if e.get("status") == "full" and + all(in_workspace(*to_world(o["tvec"])) + for o in e["objects"] if o["object_name"] != "plate")) + with open(DATA_DIR / "object_poses_combined.json") as f: + combined = json.load(f) + real_n = sum(1 for e in combined if not e["video_name"].startswith("synthetic")) + syn_n = sum(1 for e in combined if e["video_name"].startswith("synthetic")) + + print("=" * 52) + print("DATA QUALITY SUMMARY") + print("=" * 52) + print(f"UMI episodes (full): {total}") + print(f" in workspace (annular+table): {valid} ({valid/total*100:.1f}%)") + print(f" discarded: {total-valid} ({(total-valid)/total*100:.1f}%)") + print(f"Synthetic episodes: {syn_n}") + print(f"Combined base: {real_n + syn_n}") + print(f"Datagen target (×10): {(real_n + syn_n) * 10}") + print("=" * 52) + + +if __name__ == "__main__": + print_summary() + fig_pipeline() + fig_spatial_coverage() + fig_umi_filtering() + print(f"\nAll figures saved to: {OUT_DIR}") diff --git a/scripts/analyze_workspace.py b/scripts/analyze_workspace.py new file mode 100644 index 0000000..a0c697a --- /dev/null +++ b/scripts/analyze_workspace.py @@ -0,0 +1,180 @@ +"""Rigorous workspace analysis for Section 4.1. + +Method: + 1. Robot base XY from env_cfg comment: (0.35, -0.74) + 2. Franka Panda arm segment lengths from official spec to compute + maximum horizontal reach at grasp height (z = 0.13 m above table). + 3. Table boundary inferred from datagen failure observations (y ≈ 0). + 4. Valid workspace = reachable annulus ∩ table surface. + 5. Show training data distribution inside this boundary. + +Franka Panda DH parameters (official spec, meters): + d1=0.333, a2=0.000, d3=0.316, a3=0.0825, + d4=0.384, a4=0.0825, d5=0.000, d6=0.088, d_ee=0.107+0.058 + Total chain length (fully extended, horizontal): ~0.855 m +""" + +import json +import math +import sys +from pathlib import Path + +try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import matplotlib.patches as mpatches + import numpy as np +except ImportError: + print("ERROR: matplotlib / numpy not found.") + sys.exit(1) + +# ── robot & scene constants ─────────────────────────────────────────────────── +ROBOT_BASE_XY = (0.35, -0.74) # from env_cfg comment +ROBOT_BASE_Z = 0.0 # mounted at table surface level + +# Franka Panda: sum of vertical link offsets in a typical grasp configuration. +# Max horizontal reach is documented as 0.855 m (full extension). +# At grasp height we lose vertical capacity → effective horizontal reach shrinks. +FRANKA_MAX_REACH = 0.855 # m, official spec (full arm extension) +FRANKA_MIN_REACH = 0.20 # m, arm fully retracted / elbow bent back + +# Grasp height: object z (0.05) + GRASP_Z_OFFSET (0.08) = 0.13 m +GRASP_Z = 0.13 +Z_DIFF = abs(GRASP_Z - ROBOT_BASE_Z) + +# Effective horizontal reach at grasp height +# Using spherical reach model: r_h = sqrt(R² - z²) +R_MAX_H = math.sqrt(max(FRANKA_MAX_REACH**2 - Z_DIFF**2, 0)) +R_MIN_H = FRANKA_MIN_REACH # inner bound stays roughly constant + +# Table boundary (observed from datagen failures: objects at y > 0 fell off) +TABLE_Y_MAX = None # not used — table edge removed from figure +TABLE_Y_MIN = -0.70 +TABLE_X_MIN = -0.10 +TABLE_X_MAX = 1.00 + +ANCHOR_X, ANCHOR_Y = 0.40, 0.10 +PLATE_POS = (0.50, -0.40) + +DATA_DIR = Path(__file__).parent.parent / "data" / "AI-final-49" +OUT_DIR = Path(__file__).parent.parent / "data" / "figures" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +# ── load training positions ─────────────────────────────────────────────────── +def load_positions(path): + with open(path) as f: + data = json.load(f) + out = [] + for ep in data: + if ep.get("status") != "full": + continue + pos, src = {}, "synthetic" if ep["video_name"].startswith("synthetic") else "real" + for obj in ep["objects"]: + if obj["object_name"] == "plate": + continue + wx = ANCHOR_X + obj["tvec"][0] + wy = ANCHOR_Y + obj["tvec"][1] + pos[obj["object_name"]] = (wx, wy) + out.append({"pos": pos, "src": src}) + return out + +# ── workspace polygon (annulus clipped to table) ───────────────────────────── +def workspace_polygon(cx, cy, r_max, r_min, n=360): + thetas = np.linspace(0, 2 * math.pi, n) + outer = np.array([(cx + r_max * math.cos(t), cy + r_max * math.sin(t)) for t in thetas]) + inner = np.array([(cx + r_min * math.cos(t), cy + r_min * math.sin(t)) for t in thetas]) + return outer, inner + +# ── figure ──────────────────────────────────────────────────────────────────── +def fig_workspace(): + episodes = load_positions(DATA_DIR / "object_poses_combined.json") + + fig, ax = plt.subplots(figsize=(8, 7)) + + # ── reachable workspace ────────────────────────────────────────────────── + cx, cy = ROBOT_BASE_XY + outer, inner = workspace_polygon(cx, cy, R_MAX_H, R_MIN_H) + + # fill outer circle (clipped to table) + ax.fill(outer[:, 0], outer[:, 1], color="#b3e5fc", alpha=0.4, label=f"Reachable workspace (r≤{R_MAX_H:.2f} m, z={GRASP_Z} m)") + ax.plot(outer[:, 0], outer[:, 1], color="#0288d1", lw=1.5) + # subtract inner dead zone + ax.fill(inner[:, 0], inner[:, 1], color="white", alpha=1.0) + ax.plot(inner[:, 0], inner[:, 1], color="#0288d1", lw=1, linestyle=":") + + # ── training data ──────────────────────────────────────────────────────── + for ep in episodes: + clr_f = "#1565C0" if ep["src"] == "real" else "#64B5F6" + clr_k = "#BF360C" if ep["src"] == "real" else "#FFAB76" + mk_f = "^" ; mk_k = "s" + sz = 70 if ep["src"] == "real" else 35 + if "fork" in ep["pos"]: + ax.scatter(*ep["pos"]["fork"], c=clr_f, s=sz, marker=mk_f, alpha=0.85, zorder=4) + if "knife" in ep["pos"]: + ax.scatter(*ep["pos"]["knife"], c=clr_k, s=sz, marker=mk_k, alpha=0.85, zorder=4) + + # ── eval randomisation range ───────────────────────────────────────────── + for label, (dx, dy), col in [("Fork eval range", (0.55, -0.10), "#2196F3"), + ("Knife eval range", (0.50, -0.10), "#FF5722")]: + ax.add_patch(plt.Rectangle((dx - 0.05, dy - 0.05), 0.10, 0.10, + lw=1.5, edgecolor=col, facecolor=col, alpha=0.12, + linestyle="--", label=label)) + ax.plot(dx, dy, "*", ms=12, color=col, zorder=5) + + # ── robot base ─────────────────────────────────────────────────────────── + ax.plot(cx, cy, "kD", ms=10, zorder=6) + ax.plot(*PLATE_POS, "o", ms=12, color="gray", zorder=5) + + legend_elements = [ + plt.Line2D([0],[0], color="#0288d1", lw=1.5, + label=f"Reachable workspace (r≤{R_MAX_H:.2f} m at z={GRASP_Z} m)"), + plt.Line2D([0],[0], color="#0288d1", lw=1, linestyle=":", + label="Dead zone boundary (r<0.20 m)"), + plt.scatter([],[], c="#1565C0", s=70, marker="^", label="Fork — real UMI"), + plt.scatter([],[], c="#64B5F6", s=35, marker="^", label="Fork — synthetic"), + plt.scatter([],[], c="#BF360C", s=70, marker="s", label="Knife — real UMI"), + plt.scatter([],[], c="#FFAB76", s=35, marker="s", label="Knife — synthetic"), + plt.Rectangle((0,0),1,1, fc="#2196F3", alpha=0.3, label="Eval range — fork"), + plt.Rectangle((0,0),1,1, fc="#FF5722", alpha=0.3, label="Eval range — knife"), + plt.Line2D([0],[0], marker="D", ms=9, color="k", lw=0, label=f"Robot base ({cx},{cy})"), + plt.Line2D([0],[0], marker="o", ms=9, color="gray", lw=0, label="Plate (fixed)"), + ] + ax.legend(handles=legend_elements, fontsize=8, loc="lower left", + bbox_to_anchor=(0.0, -0.38), ncol=3, frameon=True) + + ax.set_xlim(-0.05, 0.95) + ax.set_ylim(-0.85, 0.25) + ax.set_aspect("equal") + ax.set_xlabel("X (m)", fontsize=11) + ax.set_ylabel("Y (m)", fontsize=11) + ax.set_title("Robot Workspace Analysis: Geometric Reach + Table Boundary", fontsize=12, fontweight="bold") + ax.grid(True, alpha=0.25) + plt.subplots_adjust(bottom=0.30) + + plt.tight_layout() + out = OUT_DIR / "fig4_workspace_analysis.png" + plt.savefig(out, dpi=150) + plt.close() + print(f"Saved: {out}") + +# ── text summary ────────────────────────────────────────────────────────────── +def print_summary(): + print("=" * 55) + print("WORKSPACE ANALYSIS SUMMARY") + print("=" * 55) + print(f"Robot base XY: ({ROBOT_BASE_XY[0]}, {ROBOT_BASE_XY[1]}) m") + print(f"Franka max reach: {FRANKA_MAX_REACH} m") + print(f"Grasp height: {GRASP_Z} m (table + GRASP_Z_OFFSET)") + print(f"Z distance (base→grasp): {Z_DIFF:.3f} m") + print(f"Horizontal reach at grasp: {R_MAX_H:.3f} m (= √({FRANKA_MAX_REACH}²−{Z_DIFF}²))") + print(f"Table y boundary: y ≤ {TABLE_Y_MAX} (empirical: datagen failures)") + print(f"Our filter used: x∈[0.22,0.80], y∈[−0.60,0.15]") + print(" Note: y upper bound 0.15 was conservative → some real episodes") + print(" still hit the table edge; synthetic data capped at y=−0.05.") + print("=" * 55) + + +if __name__ == "__main__": + print_summary() + fig_workspace() diff --git a/scripts/datagen/generate.py b/scripts/datagen/generate.py index 18959cd..8608d77 100644 --- a/scripts/datagen/generate.py +++ b/scripts/datagen/generate.py @@ -43,6 +43,53 @@ required=True, help="Path to the per-episode object_poses.json (UMI schema). Episode count = number of status=='full' entries.", ) +parser.add_argument( + "--augment_pose_factor", + type=int, + default=1, + help="Multiply the replay set by this factor using pose jitter. Example: 16 source episodes with factor 10 -> 160 replay episodes.", +) +parser.add_argument( + "--augment_global_xy_jitter", + type=float, + default=0.01, + help="Scene-level XY translation jitter in meters applied during pose augmentation.", +) +parser.add_argument( + "--augment_local_xy_jitter", + type=float, + default=0.01, + help="Per-object XY translation jitter in meters applied during pose augmentation.", +) +parser.add_argument( + "--augment_yaw_jitter_deg", + type=float, + default=0.0, + help="World-yaw jitter in degrees applied during pose augmentation.", +) +parser.add_argument( + "--augment_min_object_distance", + type=float, + default=0.05, + help="Minimum XY spacing in meters enforced between augmented objects.", +) +parser.add_argument( + "--augment_mix_objects", + action="store_true", + help="Mix object poses across different source episodes before jittering. Increases diversity but is slightly riskier than jitter-only augmentation.", +) +parser.add_argument( + "--cutlery_eval_pose_fraction", + type=float, + default=0.35, + help="For the cutlery task only: rewrite this fraction of augmented episodes so fork/knife start near the eval initial-pose distribution.", +) +parser.add_argument( + "--cutlery_eval_pose_jitter", + type=float, + default=0.05, + help="For the cutlery task only: XY jitter in meters around the eval base poses when injecting eval-like initial states.", +) parser.add_argument("--quality", action="store_true", help="Whether to enable quality render mode.") parser.add_argument("--use_lerobot_recorder", action="store_true", help="Whether to use lerobot recorder.") parser.add_argument("--lerobot_dataset_repo_id", type=str, default=None, help="Lerobot Dataset repository ID.") @@ -69,6 +116,10 @@ from simulator.datagen.state_machine.cup_stacking import CupStackingStateMachine from simulator.datagen.state_machine.cutlery_arrangement import CutleryArrangementStateMachine from simulator.datagen.state_machine.toy_blocks_collection import ToyBlocksCollectionStateMachine +from simulator.utils.object_pose_augmentation import ( + augment_episode_world_poses, + inject_cutlery_eval_pose_distribution, +) from simulator.utils.object_poses_loader import load_episode_poses # Maps gym task id → (StateMachineClass, device_type) @@ -287,6 +338,7 @@ def main(): f"Task '{task_name}' is not registered in TASK_REGISTRY.\nAvailable tasks: {list(TASK_REGISTRY.keys())}" ) SMClass, device = TASK_REGISTRY[task_name] + run_seed = args_cli.seed if args_cli.seed is not None else int(time.time()) output_dir = os.path.dirname(args_cli.dataset_file) output_file_name = os.path.splitext(os.path.basename(args_cli.dataset_file))[0] @@ -295,7 +347,7 @@ def main(): env_cfg = parse_env_cfg(task_name, device=args_cli.device, num_envs=args_cli.num_envs) env_cfg.use_teleop_device(device) - env_cfg.seed = args_cli.seed if args_cli.seed is not None else int(time.time()) + env_cfg.seed = run_seed if getattr(env_cfg, "object_pose_cfg", None) is None: raise ValueError( @@ -307,7 +359,37 @@ def main(): raise ValueError( f"No 'status==full' episodes in {args_cli.object_poses}; nothing to replay." ) - print(f"Loaded {len(episodes)} replay episodes from {args_cli.object_poses}") + base_episode_count = len(episodes) + print(f"Loaded {base_episode_count} replay episodes from {args_cli.object_poses}") + if args_cli.augment_pose_factor > 1: + episodes = augment_episode_world_poses( + episodes, + factor=args_cli.augment_pose_factor, + seed=run_seed, + global_xy_jitter=args_cli.augment_global_xy_jitter, + local_xy_jitter=args_cli.augment_local_xy_jitter, + yaw_jitter_deg=args_cli.augment_yaw_jitter_deg, + min_object_distance=args_cli.augment_min_object_distance, + mix_objects=args_cli.augment_mix_objects, + ) + print( + "Augmented replay episodes: " + f"{base_episode_count} -> {len(episodes)} " + f"(factor={args_cli.augment_pose_factor}, mix_objects={args_cli.augment_mix_objects}, seed={run_seed})" + ) + if task_name == "HCIS-CutleryArrangement-SingleArm-v0": + episodes = inject_cutlery_eval_pose_distribution( + episodes, + seed=run_seed, + eval_like_fraction=args_cli.cutlery_eval_pose_fraction, + eval_xy_jitter=args_cli.cutlery_eval_pose_jitter, + replaceable_start_index=base_episode_count, + min_object_distance=args_cli.augment_min_object_distance, + ) + print( + "Blended cutlery eval-like starts into replay set: " + f"fraction={args_cli.cutlery_eval_pose_fraction}, jitter={args_cli.cutlery_eval_pose_jitter}" + ) is_direct_env = "Direct" in task_name _configure_env_cfg(env_cfg, args_cli, is_direct_env, output_dir, output_file_name) diff --git a/scripts/gen_synthetic_poses.py b/scripts/gen_synthetic_poses.py new file mode 100644 index 0000000..10bf2f2 --- /dev/null +++ b/scripts/gen_synthetic_poses.py @@ -0,0 +1,111 @@ +"""Generate synthetic object_poses.json entries for cutlery arrangement datagen. + +Samples random fork/knife positions within the robot's reachable workspace +and outputs them in the UMI object_poses.json format so generate.py can +consume them directly without any changes. + +Anchor world pose: (0.40, 0.10, 0.0) + → tvec = (x_world - 0.40, y_world - 0.10, 0) + +Robot base: (0.35, -0.74), plate fixed at (0.50, -0.40). +""" + +import json +import math +import random +from pathlib import Path + +# ── config ────────────────────────────────────────────────────────────────── +SEED = 42 +N_SYNTHETIC = 60 # how many synthetic episodes to generate +ANCHOR_X, ANCHOR_Y = 0.40, 0.10 +PLATE_POS = (0.50, -0.40) +ROBOT_BASE = (0.35, -0.74) +R_MAX, R_MIN = 0.845, 0.20 # Franka horizontal reach at grasp height + +MIN_FORK_KNIFE_DIST = 0.08 # keep them ≥8 cm apart +MIN_PLATE_DIST = 0.12 # keep each object ≥12 cm from plate +# ──────────────────────────────────────────────────────────────────────────── + +def _dist(a, b): + return math.sqrt((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2) + + +def _in_workspace(x, y): + d = math.sqrt((x - ROBOT_BASE[0])**2 + (y - ROBOT_BASE[1])**2) + return R_MIN <= d <= R_MAX + + +def generate_synthetic(n: int, seed: int) -> list[dict]: + rng = random.Random(seed) + episodes = [] + attempts = 0 + while len(episodes) < n: + attempts += 1 + if attempts > 100_000: + print(f"[WARN] Only generated {len(episodes)}/{n} episodes after {attempts} attempts") + break + + fx = rng.uniform(ROBOT_BASE[0] - R_MAX, ROBOT_BASE[0] + R_MAX) + fy = rng.uniform(ROBOT_BASE[1] - R_MAX, ROBOT_BASE[1] + R_MAX) + kx = rng.uniform(ROBOT_BASE[0] - R_MAX, ROBOT_BASE[0] + R_MAX) + ky = rng.uniform(ROBOT_BASE[1] - R_MAX, ROBOT_BASE[1] + R_MAX) + + if not _in_workspace(fx, fy) or not _in_workspace(kx, ky): + continue + if _dist((fx, fy), (kx, ky)) < MIN_FORK_KNIFE_DIST: + continue + if _dist((fx, fy), PLATE_POS) < MIN_PLATE_DIST: + continue + if _dist((kx, ky), PLATE_POS) < MIN_PLATE_DIST: + continue + + episodes.append({ + "video_name": f"synthetic_{len(episodes):04d}", + "episode_range": [0, 100], + "objects": [ + { + "object_name": "fork", + "rvec": [0.0, 0.0, 0.0], + "tvec": [fx - ANCHOR_X, fy - ANCHOR_Y, 0.0], + }, + { + "object_name": "knife", + "rvec": [0.0, 0.0, 0.0], + "tvec": [kx - ANCHOR_X, ky - ANCHOR_Y, 0.0], + }, + { + "object_name": "plate", + "rvec": [0.0, 0.0, 0.0], + "tvec": [PLATE_POS[0] - ANCHOR_X, PLATE_POS[1] - ANCHOR_Y, 0.0], + }, + ], + "status": "full", + }) + + print(f"Generated {len(episodes)} synthetic episodes ({attempts} attempts)") + return episodes + + +def main(): + data_dir = Path(__file__).parent.parent / "data" / "AI-final-49" + real_path = data_dir / "object_poses_filtered.json" + out_path = data_dir / "object_poses_combined.json" + + with open(real_path) as f: + real_episodes = json.load(f) + print(f"Loaded {len(real_episodes)} real episodes from {real_path}") + + synthetic = generate_synthetic(N_SYNTHETIC, SEED) + + combined = real_episodes + synthetic + with open(out_path, "w") as f: + json.dump(combined, f, indent=2) + + print(f"Saved {len(combined)} total episodes → {out_path}") + print(f" real: {len(real_episodes)} synthetic: {len(synthetic)}") + print(f" × factor 10 = {len(combined) * 10} datagen episodes") + + +if __name__ == "__main__": + main() diff --git a/scripts/rollout.py b/scripts/rollout.py index 2f9ee6b..f9e6284 100644 --- a/scripts/rollout.py +++ b/scripts/rollout.py @@ -460,6 +460,40 @@ def get_camera_infos( return camera_infos +def _log_scene_state(env, label: str, step: int | None = None) -> None: + fork = env.scene["fork"] + knife = env.scene["knife"] + plate = env.scene["plate"] + robot = env.scene["robot"] + origins = env.scene.env_origins + fp = (fork.data.root_pos_w - origins)[0] + kp = (knife.data.root_pos_w - origins)[0] + pp = (plate.data.root_pos_w - origins)[0] + fd = ((fp[0] - pp[0]) ** 2 + (fp[1] - pp[1]) ** 2).sqrt().item() + kd = ((kp[0] - pp[0]) ** 2 + (kp[1] - pp[1]) ** 2).sqrt().item() + # EE position: find panda_hand body index + body_names = robot.data.body_names + ee_idx = body_names.index("panda_hand") if "panda_hand" in body_names else -1 + if ee_idx >= 0: + ee_pos = (robot.data.body_pos_w[:, ee_idx, :] - origins)[0] + ee_str = f"({ee_pos[0]:.3f}, {ee_pos[1]:.3f}, {ee_pos[2]:.3f})" + else: + ee_str = "N/A" + # Gripper: panda_finger_joint1 + joint_names = list(robot.data.joint_names) + g_idx = joint_names.index("panda_finger_joint1") if "panda_finger_joint1" in joint_names else -1 + gripper_str = f"{robot.data.joint_pos[0, g_idx].item():.4f}" if g_idx >= 0 else "N/A" + prefix = f"[step={step}] " if step is not None else "" + print( + f"{prefix}[SCENE] {label}\n" + f" plate =({pp[0]:.3f}, {pp[1]:.3f}, {pp[2]:.3f})\n" + f" fork =({fp[0]:.3f}, {fp[1]:.3f}, {fp[2]:.3f}) dist={fd:.3f} left_of_plate={'Y' if fp[0] < pp[0] else 'N'} within_15cm={'Y' if fd <= 0.15 else 'N'}\n" + f" knife =({kp[0]:.3f}, {kp[1]:.3f}, {kp[2]:.3f}) dist={kd:.3f} right_of_plate={'Y' if kp[0] > pp[0] else 'N'} within_15cm={'Y' if kd <= 0.15 else 'N'}\n" + f" EE ={ee_str} gripper(finger1)={gripper_str}", + flush=True, + ) + + def main(): task_id = resolve_task(args_cli.task) args_cli.task = task_id @@ -525,6 +559,8 @@ def main(): while max_episode_count <= 0 or episode_count <= max_episode_count: print(f"[Evaluation] Evaluating episode {episode_count}...") success, time_out = False, False + policy_call_count = 0 + _log_scene_state(env, "episode start") while simulation_app.is_running(): with torch.inference_mode(): if controller.reset_state: @@ -538,6 +574,10 @@ def main(): obs_dict["policy"], language_instruction ) actions = policy.get_action(policy_obs_dict).to(env.device) + policy_call_count += 1 + if not hasattr(policy, '_debug_printed'): + print(f"[DEBUG] first action chunk: {actions[0, 0, :].tolist()}", flush=True) + policy._debug_printed = True for action_index in range( min(args_cli.policy_action_horizon, actions.shape[0]) ): @@ -553,13 +593,17 @@ def main(): break if rate_limiter: rate_limiter.sleep(env) + if policy_call_count % 10 == 0: + _log_scene_state(env, "progress", step=policy_call_count * args_cli.policy_action_horizon) if success: + _log_scene_state(env, "SUCCESS") print(f"[Evaluation] Episode {episode_count} is successful!") episode_count += 1 success_count += 1 policy.reset() break if time_out: + _log_scene_state(env, "TIMEOUT — final positions") print(f"[Evaluation] Episode {episode_count} timed out!") episode_count += 1 policy.reset() @@ -574,6 +618,13 @@ def main(): f" [{success_count}/{max_episode_count}]" ) + fork = env.scene["fork"] + knife = env.scene["knife"] + plate = env.scene["plate"] + print(f"[DEBUG] fork pos: {fork.data.root_pos_w[0].tolist()}") + print(f"[DEBUG] knife pos: {knife.data.root_pos_w[0].tolist()}") + print(f"[DEBUG] plate pos: {plate.data.root_pos_w[0].tolist()}") + env.close() simulation_app.close() diff --git a/tests/test_object_pose_augmentation.py b/tests/test_object_pose_augmentation.py new file mode 100644 index 0000000..a9edbc9 --- /dev/null +++ b/tests/test_object_pose_augmentation.py @@ -0,0 +1,147 @@ +import math + +import pytest + +from simulator.utils.object_pose_augmentation import ( + PoseAugmentationError, + augment_episode_world_poses, + inject_cutlery_eval_pose_distribution, +) + + +def _episode(blue_xy, pink_xy): + return { + "blue_cup": ((blue_xy[0], blue_xy[1], 0.12), (1.0, 0.0, 0.0, 0.0)), + "pink_cup": ((pink_xy[0], pink_xy[1], 0.12), (1.0, 0.0, 0.0, 0.0)), + } + + +def _yaw_from_quat_wxyz(quat): + w, x, y, z = quat + return math.atan2(2.0 * (w * z + x * y), 1.0 - 2.0 * (y * y + z * z)) + + +def test_factor_one_returns_copy_with_same_content(): + episodes = [_episode((0.4, -0.2), (0.55, -0.2))] + out = augment_episode_world_poses(episodes, factor=1, seed=7) + + assert out == episodes + assert out is not episodes + assert out[0] is not episodes[0] + + +def test_factor_multiplies_episode_count(): + episodes = [ + _episode((0.40, -0.20), (0.56, -0.20)), + _episode((0.42, -0.24), (0.58, -0.22)), + ] + + out = augment_episode_world_poses( + episodes, + factor=3, + seed=11, + global_xy_jitter=0.01, + local_xy_jitter=0.005, + ) + + assert len(out) == 6 + assert out[:2] == episodes + + +def test_yaw_jitter_rotates_quaternion(): + episodes = [_episode((0.4, -0.2), (0.55, -0.2))] + + out = augment_episode_world_poses( + episodes, + factor=2, + seed=5, + yaw_jitter_deg=15.0, + ) + + assert len(out) == 2 + augmented_yaw = _yaw_from_quat_wxyz(out[1]["blue_cup"][1]) + assert abs(augmented_yaw) > 1e-6 + + +def test_mix_objects_recombines_pose_bank(): + episodes = [ + _episode((0.40, -0.20), (0.70, -0.20)), + _episode((0.48, -0.32), (0.86, -0.32)), + ] + + out = augment_episode_world_poses( + episodes, + factor=2, + seed=3, + mix_objects=True, + global_xy_jitter=0.0, + local_xy_jitter=0.0, + min_object_distance=0.05, + ) + + augmented = out[2] + blue_x = augmented["blue_cup"][0][0] + pink_x = augmented["pink_cup"][0][0] + assert blue_x in {0.40, 0.48} + assert pink_x in {0.70, 0.86} + + +def test_min_object_distance_rejects_collision_and_falls_back_to_base(): + episodes = [_episode((0.40, -0.20), (0.44, -0.20))] + + out = augment_episode_world_poses( + episodes, + factor=2, + seed=19, + mix_objects=True, + min_object_distance=0.10, + max_attempts=2, + ) + + assert out[1] == episodes[0] + + +def test_inconsistent_object_sets_raise(): + episodes = [ + _episode((0.40, -0.20), (0.56, -0.20)), + {"blue_cup": ((0.42, -0.24, 0.12), (1.0, 0.0, 0.0, 0.0))}, + ] + + with pytest.raises(PoseAugmentationError, match="object set"): + augment_episode_world_poses(episodes, factor=2, seed=1) + + +def test_inject_cutlery_eval_pose_distribution_rewrites_only_augmented_subset(): + episodes = [ + _episode((0.10, -0.20), (0.20, -0.20)), + _episode((0.30, -0.30), (0.40, -0.30)), + _episode((0.50, -0.35), (0.60, -0.35)), + ] + + out = inject_cutlery_eval_pose_distribution( + episodes, + seed=13, + eval_like_fraction=0.5, + eval_xy_jitter=0.0, + replaceable_start_index=1, + min_object_distance=0.05, + ) + + assert out[0] == episodes[0] + assert len(out) == len(episodes) + assert any( + episode["knife"][0][:2] == (0.50, -0.10) and episode["fork"][0][:2] == (0.55, -0.10) + for episode in out[1:] + ) + + +def test_inject_cutlery_eval_pose_distribution_validates_fraction(): + episodes = [_episode((0.10, -0.20), (0.20, -0.20))] + + with pytest.raises(PoseAugmentationError, match="eval_like_fraction"): + inject_cutlery_eval_pose_distribution( + episodes, + seed=1, + eval_like_fraction=1.5, + eval_xy_jitter=0.01, + )