From 15526d4266458e80a931d0c28057f3a13a82a7af Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Sat, 28 Feb 2026 22:30:16 -0800 Subject: [PATCH 01/19] Add Antim Labs sim integration module (dimsim) Add native DimSim bridge and nav blueprint for browser-based 3D simulation with LCM transport. Uses globally installed dimsim CLI (https://jsr.io/@antim/dimsim). - DimSimBridge (NativeModule) manages the dimsim subprocess - DimSimTF publishes transform tree from odom - sim-nav blueprint wires bridge + TF + voxel mapping + A* + frontier exploration Usage: dimos run sim-nav TODO: - General eval workflow integration - Test headless integration --- dimos/robot/all_blueprints.py | 2 + dimos/robot/sim/__init__.py | 0 dimos/robot/sim/blueprints/__init__.py | 0 dimos/robot/sim/blueprints/basic/__init__.py | 0 dimos/robot/sim/blueprints/basic/sim_basic.py | 111 +++++++++++ dimos/robot/sim/blueprints/nav/__init__.py | 0 dimos/robot/sim/blueprints/nav/sim_nav.py | 32 ++++ dimos/robot/sim/bridge.py | 155 +++++++++++++++ dimos/robot/sim/tf_module.py | 180 ++++++++++++++++++ 9 files changed, 480 insertions(+) create mode 100644 dimos/robot/sim/__init__.py create mode 100644 dimos/robot/sim/blueprints/__init__.py create mode 100644 dimos/robot/sim/blueprints/basic/__init__.py create mode 100644 dimos/robot/sim/blueprints/basic/sim_basic.py create mode 100644 dimos/robot/sim/blueprints/nav/__init__.py create mode 100644 dimos/robot/sim/blueprints/nav/sim_nav.py create mode 100644 dimos/robot/sim/bridge.py create mode 100644 dimos/robot/sim/tf_module.py diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 6026572388..d4e04db1eb 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -58,6 +58,8 @@ "mid360-fastlio-voxels": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels", "mid360-fastlio-voxels-native": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels_native", "phone-go2-teleop": "dimos.teleop.phone.blueprints:phone_go2_teleop", + # "sim-basic": "dimos.robot.sim.blueprints.basic.sim_basic:sim_basic", + "sim-nav": "dimos.robot.sim.blueprints.nav.sim_nav:sim_nav", "simple-phone-teleop": "dimos.teleop.phone.blueprints:simple_phone_teleop", "uintree-g1-primitive-no-nav": "dimos.robot.unitree.g1.blueprints.primitive.uintree_g1_primitive_no_nav:uintree_g1_primitive_no_nav", "unitree-g1": "dimos.robot.unitree.g1.blueprints.perceptive.unitree_g1:unitree_g1", diff --git a/dimos/robot/sim/__init__.py b/dimos/robot/sim/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dimos/robot/sim/blueprints/__init__.py b/dimos/robot/sim/blueprints/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dimos/robot/sim/blueprints/basic/__init__.py b/dimos/robot/sim/blueprints/basic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py new file mode 100644 index 0000000000..89fe2a94a5 --- /dev/null +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -0,0 +1,111 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Basic DimSim blueprint — connection + visualization.""" + +import platform +from typing import Any + +from dimos.constants import DEFAULT_CAPACITY_COLOR_IMAGE +from dimos.core.blueprints import autoconnect +from dimos.core.global_config import global_config +from dimos.core.transport import pSHMTransport +from dimos.msgs.sensor_msgs import Image +from dimos.protocol.pubsub.impl.lcmpubsub import LCM +from dimos.robot.sim.bridge import sim_bridge +from dimos.robot.sim.tf_module import sim_tf +from dimos.web.websocket_vis.websocket_vis_module import websocket_vis + +_mac_transports: dict[tuple[str, type], pSHMTransport[Image]] = { + ("color_image", Image): pSHMTransport( + "color_image", default_capacity=DEFAULT_CAPACITY_COLOR_IMAGE + ), +} + +_transports_base = ( + autoconnect() if platform.system() == "Linux" else autoconnect().transports(_mac_transports) +) + + +def _convert_camera_info(camera_info: Any) -> Any: + return camera_info.to_rerun( + image_topic="/world/color_image", + optical_frame="camera_optical", + ) + + +def _convert_global_map(grid: Any) -> Any: + return grid.to_rerun(voxel_size=0.1, mode="boxes") + + +def _convert_navigation_costmap(grid: Any) -> Any: + return grid.to_rerun( + colormap="Accent", + z_offset=0.015, + opacity=0.2, + background="#484981", + ) + + +def _static_base_link(rr: Any) -> list[Any]: + return [ + rr.Boxes3D( + half_sizes=[0.3, 0.15, 0.12], + colors=[(0, 180, 255)], + ), + rr.Transform3D(parent_frame="tf#/base_link"), + ] + + +rerun_config = { + "pubsubs": [LCM(autoconf=True)], + "visual_override": { + "world/camera_info": _convert_camera_info, + "world/global_map": _convert_global_map, + "world/navigation_costmap": _convert_navigation_costmap, + "world/pointcloud": None, + "world/depth_image": None, + }, + "static": { + "world/tf/base_link": _static_base_link, + }, +} + +match global_config.viewer_backend: + case "foxglove": + from dimos.robot.foxglove_bridge import foxglove_bridge + + with_vis = autoconnect( + _transports_base, + foxglove_bridge(shm_channels=["/color_image#sensor_msgs.Image"]), + ) + case "rerun": + from dimos.visualization.rerun.bridge import rerun_bridge + + with_vis = autoconnect(_transports_base, rerun_bridge(**rerun_config)) + case "rerun-web": + from dimos.visualization.rerun.bridge import rerun_bridge + + with_vis = autoconnect(_transports_base, rerun_bridge(viewer_mode="web", **rerun_config)) + case _: + with_vis = _transports_base + +sim_basic = autoconnect( + with_vis, + sim_bridge(), + sim_tf(), + websocket_vis(), +).global_config(n_workers=4, robot_model="dimsim") + +__all__ = ["sim_basic"] diff --git a/dimos/robot/sim/blueprints/nav/__init__.py b/dimos/robot/sim/blueprints/nav/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dimos/robot/sim/blueprints/nav/sim_nav.py b/dimos/robot/sim/blueprints/nav/sim_nav.py new file mode 100644 index 0000000000..babaecac44 --- /dev/null +++ b/dimos/robot/sim/blueprints/nav/sim_nav.py @@ -0,0 +1,32 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim navigation blueprint — basic + mapping + planning + exploration.""" + +from dimos.core.blueprints import autoconnect +from dimos.mapping.costmapper import cost_mapper +from dimos.mapping.voxels import voxel_mapper +from dimos.navigation.frontier_exploration import wavefront_frontier_explorer +from dimos.navigation.replanning_a_star.module import replanning_a_star_planner +from dimos.robot.sim.blueprints.basic.sim_basic import sim_basic + +sim_nav = autoconnect( + sim_basic, + voxel_mapper(voxel_size=0.1), + cost_mapper(), + replanning_a_star_planner(), + wavefront_frontier_explorer(), +).global_config(n_workers=6, robot_model="dimsim") + +__all__ = ["sim_nav"] diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py new file mode 100644 index 0000000000..47fd79003b --- /dev/null +++ b/dimos/robot/sim/bridge.py @@ -0,0 +1,155 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NativeModule wrapper for the DimSim bridge subprocess. + +Launches the DimSim bridge (Deno CLI) as a managed subprocess. The bridge +publishes sensor data (odom, lidar, images) directly to LCM — no Python +decode/re-encode hop. Python only handles lifecycle and TF (via DimSimTF). + +Usage:: + + from dimos.robot.sim.bridge import sim_bridge + from dimos.robot.sim.tf_module import sim_tf + from dimos.core.blueprints import autoconnect + + autoconnect(sim_bridge(), sim_tf(), some_consumer()).build().loop() +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +import shutil +from typing import TYPE_CHECKING + +from dimos import spec +from dimos.core.native_module import NativeModule, NativeModuleConfig +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from dimos.core.stream import In, Out + from dimos.msgs.geometry_msgs import PoseStamped, Twist + from dimos.msgs.sensor_msgs import CameraInfo, Image, PointCloud2 + +logger = setup_logger() + + +def _find_cli_script() -> Path | None: + """Auto-detect DimSim/dimos-cli/cli.ts relative to this repo.""" + repo_root = Path(__file__).resolve().parents[4] # dimos/dimos/robot/sim -> repo + candidate = repo_root / "DimSim" / "dimos-cli" / "cli.ts" + return candidate if candidate.exists() else None + + +def _find_deno() -> str: + """Find the deno binary.""" + return shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") + + +@dataclass(kw_only=True) +class DimSimBridgeConfig(NativeModuleConfig): + """Configuration for the DimSim bridge subprocess.""" + + # Set to deno binary — resolved in _resolve_paths(). + executable: str = "deno" + build_command: str | None = None + cwd: str | None = None + + scene: str = "apt" + port: int = 8090 + cli_script: str | None = None + + # These fields are handled via extra_args, not to_cli_args(). + cli_exclude: frozenset[str] = frozenset({"scene", "port", "cli_script"}) + + # Populated by _resolve_paths() — deno run args + dev subcommand + scene/port. + extra_args: list[str] = field(default_factory=list) + + +class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): + """NativeModule that manages the DimSim bridge subprocess. + + The bridge (Deno process) handles Browser-LCM translation and publishes + sensor data directly to LCM. Ports declared here exist for blueprint + wiring / autoconnect but data flows through LCM, not Python. + """ + + config: DimSimBridgeConfig + default_config = DimSimBridgeConfig + + # Sensor outputs (bridge publishes these directly to LCM) + odom: Out[PoseStamped] + color_image: Out[Image] + depth_image: Out[Image] + lidar: Out[PointCloud2] + pointcloud: Out[PointCloud2] + camera_info: Out[CameraInfo] + + # Control input (consumers publish cmd_vel to LCM, bridge reads it) + cmd_vel: In[Twist] + + def _resolve_paths(self) -> None: + """Resolve executable and build extra_args. + + Prefers globally installed ``dimsim`` CLI (from JSR). Falls back to + running the local ``DimSim/dimos-cli/cli.ts`` via Deno for development. + """ + dev_args = ["dev", "--scene", self.config.scene, "--port", str(self.config.port)] + + # 1. Prefer globally installed dimsim CLI (deno install jsr:@antim/dimsim) + global_dimsim = shutil.which("dimsim") + if global_dimsim: + logger.info(f"Using global dimsim CLI: {global_dimsim}") + self.config.executable = global_dimsim + self.config.extra_args = dev_args + self.config.cwd = None + return + + # 2. Fall back to local deno + cli.ts (development mode) + script = self.config.cli_script + if script and Path(script).exists(): + cli_ts = str(Path(script).resolve()) + else: + found = _find_cli_script() + if found: + cli_ts = str(found) + else: + raise FileNotFoundError( + "Cannot find DimSim. Install globally with:\n" + " deno install -gAf --unstable-net jsr:@antim/dimsim\n" + " dimsim setup && dimsim scene install apt" + ) + + self.config.executable = _find_deno() + self.config.extra_args = [ + "run", + "--allow-all", + "--unstable-net", + cli_ts, + *dev_args, + ] + self.config.cwd = None + + def _maybe_build(self) -> None: + """No build step needed for DimSim bridge.""" + + def _collect_topics(self) -> dict[str, str]: + """Bridge hardcodes LCM channel names — no topic args needed.""" + return {} + + +sim_bridge = DimSimBridge.blueprint + +__all__ = ["DimSimBridge", "DimSimBridgeConfig", "sim_bridge"] diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py new file mode 100644 index 0000000000..f677bab3d3 --- /dev/null +++ b/dimos/robot/sim/tf_module.py @@ -0,0 +1,180 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lightweight TF publisher for DimSim. + +Subscribes to odometry from the DimSim bridge (via LCM, wired by autoconnect) +and publishes the transform chain: world -> base_link -> {camera_link -> +camera_optical, lidar_link}. Also publishes CameraInfo at 1 Hz, forwards +cmd_vel to the bridge, and exposes a ``move()`` RPC. + +This module replaces the TF / camera_info / cmd_vel parts of the old +DimSimConnection while the NativeModule bridge handles sensor data directly. +""" + +from __future__ import annotations + +import math +from threading import Thread +import time + +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs import ( + PoseStamped, + Quaternion, + Transform, + Twist, + Vector3, +) +from dimos.msgs.sensor_msgs import CameraInfo +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +# DimSim captures at 960x432 with 80-degree horizontal FOV. +_DIMSIM_WIDTH = 960 +_DIMSIM_HEIGHT = 432 +_DIMSIM_FOV_DEG = 80 + + +def _camera_info_static() -> CameraInfo: + """Build CameraInfo for DimSim's virtual camera.""" + fov_rad = math.radians(_DIMSIM_FOV_DEG) + fx = (_DIMSIM_WIDTH / 2) / math.tan(fov_rad / 2) + fy = fx # square pixels + cx = _DIMSIM_WIDTH / 2.0 + cy = _DIMSIM_HEIGHT / 2.0 + + return CameraInfo( + frame_id="camera_optical", + height=_DIMSIM_HEIGHT, + width=_DIMSIM_WIDTH, + distortion_model="plumb_bob", + D=[0.0, 0.0, 0.0, 0.0, 0.0], + K=[fx, 0.0, cx, 0.0, fy, cy, 0.0, 0.0, 1.0], + R=[1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0], + P=[fx, 0.0, cx, 0.0, 0.0, fy, cy, 0.0, 0.0, 0.0, 1.0, 0.0], + binning_x=0, + binning_y=0, + ) + + +class DimSimTF(Module): + """Lightweight TF publisher for the DimSim simulator. + + Wired by autoconnect to receive odom from the bridge's LCM output. + Publishes TF transforms and camera intrinsics. Exposes ``move()`` RPC + for sending cmd_vel to the bridge. + """ + + # Odom input — autoconnect wires this to DimSimBridge.odom via LCM + odom: In[PoseStamped] + + # Outputs + camera_info: Out[CameraInfo] + cmd_vel: Out[Twist] + + _camera_info_thread: Thread | None = None + _latest_odom: PoseStamped | None = None + _odom_last_ts: float = 0.0 + _odom_count: int = 0 + + @classmethod + def _odom_to_tf(cls, odom: PoseStamped) -> list[Transform]: + """Build transform chain from odometry pose. + + Transform tree: world -> base_link -> {camera_link -> camera_optical, lidar_link} + """ + camera_link = Transform( + translation=Vector3(0.3, 0.0, 0.0), # camera 30cm forward + rotation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="base_link", + child_frame_id="camera_link", + ts=odom.ts, + ) + + camera_optical = Transform( + translation=Vector3(0.0, 0.0, 0.0), + rotation=Quaternion(-0.5, 0.5, -0.5, 0.5), + frame_id="camera_link", + child_frame_id="camera_optical", + ts=odom.ts, + ) + + lidar_link = Transform( + translation=Vector3(0.0, 0.0, 0.0), + rotation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="base_link", + child_frame_id="lidar_link", + ts=odom.ts, + ) + + return [ + Transform.from_pose("base_link", odom), + camera_link, + camera_optical, + lidar_link, + ] + + def _on_odom(self, pose: PoseStamped) -> None: + """Handle incoming odometry — publish TF transforms.""" + # Drop out-of-order messages (UDP multicast doesn't guarantee ordering) + if pose.ts <= self._odom_last_ts: + return + self._odom_last_ts = pose.ts + self._latest_odom = pose + self._odom_count += 1 + + transforms = self._odom_to_tf(pose) + self.tf.publish(*transforms) + + def _publish_camera_info_loop(self) -> None: + """Publish camera intrinsics at 1 Hz.""" + while self._camera_info_thread is not None: + self.camera_info.publish(_camera_info_static()) + time.sleep(1.0) + + @rpc + def start(self) -> None: + super().start() + + from reactivex.disposable import Disposable + + self._disposables.add(Disposable(self.odom.subscribe(self._on_odom))) + + self._camera_info_thread = Thread(target=self._publish_camera_info_loop, daemon=True) + self._camera_info_thread.start() + + logger.info("DimSimTF started — listening for odom, publishing TF + camera_info") + + @rpc + def stop(self) -> None: + thread = self._camera_info_thread + self._camera_info_thread = None + if thread and thread.is_alive(): + thread.join(timeout=1.0) + super().stop() + + @rpc + def move(self, twist: Twist, duration: float = 0.0) -> bool: + """Send movement command to the simulator via cmd_vel.""" + self.cmd_vel.publish(twist) + return True + + +sim_tf = DimSimTF.blueprint + +__all__ = ["DimSimTF", "sim_tf"] From 6d2896805b21a15536be9ac185e662a29421d846 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Sun, 1 Mar 2026 23:32:35 -0800 Subject: [PATCH 02/19] Auto-install and update dimsim CLI on dimos run sim-nav --- dimos/robot/sim/bridge.py | 75 +++++++++++++++------------------------ 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 47fd79003b..2576c08804 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -45,12 +45,7 @@ logger = setup_logger() - -def _find_cli_script() -> Path | None: - """Auto-detect DimSim/dimos-cli/cli.ts relative to this repo.""" - repo_root = Path(__file__).resolve().parents[4] # dimos/dimos/robot/sim -> repo - candidate = repo_root / "DimSim" / "dimos-cli" / "cli.ts" - return candidate if candidate.exists() else None +_DIMSIM_JSR = "jsr:@antim/dimsim" def _find_deno() -> str: @@ -69,10 +64,9 @@ class DimSimBridgeConfig(NativeModuleConfig): scene: str = "apt" port: int = 8090 - cli_script: str | None = None # These fields are handled via extra_args, not to_cli_args(). - cli_exclude: frozenset[str] = frozenset({"scene", "port", "cli_script"}) + cli_exclude: frozenset[str] = frozenset({"scene", "port"}) # Populated by _resolve_paths() — deno run args + dev subcommand + scene/port. extra_args: list[str] = field(default_factory=list) @@ -101,49 +95,38 @@ class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): cmd_vel: In[Twist] def _resolve_paths(self) -> None: - """Resolve executable and build extra_args. - - Prefers globally installed ``dimsim`` CLI (from JSR). Falls back to - running the local ``DimSim/dimos-cli/cli.ts`` via Deno for development. - """ - dev_args = ["dev", "--scene", self.config.scene, "--port", str(self.config.port)] - - # 1. Prefer globally installed dimsim CLI (deno install jsr:@antim/dimsim) - global_dimsim = shutil.which("dimsim") - if global_dimsim: - logger.info(f"Using global dimsim CLI: {global_dimsim}") - self.config.executable = global_dimsim - self.config.extra_args = dev_args - self.config.cwd = None - return - - # 2. Fall back to local deno + cli.ts (development mode) - script = self.config.cli_script - if script and Path(script).exists(): - cli_ts = str(Path(script).resolve()) - else: - found = _find_cli_script() - if found: - cli_ts = str(found) - else: - raise FileNotFoundError( - "Cannot find DimSim. Install globally with:\n" - " deno install -gAf --unstable-net jsr:@antim/dimsim\n" - " dimsim setup && dimsim scene install apt" - ) - - self.config.executable = _find_deno() + """Resolve executable and build extra_args.""" + dimsim_path = shutil.which("dimsim") or str(Path.home() / ".deno" / "bin" / "dimsim") + self.config.executable = dimsim_path self.config.extra_args = [ - "run", - "--allow-all", - "--unstable-net", - cli_ts, - *dev_args, + "dev", "--scene", self.config.scene, "--port", str(self.config.port), ] self.config.cwd = None def _maybe_build(self) -> None: - """No build step needed for DimSim bridge.""" + """Ensure dimsim CLI, core assets, and scene are latest from S3.""" + import subprocess + + deno = _find_deno() + scene = self.config.scene + + # Always install/update CLI — idempotent, pulls latest JSR version + logger.info("Ensuring dimsim CLI is up-to-date...") + subprocess.run( + [deno, "install", "-gAf", "--unstable-net", _DIMSIM_JSR], + check=True, + ) + + dimsim = shutil.which("dimsim") + if not dimsim: + raise FileNotFoundError("dimsim install failed — not found in PATH") + + # Always re-download core assets + scene (pulls latest from S3) + logger.info("Downloading latest core assets...") + subprocess.run([dimsim, "setup"], check=True) + + logger.info(f"Downloading latest scene: {scene}...") + subprocess.run([dimsim, "scene", "install", scene], check=True) def _collect_topics(self) -> dict[str, str]: """Bridge hardcodes LCM channel names — no topic args needed.""" From fe77900d1605c29a401c6cb855817aa0af72fa68 Mon Sep 17 00:00:00 2001 From: Viswa4599 <39920874+Viswa4599@users.noreply.github.com> Date: Mon, 2 Mar 2026 07:47:11 +0000 Subject: [PATCH 03/19] CI code cleanup --- dimos/robot/sim/bridge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 2576c08804..da3c16361b 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -99,7 +99,11 @@ def _resolve_paths(self) -> None: dimsim_path = shutil.which("dimsim") or str(Path.home() / ".deno" / "bin" / "dimsim") self.config.executable = dimsim_path self.config.extra_args = [ - "dev", "--scene", self.config.scene, "--port", str(self.config.port), + "dev", + "--scene", + self.config.scene, + "--port", + str(self.config.port), ] self.config.cwd = None From 97c77f706fa69ad04dc89f320ec2c33bdc83989a Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Wed, 4 Mar 2026 22:34:40 -0800 Subject: [PATCH 04/19] =?UTF-8?q?Fixes:=20-=20Remove=20out-of-order=20odom?= =?UTF-8?q?=20drop=20in=20tf=5Fmodule=20=E2=80=94=20server-side=20physics?= =?UTF-8?q?=20=20=20publishes=20at=2050Hz=20from=20a=20single=20source,=20?= =?UTF-8?q?timestamp=20filtering=20caused=20=20=20jitter=20by=20dropping?= =?UTF-8?q?=20near-identical=20timestamps=20over=20UDP=20multicast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sensor rates: - Odom: 50 Hz - Lidar: 10 Hz, 15,000 points, 4m range - RGB: 2 Hz, 960×432 JPEG - Depth: 2 Hz, 960×432 16UC1 Usage: dimos run sim-nav # normal DIMSIM_HEADLESS=1 DIMSIM_RENDER=gpu dimos run sim-nav # headless Co-Authored-By: Claude Opus 4.6 --- dimos/robot/all_blueprints.py | 2 +- dimos/robot/sim/blueprints/basic/sim_basic.py | 40 ++++--- dimos/robot/sim/bridge.py | 110 ++++++++++++++---- dimos/robot/sim/tf_module.py | 4 - 4 files changed, 116 insertions(+), 40 deletions(-) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index d4e04db1eb..307db25e0d 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -58,7 +58,7 @@ "mid360-fastlio-voxels": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels", "mid360-fastlio-voxels-native": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels_native", "phone-go2-teleop": "dimos.teleop.phone.blueprints:phone_go2_teleop", - # "sim-basic": "dimos.robot.sim.blueprints.basic.sim_basic:sim_basic", + "sim-basic": "dimos.robot.sim.blueprints.basic.sim_basic:sim_basic", "sim-nav": "dimos.robot.sim.blueprints.nav.sim_nav:sim_nav", "simple-phone-teleop": "dimos.teleop.phone.blueprints:simple_phone_teleop", "uintree-g1-primitive-no-nav": "dimos.robot.unitree.g1.blueprints.primitive.uintree_g1_primitive_no_nav:uintree_g1_primitive_no_nav", diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 89fe2a94a5..942d6d64c7 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -14,27 +14,42 @@ """Basic DimSim blueprint — connection + visualization.""" -import platform from typing import Any -from dimos.constants import DEFAULT_CAPACITY_COLOR_IMAGE from dimos.core.blueprints import autoconnect from dimos.core.global_config import global_config -from dimos.core.transport import pSHMTransport +from dimos.core.transport import JpegLcmTransport from dimos.msgs.sensor_msgs import Image -from dimos.protocol.pubsub.impl.lcmpubsub import LCM +from dimos.protocol.pubsub.encoders import DecodingError +from dimos.protocol.pubsub.impl.lcmpubsub import JpegLCM, LCM from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf from dimos.web.websocket_vis.websocket_vis_module import websocket_vis -_mac_transports: dict[tuple[str, type], pSHMTransport[Image]] = { - ("color_image", Image): pSHMTransport( - "color_image", default_capacity=DEFAULT_CAPACITY_COLOR_IMAGE - ), -} -_transports_base = ( - autoconnect() if platform.system() == "Linux" else autoconnect().transports(_mac_transports) +class _SafeJpegLCM(JpegLCM): # type: ignore[misc] + """JpegLCM that only decodes image topics, skipping everything else cheaply.""" + + _JPEG_TOPICS = frozenset({"/color_image"}) + + def decode(self, msg: bytes, topic: Any) -> Image: # type: ignore[override] + if getattr(topic, "topic", None) not in self._JPEG_TOPICS: + raise DecodingError("skip") + return super().decode(msg, topic) + + +class _SkipJpegLCM(LCM): # type: ignore[misc] + """Standard LCM that skips JPEG image topics to avoid decode errors.""" + + def decode(self, msg: bytes, topic: Any) -> Any: # type: ignore[override] + if getattr(topic, "topic", None) in _SafeJpegLCM._JPEG_TOPICS: + raise DecodingError("skip") + return super().decode(msg, topic) + + +# DimSim sends JPEG-compressed images over LCM — use JpegLcmTransport to decode. +_transports_base = autoconnect().transports( + {("color_image", Image): JpegLcmTransport("/color_image", Image)} ) @@ -69,13 +84,12 @@ def _static_base_link(rr: Any) -> list[Any]: rerun_config = { - "pubsubs": [LCM(autoconf=True)], + "pubsubs": [_SkipJpegLCM(autoconf=True), _SafeJpegLCM(autoconf=True)], "visual_override": { "world/camera_info": _convert_camera_info, "world/global_map": _convert_global_map, "world/navigation_costmap": _convert_navigation_costmap, "world/pointcloud": None, - "world/depth_image": None, }, "static": { "world/tf/base_link": _static_base_link, diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index da3c16361b..2573053f2f 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -30,6 +30,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import os from pathlib import Path import shutil from typing import TYPE_CHECKING @@ -53,6 +54,13 @@ def _find_deno() -> str: return shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") +def _find_local_cli() -> Path | None: + """Find local DimSim/dimos-cli/cli.ts for development.""" + repo_root = Path(__file__).resolve().parents[4] + candidate = repo_root / "DimSim" / "dimos-cli" / "cli.ts" + return candidate if candidate.exists() else None + + @dataclass(kw_only=True) class DimSimBridgeConfig(NativeModuleConfig): """Configuration for the DimSim bridge subprocess.""" @@ -64,9 +72,10 @@ class DimSimBridgeConfig(NativeModuleConfig): scene: str = "apt" port: int = 8090 + local: bool = False # Use local DimSim repo instead of installed CLI # These fields are handled via extra_args, not to_cli_args(). - cli_exclude: frozenset[str] = frozenset({"scene", "port"}) + cli_exclude: frozenset[str] = frozenset({"scene", "port", "local"}) # Populated by _resolve_paths() — deno run args + dev subcommand + scene/port. extra_args: list[str] = field(default_factory=list) @@ -95,41 +104,98 @@ class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): cmd_vel: In[Twist] def _resolve_paths(self) -> None: - """Resolve executable and build extra_args.""" + """Resolve executable and build extra_args. + + Set DIMSIM_LOCAL=1 to use local DimSim repo instead of installed CLI. + """ + dev_args = ["dev", "--scene", self.config.scene, "--port", str(self.config.port)] + + # DIMSIM_HEADLESS=1 → launch headless Chrome (no browser tab needed) + # Uses CPU rendering (SwiftShader) by default — no GPU required for CI. + # Set DIMSIM_RENDER=gpu for Metal/ANGLE on macOS. + if os.environ.get("DIMSIM_HEADLESS", "").strip() in ("1", "true"): + render = os.environ.get("DIMSIM_RENDER", "cpu").strip() + dev_args.extend(["--headless", "--render", render]) + + # Allow env var override: DIMSIM_LOCAL=1 dimos run sim-nav + if os.environ.get("DIMSIM_LOCAL", "").strip() in ("1", "true"): + self.config.local = True + + if self.config.local: + cli_ts = _find_local_cli() + if not cli_ts: + raise FileNotFoundError( + "Local DimSim not found. Expected DimSim/dimos-cli/cli.ts " + "next to the dimos repo." + ) + logger.info(f"Using local DimSim: {cli_ts}") + self.config.executable = _find_deno() + self.config.extra_args = [ + "run", "--allow-all", "--unstable-net", str(cli_ts), *dev_args, + ] + self.config.cwd = None + return + dimsim_path = shutil.which("dimsim") or str(Path.home() / ".deno" / "bin" / "dimsim") self.config.executable = dimsim_path - self.config.extra_args = [ - "dev", - "--scene", - self.config.scene, - "--port", - str(self.config.port), - ] + self.config.extra_args = dev_args self.config.cwd = None def _maybe_build(self) -> None: """Ensure dimsim CLI, core assets, and scene are latest from S3.""" + if self.config.local: + return # Local dev — skip install + + import json import subprocess + import urllib.request deno = _find_deno() scene = self.config.scene - # Always install/update CLI — idempotent, pulls latest JSR version - logger.info("Ensuring dimsim CLI is up-to-date...") - subprocess.run( - [deno, "install", "-gAf", "--unstable-net", _DIMSIM_JSR], - check=True, - ) - + # Check installed CLI version against S3 registry dimsim = shutil.which("dimsim") - if not dimsim: - raise FileNotFoundError("dimsim install failed — not found in PATH") - - # Always re-download core assets + scene (pulls latest from S3) - logger.info("Downloading latest core assets...") + installed_ver = None + if dimsim: + try: + result = subprocess.run( + [dimsim, "--version"], + capture_output=True, text=True, timeout=5, + ) + installed_ver = result.stdout.strip() if result.returncode == 0 else None + except Exception: + pass + + # Fetch registry version from S3 (tiny JSON, fast) + registry_ver = None + try: + with urllib.request.urlopen( + "https://dimsim-assets.s3.amazonaws.com/scenes.json", timeout=5 + ) as resp: + registry_ver = json.loads(resp.read()).get("version") + except Exception: + pass + + if not dimsim or installed_ver != registry_ver: + logger.info( + f"Updating dimsim CLI: {installed_ver or 'not installed'}" + f" → {registry_ver or 'latest'}", + ) + subprocess.run( + [deno, "install", "-gAf", "--unstable-net", _DIMSIM_JSR], + check=True, + ) + dimsim = shutil.which("dimsim") + if not dimsim: + raise FileNotFoundError("dimsim install failed — not found in PATH") + else: + logger.info(f"dimsim CLI up-to-date (v{installed_ver})") + + # setup/scene have version-aware caching (only downloads if version changed) + logger.info("Checking core assets...") subprocess.run([dimsim, "setup"], check=True) - logger.info(f"Downloading latest scene: {scene}...") + logger.info(f"Checking scene '{scene}'...") subprocess.run([dimsim, "scene", "install", scene], check=True) def _collect_topics(self) -> dict[str, str]: diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py index f677bab3d3..54d3332d2f 100644 --- a/dimos/robot/sim/tf_module.py +++ b/dimos/robot/sim/tf_module.py @@ -131,10 +131,6 @@ def _odom_to_tf(cls, odom: PoseStamped) -> list[Transform]: def _on_odom(self, pose: PoseStamped) -> None: """Handle incoming odometry — publish TF transforms.""" - # Drop out-of-order messages (UDP multicast doesn't guarantee ordering) - if pose.ts <= self._odom_last_ts: - return - self._odom_last_ts = pose.ts self._latest_odom = pose self._odom_count += 1 From 02d13424b1d5cd9f5383de21cd07885785785ac5 Mon Sep 17 00:00:00 2001 From: Viswa4599 <39920874+Viswa4599@users.noreply.github.com> Date: Thu, 5 Mar 2026 06:36:19 +0000 Subject: [PATCH 05/19] CI code cleanup --- dimos/robot/sim/blueprints/basic/sim_basic.py | 2 +- dimos/robot/sim/bridge.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 942d6d64c7..0ff400c2aa 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -21,7 +21,7 @@ from dimos.core.transport import JpegLcmTransport from dimos.msgs.sensor_msgs import Image from dimos.protocol.pubsub.encoders import DecodingError -from dimos.protocol.pubsub.impl.lcmpubsub import JpegLCM, LCM +from dimos.protocol.pubsub.impl.lcmpubsub import LCM, JpegLCM from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf from dimos.web.websocket_vis.websocket_vis_module import websocket_vis diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 2573053f2f..1df054c2e3 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -131,7 +131,11 @@ def _resolve_paths(self) -> None: logger.info(f"Using local DimSim: {cli_ts}") self.config.executable = _find_deno() self.config.extra_args = [ - "run", "--allow-all", "--unstable-net", str(cli_ts), *dev_args, + "run", + "--allow-all", + "--unstable-net", + str(cli_ts), + *dev_args, ] self.config.cwd = None return @@ -160,7 +164,9 @@ def _maybe_build(self) -> None: try: result = subprocess.run( [dimsim, "--version"], - capture_output=True, text=True, timeout=5, + capture_output=True, + text=True, + timeout=5, ) installed_ver = result.stdout.strip() if result.returncode == 0 else None except Exception: From 0dbcd7ab4106e2a98c010a8eb7a80a0fbcca0632 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Mon, 9 Mar 2026 18:39:07 -0700 Subject: [PATCH 06/19] Fix rerun camera: separate pinhole from image entity for 2D/3D compat --- dimos/robot/sim/blueprints/basic/sim_basic.py | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 0ff400c2aa..1fa4c9bc81 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -20,30 +20,20 @@ from dimos.core.global_config import global_config from dimos.core.transport import JpegLcmTransport from dimos.msgs.sensor_msgs import Image -from dimos.protocol.pubsub.encoders import DecodingError -from dimos.protocol.pubsub.impl.lcmpubsub import LCM, JpegLCM +from dimos.protocol.pubsub.impl.lcmpubsub import LCM from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf from dimos.web.websocket_vis.websocket_vis_module import websocket_vis -class _SafeJpegLCM(JpegLCM): # type: ignore[misc] - """JpegLCM that only decodes image topics, skipping everything else cheaply.""" +class _SimLCM(LCM): # type: ignore[misc] + """LCM that JPEG-decodes image topics and standard-decodes everything else.""" _JPEG_TOPICS = frozenset({"/color_image"}) - def decode(self, msg: bytes, topic: Any) -> Image: # type: ignore[override] - if getattr(topic, "topic", None) not in self._JPEG_TOPICS: - raise DecodingError("skip") - return super().decode(msg, topic) - - -class _SkipJpegLCM(LCM): # type: ignore[misc] - """Standard LCM that skips JPEG image topics to avoid decode errors.""" - def decode(self, msg: bytes, topic: Any) -> Any: # type: ignore[override] - if getattr(topic, "topic", None) in _SafeJpegLCM._JPEG_TOPICS: - raise DecodingError("skip") + if getattr(topic, "topic", None) in self._JPEG_TOPICS: + return Image.lcm_jpeg_decode(msg) return super().decode(msg, topic) @@ -54,10 +44,39 @@ def decode(self, msg: bytes, topic: Any) -> Any: # type: ignore[override] def _convert_camera_info(camera_info: Any) -> Any: - return camera_info.to_rerun( - image_topic="/world/color_image", - optical_frame="camera_optical", - ) + # Log pinhole under TF tree (3D frustum) — NOT at the image entity. + # Pinhole at the image entity blocks rerun's 2D viewer. + import rerun as rr + + fx, fy = camera_info.K[0], camera_info.K[4] + cx, cy = camera_info.K[2], camera_info.K[5] + return [ + ( + "world/tf/camera_optical", + rr.Pinhole( + focal_length=[fx, fy], + principal_point=[cx, cy], + width=camera_info.width, + height=camera_info.height, + image_plane_distance=1.0, + ), + ), + ( + "world/tf/camera_optical", + rr.Transform3D(parent_frame="tf#/camera_optical"), + ), + ] + + +def _convert_color_image(image: Any) -> Any: + # Log image at both: + # world/color_image — 2D view (no pinhole ancestor) + # world/tf/camera_optical/image — 3D view (child of pinhole) + rerun_data = image.to_rerun() + return [ + ("world/color_image", rerun_data), + ("world/tf/camera_optical/image", rerun_data), + ] def _convert_global_map(grid: Any) -> Any: @@ -84,9 +103,10 @@ def _static_base_link(rr: Any) -> list[Any]: rerun_config = { - "pubsubs": [_SkipJpegLCM(autoconf=True), _SafeJpegLCM(autoconf=True)], + "pubsubs": [_SimLCM(autoconf=True)], "visual_override": { "world/camera_info": _convert_camera_info, + "world/color_image": _convert_color_image, "world/global_map": _convert_global_map, "world/navigation_costmap": _convert_navigation_costmap, "world/pointcloud": None, From 4a46b5dc89b53a8fb3669c212f21d3f41f78576d Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Mon, 9 Mar 2026 19:13:59 -0700 Subject: [PATCH 07/19] Force Deno cache reload on dimsim CLI update --- dimos/robot/sim/bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 1df054c2e3..10545c1cf0 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -188,7 +188,7 @@ def _maybe_build(self) -> None: f" → {registry_ver or 'latest'}", ) subprocess.run( - [deno, "install", "-gAf", "--unstable-net", _DIMSIM_JSR], + [deno, "install", "-gAf", "--reload", "--unstable-net", _DIMSIM_JSR], check=True, ) dimsim = shutil.which("dimsim") From 60c0c007f5d4145d38c0d4b8cb9a5c6f815e3e8c Mon Sep 17 00:00:00 2001 From: Ivan Nikolic Date: Tue, 10 Mar 2026 17:04:03 +0800 Subject: [PATCH 08/19] sim mapper small adjustments --- dimos/robot/sim/blueprints/nav/sim_nav.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dimos/robot/sim/blueprints/nav/sim_nav.py b/dimos/robot/sim/blueprints/nav/sim_nav.py index babaecac44..fa2e829ffe 100644 --- a/dimos/robot/sim/blueprints/nav/sim_nav.py +++ b/dimos/robot/sim/blueprints/nav/sim_nav.py @@ -16,6 +16,9 @@ from dimos.core.blueprints import autoconnect from dimos.mapping.costmapper import cost_mapper +from dimos.mapping.pointclouds.occupancy import ( + HeightCostConfig, +) from dimos.mapping.voxels import voxel_mapper from dimos.navigation.frontier_exploration import wavefront_frontier_explorer from dimos.navigation.replanning_a_star.module import replanning_a_star_planner @@ -23,8 +26,8 @@ sim_nav = autoconnect( sim_basic, - voxel_mapper(voxel_size=0.1), - cost_mapper(), + voxel_mapper(voxel_size=0.1, publish_interval=0.5), + cost_mapper(algo="height_cost", config=HeightCostConfig(can_pass_under=1.5, smoothing=2.0)), replanning_a_star_planner(), wavefront_frontier_explorer(), ).global_config(n_workers=6, robot_model="dimsim") From 81ee427b39eb07457bced75c79875febd83be926 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 18:54:01 -0700 Subject: [PATCH 09/19] DimSim integration: compiled binary distribution, e2e tests, sim blueprints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - bridge.py: download compiled binary from GitHub Releases (no Deno dependency), auto-symlink to ~/.local/bin, support DIMSIM_CONNECT_ONLY and DIMSIM_CHANNELS env vars for parallel eval instances - New e2e test: test_dimsim_nav.py — 6 smoke tests for sim-nav sensor pipeline (color, depth, odom, lidar, cmd_vel control, odom rate). Runs in CI with CPU rendering, force-kills port 8090 on setup/teardown to prevent leaks - Eval tests: EVALS_DIR now reads from ~/.dimsim/evals/ (installed by dimsim setup) instead of reaching into DimSim source tree. Added force-kill cleanup to both sequential and parallel eval fixtures - New sim blueprints: sim-agentic, sim-eval, sim-parallel-eval, sim-spatial, sim-temporal-memory - Simulation developer docs at docs/development/simulation.md --- assets/temporal_memory/entities.json | 1 + assets/temporal_memory/entity_graph.db | Bin 0 -> 86016 bytes assets/temporal_memory/evidence.jsonl | 58 +++ assets/temporal_memory/frames_index.jsonl | 50 +++ assets/temporal_memory/state.json | 7 + dimos/e2e_tests/test_dimsim_eval.py | 211 ++++++++++ dimos/e2e_tests/test_dimsim_eval_parallel.py | 377 ++++++++++++++++++ dimos/e2e_tests/test_dimsim_nav.py | 251 ++++++++++++ dimos/robot/all_blueprints.py | 5 + .../robot/sim/blueprints/agentic/__init__.py | 0 .../sim/blueprints/agentic/sim_agentic.py | 35 ++ .../robot/sim/blueprints/agentic/sim_eval.py | 31 ++ .../blueprints/agentic/sim_parallel_eval.py | 72 ++++ .../blueprints/agentic/sim_temporal_memory.py | 26 ++ dimos/robot/sim/blueprints/nav/sim_spatial.py | 26 ++ dimos/robot/sim/bridge.py | 159 ++++++-- docs/development/simulation.md | 97 +++++ 17 files changed, 1370 insertions(+), 36 deletions(-) create mode 100644 assets/temporal_memory/entities.json create mode 100644 assets/temporal_memory/entity_graph.db create mode 100644 assets/temporal_memory/evidence.jsonl create mode 100644 assets/temporal_memory/frames_index.jsonl create mode 100644 assets/temporal_memory/state.json create mode 100644 dimos/e2e_tests/test_dimsim_eval.py create mode 100644 dimos/e2e_tests/test_dimsim_eval_parallel.py create mode 100644 dimos/e2e_tests/test_dimsim_nav.py create mode 100644 dimos/robot/sim/blueprints/agentic/__init__.py create mode 100644 dimos/robot/sim/blueprints/agentic/sim_agentic.py create mode 100644 dimos/robot/sim/blueprints/agentic/sim_eval.py create mode 100644 dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py create mode 100644 dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py create mode 100644 dimos/robot/sim/blueprints/nav/sim_spatial.py create mode 100644 docs/development/simulation.md diff --git a/assets/temporal_memory/entities.json b/assets/temporal_memory/entities.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/assets/temporal_memory/entities.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/assets/temporal_memory/entity_graph.db b/assets/temporal_memory/entity_graph.db new file mode 100644 index 0000000000000000000000000000000000000000..d0a6956bfc70fc3ed5dca5ac770ce70f517f102b GIT binary patch literal 86016 zcmeHw4Rl;bm1ehEQmg;ovn1jY{!uk8)K9w=}GFi z)va{5ttgq0G!r0j!Z|Sf4dDPg*_kjvCj87Y8Q9^FSqMy6es?#}6ovt$>m`Pu2} z>ayb!Wv5_Q^QB^?{qU*Wp{blTdHhK3jMb39I&s`;OteQNXPL%dVe&%bjfF1T`Eq0W z4eJ+*3DzmpfofD&r|s$d+@M9o3=r{GZ8U;i$)x|aC#v>LVU0>QN??)%Q-`iSmRqBS z)}F9c_b)#OAWh|t=1y5BPfgx<=+s&3hTK`}(CMiYlgA;u8*|5}1{+d&kg}$7XQr&< zC*Z%+$Bs2JD>eKTS+V|?W1TG(=ks$;an`Xw*T<|Qx$6#{J~m|~6T^)n6db!;bmpe# z%cUijaU;t-sI)p=nyEPDOTbyJ@NCzwR>A)D_<7dMKBs3(7mHOFcvtjBsjfS5DmQub zIMc}<1;{u^cU|sO?)c%{&D?78PDL<)@%(BzF+00VP9ELIZ;Q2dC42n$x@}@EU#Z%l zgvxX^zvQ@-(a;v$KQTsZ4DK&`FezlSv^-sj7o%YHJ^otv8bx3;a#O{grwGyBTxJ1H zV>7#dcO=-=)8l_&J8Ncc3il;SXf-LHyVY@ zHEUO$g;IHSoyPHNH8w??muo{H++vA0^+w@-=WIV*XZxZTjU>>7neyjDdj7rmvlJx?jb>n(p#m!$U6!!tA?q;9T#y}sCdaB z6+>gA_m6vu_L4IQDdYF=XyM&KgyrdC;l8wb@^OFuSL+T$j!fZ~YvmG(%W=B8xg6ua zRa18^(P)jcg~rBSELEM#I=zplBal`84aYzik;~=d9RU}cgI(j>{k5RmMp>^6!|ybk z5mX$fD1K>aFm3S{$&0ove%<5d0P~Odg+Q?DDkv$^SQ4MCs6{oeUzB9_lqG-0!d{v5 z2fMa!_uoIl>|RJDo=qOB7r(Ay^z2Wo?)*)eQ#R_HcyS7UIjfv=DzoMMa+#GpF^ttHgj3J{bx)ePWHDaHNd^7gu z$i>J+BoO{`_)o*D;iF+A^wrSAp?oOb_FUT&ZLj(rU&3keGy)m{jetf#Bk+PkV5%k9 zzP%C*`f?+M5-gKpd+}1iEtFyQ2miM#70WKpS+ltD zWK}M#P8>eDD+TqQh`(~cf+tJ>uIygLTr+$gx^A9(n}hA${R-z?zL+mASZtc+W_#n8 ze(y2Jc26rX_aJkYXS-amXJMS;Vp8^OJ%#b z;8+#d`mIb{2%UchaG1Dr>hYc@e7@0MNYCp?IM3PT3)W1z1od1lRjbaN1rQahZbHgn zbf=pz%?k!0%l0fJv7a|#(gIZXGKo_HRc;k7wi64JH{Ojx(sQ)OEi3u_Wr>7TyWt&> zaEO&P%oCQeK5Dh^4f#7$wXi z5k331K#83!5!nLlbT|_;PZbZM#gSYGW59I-awSRe*u+=tRSTLjwgszNvM%JSvx{Jb z*zhOLjC}IbsLkmQKlBrLM&IIV?LKkkX2@nID=fv46UJUBOq~B`#yN4@)453~I=KmW zd2L9IyxLe2)T#pO=84xBQzx+y(&v>OBKsuik?klujfxmeI@+7|4t-cN2qfBHbabFYkC0&xPvNq@TBa@pR_$f%_+iijv7qPQ|KDS&dElr$` z-xYzJCvGfE?185lff{s!D5(G>O9%JjgPE^TB{u$9dnR9Ws%}N7>r8Y3Ox)Jf!g^Jk zsyCFYJAwZYgC{BYMJKK_c%eX{^K$bi^Y{6D@pb@%wUs3GpiB8#=#}hCPQ70GHUKNv zAGMTOyIh@cmhNK%3ZXj|1+D{34Pfh_eujetf#BcKt`2xtT}0vZ90fJQ(gpb^jr{FV?120Hw(pFF^y zqy7Au?c>jpUHqBp<=pbu(#@ZlE&Q48 z;?LA({!DiA=kO->OpbT(=h#O69Bt>%><0cEG5IrN@Mk*4pQ$K+CL{bg9BvJ?`*Y(V z_84npkI^7|WLw!|q=h{)&Fqm5ut%z?CD0zqCHeZl!)N}B`A_Co%+H%on160QV!qvc z(7YGc|BiXayurN2_>u7~v6q7q?7(3yacO0=m&kP&o>l zRH9iW0xHoI2?RspAb`96=d1pIf_4AD&^|!fyv9tj^*>+Nf6KfV*8jhNH9xNYaozs` zSo1ruH!#HZ0`{{Vf`<+3H#0w-S|gwl&wmhC!q@+FABwO4>H5#Fwf?6xeEm-ZU;oo?0$=|V!PoyJ0bl=g0KIH;Ol?7C&SnOA`KXo-~2h6 zR)_iX*h2L2$g`2D@MGbg(C@c>uWf(u(bjLb9%*^9rL+0|z^4Mqri%Xq{!PAfzk)5+ zvRfj)&W?^?u$HVv@D^V#H3las4*u^n`zJ1Ko$I}}1Q!;CYXZDTVJ(-z%NY2j$`|3* z2HdEd1y2@({5lrW{LY;{cRO4u1TMIpguk-ubCpEz`6(pIr91OW;LpseTwGWHpH3Cz zVb?UQWw^D1wCK;*C#7Z_2P1(wxuiN+lhkTyUOmMDb0pu(!qI=f{jAQxABl*$fT zvv4=AiudJgxDQ%{3eQ^=ycbGZ5ZM><7aYiz)s7hp*MY18UVlr}Hf%!a+rg^4PUA&9 zaJf`M>BkQW+;4spyOngV_c{a$t}Ma5K`5JFK&(p8*k#=rr@DeSbrp%3X7E;7oq(Y4 zB2=BrR$VDspb&5_l&Br(i1@a*qe%N|T@ulp#M2dN5-zM_YvZ*XWW}=kyZn+cyHUhj zV0{|_h{4j}Z|ssQTDuTWxdm+HD;74%vRxPy4UZ!~4>y(za3d71&7lI=?KyiEjVDnv zHimrt?adbBSGuJlA^*-7}M2 z4_>@x$q)l)ejc=kU7Df+-dW*IGlXur#M=|xon^P4(I}u}xWJT?FBC2=<%>*y{KHDA ze1Xe*&xVk%8^v|N9!uq7C6|J}YSitkXp5Ue2YO{GnU>wgWY3f?;SD+LTjxNrhoFaC z&O2aU&@n^j0=^BoFthBIIlEjfxNAGiurH3~uh#sc!<=!T@0$ZD04)g>zdOnWfr7Bk z)I-j|1#G&s&HYZHI9!*A4%r%|nZHS}!`ilAbzv$YnIpeA#fg96k0iwFPK z^7*(pblbJg&PjVvTZhzNg}bLhqy={O8hdCm0Lla~4w<^%tn&XzhqtLUiI1c|BnfjmCZ)5mg)kLj39QO5ew1)u5JD?So8J2K|IhUGng_;6+@RB%Gys`+5e1**v zil8}6!3;&Yhw8Qo2gZy!Ibc&Jbt-_SG7BC=S;jCr^0A2JmZ(KTWO6om(5p;y+(uN~ z=vx@qF^XuR8%_|cetO2xBWxHS_Kp%#)+s7eCNZQd)k@TMwTZ#E<6dcU85rq=$+_a3 zlv7Bhzoki}s#<7{=$c~ATl3!YCl&7nP#c_;%I?|iI90f~W)rn-L68#1<9pPG0YZuy z!tABTUjZ{{j#B4xeQ=+KIXTRZ- zpSiD2zCF2Q+~QL(>RT0|tUb9@9I#!?gD(u|Uxf11`50@`=A-bk@rDr%$ku+C@ z5xOPp+l5nxT6{$q8cxKZNvGX$@7$MnjkqwAvl2Gx+LOz~QA|0=HhNaT9ze_@C{Z#U zuW#A_J+m(4fLGExERv6v)I&}d=fRB$v^sZVuhA_v>%-Voa;rQNNx~Wg~E#;uBU98%t8_}7^VTP_PWZeIk5#2s{x>m z1*)k*Gnsgzw$UHr-NMQ}o>tc&uuvS8hs zND*wOGKc5?cbY-;|Mx8T^+OMSWpkI=X&Ppb`ThG-<^k|;%nyQRfWI{VgZT#Y56%0` zZRP{u3-DU-26#Pq1w`L}t1nP^wAdN}jetf#BcKt`2xtT}0vZ90fJQ(g@b3u$<`XZu zlRqr}=;4m?c5sUP|7FSlUxxetOOKNOzbyIx8{_`}lEd8pUvilH|4XL1|G#8<6VI-L zJu>9|Z=8GoOAV9vzfto3H%{LFvgG|QN#6f5SRTX z^Ywp<-2V^L20)hF|BuoJz&LFHWN8CnG{*M}X#Jm!aCvF@pXJ{FQ+)NG;;a8uE8i1H zxA13*`~Oez^?!!1|5NPyzp>Y6e$?D+eA@V9l#>@`cqW9FUaCZlQwj9DZ1Le;#MTO*(m&m$4y!86i9u9Q-xqGDz8v}7yVD9M1RI@Z9fvpTGaL@zJ|72NF zQGrMdPT{-0Uc$&v-c`WypX3L}f06*lf0AfLkQB!kEnY;x`=8_uc>f0e|F*mSdieT( z%EvtTKY&pHF{U4nPp+}K|9>L5|9|p0&HewAmuK$( zpL_*#|Nlh54gkbWB!c_@CvUvm|3CS{<^KQ4uPgWePXzb>PyS1}|9>L5|9^5C%KiV7 zhfePQpFA{j|NlgA|NlgA|NrC=k^BEAq6P3K|Noul&*2OI)n*jF@IPwQjGK&Iv0ubK z7kg){5}S5;A?`n2K!r|YyE8NyIV`G2U;U7-)MQP<(`(~ zExVhaYkso%q2}A0$C}#$&jdaaczs|dus;xP`exI|nqJp5)0Aoo`@iCUpZ}Hqv;L&t z?|b^PjcDLZ$k)*k>I?=uFzi4&iTg&bv)}wjbCdV^e3uGKiCd8`=<*F8{3*}d!@qd; z{4?+~@DJRA{C@B}3i+mUsnPH}j@<4>zk7lYEJjGG727?FCHK zPx23(Kz^=>G~}D|04w+5R+-BEirS5j&_$1^(e9 zjGxsCL`{xPH1j4v?aabt<{N9cVZ^*2K?@$Ah{PAn0@ECeyv|f10nmD9n4G!nm zZW@w*ctQ~#vd*p}JcW6I>h>Rm6zzmKG$tmq&^3({z&3OZvJr}4!=5UNCMwzY9boJn z#Nk|K9Wi8r;j0zF(ee8_DQK_)iQ-pb3O_Gm6ygHZqeuf5s&v06^OW1L>r|?w7*|rD zqyIW_1VA!HVvMC=ytNXY^3h#E2pV zux3PC^mGl9e>lVVU1J|vr=BuJBCV7F%_HNn1lL=?lqlupPp^rK%)dVg{Ox=cN+xq7 zt=Lgei=CjE22`|v*u$3b^e2*SXdkkX*(PNq69$JcTgl#^@Z>ti1`2_)WZO5ya^}mIbDgap+2=R>QfqI|a$!7gtLk zWl*`5a!XI_L9QT`zFzy2xh8f?{;}M84P++#-jmP)g2sjhJVIwZvk{>R$v)W6vS;ln z3yOP|2ea_lvBKS_a*uiP7FCtpeY=pG_j%de=sJ1!^-8Wxjt{QFq4jNx|M=IuT8W*k zWCmM0w_f{^{JSmW=iOF1m#J5>WE|{a8S`ltbhuc(L#_OQ`p8+*d- zz}CT)3}G179Uo+hp>2xVFdV^Ja!B@ptylmxUyMTGX+n}Oeg)&>O$hkhK22D=WbN-p z2`EdLJ23kdJp!#3plX9#n4oNy2SJNc6B|K6@Xs^R;NJg%dp88a5^6~0+|q@u$ZQjv zrG8Tl16d<~e<$#Z84S3s_ZXMVcHbssBSnB`|GIuxpg^k}tX>C`gPFrfF6&W)uu93@ z2XTjlg=cbEk2NSStJat-sE$wB6~9%?gW zFlF)4GMRJ3ym$na?E7Mjor^G>tM8nI7G#3HsI32JZrq~;$u$^}n2xT~jY{rB7=#cL zNtj!Wh{zlsq0fwb^3z~-P{()(`3X1RciXxUSF$GByjgpvSHc(+{-BrN4TrUkD2Y}t z|Jo}MnSWo4gb&6yPp*eg?n?0r_BE?qyn_@Sx#Zd%KrVuzvBOgsn^eYhea|Qv`ykqY zFk6`Wd(4)&e4EbT_y4)b+k9pZ`11Qd<`2wonNOQvG(TxRZoVJR{J$xdiw#A89D7-8 z!MNS58i!&x8UEN$j9xQubQ#gu_hMg-?i;c(L8hb@76no705cn31n3K^}*iZOKbTWD%YDZrcJstUIgazO0gZr0KqH_L&P|VX@=Did=1H^ zM`r0AHkL}mHsot$#tQ)4&tclI&r6#Wi$8NSKoNe`bS9{4?VLcS-D z6AymqxzXiMjU1zQ{9tnfz2nE2>*afPlE1SrDlegT{AzNPc<{@}b@V%a`pCKb$s2QxR&1WYr`S=dv+q+47&h1*pp2kq*tiQHSTNX05Rk4{nc(Jso|^K-&4c; z-QSbrXt@iBVGjqp)=$DQu1e9w%~J8U|SWaN7$E#K2ASICj1`4eXQ$hKjY~{JYW<-m48)+7AnSgjpH?=|k5`ucM`~P#{ zANcU(Kb`b1!^wVjsz3aL=y#%Djef!WbMq1K2l$|QFRal2Q}jL2hoY~K-iK%Y`8|PX z#VF#f0plpYF@SdjqVv(S#(r3vTSkY`8a)<07)?d{qg#v*7!Mn7F&;4Pjz%KCjQlwA z50Sr%d^Ym&$Oj|;G4j^PYa_oGDMe-?rz04Nu1Y zGWP!1yJByQ-5*+q)H5{#6Fcs zsKgK__!*@^mDsBiSE@u@CH63aPX6pxi2;@9SBXBA*u@AsXVa?^{QN)tg`fW?f}j7V z11$XfKN0-=KN0-=KN0-=KM_~3a_F!KKmShzKmShzKmShzKmShzKmSh$Gx+&`BKY}# zBKY}#BKY}#BKY}#I{U!S{}aK_{}aK_{}aK_{}aK_|I>*Ae*T{be*T{be*T{be*T{b ze*T|6=kxRbMDX+fMDX+fMDX+fM8v@AeH-w9`kzKXBcKt`2xtT}0vZ90fJQ(gpb^jr zXaqC@FIEKb{J&oRzgV@a>!lIU2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc=gUjJ)L&6ONJpaGNcg|;CF#f{u#TKHEN1lyLg&zy|gnqy6 zdu{uJkG6ih^+?N;EuGEx2R;=@HdXu|@Na?ym;YDGZi)Ch!53t(maIk4J0d#bPmLcG z-oz94vSTfmoeKP5<%?Ez(Xm)?$w6x+4}3_^IK|mTON2kP zswMX8a=|WED~aCoQ@{cN9Pi98<*Tb!<>JDEQ>j*vXO-t}m#bE3o@Z?@IacXnb*@w@ zTNS5zaXC@j0&Y)tqTu(g7D}@=k5ZXKC*-Mvt{2gb5B&r%5W*LnC6Jd3e<@$d&lDUc zE>++{zB;?;6fFu;NiweFi*uzF6!lVm4r^YtoI9&dxo8)xOL=F-nlG1@n3R-0Q9HCb z?AymMJ$QFR#7h!Uc$cIw7bGI$Cd2@UZ76My zO3O~MV$J2t5ID53Dzsj(SBkF4)x~n@;sQ4kl3~!Ah1yp0#Rbc@3i%>vd)}(#7Z$6e z1(AIrf5CxlS?!pya71Jk5T!(I!zPrz9jvEUkcU zty$nMTU7}f)545XU2(uNJn8|>luMHcbVZshS~J+%cr6E6u~i$b70qFG(ax8t1=hC_u!@VIe7g)4SYXyJ#LE{E zT%Jq5V$DKBDBFcW(eOC(^U#nA)^aHitr`>n52tL;qVXhZ2KeFa2S2~TyBE|jvN>?v zGVE4?uJxVkRZT)R9j+|MmiGn10C|hXE&(J1=}qg%d(wu=0LHBpod(}J78YWF)!N1S^md4XBJww4ULRC59F@xFvGq$ zmcLr73yTXmBY6aYq*!O_A!qE_3ky8ovQvQeisEoxB06MilxF^7nW+mCi3}JN z(7`S(L%V{WO-WljVn737Jsb5)MPgy!R#uIc^H>WAFARo= znilPxA`Wp)3u{6dX|!XR0qCW29t37Et=Q!`-lcHOA7S>kybSgS?F3{gqMopeg#n4O zVOk%GBBaUihNDBqK0$n}~ z0Av7^31G;p0EPm?k2}QMRi+5|NzH|&uWgQmX*$yFu-Qlw9YeX$Q4}U+v;?gpL~L5Y zS|Ck3lp&Ig7p)GPE+0fmRM`zk@xVUsVsj9i=@(`>)kLj39QO5qTY3QU4mTz=cz*)# z?52MVXUa6=<%|P~8EWGO1GmG?iJqQe_## z=*Y(+mRq704Ux&&wE#t^M#&W1=r%6bF^Xtas?hI&?abW{ohfwp)D#(dgbm}v-cdrz zIz^n}k?dj?qEstU+tnrp-;R4_J4T25q8Yp5oRm{YrN5;~q^epl#{vjC=dF3~`ICzJ zX-t&ev)gg1aB=VxwQWI=6363v)P?~B4xeQ=+KIM;D|$^6kkb<1_^$g(^Z>dvd8bV7r)yc{wz2p*(dy#@aIJ zUztUSK4o3$84gq=%~fH9ZVCH#;gq2kUlE2jT`l31A?=QPbWNLAV~$T#*raPuE)z#F z9AI`IX6KjdfXH`Pp3eFUjHo|66c^D{gwg@^%rs4qbVVr2fOjalX1cNcrUGX3< zb|ZYPlMV=D*sBhWiZOV^A}ujvfZDpvVBVf(xIquoXclCVe5_<^fZp?y#d)WU1GT%x zVRb0jtDs{xXBPSKB0*wa7@IMsnj!!vAA8yrJaO<5D>RC5cdi%57m^!9R-y3XhwCYu zCbN))42EfdtG%xBYEEo{#A*O&vte7E6{G$VwT_jp=egTO*@R30)T7zm6y ztTCW*8fQ0(=T0VU&%j-;n*^@^L!qDh%>BkYV*f9;9R080m2W2eA^1uE(+Fq;Gy)m{ zjetf#BcKs@aUoDk?-W!2?pn8JMk>~+6vX>n@1$qbl5ZOq?DB#-vBvFr*z|?nZZ^>` zx?fdj#*bSVG^2Hw4XjjTtlKT7t!~`gsbBdeELvRG4lJ!HGzA-PT)eua&gCnxj*06x?(j)6rD1x{}$}K za8Fq74vNBP*S>eV0D5bw4(JgVXo@k<>%&%UA{Q)2Q8kn;za)!gd^P8n|VN?Q*^wQ|sOe`x0zRxpP*mqhVJW z0$``dT+Cc);CDob<>IZCAh^50focQa*m<(h$d3Zt_Us(xWJx*!7031-nqfy&Nb~*`0)DiQ3`rh_8!@x2u+frr^M57f_;<%>BNNm6ki%?uOSQ zZWy}=Njzagm-mY(`2y^y7A;mH7jlh_KQ|9|>kmFB02Hv>&$sWPL5UA;Z2O2|!t%uR z{~6!YKJfkjA@c!q#=P2$8ecOWHEPC9#;({eVxNn>GggUB#&$uxzAr}K6KRxg@!}nwy(B*xb4BVb8VTnmf+We zj|X29yfxV0`dsU0Ti@MUYCX^zY57LWV=ecz9B>J^S1q`fiH$sa4k zpG@2;gFYgk8}%CZ2Hjqee6a%mz%9s6(LgcsNmvb?D*1MwW_&y_E%0TVScYPoS~3ny zDS|>c7q6g_FLARXC}g;HJY$9bl;me2S3JQ`#RyM6VzFX3F@d3H>cG#3>0uU=_8T1M z3CWi@Df7kfGhV<%{Urau3FPOBNMl$R53rIiaa`t$d}~MFl>7rXD)M6I$rK^VC)c%D zuVb=&kT=C6aEjqg@+EGNQt z6y%~AYMK~PqyW~8s2kIhFkQ(%oMHT~v9A}uT&75*l@g$NWIUG8SVAJ@K!yNRLd$M;F5}RZO06J1)taUz z^X-ePrH^{zB}=Zv9^~Q?$ukWhwi|^=?3Vlx1*;ANVVdN68)`Q+;1N1|@Ta_E^r%9z z5B9U{S$l$bS_hv^vwt2tR=E3A?lDi^jiU7JLT*0FWOJkICc3Fh1UdV9a#; zv_=W~yHNtl5+a*1`xQL`eb0xg4Q^qAvRNJ+T1-uB2>r#U#PKq9ZU{p4iKNsqXuD> zlDiMi9SRGF7;YYGP`KKWi)Q=~Sq-5QHIiI=;0zw%&s7D18|(Weh0|0xQ@MtD4f%$L z+6);?S$wpF$Z?)wN=zpt`@R@s=ORFKxjM9j%rZe=RMsD2(0P;~xdtN=)6sRhQOTVM zgAigO33ICvft3eca$gU1jE9h)a0C8~08WT2SrcvEti97K(Igc9pqJl04!e#hiB>QF z+A9#5e_xA)55_l7uH;@HD%jVoa`6sQ=uUF&4j>o7(AeQAj7=(ITA(FrFB$vbw1F^N znEQLoR&wF`KN|a`&-{DygRuU;#T+nxX?($W#CU~qi;;jo=zkgkjetf#BcKt`2xtT} z0vZ90fJQ(g@FGAUF^6kJzP|!{7Ihm2VogaAGhr<|FpDc>`j!WuLA>A6D3~Vl%_w~E z9b{cTTk_$zMDOaDLU3Y!w;>;G!HfbQ`_?AplYEJ35*L8y8$Ft-I9Oc<=%X= zo^N;*+Y`!9oR|5coNMPR`G;S|_}NYjoC!#;hpbF7aLy}9(j$rdTs`noFW)fZ`v$kg P35D;hhc7j@PJaI%`81(Z literal 0 HcmV?d00001 diff --git a/assets/temporal_memory/evidence.jsonl b/assets/temporal_memory/evidence.jsonl new file mode 100644 index 0000000000..5b72cabf16 --- /dev/null +++ b/assets/temporal_memory/evidence.jsonl @@ -0,0 +1,58 @@ +{"window": {"start_s": 37.628, "end_s": 38.626}, "caption": "The scene shows a view from inside a room through a large window. The window frames an outdoor space featuring a bench and a green lawn under a clear blue sky. Inside the room, there's a dark flat-screen TV mounted on the wall opposite the window. A low-profile coffee table sits in front of the window, while wooden furniture is visible to the right side of the frame.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "TV"}, {"id": "E4", "type": "object", "descriptor": "coffee table"}, {"id": "E5", "type": "location", "descriptor": "outdoor space"}, {"id": "E6", "type": "object", "descriptor": "wooden furniture"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 81.84, "end_s": 84.648}, "caption": "The room features an olive green wall with a large dark flat-screen TV mounted opposite a brown couch. A wooden TV stand with a drawer is positioned underneath the TV. The floor is made of light-colored wood planks, and there is a clock on the wall above the TV, indicating time. A magazine rests on the couch, adding a touch of décor.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [{"id": "E7", "type": "object", "descriptor": "wall clock"}, {"id": "E8", "type": "object", "descriptor": "magazine"}, {"id": "E9", "type": "object", "descriptor": "TV stand"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 95.843, "end_s": 96.84}, "caption": "A wooden shelf against an olive green wall displays various items, including a few stacked books, a potted cactus, and a yellow cup. To the right, additional books are stacked, while an assortment of objects lays on the bottom shelf, including a colored circular item and a dark book. The overall lighting is soft, emphasizing the textures of the shelf and the items.", "entities_present": [], "new_entities": [{"id": "E10", "type": "object", "descriptor": "wooden shelf"}, {"id": "E11", "type": "object", "descriptor": "potted cactus"}, {"id": "E12", "type": "object", "descriptor": "stacked books"}, {"id": "E13", "type": "object", "descriptor": "yellow cup"}, {"id": "E14", "type": "object", "descriptor": "colored circular item"}, {"id": "E15", "type": "object", "descriptor": "dark book"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 29.708, "end_s": 30.711}, "caption": "The scene depicts an interior space with a view of a window looking out onto a grassy area and a bench. The room features wooden flooring, a dark green wall, a table with chairs, and a wall-mounted mirror. The lighting outside suggests dusk, as the sky transitions from blue to a darker hue.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "window"}, {"id": "E2", "type": "object", "descriptor": "grass area"}, {"id": "E3", "type": "object", "descriptor": "bench"}, {"id": "E4", "type": "object", "descriptor": "dining table"}, {"id": "E5", "type": "object", "descriptor": "chair"}, {"id": "E6", "type": "object", "descriptor": "mirror"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 42.345, "end_s": 43.345}, "caption": "The scene features a dimly lit interior room with dark green walls and wooden flooring. A window (E1) reveals an outdoor view with a grassy area (E2) and a bench (E3). Inside, a dining table (E4) is surrounded by several chairs (E5). A wall-mounted mirror (E6) reflects some of the room's elements. The atmosphere suggests evening with evening light filtering through the window.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1", "object": "E2", "confidence": 0.9, "evidence": ["visible window looking out towards the grass area"], "notes": "The window (E1) provides a view of the grassy area (E2)."}, {"type": "located_in", "subject": "E1", "object": "E4", "confidence": 0.9, "evidence": ["the window is part of the interior room containing the dining table"], "notes": "The window (E1) is part of the same room as the dining table (E4)."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 49.344, "end_s": 50.345}, "caption": "The scene shows an interior room with dark green walls. A doorway leads to another area, visible in the frames, with a carpet. A rolling cart with circular plates is positioned in the foreground, alongside a black lamp and a brown couch. In the background, there is a shelf containing books and decorative items. An abstract painting hangs on the wall.", "entities_present": [], "new_entities": [{"id": "E7", "type": "object", "descriptor": "rolling cart with plates"}, {"id": "E8", "type": "object", "descriptor": "black lamp"}, {"id": "E9", "type": "object", "descriptor": "brown couch"}, {"id": "E10", "type": "object", "descriptor": "shelf with books"}, {"id": "E11", "type": "object", "descriptor": "abstract painting"}, {"id": "E12", "type": "location", "descriptor": "interior room"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 50.345, "end_s": 111.346}, "caption": "The scene showcases an interior room with dark green walls and wooden flooring. A modern dining table (E4) surrounded by several chairs (E5) is positioned centrally, with a large window (E1) offering a view of an outdoor grassy area (E2). The window features reflections of the surroundings, contributing to the light within the room. Adjacent to the dining area, a door leads to a kitchen space with dark cabinetry. The lighting is bright, suggesting daytime. The arrangement highlights the contrast between the dark walls and the natural light coming through the window, creating a warm atmosphere.", "entities_present": [{"id": "E1", "confidence": 0.95}, {"id": "E2", "confidence": 0.85}, {"id": "E4", "confidence": 0.9}, {"id": "E5", "confidence": 0.9}], "new_entities": [{"id": "E6", "type": "object", "descriptor": "large mirror"}, {"id": "E13", "type": "object", "descriptor": "kitchen area"}], "relations": [{"type": "looks_at", "subject": "E1|window", "object": "E2|grass area", "confidence": 0.8, "evidence": ["The window (E1) is directly facing the grassy area (E2), suggesting a view."], "notes": "Inferred from the room layout showing the window's direction."}, {"type": "located_in", "subject": "E4|dining table", "object": "E12|interior room", "confidence": 0.95, "evidence": ["The dining table (E4) is seen within the interior room (E12)."], "notes": "Clearly visible as part of the room's setup."}, {"type": "adjacent_to", "subject": "E4|dining table", "object": "E13|kitchen area", "confidence": 0.9, "evidence": ["The dining area (E4) is near the doorway leading to the kitchen area (E13)"], "notes": "The proximity of both areas is evident in the frames."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.85} +{"window": {"start_s": 111.346, "end_s": 140.524}, "caption": "The scene depicts an interior room characterized by dark green walls and wooden flooring. A modern dining table (E4) is centrally positioned, surrounded by six chairs (E5). A large window (E1) allows ample light in, revealing an outdoor grassy area (E2) through its glass. Adjacent to the dining area, a doorway leads to a kitchen space (E13) that features dark cabinets and a sleek design. A large mirror (E6) is prominently displayed on one wall, reflecting the room's elements while enhancing the overall brightness of the space.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1", "object": "E2", "confidence": 0.8, "evidence": ["The window (E1) opens to the grassy area (E2), which is visible through the glass."], "notes": "The arrangement of the window suggests a direct view to the outside."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 140.524, "end_s": 155.71}, "caption": "The scene depicts an interior room with dark green walls and polished wooden flooring. A modern dining table is centrally located, surrounded by six chairs. A large window reveals a grassy area outside, illuminated by daylight. The room also features a large mirror that reflects light and space, enhancing the overall brightness. A doorway leads to a kitchen area visible to the right, containing dark cabinetry.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|window", "object": "E2|grass area", "confidence": 0.8, "evidence": ["The window provides a clear view of the grassy area."], "notes": "The orientation of the window suggests a direct line of sight."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 169.345, "end_s": 171.422}, "caption": "The scene shows a grassy area with a large tree casting a shadow. There are two potted plants on the grass, and a bench can be seen in the distance. The background has a clear blue sky, while a white wall is partially visible on the left side of the frame.", "entities_present": [{"id": "E2", "confidence": 0.9}], "new_entities": [{"id": "E14", "type": "object", "descriptor": "potted plant"}, {"id": "E15", "type": "object", "descriptor": "large tree"}, {"id": "E16", "type": "object", "descriptor": "bench"}], "relations": [{"type": "looks_at", "subject": "E14|potted plant", "object": "E2|grass area", "confidence": 0.6, "evidence": ["Present in both frames, with potted plants placed on the grass."], "notes": "Potted plants are positioned directly on the grassy area."}, {"type": "is_near", "subject": "E15|large tree", "object": "E16|bench", "confidence": 0.7, "evidence": ["The tree stands adjacent to the bench."], "notes": "The tree and bench are both within the visible grassy area."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.8} +{"window": {"start_s": 171.422, "end_s": 222.66}, "caption": "The scene depicts an outdoor grassy area (E2) with a large tree (E15) casting a dark shadow. Two potted plants (E14) are positioned on the grass, adding greenery, while a bench (E16) is seen in the background, suggesting a space for relaxation. The clear blue sky emphasizes a tranquil atmosphere, complemented by a white wall partially visible on the left side.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E14", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}, {"id": "E16", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 222.66, "end_s": 591.381}, "caption": "The scene features a grassy area (E2) with a large tree (E15) on the left side, casting a dark shadow over the ground. The clear blue sky contrasts with the green grass, creating a serene outdoor atmosphere. A portion of a white wall is visible, framing the scene. The grassy area suggests a place for relaxation and enjoyment of nature.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 591.381, "end_s": 697.334}, "caption": "The scene features a grassy area (E2) bordered by a white wall, creating a tranquil outdoor setting. Bright blue skies extend above, enhancing the serenity of the environment. A potted plant (E14) is partially visible in the foreground, while a large tree (E15) stands nearby, providing shade over the ground. In the background, hints of furniture, including a bench (E16) and a dining area (not fully visible), can be observed, suggesting a space designed for relaxation and outdoor enjoyment.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E14", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}, {"id": "E16", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "overlooks", "subject": "E15|large tree", "object": "E2|grass area", "confidence": 1.0, "evidence": ["The tree's shadow extends over the grassy area in the second frame."], "notes": "The tree is positioned to cast shade on the grass."}, {"type": "near", "subject": "E14|potted plant", "object": "E16|bench", "confidence": 0.7, "evidence": ["Both the potted plant and bench are visible in the composite view."], "notes": "The proximity suggests a cozy area for rest."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 130.787, "end_s": 131.278}, "caption": "The interior of a dimly lit room features a dark wall with a large clock on the left side. A wooden floor stretches beneath a low, black TV stand situated in the center, with a large screen mounted on the wall above. To the right, a glass door offers a view of an outdoor area, revealing a green lawn and a fence visible through the transparent pane. The lighting outside suggests it is either dawn or dusk.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large wall clock"}, {"id": "E2", "type": "object", "descriptor": "black TV stand"}, {"id": "E3", "type": "screen", "descriptor": "large TV screen"}, {"id": "E4", "type": "location", "descriptor": "glass door with outdoor view"}, {"id": "E5", "type": "other", "descriptor": "green lawn visible outside"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 143.277, "end_s": 144.333}, "caption": "The interior of a room features a sectional brown couch positioned against a dark green wall, with a painting adorned with colorful abstract designs hanging above. A white notepad sits on the couch. To the side, a floor lamp with a red shade adds a pop of color, alongside a wooden coffee table with a glass top, suggestive of a cozy living area.", "entities_present": [], "new_entities": [{"id": "E6", "type": "object", "descriptor": "brown sectional couch"}, {"id": "E7", "type": "object", "descriptor": "colorful abstract painting"}, {"id": "E8", "type": "object", "descriptor": "white notepad"}, {"id": "E9", "type": "object", "descriptor": "red floor lamp"}, {"id": "E10", "type": "object", "descriptor": "wooden coffee table"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames; possible presence of individuals is unconfirmed."], "confidence": 0.9} +{"window": {"start_s": 643.286, "end_s": 644.281}, "caption": "The interior of a room has a dark green wall with a colorful abstract painting (E7) hanging above a brown sectional couch (E6). A white notepad (E8) rests on the couch. Nearby, a red floor lamp (E9) stands beside a wooden coffee table (E10). In the background, a glass door (E4) leads to an outdoor view featuring a green lawn (E5). The lighting indicates it could be dusk as it creates a warm atmosphere in the space.", "entities_present": [{"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 653.281, "end_s": 653.781}, "caption": "The room features a green wall with an abstract painting (E7) above a brown sectional couch (E6). A white notepad (E8) is placed on the couch, and a red floor lamp (E9) stands beside a wooden coffee table (E10). The scene includes a glass door (E4) that opens to a view of a green lawn (E5). Across the room, a dark storage cabinet is visible under bright lighting.", "entities_present": [{"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [{"id": "E11", "type": "object", "descriptor": "dark storage cabinet"}], "relations": [{"type": "looks_at", "subject": "E11|dark storage cabinet", "object": "E4|glass door", "confidence": 0.6, "evidence": ["The cabinet is positioned in the foreground towards the glass door."], "notes": "The cabinet's orientation suggests attention towards the door."}], "on_screen_text": [], "uncertainties": ["The presence or details of other possible objects or decor."], "confidence": 0.9} +{"window": {"start_s": 700.281, "end_s": 701.286}, "caption": "The interior of a dimly lit room showcases a dark green wall adorned with a colorful abstract painting (E7) above a brown sectional couch (E6). A white notepad (E8) is placed on the couch, while a red floor lamp (E9) stands nearby. A dark TV stand (E2) is positioned across from a large TV screen (E3), reflecting soft lighting that complements the warm wood floor. The room opens up through a glass door (E4) leading to an outdoor area where a green lawn (E5) is visible, suggesting a tranquil ambience as daylight fades.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E11", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 711.781, "end_s": 712.781}, "caption": "The corner of a room is depicted, showcasing a dark green wall with a bookshelf displaying various objects including books and a plant. The floor is wooden, and the adjacent area features a black cabinet. The space is well-lit, revealing the minimalist decor.", "entities_present": [], "new_entities": [{"id": "E12", "type": "object", "descriptor": "bookshelf"}, {"id": "E13", "type": "object", "descriptor": "plant"}, {"id": "E14", "type": "object", "descriptor": "books"}, {"id": "E15", "type": "object", "descriptor": "black cabinet"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 0.9} +{"window": {"start_s": 722.296, "end_s": 722.78}, "caption": "The scene shows an interior with a desk featuring a laptop and a small white notepad, located adjacent to a bed and a wall with a vibrant blue paint. On the left, a dark green wall contains an outline of a table amidst dim lighting, hinting at a dining area. The juxtaposition of the brightly colored room and the dark space brings a contrasting ambiance to the overall look.", "entities_present": [], "new_entities": [{"id": "E16", "type": "object", "descriptor": "desk with laptop and notepad"}, {"id": "E17", "type": "object", "descriptor": "bed with pink bedding"}, {"id": "E18", "type": "location", "descriptor": "room with dark green wall"}], "relations": [{"type": "look_at", "subject": "E16|desk with laptop and notepad", "object": "E17|bed with pink bedding", "confidence": 0.7, "evidence": ["The positioning of the desk is oriented towards the bed."], "notes": "Desk appears aligned in view of the bed."}], "on_screen_text": [], "uncertainties": ["The specific use of the desk setup."], "confidence": 0.8} +{"window": {"start_s": 736.282, "end_s": 737.791}, "caption": "The scene depicts a modern interior featuring a desk with a laptop (E16) positioned to the left. In the background, a bed with pink bedding (E17) is visible against a bright blue wall, adorned with three artwork pieces. A tall lamp stands between the desk and the bed, contributing to the warm lighting of the room, while an abstract painting complementing the overall decor hangs on the wall.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}], "new_entities": [{"id": "E19", "type": "object", "descriptor": "tall lamp"}, {"id": "E20", "type": "object", "descriptor": "three artwork pieces on blue wall"}], "relations": [{"type": "looks_at", "subject": "E19|tall lamp", "object": "E16|desk with laptop", "confidence": 0.6, "evidence": ["The lamp is positioned near the desk, indicating a potential relationship between the two."], "notes": "Lamp's position suggests it provides light for the desk area."}, {"type": "located_behind", "subject": "E17|bed with pink bedding", "object": "E19|tall lamp", "confidence": 0.8, "evidence": ["The bed is in the background relative to the lamp."], "notes": "The bed is further back in the space compared to the lamp."}, {"type": "adorns", "subject": "E20|three artwork pieces on blue wall", "object": "E17|bed with pink bedding", "confidence": 0.9, "evidence": ["The artworks are directly above the bed."], "notes": "Art pieces enhance the decor around the bed."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 750.292, "end_s": 750.791}, "caption": "The scene shows a modern room with a sleek desk featuring a laptop, located on the left side. A bed with pink bedding is situated against a vibrant blue wall adorned with three pieces of artwork. A tall lamp provides lighting for both the desk area and the space near the bed. The arrangement emphasizes a cozy yet stylish atmosphere, integrating the desk and bed into a harmonious layout.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "provides_light_for", "subject": "E19|tall lamp", "object": "E16|desk with laptop", "confidence": 0.8, "evidence": ["Visible proximity and lighting effect in both frames"], "notes": "The lamp is positioned near the desk and illuminates the workspace."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 760.285, "end_s": 762.29}, "caption": "The frames depict a modern bedroom featuring a sleek desk with a laptop (E16) positioned to the left. A bed adorned with pink bedding (E17) is set against a vibrant blue wall that showcases three pieces of artwork (E20). A tall lamp (E19) illuminates the space, enhancing the cozy atmosphere created by the desk and bed's arrangement.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 769.781, "end_s": 770.294}, "caption": "The room features a desk with a laptop (E16) on the left and a bed with pink bedding (E17) against a vibrant blue wall adorned with three pieces of artwork (E20). A tall lamp (E19) provides light in the space, enhancing the inviting ambiance. The sleek layout emphasizes both workspace functionality and relaxation.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 58.337, "end_s": 58.846}, "caption": "The scene features an interior room with dark green walls, showcasing a dining table surrounded by wooden chairs, a wall-mounted mirror, and a bookshelf filled with various items. A gold-framed painting hangs on the wall adjacent to a doorway leading to another room, while a small cart with plates and a lamp is positioned on the right.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dining table"}, {"id": "E2", "type": "object", "descriptor": "wooden chairs"}, {"id": "E3", "type": "object", "descriptor": "mirror"}, {"id": "E4", "type": "object", "descriptor": "bookshelf"}, {"id": "E5", "type": "object", "descriptor": "painting"}, {"id": "E6", "type": "object", "descriptor": "cart"}, {"id": "E7", "type": "object", "descriptor": "plates"}, {"id": "E8", "type": "object", "descriptor": "lamp"}, {"id": "E9", "type": "location", "descriptor": "interior room"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 68.291, "end_s": 68.798}, "caption": "The scene shows an interior room (E9) with dark green walls and wooden flooring. There's a doorway leading to another room, which contains a black kitchen setup visible through the opening. A mirror (E3) is seen on the left wall, and a bookshelf (E4) is situated on the right, decorated with various items like books and a yellow object. The floor tiles of the adjacent room are gray, contrasting with the wooden floor in the current view.", "entities_present": [{"id": "E4", "confidence": 0.9}, {"id": "E3", "confidence": 0.8}], "new_entities": [{"id": "E10", "type": "object", "descriptor": "kitchen area visible through doorway"}, {"id": "E11", "type": "object", "descriptor": "yellow object on shelf"}], "relations": [{"type": "scene_change", "subject": "E4|bookshelf", "object": "E10|kitchen area", "confidence": 0.7, "evidence": ["The doorway provides a view into the kitchen setup."], "notes": "This indicates a transition to another space."}], "on_screen_text": [], "uncertainties": ["The exact nature of the yellow object on the shelf is unclear."], "confidence": 0.85} +{"window": {"start_s": 77.79, "end_s": 78.298}, "caption": "The frames depict an interior room with dark green walls and wooden flooring. A dining table (E1) with a polished surface is central to the scene, surrounded by wooden chairs (E2). A mirror (E3) on the left wall reflects part of the room, while the right side contains a visible doorway leading to a kitchen area. Light filters in through a window on the far wall, displaying a scenic view outside. The ambiance is cozy and inviting, with careful arrangement of furniture enhancing the living space.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [{"id": "E12", "type": "location", "descriptor": "window with outside view"}, {"id": "E13", "type": "object", "descriptor": "doorway leading to kitchen area"}], "relations": [{"type": "looks_at", "subject": "E1|dining table", "object": "E2|wooden chairs", "confidence": 0.7, "evidence": ["In both frames, the table and chairs are part of the same visual arrangement."], "notes": "The dining table is central, suggesting a focal point of interaction."}, {"type": "leads_to", "subject": "E13|doorway leading to kitchen area", "object": "E9|interior room", "confidence": 0.8, "evidence": ["Frame shows a clear transition area between the kitchen and room."], "notes": "The doorway connects the two spaces."}], "on_screen_text": [], "uncertainties": ["The specific contents or details of the kitchen area are unclear."], "confidence": 0.9} +{"window": {"start_s": 88.799, "end_s": 89.291}, "caption": "The frames depict an interior kitchen area viewed through an open doorway, showcasing dark walls and sleek, modern cabinets. The kitchen has a countertop with a microwave and another appliance with illuminated buttons, reflecting the ambient light from outside. A clear view of the green lawn is visible through the doorway, with a bright blue sky in the background, suggesting a pleasant outdoor environment.", "entities_present": [{"id": "E10", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E13|doorway leading to kitchen area", "object": "E12|window with outside view", "confidence": 0.7, "evidence": ["The doorway opens directly to the lawn visible through the kitchen, creating a line of sight."], "notes": "The doorway likely frames the view to the outside."}, {"type": "contextual_change", "subject": "E10|kitchen area visible through doorway", "object": "E12|window with outside view", "confidence": 0.8, "evidence": ["Both the kitchen area and the outside view are visible within the same spatial frame."], "notes": "Transition from the indoor kitchen to the outdoor view."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 97.798, "end_s": 99.29}, "caption": "The scene transitions between two perspectives of an interior space. In the first frame, a doorframe provides a view into the room, revealing a wall painted dark green and a wooden floor. Light reflects off the floor, enhancing the warm tones of the space. The second frame shifts to a view from within a kitchen area; a large black refrigerator stands against the wall, while through an open doorway, the green wall of the living area and some furniture, including a cart, are visible. Both frames convey a cozy, modern atmosphere with dark colors and natural wood elements.", "entities_present": [{"id": "E10", "confidence": 1.0}], "new_entities": [{"id": "E14", "type": "object", "descriptor": "black refrigerator"}, {"id": "E15", "type": "other", "descriptor": "doorway"}, {"id": "E16", "type": "object", "descriptor": "cart"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 107.291, "end_s": 108.291}, "caption": "The frames depict a modern kitchen area with dark cabinets and appliances against a dark wall. A black refrigerator is visible alongside a microwave and another appliance, with a sink and countertop completing the setup. The floor features a patterned tile design. Outside the window, a green space is visible, contrasting with the kitchen's dark tones.", "entities_present": [{"id": "E10", "confidence": 0.8}, {"id": "E14", "confidence": 0.9}, {"id": "E12", "confidence": 0.7}], "new_entities": [{"id": "E17", "type": "object", "descriptor": "microwave"}, {"id": "E18", "type": "object", "descriptor": "sink"}, {"id": "E19", "type": "object", "descriptor": "countertop"}, {"id": "E20", "type": "location", "descriptor": "window to outside"}], "relations": [{"type": "located_in", "subject": "E10|kitchen area visible through doorway", "object": "E12|window with outside view", "confidence": 0.9, "evidence": ["Both elements are visible within the kitchen context."], "notes": "The window provides an external view from the kitchen."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} +{"window": {"start_s": 117.791, "end_s": 119.791}, "caption": "The view captures a corner of an interior space, with a stark black wall on one side leading to a visible green exterior through a window outside, presumably within a garden. The floor is tiled with a small black-and-white pattern. Though details of the room are obscured, there is a suggestion of a well-furnished area beyond the wall, indicated by soft lighting and a glimpse of the landscape outside.", "entities_present": [], "new_entities": [{"id": "E21", "type": "location", "descriptor": "view of green lawn outside"}, {"id": "E22", "type": "object", "descriptor": "black wall"}], "relations": [], "on_screen_text": [], "uncertainties": ["The exact nature of the room beyond the black wall is unclear."], "confidence": 0.7} +{"window": {"start_s": 126.292, "end_s": 126.791}, "caption": "The scene depicts a dark interior space featuring a black refrigerator (E14) and a doorway (E13) leading towards a kitchen area (E10). The walls are painted in a deep shade, enhancing the contrast with the light-colored flooring. The kitchen visible through the doorway showcases a modern design, emphasizing sleek furniture and appliances.", "entities_present": [{"id": "E10", "confidence": 0.9}, {"id": "E14", "confidence": 0.9}, {"id": "E13", "confidence": 0.9}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 133.804, "end_s": 134.79}, "caption": "The kitchen area features modern dark cabinets, appliances, and a contrasting white countertop. Among the visible kitchen equipment are a microwave (E17) and a sink (E18). The sleek black refrigerator (E14) is also present, against a backdrop of dark walls (E22). Light reflects off the white tile floor, emphasizing the contemporary design. The framed layout appears well-coordinated, with items meticulously arranged.", "entities_present": [{"id": "E14", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E18", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}, {"id": "E22", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 1.0} +{"window": {"start_s": 141.291, "end_s": 141.791}, "caption": "The scene shows an interior view from a kitchen doorway that opens to an outdoor green lawn. The left side of the frame features a dark wall, while on the right, a kitchen area is visible with a coffee machine and cabinetry. The doorway leads outside to a grass area under a clear sky.", "entities_present": [{"id": "E10", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 149.796, "end_s": 150.295}, "caption": "The scene captures a clear, expansive sky with a subtle gradient from blue to lighter shades indicating a bright atmosphere. A lush, green lawn is bordered by a low, light-colored wall designed to provide a boundary. The left side features a large tree, visible near the top of the frame, displaying dense foliage that casts shadow over the grass.", "entities_present": [], "new_entities": [{"id": "E23", "type": "location", "descriptor": "outdoor lawn"}, {"id": "E24", "type": "object", "descriptor": "tree"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 158.292, "end_s": 159.795}, "caption": "The frames show a view from the corner of a space with a green lawn outside, bordered by a low wall. The walls of the interior space are grey, and the floor is covered with a green surface that resembles grass. The sky above transitions from blue to light blue, indicating a clear day. There are no noticeable objects or people within the interior.", "entities_present": [{"id": "E20", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 164.791, "end_s": 165.291}, "caption": "The scene depicts an outdoor lawn area featuring a large tree (E24) that offers shade. In the background, a white building with glass windows can be seen, and a low light-colored wall borders the lawn. The sky appears bright and clear, enhancing the open-feel of the outdoor space.", "entities_present": [{"id": "E24", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [{"id": "E25", "type": "location", "descriptor": "outdoor lawn area"}, {"id": "E26", "type": "object", "descriptor": "building with glass windows"}, {"id": "E27", "type": "location", "descriptor": "clear sky"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 172.344, "end_s": 173.291}, "caption": "The scene depicts a modern outdoor space featuring a white building with a glass door facing a green lawn. There is a wooden bench positioned nearby, with potted plants in shades of red visible at the far right. The clear blue sky provides a bright backdrop.", "entities_present": [{"id": "E23", "confidence": 0.9}, {"id": "E26", "confidence": 0.8}], "new_entities": [{"id": "E28", "type": "object", "descriptor": "wooden bench"}, {"id": "E29", "type": "object", "descriptor": "potted plants"}], "relations": [{"type": "near", "subject": "E28|wooden bench", "object": "E29|potted plants", "confidence": 0.85, "evidence": ["Both entities are present in the scene, with the bench adjacent to the plants."], "notes": "The proximity suggests they are part of the same outdoor setup."}, {"type": "faces", "subject": "E26|white building", "object": "E23|outdoor lawn", "confidence": 0.9, "evidence": ["The glass door of the white building directly overlooks the green lawn."], "notes": "The orientation of the building suggests it is designed to interact with the lawn."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 184.291, "end_s": 186.291}, "caption": "The scene displays an outdoor lawn area with a clear blue sky above and a low wall in the distance. In the foreground, there are two potted plants, one red and one in a terracotta color, placed on the green grass. The grassy area is bordered by a gray wall, suggesting a manicured outdoor space.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E20", "confidence": 0.9}], "new_entities": [{"id": "E30", "type": "object", "descriptor": "red potted plant"}, {"id": "E31", "type": "object", "descriptor": "terracotta potted plant"}, {"id": "E32", "type": "location", "descriptor": "green lawn area"}], "relations": [], "on_screen_text": [], "uncertainties": ["The exact role or visibility of the gray wall and the type of surface is unclear."], "confidence": 0.9} +{"window": {"start_s": 192.791, "end_s": 194.793}, "caption": "The scene features a modern outdoor space with a green lawn area bordered by a low gray wall. A wooden bench sits on the lawn, while two potted plants, one red and the other terracotta, are arranged nearby. In the background, a white building with large glass windows is visible. Above, the sky is clear and blue.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E28", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}, {"id": "E26", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 205.295, "end_s": 205.79}, "caption": "The scene shows an outdoor area featuring a wooden bench (E28) with orange slats placed on green grass. Nearby, there are two potted plants, one red (E30) and one terracotta (E31). The background highlights a clear blue sky (E27) above and a low gray wall surrounding the space, creating an open atmosphere.", "entities_present": [{"id": "E28", "confidence": 0.9}, {"id": "E30", "confidence": 0.9}, {"id": "E31", "confidence": 0.9}, {"id": "E27", "confidence": 0.9}, {"id": "E23", "confidence": 0.9}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} +{"window": {"start_s": 214.793, "end_s": 216.293}, "caption": "The scene features an outdoor space with a clear blue sky visible above. In the foreground, a green lawn area is present, along with two round potted plants: one is light terracotta and the other is dark brown. A wooden bench with orange slats is also seen in the scene. The background includes a blank white wall, contrasting with the vibrant outdoor elements.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E29", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E28", "confidence": 1.0}], "new_entities": [{"id": "E33", "type": "object", "descriptor": "terracotta potted plant"}, {"id": "E34", "type": "object", "descriptor": "dark brown potted plant"}, {"id": "E35", "type": "object", "descriptor": "wooden bench with orange slats"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 226.32, "end_s": 226.81}, "caption": "The scene depicts an outdoor area with a well-maintained green lawn (E23) bordered by a blank white wall on one side. In the foreground, a black watering can (E36) is positioned near the edge of the grassy area. The sky above is a clear blue (E27), with a solid structure visible in the background, likely a building with glass windows (E26).", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [{"id": "E36", "type": "object", "descriptor": "black watering can"}, {"id": "E26", "type": "object", "descriptor": "building with glass windows"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 233.29, "end_s": 234.302}, "caption": "The scene features a flat, green lawn (E23) illuminated by small lights along the ground, leading up to a modern white building (E26) with large glass windows that reflect the interior. A sizeable black tree (E24) stands on the left side against a bright blue sky (E27). The outdoor area appears well-maintained, emphasizing the contrast between the vibrant greenery and the stark wall of the building.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E24", "confidence": 1.0}, {"id": "E26", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [{"id": "E37", "type": "object", "descriptor": "small ground lights"}, {"id": "E38", "type": "location", "descriptor": "lawn area near building"}], "relations": [{"type": "illuminates", "subject": "E37|small ground lights", "object": "E23|lawn area", "confidence": 0.9, "evidence": ["E37 is positioned along the foreground of E23 in the frames."], "notes": "The lights are visible and illuminating the lawn."}, {"type": "located_at", "subject": "E24|tree", "object": "E38|lawn area near building", "confidence": 0.9, "evidence": ["E24 is seen directly on the lawn adjacent to the building (E26)."], "notes": "The tree is firmly placed in the lawn area."}, {"type": "reflected_in", "subject": "E26|building", "object": "E27|blue sky", "confidence": 0.8, "evidence": ["The windows of E26 show a reflection of E27."], "notes": "The clear sky can be seen reflected in the glass."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} +{"window": {"start_s": 287.301, "end_s": 288.298}, "caption": "The scene showcases a view through a large glass window looking into an interior room with dark green walls and wooden flooring. Inside, there is a dining table surrounded by wooden chairs, with a visible bookshelf and lamp enhancing the space's decor. The second frame reveals a painting on the wall, adding visual interest. Outside the window, a green lawn is bordered by a white wall and features small ground lights illuminating the area. A large tree stands nearby under a clear blue sky.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}, {"id": "E24", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 296.291, "end_s": 297.291}, "caption": "The scene depicts a small outdoor area enclosed by a white wall on one side. There is a black watering can (E36) placed on the grass alongside a wooden bench (E35) with orange slats, both set against a clear blue sky (E27). The grass is a vibrant green, and the area has a minimalistic aesthetic reminiscent of a small garden space.", "entities_present": [{"id": "E35", "confidence": 1.0}, {"id": "E36", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 388.291, "end_s": 389.295}, "caption": "The scene depicts an outdoor lawn area (E23) featuring a wooden bench (E35) with orange slats situated against a backdrop of clear blue sky (E27). To the right, there are two red potted plants (E30, E31) on the green grass. The area has minimal furnishings, characterized by an open space that emphasizes the vibrant greenery.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E35", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 396.874, "end_s": 397.791}, "caption": "The scene presents a view from an interior room with dark green walls and wooden floors, leading towards a clear sky and outdoor lawn. Visible through a glass window, a green lawn (E23) is illuminated by small ground lights (E37), bordered by a large tree (E24) and accompanied by a few potted plants (E30, E31). The exterior also features a blank white wall and a clear blue sky (E27).", "entities_present": [{"id": "E1", "confidence": 0.8}, {"id": "E2", "confidence": 0.8}, {"id": "E23", "confidence": 0.9}, {"id": "E24", "confidence": 0.9}, {"id": "E27", "confidence": 0.9}, {"id": "E37", "confidence": 0.8}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": ["No new salient entities identified in these frames."], "confidence": 0.9} +{"window": {"start_s": 57.189, "end_s": 60.19}, "caption": "The scene presents a dimly lit interior with a large window showing a serene outdoor view. Through the window, there is a clear blue sky, green grass, and a bench placed outside. In the foreground, a wooden table with red placemats and wooden chairs is visible, along with a dark furniture piece that could be a side table or cabinet. A reflection can be seen in the glass surface of the table.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "wooden table"}, {"id": "E4", "type": "object", "descriptor": "red placemat"}, {"id": "E5", "type": "object", "descriptor": "wooden chairs"}, {"id": "E6", "type": "object", "descriptor": "dark furniture piece"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 43.626, "end_s": 45.628}, "caption": "The frames display an interior view of a room with dark green walls and wooden flooring. A table with dark surface and red placemats is visible in the foreground, while an open window shows a clear sky and a grassy area outside. A bench is positioned in the outside view, emphasizing the peaceful outdoor setting beyond the room's interior.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark surface dining table"}, {"id": "E2", "type": "object", "descriptor": "red placemat"}, {"id": "E3", "type": "location", "descriptor": "outdoor grassy area"}, {"id": "E4", "type": "object", "descriptor": "wooden bench"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 44.633, "end_s": 45.634}, "caption": "A view from inside a room with dark green walls, looking through a glass door that frames an outdoor scene. In the foreground, there is a dark wooden table with red placemats and a set of chairs. Outside, a bench is visible in the sunlight, against a blue sky and some green grass. The room has minimal decor, with a large black television screen mounted on the wall and a mirror hanging nearby.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark wooden table"}, {"id": "E2", "type": "object", "descriptor": "red placemats"}, {"id": "E3", "type": "object", "descriptor": "outdoor bench"}, {"id": "E4", "type": "location", "descriptor": "room with green walls"}, {"id": "E5", "type": "screen", "descriptor": "black television"}, {"id": "E6", "type": "object", "descriptor": "mirror on the wall"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames.", "No significant text is visible."], "confidence": 0.8} +{"window": {"start_s": 41.368, "end_s": 42.37}, "caption": "The frames show a dimly lit room with dark-colored walls. In the first frame, there is a large window providing a view of an outdoor area, which includes a flat lawn and a bench in the distance. Adjacent to the window on the left is a dark TV screen mounted on the wall. The floor is wooden. The second frame shifts perspective to a dining table set with plates and chairs, facing the same window, revealing the lawn and a reflective surface on the wall. The two frames depict an indoor space looking out to the outside.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "window"}, {"id": "E2", "type": "object", "descriptor": "TV screen"}, {"id": "E3", "type": "object", "descriptor": "dining table"}, {"id": "E4", "type": "object", "descriptor": "chairs"}, {"id": "E5", "type": "location", "descriptor": "outdoor lawn"}], "relations": [{"type": "scene_change", "subject": "E1|window", "object": "E5|outdoor lawn", "confidence": 0.9, "evidence": ["The same outdoor view is visible through the window in both frames."], "notes": "Both frames depict the view from the same window."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 0.215, "end_s": 1.215}, "caption": "The scene features a dark green interior room with a visible window showing a clear blue sky and a grassy area outside. A bench can be seen outside, prominently framed by the window. Inside the room, there's a wooden dining table with placemats, and a TV screen is mounted on the wall opposite the window. The lighting is natural, coming from the outdoor sunlight, which adds contrast to the deep green walls.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark green wall"}, {"id": "E2", "type": "object", "descriptor": "window"}, {"id": "E3", "type": "object", "descriptor": "bench"}, {"id": "E4", "type": "object", "descriptor": "wooden dining table"}, {"id": "E5", "type": "object", "descriptor": "TV screen"}, {"id": "E6", "type": "object", "descriptor": "placemats"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 10.376, "end_s": 11.682}, "caption": "The frames depict two interior settings featuring a dining area and a living space. The first frame shows a dark green wall, a wooden dining table set with red placemats, and a view of an outdoor landscape through a window revealing a clear sky. The second frame presents a similar wall color, a large wall-mounted screen, and a view through a window that includes a park bench and green grass, with the interior focusing on a low, dark wooden table.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "wooden dining table with red placemats"}, {"id": "E2", "type": "object", "descriptor": "dark wooden low table"}, {"id": "E3", "type": "screen", "descriptor": "wall-mounted screen"}, {"id": "E4", "type": "location", "descriptor": "view of outdoor area with park bench in the background"}, {"id": "E5", "type": "location", "descriptor": "view of outdoor area with green grass and clear sky"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames.", "Specifics about the objects on the tables are unclear."], "confidence": 0.9} +{"window": {"start_s": 11.213, "end_s": 12.754}, "caption": "The scene features a dining area with a dark green wall and a wooden dining table equipped with red placemats. A large window allows a view of an outdoor setting, which is grassy and includes a bench visible through the window. A TV screen is mounted on the wall, adjacent to the window. The room is well-lit, with natural light entering from outside, creating a vibrant atmosphere.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}], "new_entities": [{"id": "E7", "type": "object", "descriptor": "outdoor bench"}, {"id": "E8", "type": "location", "descriptor": "outdoor grassy area"}], "relations": [{"type": "looks_at", "subject": "E2", "object": "E8", "confidence": 0.7, "evidence": ["The window (E2) is facing towards the outdoor area (E8) which is visible in the frame."], "notes": "The window appears to direct the viewer's gaze towards the outdoor space."}], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 0.9} +{"window": {"start_s": 37.649, "end_s": 39.143}, "caption": "The scene depicts an interior space with a view through a large window. In the window, a clear blue sky and a bench in an outdoor area are visible. Inside, there is a dark-colored cabinet on the left, empty space in front of it, and a dining table with several chairs on the right. The walls are painted dark green, and there is a reflection of the room visible on the window glass.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "dark cabinet"}, {"id": "E4", "type": "object", "descriptor": "dining table"}, {"id": "E5", "type": "object", "descriptor": "chairs"}], "relations": [{"type": "scene_change", "subject": "E1|large window", "object": "E2|bench", "confidence": 0.9, "evidence": ["Both frames display the window looking out towards the bench."], "notes": "The window shows the outdoor view prominently."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} +{"window": {"start_s": 48.644, "end_s": 50.16}, "caption": "The scene depicts an interior room with dark green walls featuring a large window (E1) that opens up to a clear blue sky and a view of an outdoor bench (E2) visible through the glass. Inside, a dark cabinet (E3) is located on the left, with a dining table (E4) and several chairs (E5) positioned to the right. The lighting highlights the wooden floor, and there is an empty space in front of the cabinet and behind the table, indicating a spacious design.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|large window", "object": "E2|bench", "confidence": 0.8, "evidence": ["The window (E1) directly displays the outdoor area, where the bench (E2) is visible."], "notes": "The window is oriented towards the outdoor view."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} +{"window": {"start_s": 58.152, "end_s": 59.644}, "caption": "The scene depicts an interior room with dark green walls, featuring a large window (E1) that opens to an outdoor view. Inside, there is a dark cabinet (E3) on the left side and a dining table (E4) with several chairs (E5) on the right. The atmosphere is bright due to natural light coming through the window, illuminating the wooden floor and creating a spacious feeling.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|large window", "object": "E2|unknown", "confidence": 0.8, "evidence": ["In both frames, the large window is open, showing an outdoor view."], "notes": "The window directly looks out, suggesting a connection to the outside."}], "on_screen_text": [], "uncertainties": ["Presence of any people is unclear."], "confidence": 1.0} diff --git a/assets/temporal_memory/frames_index.jsonl b/assets/temporal_memory/frames_index.jsonl new file mode 100644 index 0000000000..c7b2b7a0ae --- /dev/null +++ b/assets/temporal_memory/frames_index.jsonl @@ -0,0 +1,50 @@ +{"frame_index": 57, "timestamp_s": 70.68190789222717, "timestamp": "01:10.682"} +{"frame_index": 58, "timestamp_s": 72.17690801620483, "timestamp": "01:12.177"} +{"frame_index": 59, "timestamp_s": 72.67690801620483, "timestamp": "01:12.677"} +{"frame_index": 60, "timestamp_s": 74.17690801620483, "timestamp": "01:14.177"} +{"frame_index": 61, "timestamp_s": 74.67590808868408, "timestamp": "01:14.676"} +{"frame_index": 62, "timestamp_s": 76.17990803718567, "timestamp": "01:16.180"} +{"frame_index": 63, "timestamp_s": 76.67690801620483, "timestamp": "01:16.677"} +{"frame_index": 64, "timestamp_s": 77.67790794372559, "timestamp": "01:17.678"} +{"frame_index": 65, "timestamp_s": 78.67690801620483, "timestamp": "01:18.677"} +{"frame_index": 66, "timestamp_s": 79.67690801620483, "timestamp": "01:19.677"} +{"frame_index": 67, "timestamp_s": 80.67590808868408, "timestamp": "01:20.676"} +{"frame_index": 68, "timestamp_s": 81.67690801620483, "timestamp": "01:21.677"} +{"frame_index": 69, "timestamp_s": 82.67590808868408, "timestamp": "01:22.676"} +{"frame_index": 70, "timestamp_s": 84.17690801620483, "timestamp": "01:24.177"} +{"frame_index": 71, "timestamp_s": 84.67590808868408, "timestamp": "01:24.676"} +{"frame_index": 72, "timestamp_s": 85.67690801620483, "timestamp": "01:25.677"} +{"frame_index": 73, "timestamp_s": 87.17590808868408, "timestamp": "01:27.176"} +{"frame_index": 74, "timestamp_s": 88.17690801620483, "timestamp": "01:28.177"} +{"frame_index": 75, "timestamp_s": 88.67690801620483, "timestamp": "01:28.677"} +{"frame_index": 76, "timestamp_s": 89.67590808868408, "timestamp": "01:29.676"} +{"frame_index": 77, "timestamp_s": 90.67590808868408, "timestamp": "01:30.676"} +{"frame_index": 78, "timestamp_s": 91.67690801620483, "timestamp": "01:31.677"} +{"frame_index": 79, "timestamp_s": 92.67890787124634, "timestamp": "01:32.679"} +{"frame_index": 80, "timestamp_s": 93.67690801620483, "timestamp": "01:33.677"} +{"frame_index": 81, "timestamp_s": 94.67590808868408, "timestamp": "01:34.676"} +{"frame_index": 82, "timestamp_s": 95.67590808868408, "timestamp": "01:35.676"} +{"frame_index": 83, "timestamp_s": 96.67890787124634, "timestamp": "01:36.679"} +{"frame_index": 84, "timestamp_s": 97.67690801620483, "timestamp": "01:37.677"} +{"frame_index": 85, "timestamp_s": 98.67690801620483, "timestamp": "01:38.677"} +{"frame_index": 86, "timestamp_s": 100.67690801620483, "timestamp": "01:40.677"} +{"frame_index": 87, "timestamp_s": 101.17590808868408, "timestamp": "01:41.176"} +{"frame_index": 88, "timestamp_s": 102.17990803718567, "timestamp": "01:42.180"} +{"frame_index": 89, "timestamp_s": 104.17590808868408, "timestamp": "01:44.176"} +{"frame_index": 90, "timestamp_s": 104.67690801620483, "timestamp": "01:44.677"} +{"frame_index": 91, "timestamp_s": 105.67690801620483, "timestamp": "01:45.677"} +{"frame_index": 92, "timestamp_s": 106.67590808868408, "timestamp": "01:46.676"} +{"frame_index": 93, "timestamp_s": 107.67690801620483, "timestamp": "01:47.677"} +{"frame_index": 94, "timestamp_s": 109.17590808868408, "timestamp": "01:49.176"} +{"frame_index": 95, "timestamp_s": 110.17690801620483, "timestamp": "01:50.177"} +{"frame_index": 96, "timestamp_s": 111.17890787124634, "timestamp": "01:51.179"} +{"frame_index": 97, "timestamp_s": 112.17690801620483, "timestamp": "01:52.177"} +{"frame_index": 98, "timestamp_s": 113.67690801620483, "timestamp": "01:53.677"} +{"frame_index": 99, "timestamp_s": 114.67590808868408, "timestamp": "01:54.676"} +{"frame_index": 100, "timestamp_s": 115.67590808868408, "timestamp": "01:55.676"} +{"frame_index": 101, "timestamp_s": 117.17690801620483, "timestamp": "01:57.177"} +{"frame_index": 102, "timestamp_s": 118.17690801620483, "timestamp": "01:58.177"} +{"frame_index": 103, "timestamp_s": 119.17590808868408, "timestamp": "01:59.176"} +{"frame_index": 104, "timestamp_s": 120.17590808868408, "timestamp": "02:00.176"} +{"frame_index": 105, "timestamp_s": 121.17590808868408, "timestamp": "02:01.176"} +{"frame_index": 106, "timestamp_s": 122.67690801620483, "timestamp": "02:02.677"} diff --git a/assets/temporal_memory/state.json b/assets/temporal_memory/state.json new file mode 100644 index 0000000000..70deefe154 --- /dev/null +++ b/assets/temporal_memory/state.json @@ -0,0 +1,7 @@ +{ + "entity_roster": [], + "rolling_summary": "", + "chunk_buffer": [], + "next_summary_at_s": 10.0, + "last_present": [] +} \ No newline at end of file diff --git a/dimos/e2e_tests/test_dimsim_eval.py b/dimos/e2e_tests/test_dimsim_eval.py new file mode 100644 index 0000000000..cc99428252 --- /dev/null +++ b/dimos/e2e_tests/test_dimsim_eval.py @@ -0,0 +1,211 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sequential eval tests — 1 dimos instance, run workflows one at a time. + +Uses sim-eval blueprint (includes rerun/browser for manual observation). +Run individual evals or all of them: + + pytest test_sim_eval_sequential.py -v -s -m slow + pytest test_sim_eval_sequential.py::TestSimEvalSequential::test_go_to_tv -v -s -m slow +""" + +import json +import os +import signal +import socket +import subprocess +import time +from pathlib import Path + +import pytest +import websocket + +from dimos.e2e_tests.dimos_cli_call import DimosCliCall +from dimos.e2e_tests.lcm_spy import LcmSpy + +PORT = 8090 +EVALS_DIR = Path.home() / ".dimsim" / "evals" + + +def _force_kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + pids = result.stdout.strip().split() + for pid in pids: + if pid: + try: + os.kill(int(pid), signal.SIGKILL) + except (ProcessLookupError, ValueError): + pass + except Exception: + pass + + +def _wait_for_port(port: int, timeout: float = 120) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=2): + return True + except OSError: + time.sleep(1) + return False + + +def _wait_for_port_free(port: int, timeout: float = 30) -> bool: + """Wait until nothing is listening on *port*.""" + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=1): + time.sleep(1) + except OSError: + return True + return False + + +class EvalClient: + """Talks to the browser eval harness via the bridge WebSocket.""" + + def __init__(self, port: int = PORT): + self.ws = websocket.WebSocket() + self.ws.connect(f"ws://localhost:{port}") + + def _send(self, msg: dict) -> None: + self.ws.send(json.dumps(msg)) + + def _wait_for(self, msg_type: str, timeout: float = 120) -> dict: + self.ws.settimeout(timeout) + while True: + raw = self.ws.recv() + if isinstance(raw, bytes): + continue + msg = json.loads(raw) + if msg.get("type") == msg_type: + return msg + + def wait_for_harness(self, timeout: float = 60) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + self._send({"type": "ping"}) + self.ws.settimeout(3) + raw = self.ws.recv() + if isinstance(raw, str): + msg = json.loads(raw) + if msg.get("type") == "pong": + return True + except (websocket.WebSocketTimeoutException, Exception): + time.sleep(1) + return False + + def run_workflow(self, workflow: dict) -> dict: + """Send loadEnv + startWorkflow, wait for workflowComplete.""" + timeout = (workflow.get("timeoutSec", 120) + 30) + self._send({"type": "loadEnv", "scene": workflow.get("environment", "apt")}) + self._wait_for("envReady", timeout=30) + self._send({"type": "startWorkflow", "workflow": workflow}) + return self._wait_for("workflowComplete", timeout=timeout) + + def close(self): + self.ws.close() + + +def _load_workflow(env: str, name: str) -> dict: + path = EVALS_DIR / env / f"{name}.json" + return json.loads(path.read_text()) + + +@pytest.fixture(scope="class") +def sim_eval(): + """Start dimos sim-eval headless, tear down after.""" + _force_kill_port(PORT) + assert _wait_for_port_free(PORT, timeout=10), ( + f"Port {PORT} still in use after force-kill" + ) + log_dir = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") + if log_dir: + log_path = Path(log_dir) + log_path.mkdir(parents=True, exist_ok=True) + log_file = open(log_path / "dimos-sequential.log", "w") + print(f"\n dimos logs → {log_path}/dimos-sequential.log") + else: + log_file = None + + spy = LcmSpy() + spy.save_topic("/color_image#sensor_msgs.Image") + spy.save_topic("/odom#geometry_msgs.PoseStamped") + spy.start() + + env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu"} + call = DimosCliCall() + call.demo_args = ["sim-eval"] + call.process = subprocess.Popen( + ["dimos", "--simulation", "run", "sim-eval"], + env=env, + stdout=log_file or subprocess.DEVNULL, + stderr=log_file or subprocess.DEVNULL, + ) + + try: + assert _wait_for_port(PORT, timeout=120), f"Bridge not ready on port {PORT}" + spy.wait_for_saved_topic("/color_image#sensor_msgs.Image", timeout=60.0) + spy.wait_for_saved_topic("/odom#geometry_msgs.PoseStamped", timeout=60.0) + + yield call + finally: + call.stop() + spy.stop() + if log_file: + log_file.close() + _force_kill_port(PORT) + + +@pytest.fixture(scope="class") +def eval_client(sim_eval): + """Connect to bridge WS and wait for eval harness.""" + client = EvalClient(PORT) + assert client.wait_for_harness(timeout=60), "Eval harness not responding" + yield client + client.close() + + +@pytest.mark.skipif_in_ci +@pytest.mark.slow +class TestSimEvalSequential: + """Run DimSim evals sequentially against a live dimos sim-eval instance.""" + + def _run_and_assert(self, eval_client: EvalClient, env: str, workflow_name: str) -> None: + workflow = _load_workflow(env, workflow_name) + result = eval_client.run_workflow(workflow) + scores = result.get("rubricScores", {}) + od = scores.get("objectDistance", {}) + passed = od.get("pass", False) + details = od.get("details", result.get("reason", "unknown")) + print(f" {workflow_name}: {'PASS' if passed else 'FAIL'} — {details}") + assert passed, f"Eval '{workflow_name}' failed: {details}" + + def test_go_to_tv(self, eval_client) -> None: + self._run_and_assert(eval_client, "apt", "television") + + def test_go_to_couch(self, eval_client) -> None: + self._run_and_assert(eval_client, "apt", "go-to-couch") + + def test_go_to_kitchen(self, eval_client) -> None: + self._run_and_assert(eval_client, "apt", "go-to-kitchen") diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py new file mode 100644 index 0000000000..3c204fb95a --- /dev/null +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -0,0 +1,377 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parallel eval tests for DimSim simulation. + +3 dimos instances + 3 headless browser pages, 1 eval workflow each. +Runs all workflows concurrently, cutting wall-clock time to ~1 min. + + pytest test_sim_eval.py -v -s -m slow +""" + +import json +import os +import signal +import socket +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +import pytest +import websocket + +from dimos.e2e_tests.dimos_cli_call import DimosCliCall + +PORT = 8090 +EVALS_DIR = Path.home() / ".dimsim" / "evals" +NUM_CHANNELS = 3 +LCM_BASE_PORT = 7667 + + +def _force_kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + pids = result.stdout.strip().split() + for pid in pids: + if pid: + try: + os.kill(int(pid), signal.SIGKILL) + except (ProcessLookupError, ValueError): + pass + except Exception: + pass + + +def _wait_for_port(port: int, timeout: float = 120) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=2): + return True + except OSError: + time.sleep(1) + return False + + +def _wait_for_all_pages(log_path: Path, num_pages: int, timeout: float = 90) -> None: + """Wait until dimsim's log shows all headless pages are sending sensor data. + + Checks for sensor messages from the last channel (page-N-1) which proves + all pages have loaded their Three.js scenes and are rendering. + """ + last_channel = f"page-{num_pages - 1}" + marker = f"bridge:{last_channel}] sensor" + deadline = time.time() + timeout + while time.time() < deadline: + try: + text = log_path.read_text() + if marker in text: + print(f" All {num_pages} headless pages confirmed ready (sensor data flowing)") + return + except FileNotFoundError: + pass + time.sleep(2) + print(f" WARNING: page-{num_pages - 1} sensor data not seen after {timeout}s, proceeding anyway") + + +def _wait_for_port_free(port: int, timeout: float = 30) -> bool: + """Wait until nothing is listening on *port*.""" + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=1): + time.sleep(1) # still occupied + except OSError: + return True # connection refused → port is free + return False + + +def _load_workflow(env: str, name: str) -> dict: + path = EVALS_DIR / env / f"{name}.json" + return json.loads(path.read_text()) + + +# ── Eval clients ───────────────────────────────────────────────────────────── + + +class EvalClient: + """Talks to the browser eval harness via the bridge WebSocket.""" + + def __init__(self, port: int = PORT): + self.ws = websocket.WebSocket() + self.ws.connect(f"ws://localhost:{port}") + + def _send(self, msg: dict) -> None: + self.ws.send(json.dumps(msg)) + + def _wait_for(self, msg_type: str, timeout: float = 120) -> dict: + self.ws.settimeout(timeout) + while True: + raw = self.ws.recv() + if isinstance(raw, bytes): + continue + msg = json.loads(raw) + if msg.get("type") == msg_type: + return msg + + def wait_for_harness(self, timeout: float = 60) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + self._send({"type": "ping"}) + self.ws.settimeout(3) + raw = self.ws.recv() + if isinstance(raw, str): + msg = json.loads(raw) + if msg.get("type") == "pong": + return True + except (websocket.WebSocketTimeoutException, Exception): + time.sleep(1) + return False + + def run_workflow(self, workflow: dict) -> dict: + """Send loadEnv + startWorkflow, wait for workflowComplete.""" + timeout = (workflow.get("timeoutSec", 120) + 30) + self._send({"type": "loadEnv", "scene": workflow.get("environment", "apt")}) + self._wait_for("envReady", timeout=30) + self._send({"type": "startWorkflow", "workflow": workflow}) + return self._wait_for("workflowComplete", timeout=timeout) + + def close(self): + self.ws.close() + + +class ChannelEvalClient(EvalClient): + """EvalClient that connects to a specific channel's control WS. + + Inherits run_workflow (loadEnv + startWorkflow) from EvalClient. + Only overrides _send/_wait_for for channel routing. + """ + + def __init__(self, port: int, channel: str): + self._port = port + self.channel = channel + self.ws = websocket.WebSocket() + self.ws.connect(f"ws://localhost:{port}?channel={channel}") + + def wait_for_harness(self, timeout: float = 60) -> bool: + """Override: drain pose/sensor messages to find the pong. + + The bridge sends pose updates at 50Hz to all control clients. + The base class reads only one message per loop and misses the pong. + """ + deadline = time.time() + timeout + while time.time() < deadline: + try: + self._send({"type": "ping"}) + self._wait_for("pong", timeout=5) + return True + except Exception: + time.sleep(1) + return False + + def _send(self, msg: dict) -> None: + msg["channel"] = self.channel + self.ws.send(json.dumps(msg)) + + def _wait_for(self, msg_type: str, timeout: float = 120) -> dict: + self.ws.settimeout(timeout) + while True: + raw = self.ws.recv() + if isinstance(raw, bytes): + continue + if not raw: + continue + try: + msg = json.loads(raw) + except json.JSONDecodeError: + continue + msg_ch = msg.get("channel", "") + if msg.get("type") == msg_type and (not msg_ch or msg_ch == self.channel): + return msg + + +# ── Fixtures ───────────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="class") +def parallel_env(): + """Start dimos-0 (launches dimsim with 3 channels), then dimos-1/2 (connect-only).""" + _force_kill_port(PORT) + assert _wait_for_port_free(PORT, timeout=10), ( + f"Port {PORT} still in use after force-kill" + ) + + calls: list[DimosCliCall] = [] + log_files: list = [] + base_env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu"} + log_dir_env = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") + log_dir = Path(log_dir_env) if log_dir_env else None + if log_dir: + log_dir.mkdir(parents=True, exist_ok=True) + + # dimos-0: launches dimsim normally with --channels 3 + env0 = { + **base_env, + "LCM_DEFAULT_URL": f"udpm://239.255.76.67:{LCM_BASE_PORT}?ttl=0", + "DIMSIM_CHANNELS": str(NUM_CHANNELS), + "EVAL_INSTANCE_ID": "0", + } + log0 = open(log_dir / "dimos-0.log", "w") if log_dir else None + log_files.append(log0) + call0 = DimosCliCall() + call0.demo_args = ["sim-parallel-eval"] + call0.process = subprocess.Popen( + ["dimos", "--simulation", "run", "sim-parallel-eval"], + env=env0, + stdout=log0 or subprocess.DEVNULL, + stderr=log0 or subprocess.DEVNULL, + ) + calls.append(call0) + + try: + # Wait for dimsim bridge + all headless pages before starting dimos-1/2 + assert _wait_for_port(PORT, timeout=120), f"Bridge not ready on port {PORT}" + if log_dir: + _wait_for_all_pages(log_dir / "dimos-0.log", NUM_CHANNELS, timeout=90) + else: + time.sleep(30) # No log file to check — wait a fixed period for pages to load + + # dimos-1 and dimos-2: connect-only (skip dimsim launch) + for i in range(1, NUM_CHANNELS): + env_i = { + **base_env, + "DIMSIM_CONNECT_ONLY": "1", + "LCM_DEFAULT_URL": f"udpm://239.255.76.67:{LCM_BASE_PORT + i}?ttl=0", + "EVAL_INSTANCE_ID": str(i), + } + log_i = open(log_dir / f"dimos-{i}.log", "w") if log_dir else None + log_files.append(log_i) + call_i = DimosCliCall() + call_i.demo_args = ["sim-parallel-eval"] + call_i.process = subprocess.Popen( + ["dimos", "--simulation", "run", "sim-parallel-eval"], + env=env_i, + stdout=log_i or subprocess.DEVNULL, + stderr=log_i or subprocess.DEVNULL, + ) + calls.append(call_i) + + # Give connect-only instances time to start and establish LCM connections + time.sleep(10) + + if log_dir: + print(f"\n dimos logs → {log_dir}/dimos-{{0,1,2}}.log") + yield calls + finally: + # Teardown: stop in reverse (connect-only first, then the one that owns dimsim) + for call in reversed(calls): + call.stop() + for f in log_files: + if f: + f.close() + _force_kill_port(PORT) + + +def _connect_channel(port: int, channel: str, timeout: float = 120) -> ChannelEvalClient: + """Connect to a channel with retries (bridge may be under load).""" + deadline = time.time() + timeout + last_err = None + while time.time() < deadline: + try: + client = ChannelEvalClient(port, channel) + if client.wait_for_harness(timeout=min(30, deadline - time.time())): + return client + client.close() + except Exception as e: + last_err = e + time.sleep(2) + raise ConnectionError(f"Could not connect to {channel} after {timeout}s: {last_err}") + + +@pytest.fixture(scope="class") +def parallel_eval_clients(parallel_env): + """Create one ChannelEvalClient per channel, wait for each harness.""" + clients: list[ChannelEvalClient] = [] + for i in range(NUM_CHANNELS): + channel = f"page-{i}" + client = _connect_channel(PORT, channel, timeout=120) + print(f" {channel}: harness ready") + clients.append(client) + yield clients + for client in clients: + client.close() + + +# ── Test ───────────────────────────────────────────────────────────────────── + + +@pytest.mark.skipif_in_ci +@pytest.mark.slow +class TestSimEvalParallel: + """Run DimSim evals in parallel — 3 dimos instances, 3 browser pages.""" + + WORKFLOWS = [ + ("apt", "television"), + ("apt", "go-to-couch"), + ("apt", "go-to-kitchen"), + ] + + def test_parallel_evals(self, parallel_eval_clients: list[ChannelEvalClient]) -> None: + """Run all 3 eval workflows concurrently, one per channel.""" + results: dict[str, dict] = {} + errors: dict[str, str] = {} + + def run_one(idx: int) -> tuple[str, dict]: + # Stagger starts so loadEnv doesn't hammer the bridge simultaneously + time.sleep(idx * 3) + env, name = self.WORKFLOWS[idx] + workflow = _load_workflow(env, name) + result = parallel_eval_clients[idx].run_workflow(workflow) + return name, result + + with ThreadPoolExecutor(max_workers=NUM_CHANNELS) as pool: + futures = {pool.submit(run_one, i): i for i in range(NUM_CHANNELS)} + for future in as_completed(futures): + try: + name, result = future.result() + results[name] = result + except Exception as exc: + idx = futures[future] + errors[self.WORKFLOWS[idx][1]] = str(exc) + + # Report and assert all results + failures: list[str] = [] + + for name, result in results.items(): + scores = result.get("rubricScores", {}) + od = scores.get("objectDistance", {}) + passed = od.get("pass", False) + details = od.get("details", result.get("reason", "unknown")) + print(f" {name}: {'PASS' if passed else 'FAIL'} — {details}") + if not passed: + failures.append(f"{name}: {details}") + + for name, err in errors.items(): + print(f" {name}: ERROR — {err}") + failures.append(f"{name}: ERROR — {err}") + + assert not failures, ( + f"{len(failures)}/{NUM_CHANNELS} evals failed:\n" + "\n".join(f" - {f}" for f in failures) + ) diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py new file mode 100644 index 0000000000..141b41aac7 --- /dev/null +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -0,0 +1,251 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""E2E smoke tests for sim-nav — verifies the full sensor + control pipeline. + +Starts dimos sim-nav headless (GPU rendering), then checks that all LCM +topics are publishing and that cmd_vel actually moves the robot. + + pytest test_sim_nav.py -v -s -m slow +""" + +import math +import os +import signal +import socket +import subprocess +import time +from pathlib import Path + +import pytest + +from dimos.e2e_tests.dimos_cli_call import DimosCliCall +from dimos.e2e_tests.lcm_spy import LcmSpy +from dimos.msgs.geometry_msgs import PoseStamped, Twist, Vector3 +from dimos.msgs.sensor_msgs import Image, PointCloud2 + +BRIDGE_PORT = 8090 + + +def _force_kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + pids = result.stdout.strip().split() + for pid in pids: + if pid: + try: + os.kill(int(pid), signal.SIGKILL) + except (ProcessLookupError, ValueError): + pass + except Exception: + pass + + +def _wait_for_port(port: int, timeout: float = 120) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=2): + return True + except OSError: + time.sleep(1) + return False + + +def _wait_for_port_free(port: int, timeout: float = 30) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("localhost", port), timeout=1): + time.sleep(1) + except OSError: + return True + return False + + +@pytest.fixture(scope="class") +def sim_nav(): + """Start dimos sim-nav headless with GPU rendering, tear down after.""" + # Kill any leftover processes from a previous crashed run + _force_kill_port(BRIDGE_PORT) + assert _wait_for_port_free(BRIDGE_PORT, timeout=10), ( + f"Port {BRIDGE_PORT} still in use after force-kill" + ) + + log_dir = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") + if log_dir: + log_path = Path(log_dir) + log_path.mkdir(parents=True, exist_ok=True) + log_file = open(log_path / "dimos-sim-nav.log", "w") + print(f"\n dimos logs → {log_path}/dimos-sim-nav.log") + else: + log_file = None + + render = os.environ.get("DIMSIM_RENDER", "cpu") + env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": render} + call = DimosCliCall() + call.demo_args = ["sim-nav"] + call.process = subprocess.Popen( + ["dimos", "--simulation", "run", "sim-nav"], + env=env, + stdout=log_file or subprocess.DEVNULL, + stderr=log_file or subprocess.DEVNULL, + ) + + try: + assert _wait_for_port(BRIDGE_PORT, timeout=120), ( + f"Bridge not ready on port {BRIDGE_PORT}" + ) + yield call + finally: + call.stop() + if log_file: + log_file.close() + # Ensure port is freed even if stop() doesn't fully clean up + _force_kill_port(BRIDGE_PORT) + + +@pytest.fixture(scope="class") +def spy(sim_nav): + """LCM spy that subscribes to all topics and waits for initial sensor data.""" + s = LcmSpy() + s.save_topic("/color_image#sensor_msgs.Image") + s.save_topic("/depth_image#sensor_msgs.Image") + s.save_topic("/odom#geometry_msgs.PoseStamped") + s.save_topic("/lidar#sensor_msgs.PointCloud2") + s.start() + + # Wait for at least one message on color_image before tests start + s.wait_for_saved_topic("/color_image#sensor_msgs.Image", timeout=60.0) + + yield s + + s.stop() + + +class TestSimNav: + """Smoke tests for the sim-nav pipeline — sensors, control, and data integrity.""" + + def test_color_image_publishing(self, spy: LcmSpy) -> None: + """Verify /color_image is publishing and decodable.""" + spy.wait_for_saved_topic("/color_image#sensor_msgs.Image", timeout=30) + msgs = spy.messages.get("/color_image#sensor_msgs.Image", []) + assert len(msgs) > 0, "No color_image messages received" + + # Decode the latest image — DimSim sends JPEG encoding + img = Image.lcm_jpeg_decode(msgs[-1]) + assert img.width > 0, f"Image width is {img.width}" + assert img.height > 0, f"Image height is {img.height}" + print(f" color_image: {img.width}x{img.height}, {len(msgs)} messages") + + def test_depth_image_publishing(self, spy: LcmSpy) -> None: + """Verify /depth_image is publishing and decodable.""" + spy.wait_for_saved_topic("/depth_image#sensor_msgs.Image", timeout=30) + msgs = spy.messages.get("/depth_image#sensor_msgs.Image", []) + assert len(msgs) > 0, "No depth_image messages received" + + # Depth images use raw encoding (16UC1), not JPEG + img = Image.lcm_decode(msgs[-1]) + assert img.width > 0, f"Depth image width is {img.width}" + assert img.height > 0, f"Depth image height is {img.height}" + print(f" depth_image: {img.width}x{img.height}, {len(msgs)} messages") + + def test_odom_publishing(self, spy: LcmSpy) -> None: + """Verify /odom is publishing with sane values.""" + spy.wait_for_saved_topic("/odom#geometry_msgs.PoseStamped", timeout=30) + msgs = spy.messages.get("/odom#geometry_msgs.PoseStamped", []) + assert len(msgs) > 0, "No odom messages received" + + pose = PoseStamped.lcm_decode(msgs[-1]) + pos = pose.position + # Position should be finite (not NaN or inf) + assert math.isfinite(pos.x), f"odom x is {pos.x}" + assert math.isfinite(pos.y), f"odom y is {pos.y}" + assert math.isfinite(pos.z), f"odom z is {pos.z}" + # Position should be within reasonable bounds (scene is ~50m) + assert abs(pos.x) < 100, f"odom x out of bounds: {pos.x}" + assert abs(pos.y) < 100, f"odom y out of bounds: {pos.y}" + assert abs(pos.z) < 100, f"odom z out of bounds: {pos.z}" + print(f" odom: ({pos.x:.2f}, {pos.y:.2f}, {pos.z:.2f}), {len(msgs)} messages") + + def test_lidar_publishing(self, spy: LcmSpy) -> None: + """Verify /lidar is publishing with non-empty point cloud.""" + spy.wait_for_saved_topic("/lidar#sensor_msgs.PointCloud2", timeout=30) + msgs = spy.messages.get("/lidar#sensor_msgs.PointCloud2", []) + assert len(msgs) > 0, "No lidar messages received" + + cloud = PointCloud2.lcm_decode(msgs[-1]) + num_points = len(cloud) + assert num_points > 0, "Lidar point cloud is empty" + print(f" lidar: {num_points} points, {len(msgs)} messages") + + def test_cmd_vel_moves_robot(self, spy: LcmSpy) -> None: + """Publish cmd_vel and verify odom position changes.""" + # Record initial position + spy.wait_for_saved_topic("/odom#geometry_msgs.PoseStamped", timeout=30) + initial_msgs = spy.messages.get("/odom#geometry_msgs.PoseStamped", []) + assert len(initial_msgs) > 0, "No initial odom" + initial_pose = PoseStamped.lcm_decode(initial_msgs[-1]) + initial_pos = initial_pose.position + print(f" initial position: ({initial_pos.x:.2f}, {initial_pos.y:.2f}, {initial_pos.z:.2f})") + + # Send forward velocity for 3 seconds + twist = Twist( + linear=Vector3(0.5, 0.0, 0.0), + angular=Vector3(0.0, 0.0, 0.0), + ) + deadline = time.time() + 3.0 + while time.time() < deadline: + spy.publish("/cmd_vel#geometry_msgs.Twist", twist) + time.sleep(0.05) # 20 Hz + + # Wait a bit for physics to settle and odom to update + time.sleep(1.0) + + # Check position changed + final_msgs = spy.messages.get("/odom#geometry_msgs.PoseStamped", []) + final_pose = PoseStamped.lcm_decode(final_msgs[-1]) + final_pos = final_pose.position + + dx = final_pos.x - initial_pos.x + dy = final_pos.y - initial_pos.y + distance = math.sqrt(dx * dx + dy * dy) + print(f" final position: ({final_pos.x:.2f}, {final_pos.y:.2f}, {final_pos.z:.2f})") + print(f" distance moved: {distance:.2f}m") + + assert distance > 0.1, ( + f"Robot didn't move after cmd_vel: distance={distance:.3f}m " + f"(initial=({initial_pos.x:.2f},{initial_pos.y:.2f}), " + f"final=({final_pos.x:.2f},{final_pos.y:.2f}))" + ) + + def test_odom_rate(self, spy: LcmSpy) -> None: + """Verify odom publishes at a reasonable rate (>5 Hz).""" + # Clear existing messages and collect fresh ones + topic = "/odom#geometry_msgs.PoseStamped" + with spy._messages_lock: + spy.messages[topic] = [] + + time.sleep(2.0) + + with spy._messages_lock: + count = len(spy.messages.get(topic, [])) + + rate = count / 2.0 + print(f" odom rate: {rate:.1f} Hz ({count} messages in 2s)") + assert rate > 5, f"Odom rate too low: {rate:.1f} Hz (expected >5 Hz)" diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 307db25e0d..dc1a9cbab4 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -58,8 +58,13 @@ "mid360-fastlio-voxels": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels", "mid360-fastlio-voxels-native": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio_voxels_native", "phone-go2-teleop": "dimos.teleop.phone.blueprints:phone_go2_teleop", + "sim-agentic": "dimos.robot.sim.blueprints.agentic.sim_agentic:sim_agentic", "sim-basic": "dimos.robot.sim.blueprints.basic.sim_basic:sim_basic", + "sim-eval": "dimos.robot.sim.blueprints.agentic.sim_eval:sim_eval", + "sim-parallel-eval": "dimos.robot.sim.blueprints.agentic.sim_parallel_eval:sim_parallel_eval", "sim-nav": "dimos.robot.sim.blueprints.nav.sim_nav:sim_nav", + "sim-spatial": "dimos.robot.sim.blueprints.nav.sim_spatial:sim_spatial", + "sim-temporal-memory": "dimos.robot.sim.blueprints.agentic.sim_temporal_memory:sim_temporal_memory", "simple-phone-teleop": "dimos.teleop.phone.blueprints:simple_phone_teleop", "uintree-g1-primitive-no-nav": "dimos.robot.unitree.g1.blueprints.primitive.uintree_g1_primitive_no_nav:uintree_g1_primitive_no_nav", "unitree-g1": "dimos.robot.unitree.g1.blueprints.perceptive.unitree_g1:unitree_g1", diff --git a/dimos/robot/sim/blueprints/agentic/__init__.py b/dimos/robot/sim/blueprints/agentic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dimos/robot/sim/blueprints/agentic/sim_agentic.py b/dimos/robot/sim/blueprints/agentic/sim_agentic.py new file mode 100644 index 0000000000..e468267101 --- /dev/null +++ b/dimos/robot/sim/blueprints/agentic/sim_agentic.py @@ -0,0 +1,35 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim agentic blueprint — spatial + agent + skills + human input.""" + +from dimos.agents.agent import agent +from dimos.agents.skills.navigation import navigation_skill +from dimos.agents.skills.person_follow import person_follow_skill +from dimos.agents.skills.speak_skill import speak_skill +from dimos.agents.web_human_input import web_input +from dimos.core.blueprints import autoconnect +from dimos.robot.sim.blueprints.nav.sim_spatial import sim_spatial +from dimos.robot.sim.tf_module import _camera_info_static + +sim_agentic = autoconnect( + sim_spatial, + agent(), + navigation_skill(), + person_follow_skill(camera_info=_camera_info_static()), + web_input(), + speak_skill(), +) + +__all__ = ["sim_agentic"] diff --git a/dimos/robot/sim/blueprints/agentic/sim_eval.py b/dimos/robot/sim/blueprints/agentic/sim_eval.py new file mode 100644 index 0000000000..226529cd93 --- /dev/null +++ b/dimos/robot/sim/blueprints/agentic/sim_eval.py @@ -0,0 +1,31 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim sequential eval blueprint — full stack with visualization. + +Same as sim_temporal_memory. Use this for single-instance eval runs where +you want rerun + browser open to watch the agent. + +For parallel (headless, multi-instance) evals, use sim-parallel-eval instead. + +Usage: + dimos --simulation run sim-eval +""" + +from dimos.core.blueprints import autoconnect +from dimos.robot.sim.blueprints.agentic.sim_temporal_memory import sim_temporal_memory + +sim_eval = autoconnect(sim_temporal_memory) + +__all__ = ["sim_eval"] diff --git a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py new file mode 100644 index 0000000000..2f7c575c5f --- /dev/null +++ b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim parallel eval blueprint — full temporal memory stack without visualization. + +Same as sim_temporal_memory but replaces sim_basic with a headless variant +that skips rerun and websocket_vis. This avoids port conflicts when running +multiple dimos instances in parallel for eval. + +Each instance gets isolated ChromaDB/visual-memory paths derived from +EVAL_INSTANCE_ID (defaults to "0"). + +Usage: + EVAL_INSTANCE_ID=0 dimos --simulation run sim-parallel-eval + EVAL_INSTANCE_ID=1 dimos --simulation run sim-parallel-eval +""" + +import os +from dimos.core.blueprints import autoconnect +from dimos.core.transport import JpegLcmTransport +from dimos.mapping.costmapper import cost_mapper +from dimos.mapping.voxels import voxel_mapper +from dimos.msgs.sensor_msgs import Image +from dimos.navigation.frontier_exploration import wavefront_frontier_explorer +from dimos.navigation.replanning_a_star.module import replanning_a_star_planner +from dimos.perception.experimental.temporal_memory import temporal_memory +from dimos.perception.spatial_perception import spatial_memory +from dimos.robot.sim.bridge import sim_bridge +from dimos.robot.sim.tf_module import sim_tf +from dimos.utils.data import DIMOS_PROJECT_ROOT + +# Per-instance isolation: separate ChromaDB + visual memory paths +_instance_id = os.environ.get("EVAL_INSTANCE_ID", "0") +_instance_dir = DIMOS_PROJECT_ROOT / "assets" / "output" / "memory" / f"eval_{_instance_id}" +_db_path = str(_instance_dir / "chromadb_data") +_visual_memory_path = str(_instance_dir / "visual_memory.pkl") + +# Headless sim_basic: LCM transports + bridge + TF +# No rerun, websocket_vis, web_input, speak_skill, or agent +# (these bind fixed ports that conflict when running multiple instances) +_transports = autoconnect().transports( + {("color_image", Image): JpegLcmTransport("/color_image", Image)} +) + +sim_parallel_eval = autoconnect( + # sim_basic (headless — no visualization) + _transports, + sim_bridge(), + sim_tf(), + # sim_nav + voxel_mapper(voxel_size=0.1), + cost_mapper(), + replanning_a_star_planner(), + wavefront_frontier_explorer(), + # sim_spatial (isolated DB per instance) + spatial_memory(db_path=_db_path, visual_memory_path=_visual_memory_path, new_memory=True), + # temporal_memory + temporal_memory(), +).global_config(n_workers=8, robot_model="dimsim") + +__all__ = ["sim_parallel_eval"] diff --git a/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py b/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py new file mode 100644 index 0000000000..619d19912f --- /dev/null +++ b/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim temporal memory blueprint — agentic + temporal memory.""" + +from dimos.core.blueprints import autoconnect +from dimos.perception.experimental.temporal_memory import temporal_memory +from dimos.robot.sim.blueprints.agentic.sim_agentic import sim_agentic + +sim_temporal_memory = autoconnect( + sim_agentic, + temporal_memory(), +) + +__all__ = ["sim_temporal_memory"] diff --git a/dimos/robot/sim/blueprints/nav/sim_spatial.py b/dimos/robot/sim/blueprints/nav/sim_spatial.py new file mode 100644 index 0000000000..4b252b77db --- /dev/null +++ b/dimos/robot/sim/blueprints/nav/sim_spatial.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimSim spatial blueprint — nav + spatial memory.""" + +from dimos.core.blueprints import autoconnect +from dimos.perception.spatial_perception import spatial_memory +from dimos.robot.sim.blueprints.nav.sim_nav import sim_nav + +sim_spatial = autoconnect( + sim_nav, + spatial_memory(), +).global_config(n_workers=8) + +__all__ = ["sim_spatial"] diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 10545c1cf0..961bb9f3e4 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -14,7 +14,8 @@ """NativeModule wrapper for the DimSim bridge subprocess. -Launches the DimSim bridge (Deno CLI) as a managed subprocess. The bridge +Launches the DimSim bridge as a managed subprocess. On first run, downloads +a compiled binary from GitHub Releases — no Deno runtime required. The bridge publishes sensor data (odom, lidar, images) directly to LCM — no Python decode/re-encode hop. Python only handles lifecycle and TF (via DimSimTF). @@ -46,11 +47,17 @@ logger = setup_logger() -_DIMSIM_JSR = "jsr:@antim/dimsim" +_GITHUB_REPO = "Antim-Labs/DimSim" +_RELEASES_API = f"https://api.github.com/repos/{_GITHUB_REPO}/releases/latest" + + +def _dimsim_bin() -> Path: + """Path to the dimsim compiled binary.""" + return Path.home() / ".dimsim" / "bin" / "dimsim" def _find_deno() -> str: - """Find the deno binary.""" + """Find the deno binary (only needed for local dev mode).""" return shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") @@ -65,8 +72,8 @@ def _find_local_cli() -> Path | None: class DimSimBridgeConfig(NativeModuleConfig): """Configuration for the DimSim bridge subprocess.""" - # Set to deno binary — resolved in _resolve_paths(). - executable: str = "deno" + # Resolved in _resolve_paths() — compiled binary or deno (local dev). + executable: str = "dimsim" build_command: str | None = None cwd: str | None = None @@ -103,6 +110,17 @@ class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): # Control input (consumers publish cmd_vel to LCM, bridge reads it) cmd_vel: In[Twist] + def start(self) -> None: + """Start the DimSim bridge subprocess. + + Set DIMSIM_CONNECT_ONLY=1 to skip subprocess launch — used when + connecting to a shared dimsim server already started by another instance. + """ + if os.environ.get("DIMSIM_CONNECT_ONLY", "").strip() in ("1", "true"): + logger.info("DIMSIM_CONNECT_ONLY: skipping subprocess, connecting to existing server") + return + super().start() + def _resolve_paths(self) -> None: """Resolve executable and build extra_args. @@ -117,6 +135,11 @@ def _resolve_paths(self) -> None: render = os.environ.get("DIMSIM_RENDER", "cpu").strip() dev_args.extend(["--headless", "--render", render]) + # DIMSIM_CHANNELS=N → multi-page mode (N browser pages for parallel evals) + channels = os.environ.get("DIMSIM_CHANNELS", "").strip() + if channels: + dev_args.extend(["--channels", channels]) + # Allow env var override: DIMSIM_LOCAL=1 dimos run sim-nav if os.environ.get("DIMSIM_LOCAL", "").strip() in ("1", "true"): self.config.local = True @@ -140,69 +163,133 @@ def _resolve_paths(self) -> None: self.config.cwd = None return - dimsim_path = shutil.which("dimsim") or str(Path.home() / ".deno" / "bin" / "dimsim") + dimsim_path = shutil.which("dimsim") or str(_dimsim_bin()) self.config.executable = dimsim_path self.config.extra_args = dev_args self.config.cwd = None def _maybe_build(self) -> None: - """Ensure dimsim CLI, core assets, and scene are latest from S3.""" + """Ensure dimsim binary, core assets, and scene are installed. + + Tries compiled binary from GitHub Releases first. Falls back to + existing dimsim in PATH (e.g. installed via deno) for older releases + that don't ship compiled binaries yet. + """ if self.config.local: return # Local dev — skip install import json + import platform + import stat import subprocess import urllib.request - deno = _find_deno() scene = self.config.scene + dimsim = _dimsim_bin() + dimsim.parent.mkdir(parents=True, exist_ok=True) - # Check installed CLI version against S3 registry - dimsim = shutil.which("dimsim") + # Check installed version (compiled binary or PATH) + dimsim_path = str(dimsim) if dimsim.exists() else shutil.which("dimsim") installed_ver = None - if dimsim: + if dimsim_path: try: result = subprocess.run( - [dimsim, "--version"], - capture_output=True, - text=True, - timeout=5, + [dimsim_path, "--version"], + capture_output=True, text=True, timeout=5, ) installed_ver = result.stdout.strip() if result.returncode == 0 else None except Exception: pass - # Fetch registry version from S3 (tiny JSON, fast) - registry_ver = None + # Fetch latest version from GitHub Releases + latest_ver = None + release_tag = None try: - with urllib.request.urlopen( - "https://dimsim-assets.s3.amazonaws.com/scenes.json", timeout=5 - ) as resp: - registry_ver = json.loads(resp.read()).get("version") + req = urllib.request.Request( + _RELEASES_API, + headers={"Accept": "application/vnd.github.v3+json"}, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read()) + release_tag = data["tag_name"] + latest_ver = release_tag.lstrip("v") except Exception: pass - if not dimsim or installed_ver != registry_ver: - logger.info( - f"Updating dimsim CLI: {installed_ver or 'not installed'}" - f" → {registry_ver or 'latest'}", - ) - subprocess.run( - [deno, "install", "-gAf", "--reload", "--unstable-net", _DIMSIM_JSR], - check=True, - ) - dimsim = shutil.which("dimsim") - if not dimsim: - raise FileNotFoundError("dimsim install failed — not found in PATH") + if not dimsim_path or installed_ver != latest_ver: + # Try downloading compiled binary from GitHub Releases + downloaded = False + if release_tag: + system = platform.system().lower() + machine = platform.machine().lower() + if system == "darwin" and machine in ("arm64", "aarch64"): + binary_name = "dimsim-darwin-arm64" + elif system == "darwin": + binary_name = "dimsim-darwin-x64" + elif system == "linux" and machine in ("x86_64", "amd64"): + binary_name = "dimsim-linux-x64" + else: + binary_name = None + + if binary_name: + url = ( + f"https://github.com/{_GITHUB_REPO}/releases/download" + f"/{release_tag}/{binary_name}" + ) + try: + logger.info(f"Downloading dimsim {latest_ver} for {system}/{machine}...") + urllib.request.urlretrieve(url, str(dimsim)) + dimsim.chmod(dimsim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + # macOS quarantines downloaded binaries — clear all xattrs + if system == "darwin": + subprocess.run( + ["xattr", "-c", str(dimsim)], + capture_output=True, + ) + dimsim_path = str(dimsim) + downloaded = True + logger.info("dimsim binary installed.") + except Exception as exc: + logger.warning(f"Compiled binary not available ({exc}), trying deno fallback...") + + # Fallback: install via deno if compiled binary not available + if not downloaded and not dimsim_path: + deno = shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") + logger.info("Installing dimsim via deno (compiled binary not yet available)...") + subprocess.run( + [deno, "install", "-gAf", "--reload", "--unstable-net", "jsr:@antim/dimsim"], + check=True, + ) + dimsim_path = shutil.which("dimsim") + if not dimsim_path: + dimsim_path = str(Path.home() / ".deno" / "bin" / "dimsim") else: - logger.info(f"dimsim CLI up-to-date (v{installed_ver})") + logger.info(f"dimsim up-to-date (v{installed_ver})") + + if not dimsim_path: + raise FileNotFoundError("dimsim not found — install Deno and retry, or wait for next release with compiled binaries") + + # Symlink to ~/.local/bin so `dimsim` is on PATH for eval authoring + local_bin = Path.home() / ".local" / "bin" + local_bin.mkdir(parents=True, exist_ok=True) + symlink = local_bin / "dimsim" + if dimsim_path: + try: + target = Path(dimsim_path).resolve() + if symlink.is_symlink() and symlink.resolve() != target: + symlink.unlink() + if not symlink.exists(): + symlink.symlink_to(target) + logger.info(f"Symlinked dimsim → {symlink}") + except OSError: + pass # no permission # setup/scene have version-aware caching (only downloads if version changed) logger.info("Checking core assets...") - subprocess.run([dimsim, "setup"], check=True) + subprocess.run([dimsim_path, "setup"], check=True) logger.info(f"Checking scene '{scene}'...") - subprocess.run([dimsim, "scene", "install", scene], check=True) + subprocess.run([dimsim_path, "scene", "install", scene], check=True) def _collect_topics(self) -> dict[str, str]: """Bridge hardcodes LCM channel names — no topic args needed.""" diff --git a/docs/development/simulation.md b/docs/development/simulation.md new file mode 100644 index 0000000000..2d04a8bf6f --- /dev/null +++ b/docs/development/simulation.md @@ -0,0 +1,97 @@ +# Simulation + +DimSim is a browser-based 3D simulator that provides the same sensor and control interface as physical robots. When you run any `sim-*` blueprint, dimos automatically downloads and launches DimSim — no manual setup or external dependencies required. + +## Running + +```bash +# Basic sim — camera feed + visualization +dimos --simulation run sim-basic + +# With navigation stack +dimos --simulation run sim-nav + +# Full agentic stack (navigation + VLM agent + skills) +dimos --simulation run sim-agentic +``` + +This opens a browser tab with the 3D environment. The robot is controlled via the same `cmd_vel` topic as physical hardware. + +## Blueprints + +| Blueprint | What it includes | +|-----------|-----------------| +| `sim-basic` | Sensor bridge + TF + visualization | +| `sim-nav` | + voxel mapper, costmap, A* planner, frontier explorer | +| `sim-spatial` | + spatial memory | +| `sim-agentic` | + VLM agent, navigation/speak/follow skills, web input | +| `sim-eval` | Agentic + temporal memory, for running eval workflows | +| `sim-parallel-eval` | Headless parallel eval (3 concurrent instances) | +| `sim-temporal-memory` | Full agentic stack with temporal memory | + +Each blueprint builds on the previous one. Pick the one that matches the modules you need. + +## Sensor Topics + +DimSim publishes the same topics as physical robots: + +| Topic | Message Type | Notes | +|-------|-------------|-------| +| `/odom` | `PoseStamped` | Robot pose in world frame | +| `/color_image` | `Image` | JPEG, 960x432 | +| `/depth_image` | `Image` | Depth buffer | +| `/lidar` | `PointCloud2` | Simulated 2D LiDAR | +| `/pointcloud` | `PointCloud2` | 3D point cloud | +| `/camera_info` | `CameraInfo` | Camera intrinsics (1 Hz) | +| `/cmd_vel` | `Twist` | Velocity commands (input) | + +Transform tree: `world -> base_link -> camera_link -> camera_optical`, plus `base_link -> lidar_link`. + +## Headless Mode + +To run without a browser window (for CI or automated testing): + +```bash +DIMSIM_HEADLESS=1 dimos --simulation run sim-basic +``` + +Add `DIMSIM_RENDER=gpu` on macOS for faster rendering (uses Metal). Default is `cpu` (SwiftShader, works without GPU). + +## Running Evals + +Evals test navigation accuracy by sending the agent to target locations and scoring distance. + +### Sequential + +```bash +pytest dimos/e2e_tests/test_dimsim_eval.py -v -s -m slow +``` + +### Parallel (3 concurrent workflows) + +```bash +pytest dimos/e2e_tests/test_dimsim_eval_parallel.py -v -s -m slow +``` + +### Single eval + +```bash +pytest dimos/e2e_tests/test_dimsim_eval.py::TestSimEvalSequential::test_go_to_tv -v -s -m slow +``` + +### With log capture + +```bash +DIMSIM_EVAL_LOG_DIR=./logs pytest dimos/e2e_tests/test_dimsim_eval.py -v -s -m slow +``` + +## Environment Variables + +| Variable | Purpose | Default | +|----------|---------|---------| +| `DIMSIM_HEADLESS` | Run headless (no browser window) | unset (browser opens) | +| `DIMSIM_RENDER` | Headless rendering: `gpu` or `cpu` | `cpu` | +| `DIMSIM_CHANNELS` | Number of parallel browser pages | unset | +| `DIMSIM_CONNECT_ONLY` | Skip DimSim launch, connect to existing server | unset | +| `EVAL_INSTANCE_ID` | Isolate memory paths per parallel instance | `0` | +| `DIMSIM_EVAL_LOG_DIR` | Directory for subprocess logs (eval tests) | unset | From 6131832580d7466a1f8ffbfa88aabb947fa70972 Mon Sep 17 00:00:00 2001 From: Viswa4599 <39920874+Viswa4599@users.noreply.github.com> Date: Sat, 28 Mar 2026 01:59:36 +0000 Subject: [PATCH 10/19] CI code cleanup --- assets/temporal_memory/entities.json | 2 +- assets/temporal_memory/state.json | 2 +- dimos/e2e_tests/test_dimsim_eval.py | 12 +++++----- dimos/e2e_tests/test_dimsim_eval_parallel.py | 22 ++++++++++--------- dimos/e2e_tests/test_dimsim_nav.py | 14 +++++++----- .../blueprints/agentic/sim_parallel_eval.py | 1 + dimos/robot/sim/bridge.py | 16 ++++++++++---- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/assets/temporal_memory/entities.json b/assets/temporal_memory/entities.json index 0637a088a0..fe51488c70 100644 --- a/assets/temporal_memory/entities.json +++ b/assets/temporal_memory/entities.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/assets/temporal_memory/state.json b/assets/temporal_memory/state.json index 70deefe154..ff9b6ccb03 100644 --- a/assets/temporal_memory/state.json +++ b/assets/temporal_memory/state.json @@ -4,4 +4,4 @@ "chunk_buffer": [], "next_summary_at_s": 10.0, "last_present": [] -} \ No newline at end of file +} diff --git a/dimos/e2e_tests/test_dimsim_eval.py b/dimos/e2e_tests/test_dimsim_eval.py index cc99428252..dbea3cb28d 100644 --- a/dimos/e2e_tests/test_dimsim_eval.py +++ b/dimos/e2e_tests/test_dimsim_eval.py @@ -23,11 +23,11 @@ import json import os +from pathlib import Path import signal import socket import subprocess import time -from pathlib import Path import pytest import websocket @@ -44,7 +44,9 @@ def _force_kill_port(port: int) -> None: try: result = subprocess.run( ["lsof", "-ti", f":{port}"], - capture_output=True, text=True, timeout=5, + capture_output=True, + text=True, + timeout=5, ) pids = result.stdout.strip().split() for pid in pids: @@ -117,7 +119,7 @@ def wait_for_harness(self, timeout: float = 60) -> bool: def run_workflow(self, workflow: dict) -> dict: """Send loadEnv + startWorkflow, wait for workflowComplete.""" - timeout = (workflow.get("timeoutSec", 120) + 30) + timeout = workflow.get("timeoutSec", 120) + 30 self._send({"type": "loadEnv", "scene": workflow.get("environment", "apt")}) self._wait_for("envReady", timeout=30) self._send({"type": "startWorkflow", "workflow": workflow}) @@ -136,9 +138,7 @@ def _load_workflow(env: str, name: str) -> dict: def sim_eval(): """Start dimos sim-eval headless, tear down after.""" _force_kill_port(PORT) - assert _wait_for_port_free(PORT, timeout=10), ( - f"Port {PORT} still in use after force-kill" - ) + assert _wait_for_port_free(PORT, timeout=10), f"Port {PORT} still in use after force-kill" log_dir = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") if log_dir: log_path = Path(log_dir) diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py index 3c204fb95a..76c65d6dd9 100644 --- a/dimos/e2e_tests/test_dimsim_eval_parallel.py +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -20,14 +20,14 @@ pytest test_sim_eval.py -v -s -m slow """ +from concurrent.futures import ThreadPoolExecutor, as_completed import json import os +from pathlib import Path import signal import socket import subprocess import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path import pytest import websocket @@ -45,7 +45,9 @@ def _force_kill_port(port: int) -> None: try: result = subprocess.run( ["lsof", "-ti", f":{port}"], - capture_output=True, text=True, timeout=5, + capture_output=True, + text=True, + timeout=5, ) pids = result.stdout.strip().split() for pid in pids: @@ -87,7 +89,9 @@ def _wait_for_all_pages(log_path: Path, num_pages: int, timeout: float = 90) -> except FileNotFoundError: pass time.sleep(2) - print(f" WARNING: page-{num_pages - 1} sensor data not seen after {timeout}s, proceeding anyway") + print( + f" WARNING: page-{num_pages - 1} sensor data not seen after {timeout}s, proceeding anyway" + ) def _wait_for_port_free(port: int, timeout: float = 30) -> bool: @@ -147,7 +151,7 @@ def wait_for_harness(self, timeout: float = 60) -> bool: def run_workflow(self, workflow: dict) -> dict: """Send loadEnv + startWorkflow, wait for workflowComplete.""" - timeout = (workflow.get("timeoutSec", 120) + 30) + timeout = workflow.get("timeoutSec", 120) + 30 self._send({"type": "loadEnv", "scene": workflow.get("environment", "apt")}) self._wait_for("envReady", timeout=30) self._send({"type": "startWorkflow", "workflow": workflow}) @@ -214,9 +218,7 @@ def _wait_for(self, msg_type: str, timeout: float = 120) -> dict: def parallel_env(): """Start dimos-0 (launches dimsim with 3 channels), then dimos-1/2 (connect-only).""" _force_kill_port(PORT) - assert _wait_for_port_free(PORT, timeout=10), ( - f"Port {PORT} still in use after force-kill" - ) + assert _wait_for_port_free(PORT, timeout=10), f"Port {PORT} still in use after force-kill" calls: list[DimosCliCall] = [] log_files: list = [] @@ -372,6 +374,6 @@ def run_one(idx: int) -> tuple[str, dict]: print(f" {name}: ERROR — {err}") failures.append(f"{name}: ERROR — {err}") - assert not failures, ( - f"{len(failures)}/{NUM_CHANNELS} evals failed:\n" + "\n".join(f" - {f}" for f in failures) + assert not failures, f"{len(failures)}/{NUM_CHANNELS} evals failed:\n" + "\n".join( + f" - {f}" for f in failures ) diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index 141b41aac7..b26b47f3a8 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -22,11 +22,11 @@ import math import os +from pathlib import Path import signal import socket import subprocess import time -from pathlib import Path import pytest @@ -43,7 +43,9 @@ def _force_kill_port(port: int) -> None: try: result = subprocess.run( ["lsof", "-ti", f":{port}"], - capture_output=True, text=True, timeout=5, + capture_output=True, + text=True, + timeout=5, ) pids = result.stdout.strip().split() for pid in pids: @@ -108,9 +110,7 @@ def sim_nav(): ) try: - assert _wait_for_port(BRIDGE_PORT, timeout=120), ( - f"Bridge not ready on port {BRIDGE_PORT}" - ) + assert _wait_for_port(BRIDGE_PORT, timeout=120), f"Bridge not ready on port {BRIDGE_PORT}" yield call finally: call.stop() @@ -202,7 +202,9 @@ def test_cmd_vel_moves_robot(self, spy: LcmSpy) -> None: assert len(initial_msgs) > 0, "No initial odom" initial_pose = PoseStamped.lcm_decode(initial_msgs[-1]) initial_pos = initial_pose.position - print(f" initial position: ({initial_pos.x:.2f}, {initial_pos.y:.2f}, {initial_pos.z:.2f})") + print( + f" initial position: ({initial_pos.x:.2f}, {initial_pos.y:.2f}, {initial_pos.z:.2f})" + ) # Send forward velocity for 3 seconds twist = Twist( diff --git a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py index 2f7c575c5f..f148e8ada6 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py +++ b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py @@ -27,6 +27,7 @@ """ import os + from dimos.core.blueprints import autoconnect from dimos.core.transport import JpegLcmTransport from dimos.mapping.costmapper import cost_mapper diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 961bb9f3e4..2d1329b08f 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -195,7 +195,9 @@ def _maybe_build(self) -> None: try: result = subprocess.run( [dimsim_path, "--version"], - capture_output=True, text=True, timeout=5, + capture_output=True, + text=True, + timeout=5, ) installed_ver = result.stdout.strip() if result.returncode == 0 else None except Exception: @@ -239,7 +241,9 @@ def _maybe_build(self) -> None: try: logger.info(f"Downloading dimsim {latest_ver} for {system}/{machine}...") urllib.request.urlretrieve(url, str(dimsim)) - dimsim.chmod(dimsim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + dimsim.chmod( + dimsim.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH + ) # macOS quarantines downloaded binaries — clear all xattrs if system == "darwin": subprocess.run( @@ -250,7 +254,9 @@ def _maybe_build(self) -> None: downloaded = True logger.info("dimsim binary installed.") except Exception as exc: - logger.warning(f"Compiled binary not available ({exc}), trying deno fallback...") + logger.warning( + f"Compiled binary not available ({exc}), trying deno fallback..." + ) # Fallback: install via deno if compiled binary not available if not downloaded and not dimsim_path: @@ -267,7 +273,9 @@ def _maybe_build(self) -> None: logger.info(f"dimsim up-to-date (v{installed_ver})") if not dimsim_path: - raise FileNotFoundError("dimsim not found — install Deno and retry, or wait for next release with compiled binaries") + raise FileNotFoundError( + "dimsim not found — install Deno and retry, or wait for next release with compiled binaries" + ) # Symlink to ~/.local/bin so `dimsim` is on PATH for eval authoring local_bin = Path.home() / ".local" / "bin" From 3bd9c2ca6303c38645553be005a1bd3bf4047689 Mon Sep 17 00:00:00 2001 From: Viswajit <39920874+Viswa4599@users.noreply.github.com> Date: Fri, 27 Mar 2026 22:07:30 -0400 Subject: [PATCH 11/19] Update dimos/e2e_tests/test_dimsim_eval_parallel.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- dimos/e2e_tests/test_dimsim_eval_parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py index 76c65d6dd9..ec98445679 100644 --- a/dimos/e2e_tests/test_dimsim_eval_parallel.py +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -17,7 +17,7 @@ 3 dimos instances + 3 headless browser pages, 1 eval workflow each. Runs all workflows concurrently, cutting wall-clock time to ~1 min. - pytest test_sim_eval.py -v -s -m slow + pytest test_dimsim_eval_parallel.py -v -s -m slow """ from concurrent.futures import ThreadPoolExecutor, as_completed From 71d571faa696f841d113a48a337f948545135c4f Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 19:29:39 -0700 Subject: [PATCH 12/19] Fix PR review issues: viewer config bug, missing @rpc, process teardown, untrack generated artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix global_config.viewer_backend → global_config.viewer (AttributeError) - Add @rpc decorator to DimSimBridge.start() override - Update self.config.executable after binary download in _maybe_build() - Add start_new_session=True to all test subprocess.Popen calls - Implement duration-based motion in DimSimTF.move() - Fix docstring filenames to match actual test file names - Untrack assets/temporal_memory/ and add to .gitignore --- .gitignore | 1 + assets/temporal_memory/entities.json | 1 - assets/temporal_memory/entity_graph.db | Bin 86016 -> 0 bytes assets/temporal_memory/evidence.jsonl | 58 ------------------ assets/temporal_memory/frames_index.jsonl | 50 --------------- assets/temporal_memory/state.json | 7 --- dimos/e2e_tests/test_dimsim_eval.py | 5 +- dimos/e2e_tests/test_dimsim_eval_parallel.py | 4 +- dimos/e2e_tests/test_dimsim_nav.py | 3 +- dimos/robot/sim/blueprints/basic/sim_basic.py | 2 +- dimos/robot/sim/bridge.py | 3 + dimos/robot/sim/tf_module.py | 16 ++++- 12 files changed, 27 insertions(+), 123 deletions(-) delete mode 100644 assets/temporal_memory/entities.json delete mode 100644 assets/temporal_memory/entity_graph.db delete mode 100644 assets/temporal_memory/evidence.jsonl delete mode 100644 assets/temporal_memory/frames_index.jsonl delete mode 100644 assets/temporal_memory/state.json diff --git a/.gitignore b/.gitignore index 4045db012e..d4c7adc989 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ __pycache__ /assets/saved_maps/ /assets/model-cache/ /assets/agent/memory.txt +/assets/temporal_memory/ .bash_history diff --git a/assets/temporal_memory/entities.json b/assets/temporal_memory/entities.json deleted file mode 100644 index fe51488c70..0000000000 --- a/assets/temporal_memory/entities.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/assets/temporal_memory/entity_graph.db b/assets/temporal_memory/entity_graph.db deleted file mode 100644 index d0a6956bfc70fc3ed5dca5ac770ce70f517f102b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 86016 zcmeHw4Rl;bm1ehEQmg;ovn1jY{!uk8)K9w=}GFi z)va{5ttgq0G!r0j!Z|Sf4dDPg*_kjvCj87Y8Q9^FSqMy6es?#}6ovt$>m`Pu2} z>ayb!Wv5_Q^QB^?{qU*Wp{blTdHhK3jMb39I&s`;OteQNXPL%dVe&%bjfF1T`Eq0W z4eJ+*3DzmpfofD&r|s$d+@M9o3=r{GZ8U;i$)x|aC#v>LVU0>QN??)%Q-`iSmRqBS z)}F9c_b)#OAWh|t=1y5BPfgx<=+s&3hTK`}(CMiYlgA;u8*|5}1{+d&kg}$7XQr&< zC*Z%+$Bs2JD>eKTS+V|?W1TG(=ks$;an`Xw*T<|Qx$6#{J~m|~6T^)n6db!;bmpe# z%cUijaU;t-sI)p=nyEPDOTbyJ@NCzwR>A)D_<7dMKBs3(7mHOFcvtjBsjfS5DmQub zIMc}<1;{u^cU|sO?)c%{&D?78PDL<)@%(BzF+00VP9ELIZ;Q2dC42n$x@}@EU#Z%l zgvxX^zvQ@-(a;v$KQTsZ4DK&`FezlSv^-sj7o%YHJ^otv8bx3;a#O{grwGyBTxJ1H zV>7#dcO=-=)8l_&J8Ncc3il;SXf-LHyVY@ zHEUO$g;IHSoyPHNH8w??muo{H++vA0^+w@-=WIV*XZxZTjU>>7neyjDdj7rmvlJx?jb>n(p#m!$U6!!tA?q;9T#y}sCdaB z6+>gA_m6vu_L4IQDdYF=XyM&KgyrdC;l8wb@^OFuSL+T$j!fZ~YvmG(%W=B8xg6ua zRa18^(P)jcg~rBSELEM#I=zplBal`84aYzik;~=d9RU}cgI(j>{k5RmMp>^6!|ybk z5mX$fD1K>aFm3S{$&0ove%<5d0P~Odg+Q?DDkv$^SQ4MCs6{oeUzB9_lqG-0!d{v5 z2fMa!_uoIl>|RJDo=qOB7r(Ay^z2Wo?)*)eQ#R_HcyS7UIjfv=DzoMMa+#GpF^ttHgj3J{bx)ePWHDaHNd^7gu z$i>J+BoO{`_)o*D;iF+A^wrSAp?oOb_FUT&ZLj(rU&3keGy)m{jetf#Bk+PkV5%k9 zzP%C*`f?+M5-gKpd+}1iEtFyQ2miM#70WKpS+ltD zWK}M#P8>eDD+TqQh`(~cf+tJ>uIygLTr+$gx^A9(n}hA${R-z?zL+mASZtc+W_#n8 ze(y2Jc26rX_aJkYXS-amXJMS;Vp8^OJ%#b z;8+#d`mIb{2%UchaG1Dr>hYc@e7@0MNYCp?IM3PT3)W1z1od1lRjbaN1rQahZbHgn zbf=pz%?k!0%l0fJv7a|#(gIZXGKo_HRc;k7wi64JH{Ojx(sQ)OEi3u_Wr>7TyWt&> zaEO&P%oCQeK5Dh^4f#7$wXi z5k331K#83!5!nLlbT|_;PZbZM#gSYGW59I-awSRe*u+=tRSTLjwgszNvM%JSvx{Jb z*zhOLjC}IbsLkmQKlBrLM&IIV?LKkkX2@nID=fv46UJUBOq~B`#yN4@)453~I=KmW zd2L9IyxLe2)T#pO=84xBQzx+y(&v>OBKsuik?klujfxmeI@+7|4t-cN2qfBHbabFYkC0&xPvNq@TBa@pR_$f%_+iijv7qPQ|KDS&dElr$` z-xYzJCvGfE?185lff{s!D5(G>O9%JjgPE^TB{u$9dnR9Ws%}N7>r8Y3Ox)Jf!g^Jk zsyCFYJAwZYgC{BYMJKK_c%eX{^K$bi^Y{6D@pb@%wUs3GpiB8#=#}hCPQ70GHUKNv zAGMTOyIh@cmhNK%3ZXj|1+D{34Pfh_eujetf#BcKt`2xtT}0vZ90fJQ(gpb^jr{FV?120Hw(pFF^y zqy7Au?c>jpUHqBp<=pbu(#@ZlE&Q48 z;?LA({!DiA=kO->OpbT(=h#O69Bt>%><0cEG5IrN@Mk*4pQ$K+CL{bg9BvJ?`*Y(V z_84npkI^7|WLw!|q=h{)&Fqm5ut%z?CD0zqCHeZl!)N}B`A_Co%+H%on160QV!qvc z(7YGc|BiXayurN2_>u7~v6q7q?7(3yacO0=m&kP&o>l zRH9iW0xHoI2?RspAb`96=d1pIf_4AD&^|!fyv9tj^*>+Nf6KfV*8jhNH9xNYaozs` zSo1ruH!#HZ0`{{Vf`<+3H#0w-S|gwl&wmhC!q@+FABwO4>H5#Fwf?6xeEm-ZU;oo?0$=|V!PoyJ0bl=g0KIH;Ol?7C&SnOA`KXo-~2h6 zR)_iX*h2L2$g`2D@MGbg(C@c>uWf(u(bjLb9%*^9rL+0|z^4Mqri%Xq{!PAfzk)5+ zvRfj)&W?^?u$HVv@D^V#H3las4*u^n`zJ1Ko$I}}1Q!;CYXZDTVJ(-z%NY2j$`|3* z2HdEd1y2@({5lrW{LY;{cRO4u1TMIpguk-ubCpEz`6(pIr91OW;LpseTwGWHpH3Cz zVb?UQWw^D1wCK;*C#7Z_2P1(wxuiN+lhkTyUOmMDb0pu(!qI=f{jAQxABl*$fT zvv4=AiudJgxDQ%{3eQ^=ycbGZ5ZM><7aYiz)s7hp*MY18UVlr}Hf%!a+rg^4PUA&9 zaJf`M>BkQW+;4spyOngV_c{a$t}Ma5K`5JFK&(p8*k#=rr@DeSbrp%3X7E;7oq(Y4 zB2=BrR$VDspb&5_l&Br(i1@a*qe%N|T@ulp#M2dN5-zM_YvZ*XWW}=kyZn+cyHUhj zV0{|_h{4j}Z|ssQTDuTWxdm+HD;74%vRxPy4UZ!~4>y(za3d71&7lI=?KyiEjVDnv zHimrt?adbBSGuJlA^*-7}M2 z4_>@x$q)l)ejc=kU7Df+-dW*IGlXur#M=|xon^P4(I}u}xWJT?FBC2=<%>*y{KHDA ze1Xe*&xVk%8^v|N9!uq7C6|J}YSitkXp5Ue2YO{GnU>wgWY3f?;SD+LTjxNrhoFaC z&O2aU&@n^j0=^BoFthBIIlEjfxNAGiurH3~uh#sc!<=!T@0$ZD04)g>zdOnWfr7Bk z)I-j|1#G&s&HYZHI9!*A4%r%|nZHS}!`ilAbzv$YnIpeA#fg96k0iwFPK z^7*(pblbJg&PjVvTZhzNg}bLhqy={O8hdCm0Lla~4w<^%tn&XzhqtLUiI1c|BnfjmCZ)5mg)kLj39QO5ew1)u5JD?So8J2K|IhUGng_;6+@RB%Gys`+5e1**v zil8}6!3;&Yhw8Qo2gZy!Ibc&Jbt-_SG7BC=S;jCr^0A2JmZ(KTWO6om(5p;y+(uN~ z=vx@qF^XuR8%_|cetO2xBWxHS_Kp%#)+s7eCNZQd)k@TMwTZ#E<6dcU85rq=$+_a3 zlv7Bhzoki}s#<7{=$c~ATl3!YCl&7nP#c_;%I?|iI90f~W)rn-L68#1<9pPG0YZuy z!tABTUjZ{{j#B4xeQ=+KIXTRZ- zpSiD2zCF2Q+~QL(>RT0|tUb9@9I#!?gD(u|Uxf11`50@`=A-bk@rDr%$ku+C@ z5xOPp+l5nxT6{$q8cxKZNvGX$@7$MnjkqwAvl2Gx+LOz~QA|0=HhNaT9ze_@C{Z#U zuW#A_J+m(4fLGExERv6v)I&}d=fRB$v^sZVuhA_v>%-Voa;rQNNx~Wg~E#;uBU98%t8_}7^VTP_PWZeIk5#2s{x>m z1*)k*Gnsgzw$UHr-NMQ}o>tc&uuvS8hs zND*wOGKc5?cbY-;|Mx8T^+OMSWpkI=X&Ppb`ThG-<^k|;%nyQRfWI{VgZT#Y56%0` zZRP{u3-DU-26#Pq1w`L}t1nP^wAdN}jetf#BcKt`2xtT}0vZ90fJQ(g@b3u$<`XZu zlRqr}=;4m?c5sUP|7FSlUxxetOOKNOzbyIx8{_`}lEd8pUvilH|4XL1|G#8<6VI-L zJu>9|Z=8GoOAV9vzfto3H%{LFvgG|QN#6f5SRTX z^Ywp<-2V^L20)hF|BuoJz&LFHWN8CnG{*M}X#Jm!aCvF@pXJ{FQ+)NG;;a8uE8i1H zxA13*`~Oez^?!!1|5NPyzp>Y6e$?D+eA@V9l#>@`cqW9FUaCZlQwj9DZ1Le;#MTO*(m&m$4y!86i9u9Q-xqGDz8v}7yVD9M1RI@Z9fvpTGaL@zJ|72NF zQGrMdPT{-0Uc$&v-c`WypX3L}f06*lf0AfLkQB!kEnY;x`=8_uc>f0e|F*mSdieT( z%EvtTKY&pHF{U4nPp+}K|9>L5|9|p0&HewAmuK$( zpL_*#|Nlh54gkbWB!c_@CvUvm|3CS{<^KQ4uPgWePXzb>PyS1}|9>L5|9^5C%KiV7 zhfePQpFA{j|NlgA|NlgA|NrC=k^BEAq6P3K|Noul&*2OI)n*jF@IPwQjGK&Iv0ubK z7kg){5}S5;A?`n2K!r|YyE8NyIV`G2U;U7-)MQP<(`(~ zExVhaYkso%q2}A0$C}#$&jdaaczs|dus;xP`exI|nqJp5)0Aoo`@iCUpZ}Hqv;L&t z?|b^PjcDLZ$k)*k>I?=uFzi4&iTg&bv)}wjbCdV^e3uGKiCd8`=<*F8{3*}d!@qd; z{4?+~@DJRA{C@B}3i+mUsnPH}j@<4>zk7lYEJjGG727?FCHK zPx23(Kz^=>G~}D|04w+5R+-BEirS5j&_$1^(e9 zjGxsCL`{xPH1j4v?aabt<{N9cVZ^*2K?@$Ah{PAn0@ECeyv|f10nmD9n4G!nm zZW@w*ctQ~#vd*p}JcW6I>h>Rm6zzmKG$tmq&^3({z&3OZvJr}4!=5UNCMwzY9boJn z#Nk|K9Wi8r;j0zF(ee8_DQK_)iQ-pb3O_Gm6ygHZqeuf5s&v06^OW1L>r|?w7*|rD zqyIW_1VA!HVvMC=ytNXY^3h#E2pV zux3PC^mGl9e>lVVU1J|vr=BuJBCV7F%_HNn1lL=?lqlupPp^rK%)dVg{Ox=cN+xq7 zt=Lgei=CjE22`|v*u$3b^e2*SXdkkX*(PNq69$JcTgl#^@Z>ti1`2_)WZO5ya^}mIbDgap+2=R>QfqI|a$!7gtLk zWl*`5a!XI_L9QT`zFzy2xh8f?{;}M84P++#-jmP)g2sjhJVIwZvk{>R$v)W6vS;ln z3yOP|2ea_lvBKS_a*uiP7FCtpeY=pG_j%de=sJ1!^-8Wxjt{QFq4jNx|M=IuT8W*k zWCmM0w_f{^{JSmW=iOF1m#J5>WE|{a8S`ltbhuc(L#_OQ`p8+*d- zz}CT)3}G179Uo+hp>2xVFdV^Ja!B@ptylmxUyMTGX+n}Oeg)&>O$hkhK22D=WbN-p z2`EdLJ23kdJp!#3plX9#n4oNy2SJNc6B|K6@Xs^R;NJg%dp88a5^6~0+|q@u$ZQjv zrG8Tl16d<~e<$#Z84S3s_ZXMVcHbssBSnB`|GIuxpg^k}tX>C`gPFrfF6&W)uu93@ z2XTjlg=cbEk2NSStJat-sE$wB6~9%?gW zFlF)4GMRJ3ym$na?E7Mjor^G>tM8nI7G#3HsI32JZrq~;$u$^}n2xT~jY{rB7=#cL zNtj!Wh{zlsq0fwb^3z~-P{()(`3X1RciXxUSF$GByjgpvSHc(+{-BrN4TrUkD2Y}t z|Jo}MnSWo4gb&6yPp*eg?n?0r_BE?qyn_@Sx#Zd%KrVuzvBOgsn^eYhea|Qv`ykqY zFk6`Wd(4)&e4EbT_y4)b+k9pZ`11Qd<`2wonNOQvG(TxRZoVJR{J$xdiw#A89D7-8 z!MNS58i!&x8UEN$j9xQubQ#gu_hMg-?i;c(L8hb@76no705cn31n3K^}*iZOKbTWD%YDZrcJstUIgazO0gZr0KqH_L&P|VX@=Did=1H^ zM`r0AHkL}mHsot$#tQ)4&tclI&r6#Wi$8NSKoNe`bS9{4?VLcS-D z6AymqxzXiMjU1zQ{9tnfz2nE2>*afPlE1SrDlegT{AzNPc<{@}b@V%a`pCKb$s2QxR&1WYr`S=dv+q+47&h1*pp2kq*tiQHSTNX05Rk4{nc(Jso|^K-&4c; z-QSbrXt@iBVGjqp)=$DQu1e9w%~J8U|SWaN7$E#K2ASICj1`4eXQ$hKjY~{JYW<-m48)+7AnSgjpH?=|k5`ucM`~P#{ zANcU(Kb`b1!^wVjsz3aL=y#%Djef!WbMq1K2l$|QFRal2Q}jL2hoY~K-iK%Y`8|PX z#VF#f0plpYF@SdjqVv(S#(r3vTSkY`8a)<07)?d{qg#v*7!Mn7F&;4Pjz%KCjQlwA z50Sr%d^Ym&$Oj|;G4j^PYa_oGDMe-?rz04Nu1Y zGWP!1yJByQ-5*+q)H5{#6Fcs zsKgK__!*@^mDsBiSE@u@CH63aPX6pxi2;@9SBXBA*u@AsXVa?^{QN)tg`fW?f}j7V z11$XfKN0-=KN0-=KN0-=KM_~3a_F!KKmShzKmShzKmShzKmShzKmSh$Gx+&`BKY}# zBKY}#BKY}#BKY}#I{U!S{}aK_{}aK_{}aK_{}aK_|I>*Ae*T{be*T{be*T{be*T{b ze*T|6=kxRbMDX+fMDX+fMDX+fM8v@AeH-w9`kzKXBcKt`2xtT}0vZ90fJQ(gpb^jr zXaqC@FIEKb{J&oRzgV@a>!lIU2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc=gUjJ)L&6ONJpaGNcg|;CF#f{u#TKHEN1lyLg&zy|gnqy6 zdu{uJkG6ih^+?N;EuGEx2R;=@HdXu|@Na?ym;YDGZi)Ch!53t(maIk4J0d#bPmLcG z-oz94vSTfmoeKP5<%?Ez(Xm)?$w6x+4}3_^IK|mTON2kP zswMX8a=|WED~aCoQ@{cN9Pi98<*Tb!<>JDEQ>j*vXO-t}m#bE3o@Z?@IacXnb*@w@ zTNS5zaXC@j0&Y)tqTu(g7D}@=k5ZXKC*-Mvt{2gb5B&r%5W*LnC6Jd3e<@$d&lDUc zE>++{zB;?;6fFu;NiweFi*uzF6!lVm4r^YtoI9&dxo8)xOL=F-nlG1@n3R-0Q9HCb z?AymMJ$QFR#7h!Uc$cIw7bGI$Cd2@UZ76My zO3O~MV$J2t5ID53Dzsj(SBkF4)x~n@;sQ4kl3~!Ah1yp0#Rbc@3i%>vd)}(#7Z$6e z1(AIrf5CxlS?!pya71Jk5T!(I!zPrz9jvEUkcU zty$nMTU7}f)545XU2(uNJn8|>luMHcbVZshS~J+%cr6E6u~i$b70qFG(ax8t1=hC_u!@VIe7g)4SYXyJ#LE{E zT%Jq5V$DKBDBFcW(eOC(^U#nA)^aHitr`>n52tL;qVXhZ2KeFa2S2~TyBE|jvN>?v zGVE4?uJxVkRZT)R9j+|MmiGn10C|hXE&(J1=}qg%d(wu=0LHBpod(}J78YWF)!N1S^md4XBJww4ULRC59F@xFvGq$ zmcLr73yTXmBY6aYq*!O_A!qE_3ky8ovQvQeisEoxB06MilxF^7nW+mCi3}JN z(7`S(L%V{WO-WljVn737Jsb5)MPgy!R#uIc^H>WAFARo= znilPxA`Wp)3u{6dX|!XR0qCW29t37Et=Q!`-lcHOA7S>kybSgS?F3{gqMopeg#n4O zVOk%GBBaUihNDBqK0$n}~ z0Av7^31G;p0EPm?k2}QMRi+5|NzH|&uWgQmX*$yFu-Qlw9YeX$Q4}U+v;?gpL~L5Y zS|Ck3lp&Ig7p)GPE+0fmRM`zk@xVUsVsj9i=@(`>)kLj39QO5qTY3QU4mTz=cz*)# z?52MVXUa6=<%|P~8EWGO1GmG?iJqQe_## z=*Y(+mRq704Ux&&wE#t^M#&W1=r%6bF^Xtas?hI&?abW{ohfwp)D#(dgbm}v-cdrz zIz^n}k?dj?qEstU+tnrp-;R4_J4T25q8Yp5oRm{YrN5;~q^epl#{vjC=dF3~`ICzJ zX-t&ev)gg1aB=VxwQWI=6363v)P?~B4xeQ=+KIM;D|$^6kkb<1_^$g(^Z>dvd8bV7r)yc{wz2p*(dy#@aIJ zUztUSK4o3$84gq=%~fH9ZVCH#;gq2kUlE2jT`l31A?=QPbWNLAV~$T#*raPuE)z#F z9AI`IX6KjdfXH`Pp3eFUjHo|66c^D{gwg@^%rs4qbVVr2fOjalX1cNcrUGX3< zb|ZYPlMV=D*sBhWiZOV^A}ujvfZDpvVBVf(xIquoXclCVe5_<^fZp?y#d)WU1GT%x zVRb0jtDs{xXBPSKB0*wa7@IMsnj!!vAA8yrJaO<5D>RC5cdi%57m^!9R-y3XhwCYu zCbN))42EfdtG%xBYEEo{#A*O&vte7E6{G$VwT_jp=egTO*@R30)T7zm6y ztTCW*8fQ0(=T0VU&%j-;n*^@^L!qDh%>BkYV*f9;9R080m2W2eA^1uE(+Fq;Gy)m{ zjetf#BcKs@aUoDk?-W!2?pn8JMk>~+6vX>n@1$qbl5ZOq?DB#-vBvFr*z|?nZZ^>` zx?fdj#*bSVG^2Hw4XjjTtlKT7t!~`gsbBdeELvRG4lJ!HGzA-PT)eua&gCnxj*06x?(j)6rD1x{}$}K za8Fq74vNBP*S>eV0D5bw4(JgVXo@k<>%&%UA{Q)2Q8kn;za)!gd^P8n|VN?Q*^wQ|sOe`x0zRxpP*mqhVJW z0$``dT+Cc);CDob<>IZCAh^50focQa*m<(h$d3Zt_Us(xWJx*!7031-nqfy&Nb~*`0)DiQ3`rh_8!@x2u+frr^M57f_;<%>BNNm6ki%?uOSQ zZWy}=Njzagm-mY(`2y^y7A;mH7jlh_KQ|9|>kmFB02Hv>&$sWPL5UA;Z2O2|!t%uR z{~6!YKJfkjA@c!q#=P2$8ecOWHEPC9#;({eVxNn>GggUB#&$uxzAr}K6KRxg@!}nwy(B*xb4BVb8VTnmf+We zj|X29yfxV0`dsU0Ti@MUYCX^zY57LWV=ecz9B>J^S1q`fiH$sa4k zpG@2;gFYgk8}%CZ2Hjqee6a%mz%9s6(LgcsNmvb?D*1MwW_&y_E%0TVScYPoS~3ny zDS|>c7q6g_FLARXC}g;HJY$9bl;me2S3JQ`#RyM6VzFX3F@d3H>cG#3>0uU=_8T1M z3CWi@Df7kfGhV<%{Urau3FPOBNMl$R53rIiaa`t$d}~MFl>7rXD)M6I$rK^VC)c%D zuVb=&kT=C6aEjqg@+EGNQt z6y%~AYMK~PqyW~8s2kIhFkQ(%oMHT~v9A}uT&75*l@g$NWIUG8SVAJ@K!yNRLd$M;F5}RZO06J1)taUz z^X-ePrH^{zB}=Zv9^~Q?$ukWhwi|^=?3Vlx1*;ANVVdN68)`Q+;1N1|@Ta_E^r%9z z5B9U{S$l$bS_hv^vwt2tR=E3A?lDi^jiU7JLT*0FWOJkICc3Fh1UdV9a#; zv_=W~yHNtl5+a*1`xQL`eb0xg4Q^qAvRNJ+T1-uB2>r#U#PKq9ZU{p4iKNsqXuD> zlDiMi9SRGF7;YYGP`KKWi)Q=~Sq-5QHIiI=;0zw%&s7D18|(Weh0|0xQ@MtD4f%$L z+6);?S$wpF$Z?)wN=zpt`@R@s=ORFKxjM9j%rZe=RMsD2(0P;~xdtN=)6sRhQOTVM zgAigO33ICvft3eca$gU1jE9h)a0C8~08WT2SrcvEti97K(Igc9pqJl04!e#hiB>QF z+A9#5e_xA)55_l7uH;@HD%jVoa`6sQ=uUF&4j>o7(AeQAj7=(ITA(FrFB$vbw1F^N znEQLoR&wF`KN|a`&-{DygRuU;#T+nxX?($W#CU~qi;;jo=zkgkjetf#BcKt`2xtT} z0vZ90fJQ(g@FGAUF^6kJzP|!{7Ihm2VogaAGhr<|FpDc>`j!WuLA>A6D3~Vl%_w~E z9b{cTTk_$zMDOaDLU3Y!w;>;G!HfbQ`_?AplYEJ35*L8y8$Ft-I9Oc<=%X= zo^N;*+Y`!9oR|5coNMPR`G;S|_}NYjoC!#;hpbF7aLy}9(j$rdTs`noFW)fZ`v$kg P35D;hhc7j@PJaI%`81(Z diff --git a/assets/temporal_memory/evidence.jsonl b/assets/temporal_memory/evidence.jsonl deleted file mode 100644 index 5b72cabf16..0000000000 --- a/assets/temporal_memory/evidence.jsonl +++ /dev/null @@ -1,58 +0,0 @@ -{"window": {"start_s": 37.628, "end_s": 38.626}, "caption": "The scene shows a view from inside a room through a large window. The window frames an outdoor space featuring a bench and a green lawn under a clear blue sky. Inside the room, there's a dark flat-screen TV mounted on the wall opposite the window. A low-profile coffee table sits in front of the window, while wooden furniture is visible to the right side of the frame.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "TV"}, {"id": "E4", "type": "object", "descriptor": "coffee table"}, {"id": "E5", "type": "location", "descriptor": "outdoor space"}, {"id": "E6", "type": "object", "descriptor": "wooden furniture"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 81.84, "end_s": 84.648}, "caption": "The room features an olive green wall with a large dark flat-screen TV mounted opposite a brown couch. A wooden TV stand with a drawer is positioned underneath the TV. The floor is made of light-colored wood planks, and there is a clock on the wall above the TV, indicating time. A magazine rests on the couch, adding a touch of décor.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [{"id": "E7", "type": "object", "descriptor": "wall clock"}, {"id": "E8", "type": "object", "descriptor": "magazine"}, {"id": "E9", "type": "object", "descriptor": "TV stand"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 95.843, "end_s": 96.84}, "caption": "A wooden shelf against an olive green wall displays various items, including a few stacked books, a potted cactus, and a yellow cup. To the right, additional books are stacked, while an assortment of objects lays on the bottom shelf, including a colored circular item and a dark book. The overall lighting is soft, emphasizing the textures of the shelf and the items.", "entities_present": [], "new_entities": [{"id": "E10", "type": "object", "descriptor": "wooden shelf"}, {"id": "E11", "type": "object", "descriptor": "potted cactus"}, {"id": "E12", "type": "object", "descriptor": "stacked books"}, {"id": "E13", "type": "object", "descriptor": "yellow cup"}, {"id": "E14", "type": "object", "descriptor": "colored circular item"}, {"id": "E15", "type": "object", "descriptor": "dark book"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 29.708, "end_s": 30.711}, "caption": "The scene depicts an interior space with a view of a window looking out onto a grassy area and a bench. The room features wooden flooring, a dark green wall, a table with chairs, and a wall-mounted mirror. The lighting outside suggests dusk, as the sky transitions from blue to a darker hue.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "window"}, {"id": "E2", "type": "object", "descriptor": "grass area"}, {"id": "E3", "type": "object", "descriptor": "bench"}, {"id": "E4", "type": "object", "descriptor": "dining table"}, {"id": "E5", "type": "object", "descriptor": "chair"}, {"id": "E6", "type": "object", "descriptor": "mirror"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 42.345, "end_s": 43.345}, "caption": "The scene features a dimly lit interior room with dark green walls and wooden flooring. A window (E1) reveals an outdoor view with a grassy area (E2) and a bench (E3). Inside, a dining table (E4) is surrounded by several chairs (E5). A wall-mounted mirror (E6) reflects some of the room's elements. The atmosphere suggests evening with evening light filtering through the window.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1", "object": "E2", "confidence": 0.9, "evidence": ["visible window looking out towards the grass area"], "notes": "The window (E1) provides a view of the grassy area (E2)."}, {"type": "located_in", "subject": "E1", "object": "E4", "confidence": 0.9, "evidence": ["the window is part of the interior room containing the dining table"], "notes": "The window (E1) is part of the same room as the dining table (E4)."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 49.344, "end_s": 50.345}, "caption": "The scene shows an interior room with dark green walls. A doorway leads to another area, visible in the frames, with a carpet. A rolling cart with circular plates is positioned in the foreground, alongside a black lamp and a brown couch. In the background, there is a shelf containing books and decorative items. An abstract painting hangs on the wall.", "entities_present": [], "new_entities": [{"id": "E7", "type": "object", "descriptor": "rolling cart with plates"}, {"id": "E8", "type": "object", "descriptor": "black lamp"}, {"id": "E9", "type": "object", "descriptor": "brown couch"}, {"id": "E10", "type": "object", "descriptor": "shelf with books"}, {"id": "E11", "type": "object", "descriptor": "abstract painting"}, {"id": "E12", "type": "location", "descriptor": "interior room"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 50.345, "end_s": 111.346}, "caption": "The scene showcases an interior room with dark green walls and wooden flooring. A modern dining table (E4) surrounded by several chairs (E5) is positioned centrally, with a large window (E1) offering a view of an outdoor grassy area (E2). The window features reflections of the surroundings, contributing to the light within the room. Adjacent to the dining area, a door leads to a kitchen space with dark cabinetry. The lighting is bright, suggesting daytime. The arrangement highlights the contrast between the dark walls and the natural light coming through the window, creating a warm atmosphere.", "entities_present": [{"id": "E1", "confidence": 0.95}, {"id": "E2", "confidence": 0.85}, {"id": "E4", "confidence": 0.9}, {"id": "E5", "confidence": 0.9}], "new_entities": [{"id": "E6", "type": "object", "descriptor": "large mirror"}, {"id": "E13", "type": "object", "descriptor": "kitchen area"}], "relations": [{"type": "looks_at", "subject": "E1|window", "object": "E2|grass area", "confidence": 0.8, "evidence": ["The window (E1) is directly facing the grassy area (E2), suggesting a view."], "notes": "Inferred from the room layout showing the window's direction."}, {"type": "located_in", "subject": "E4|dining table", "object": "E12|interior room", "confidence": 0.95, "evidence": ["The dining table (E4) is seen within the interior room (E12)."], "notes": "Clearly visible as part of the room's setup."}, {"type": "adjacent_to", "subject": "E4|dining table", "object": "E13|kitchen area", "confidence": 0.9, "evidence": ["The dining area (E4) is near the doorway leading to the kitchen area (E13)"], "notes": "The proximity of both areas is evident in the frames."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.85} -{"window": {"start_s": 111.346, "end_s": 140.524}, "caption": "The scene depicts an interior room characterized by dark green walls and wooden flooring. A modern dining table (E4) is centrally positioned, surrounded by six chairs (E5). A large window (E1) allows ample light in, revealing an outdoor grassy area (E2) through its glass. Adjacent to the dining area, a doorway leads to a kitchen space (E13) that features dark cabinets and a sleek design. A large mirror (E6) is prominently displayed on one wall, reflecting the room's elements while enhancing the overall brightness of the space.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1", "object": "E2", "confidence": 0.8, "evidence": ["The window (E1) opens to the grassy area (E2), which is visible through the glass."], "notes": "The arrangement of the window suggests a direct view to the outside."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 140.524, "end_s": 155.71}, "caption": "The scene depicts an interior room with dark green walls and polished wooden flooring. A modern dining table is centrally located, surrounded by six chairs. A large window reveals a grassy area outside, illuminated by daylight. The room also features a large mirror that reflects light and space, enhancing the overall brightness. A doorway leads to a kitchen area visible to the right, containing dark cabinetry.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|window", "object": "E2|grass area", "confidence": 0.8, "evidence": ["The window provides a clear view of the grassy area."], "notes": "The orientation of the window suggests a direct line of sight."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 169.345, "end_s": 171.422}, "caption": "The scene shows a grassy area with a large tree casting a shadow. There are two potted plants on the grass, and a bench can be seen in the distance. The background has a clear blue sky, while a white wall is partially visible on the left side of the frame.", "entities_present": [{"id": "E2", "confidence": 0.9}], "new_entities": [{"id": "E14", "type": "object", "descriptor": "potted plant"}, {"id": "E15", "type": "object", "descriptor": "large tree"}, {"id": "E16", "type": "object", "descriptor": "bench"}], "relations": [{"type": "looks_at", "subject": "E14|potted plant", "object": "E2|grass area", "confidence": 0.6, "evidence": ["Present in both frames, with potted plants placed on the grass."], "notes": "Potted plants are positioned directly on the grassy area."}, {"type": "is_near", "subject": "E15|large tree", "object": "E16|bench", "confidence": 0.7, "evidence": ["The tree stands adjacent to the bench."], "notes": "The tree and bench are both within the visible grassy area."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.8} -{"window": {"start_s": 171.422, "end_s": 222.66}, "caption": "The scene depicts an outdoor grassy area (E2) with a large tree (E15) casting a dark shadow. Two potted plants (E14) are positioned on the grass, adding greenery, while a bench (E16) is seen in the background, suggesting a space for relaxation. The clear blue sky emphasizes a tranquil atmosphere, complemented by a white wall partially visible on the left side.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E14", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}, {"id": "E16", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 222.66, "end_s": 591.381}, "caption": "The scene features a grassy area (E2) with a large tree (E15) on the left side, casting a dark shadow over the ground. The clear blue sky contrasts with the green grass, creating a serene outdoor atmosphere. A portion of a white wall is visible, framing the scene. The grassy area suggests a place for relaxation and enjoyment of nature.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 591.381, "end_s": 697.334}, "caption": "The scene features a grassy area (E2) bordered by a white wall, creating a tranquil outdoor setting. Bright blue skies extend above, enhancing the serenity of the environment. A potted plant (E14) is partially visible in the foreground, while a large tree (E15) stands nearby, providing shade over the ground. In the background, hints of furniture, including a bench (E16) and a dining area (not fully visible), can be observed, suggesting a space designed for relaxation and outdoor enjoyment.", "entities_present": [{"id": "E2", "confidence": 1.0}, {"id": "E14", "confidence": 1.0}, {"id": "E15", "confidence": 1.0}, {"id": "E16", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "overlooks", "subject": "E15|large tree", "object": "E2|grass area", "confidence": 1.0, "evidence": ["The tree's shadow extends over the grassy area in the second frame."], "notes": "The tree is positioned to cast shade on the grass."}, {"type": "near", "subject": "E14|potted plant", "object": "E16|bench", "confidence": 0.7, "evidence": ["Both the potted plant and bench are visible in the composite view."], "notes": "The proximity suggests a cozy area for rest."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 130.787, "end_s": 131.278}, "caption": "The interior of a dimly lit room features a dark wall with a large clock on the left side. A wooden floor stretches beneath a low, black TV stand situated in the center, with a large screen mounted on the wall above. To the right, a glass door offers a view of an outdoor area, revealing a green lawn and a fence visible through the transparent pane. The lighting outside suggests it is either dawn or dusk.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large wall clock"}, {"id": "E2", "type": "object", "descriptor": "black TV stand"}, {"id": "E3", "type": "screen", "descriptor": "large TV screen"}, {"id": "E4", "type": "location", "descriptor": "glass door with outdoor view"}, {"id": "E5", "type": "other", "descriptor": "green lawn visible outside"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 143.277, "end_s": 144.333}, "caption": "The interior of a room features a sectional brown couch positioned against a dark green wall, with a painting adorned with colorful abstract designs hanging above. A white notepad sits on the couch. To the side, a floor lamp with a red shade adds a pop of color, alongside a wooden coffee table with a glass top, suggestive of a cozy living area.", "entities_present": [], "new_entities": [{"id": "E6", "type": "object", "descriptor": "brown sectional couch"}, {"id": "E7", "type": "object", "descriptor": "colorful abstract painting"}, {"id": "E8", "type": "object", "descriptor": "white notepad"}, {"id": "E9", "type": "object", "descriptor": "red floor lamp"}, {"id": "E10", "type": "object", "descriptor": "wooden coffee table"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames; possible presence of individuals is unconfirmed."], "confidence": 0.9} -{"window": {"start_s": 643.286, "end_s": 644.281}, "caption": "The interior of a room has a dark green wall with a colorful abstract painting (E7) hanging above a brown sectional couch (E6). A white notepad (E8) rests on the couch. Nearby, a red floor lamp (E9) stands beside a wooden coffee table (E10). In the background, a glass door (E4) leads to an outdoor view featuring a green lawn (E5). The lighting indicates it could be dusk as it creates a warm atmosphere in the space.", "entities_present": [{"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 653.281, "end_s": 653.781}, "caption": "The room features a green wall with an abstract painting (E7) above a brown sectional couch (E6). A white notepad (E8) is placed on the couch, and a red floor lamp (E9) stands beside a wooden coffee table (E10). The scene includes a glass door (E4) that opens to a view of a green lawn (E5). Across the room, a dark storage cabinet is visible under bright lighting.", "entities_present": [{"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [{"id": "E11", "type": "object", "descriptor": "dark storage cabinet"}], "relations": [{"type": "looks_at", "subject": "E11|dark storage cabinet", "object": "E4|glass door", "confidence": 0.6, "evidence": ["The cabinet is positioned in the foreground towards the glass door."], "notes": "The cabinet's orientation suggests attention towards the door."}], "on_screen_text": [], "uncertainties": ["The presence or details of other possible objects or decor."], "confidence": 0.9} -{"window": {"start_s": 700.281, "end_s": 701.286}, "caption": "The interior of a dimly lit room showcases a dark green wall adorned with a colorful abstract painting (E7) above a brown sectional couch (E6). A white notepad (E8) is placed on the couch, while a red floor lamp (E9) stands nearby. A dark TV stand (E2) is positioned across from a large TV screen (E3), reflecting soft lighting that complements the warm wood floor. The room opens up through a glass door (E4) leading to an outdoor area where a green lawn (E5) is visible, suggesting a tranquil ambience as daylight fades.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}, {"id": "E7", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E9", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}, {"id": "E11", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 711.781, "end_s": 712.781}, "caption": "The corner of a room is depicted, showcasing a dark green wall with a bookshelf displaying various objects including books and a plant. The floor is wooden, and the adjacent area features a black cabinet. The space is well-lit, revealing the minimalist decor.", "entities_present": [], "new_entities": [{"id": "E12", "type": "object", "descriptor": "bookshelf"}, {"id": "E13", "type": "object", "descriptor": "plant"}, {"id": "E14", "type": "object", "descriptor": "books"}, {"id": "E15", "type": "object", "descriptor": "black cabinet"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 0.9} -{"window": {"start_s": 722.296, "end_s": 722.78}, "caption": "The scene shows an interior with a desk featuring a laptop and a small white notepad, located adjacent to a bed and a wall with a vibrant blue paint. On the left, a dark green wall contains an outline of a table amidst dim lighting, hinting at a dining area. The juxtaposition of the brightly colored room and the dark space brings a contrasting ambiance to the overall look.", "entities_present": [], "new_entities": [{"id": "E16", "type": "object", "descriptor": "desk with laptop and notepad"}, {"id": "E17", "type": "object", "descriptor": "bed with pink bedding"}, {"id": "E18", "type": "location", "descriptor": "room with dark green wall"}], "relations": [{"type": "look_at", "subject": "E16|desk with laptop and notepad", "object": "E17|bed with pink bedding", "confidence": 0.7, "evidence": ["The positioning of the desk is oriented towards the bed."], "notes": "Desk appears aligned in view of the bed."}], "on_screen_text": [], "uncertainties": ["The specific use of the desk setup."], "confidence": 0.8} -{"window": {"start_s": 736.282, "end_s": 737.791}, "caption": "The scene depicts a modern interior featuring a desk with a laptop (E16) positioned to the left. In the background, a bed with pink bedding (E17) is visible against a bright blue wall, adorned with three artwork pieces. A tall lamp stands between the desk and the bed, contributing to the warm lighting of the room, while an abstract painting complementing the overall decor hangs on the wall.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}], "new_entities": [{"id": "E19", "type": "object", "descriptor": "tall lamp"}, {"id": "E20", "type": "object", "descriptor": "three artwork pieces on blue wall"}], "relations": [{"type": "looks_at", "subject": "E19|tall lamp", "object": "E16|desk with laptop", "confidence": 0.6, "evidence": ["The lamp is positioned near the desk, indicating a potential relationship between the two."], "notes": "Lamp's position suggests it provides light for the desk area."}, {"type": "located_behind", "subject": "E17|bed with pink bedding", "object": "E19|tall lamp", "confidence": 0.8, "evidence": ["The bed is in the background relative to the lamp."], "notes": "The bed is further back in the space compared to the lamp."}, {"type": "adorns", "subject": "E20|three artwork pieces on blue wall", "object": "E17|bed with pink bedding", "confidence": 0.9, "evidence": ["The artworks are directly above the bed."], "notes": "Art pieces enhance the decor around the bed."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 750.292, "end_s": 750.791}, "caption": "The scene shows a modern room with a sleek desk featuring a laptop, located on the left side. A bed with pink bedding is situated against a vibrant blue wall adorned with three pieces of artwork. A tall lamp provides lighting for both the desk area and the space near the bed. The arrangement emphasizes a cozy yet stylish atmosphere, integrating the desk and bed into a harmonious layout.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "provides_light_for", "subject": "E19|tall lamp", "object": "E16|desk with laptop", "confidence": 0.8, "evidence": ["Visible proximity and lighting effect in both frames"], "notes": "The lamp is positioned near the desk and illuminates the workspace."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 760.285, "end_s": 762.29}, "caption": "The frames depict a modern bedroom featuring a sleek desk with a laptop (E16) positioned to the left. A bed adorned with pink bedding (E17) is set against a vibrant blue wall that showcases three pieces of artwork (E20). A tall lamp (E19) illuminates the space, enhancing the cozy atmosphere created by the desk and bed's arrangement.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 769.781, "end_s": 770.294}, "caption": "The room features a desk with a laptop (E16) on the left and a bed with pink bedding (E17) against a vibrant blue wall adorned with three pieces of artwork (E20). A tall lamp (E19) provides light in the space, enhancing the inviting ambiance. The sleek layout emphasizes both workspace functionality and relaxation.", "entities_present": [{"id": "E16", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E20", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 58.337, "end_s": 58.846}, "caption": "The scene features an interior room with dark green walls, showcasing a dining table surrounded by wooden chairs, a wall-mounted mirror, and a bookshelf filled with various items. A gold-framed painting hangs on the wall adjacent to a doorway leading to another room, while a small cart with plates and a lamp is positioned on the right.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dining table"}, {"id": "E2", "type": "object", "descriptor": "wooden chairs"}, {"id": "E3", "type": "object", "descriptor": "mirror"}, {"id": "E4", "type": "object", "descriptor": "bookshelf"}, {"id": "E5", "type": "object", "descriptor": "painting"}, {"id": "E6", "type": "object", "descriptor": "cart"}, {"id": "E7", "type": "object", "descriptor": "plates"}, {"id": "E8", "type": "object", "descriptor": "lamp"}, {"id": "E9", "type": "location", "descriptor": "interior room"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 68.291, "end_s": 68.798}, "caption": "The scene shows an interior room (E9) with dark green walls and wooden flooring. There's a doorway leading to another room, which contains a black kitchen setup visible through the opening. A mirror (E3) is seen on the left wall, and a bookshelf (E4) is situated on the right, decorated with various items like books and a yellow object. The floor tiles of the adjacent room are gray, contrasting with the wooden floor in the current view.", "entities_present": [{"id": "E4", "confidence": 0.9}, {"id": "E3", "confidence": 0.8}], "new_entities": [{"id": "E10", "type": "object", "descriptor": "kitchen area visible through doorway"}, {"id": "E11", "type": "object", "descriptor": "yellow object on shelf"}], "relations": [{"type": "scene_change", "subject": "E4|bookshelf", "object": "E10|kitchen area", "confidence": 0.7, "evidence": ["The doorway provides a view into the kitchen setup."], "notes": "This indicates a transition to another space."}], "on_screen_text": [], "uncertainties": ["The exact nature of the yellow object on the shelf is unclear."], "confidence": 0.85} -{"window": {"start_s": 77.79, "end_s": 78.298}, "caption": "The frames depict an interior room with dark green walls and wooden flooring. A dining table (E1) with a polished surface is central to the scene, surrounded by wooden chairs (E2). A mirror (E3) on the left wall reflects part of the room, while the right side contains a visible doorway leading to a kitchen area. Light filters in through a window on the far wall, displaying a scenic view outside. The ambiance is cozy and inviting, with careful arrangement of furniture enhancing the living space.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [{"id": "E12", "type": "location", "descriptor": "window with outside view"}, {"id": "E13", "type": "object", "descriptor": "doorway leading to kitchen area"}], "relations": [{"type": "looks_at", "subject": "E1|dining table", "object": "E2|wooden chairs", "confidence": 0.7, "evidence": ["In both frames, the table and chairs are part of the same visual arrangement."], "notes": "The dining table is central, suggesting a focal point of interaction."}, {"type": "leads_to", "subject": "E13|doorway leading to kitchen area", "object": "E9|interior room", "confidence": 0.8, "evidence": ["Frame shows a clear transition area between the kitchen and room."], "notes": "The doorway connects the two spaces."}], "on_screen_text": [], "uncertainties": ["The specific contents or details of the kitchen area are unclear."], "confidence": 0.9} -{"window": {"start_s": 88.799, "end_s": 89.291}, "caption": "The frames depict an interior kitchen area viewed through an open doorway, showcasing dark walls and sleek, modern cabinets. The kitchen has a countertop with a microwave and another appliance with illuminated buttons, reflecting the ambient light from outside. A clear view of the green lawn is visible through the doorway, with a bright blue sky in the background, suggesting a pleasant outdoor environment.", "entities_present": [{"id": "E10", "confidence": 1.0}, {"id": "E13", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E13|doorway leading to kitchen area", "object": "E12|window with outside view", "confidence": 0.7, "evidence": ["The doorway opens directly to the lawn visible through the kitchen, creating a line of sight."], "notes": "The doorway likely frames the view to the outside."}, {"type": "contextual_change", "subject": "E10|kitchen area visible through doorway", "object": "E12|window with outside view", "confidence": 0.8, "evidence": ["Both the kitchen area and the outside view are visible within the same spatial frame."], "notes": "Transition from the indoor kitchen to the outdoor view."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 97.798, "end_s": 99.29}, "caption": "The scene transitions between two perspectives of an interior space. In the first frame, a doorframe provides a view into the room, revealing a wall painted dark green and a wooden floor. Light reflects off the floor, enhancing the warm tones of the space. The second frame shifts to a view from within a kitchen area; a large black refrigerator stands against the wall, while through an open doorway, the green wall of the living area and some furniture, including a cart, are visible. Both frames convey a cozy, modern atmosphere with dark colors and natural wood elements.", "entities_present": [{"id": "E10", "confidence": 1.0}], "new_entities": [{"id": "E14", "type": "object", "descriptor": "black refrigerator"}, {"id": "E15", "type": "other", "descriptor": "doorway"}, {"id": "E16", "type": "object", "descriptor": "cart"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 107.291, "end_s": 108.291}, "caption": "The frames depict a modern kitchen area with dark cabinets and appliances against a dark wall. A black refrigerator is visible alongside a microwave and another appliance, with a sink and countertop completing the setup. The floor features a patterned tile design. Outside the window, a green space is visible, contrasting with the kitchen's dark tones.", "entities_present": [{"id": "E10", "confidence": 0.8}, {"id": "E14", "confidence": 0.9}, {"id": "E12", "confidence": 0.7}], "new_entities": [{"id": "E17", "type": "object", "descriptor": "microwave"}, {"id": "E18", "type": "object", "descriptor": "sink"}, {"id": "E19", "type": "object", "descriptor": "countertop"}, {"id": "E20", "type": "location", "descriptor": "window to outside"}], "relations": [{"type": "located_in", "subject": "E10|kitchen area visible through doorway", "object": "E12|window with outside view", "confidence": 0.9, "evidence": ["Both elements are visible within the kitchen context."], "notes": "The window provides an external view from the kitchen."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} -{"window": {"start_s": 117.791, "end_s": 119.791}, "caption": "The view captures a corner of an interior space, with a stark black wall on one side leading to a visible green exterior through a window outside, presumably within a garden. The floor is tiled with a small black-and-white pattern. Though details of the room are obscured, there is a suggestion of a well-furnished area beyond the wall, indicated by soft lighting and a glimpse of the landscape outside.", "entities_present": [], "new_entities": [{"id": "E21", "type": "location", "descriptor": "view of green lawn outside"}, {"id": "E22", "type": "object", "descriptor": "black wall"}], "relations": [], "on_screen_text": [], "uncertainties": ["The exact nature of the room beyond the black wall is unclear."], "confidence": 0.7} -{"window": {"start_s": 126.292, "end_s": 126.791}, "caption": "The scene depicts a dark interior space featuring a black refrigerator (E14) and a doorway (E13) leading towards a kitchen area (E10). The walls are painted in a deep shade, enhancing the contrast with the light-colored flooring. The kitchen visible through the doorway showcases a modern design, emphasizing sleek furniture and appliances.", "entities_present": [{"id": "E10", "confidence": 0.9}, {"id": "E14", "confidence": 0.9}, {"id": "E13", "confidence": 0.9}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 133.804, "end_s": 134.79}, "caption": "The kitchen area features modern dark cabinets, appliances, and a contrasting white countertop. Among the visible kitchen equipment are a microwave (E17) and a sink (E18). The sleek black refrigerator (E14) is also present, against a backdrop of dark walls (E22). Light reflects off the white tile floor, emphasizing the contemporary design. The framed layout appears well-coordinated, with items meticulously arranged.", "entities_present": [{"id": "E14", "confidence": 1.0}, {"id": "E17", "confidence": 1.0}, {"id": "E18", "confidence": 1.0}, {"id": "E19", "confidence": 1.0}, {"id": "E22", "confidence": 1.0}, {"id": "E10", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 1.0} -{"window": {"start_s": 141.291, "end_s": 141.791}, "caption": "The scene shows an interior view from a kitchen doorway that opens to an outdoor green lawn. The left side of the frame features a dark wall, while on the right, a kitchen area is visible with a coffee machine and cabinetry. The doorway leads outside to a grass area under a clear sky.", "entities_present": [{"id": "E10", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 149.796, "end_s": 150.295}, "caption": "The scene captures a clear, expansive sky with a subtle gradient from blue to lighter shades indicating a bright atmosphere. A lush, green lawn is bordered by a low, light-colored wall designed to provide a boundary. The left side features a large tree, visible near the top of the frame, displaying dense foliage that casts shadow over the grass.", "entities_present": [], "new_entities": [{"id": "E23", "type": "location", "descriptor": "outdoor lawn"}, {"id": "E24", "type": "object", "descriptor": "tree"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 158.292, "end_s": 159.795}, "caption": "The frames show a view from the corner of a space with a green lawn outside, bordered by a low wall. The walls of the interior space are grey, and the floor is covered with a green surface that resembles grass. The sky above transitions from blue to light blue, indicating a clear day. There are no noticeable objects or people within the interior.", "entities_present": [{"id": "E20", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 164.791, "end_s": 165.291}, "caption": "The scene depicts an outdoor lawn area featuring a large tree (E24) that offers shade. In the background, a white building with glass windows can be seen, and a low light-colored wall borders the lawn. The sky appears bright and clear, enhancing the open-feel of the outdoor space.", "entities_present": [{"id": "E24", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [{"id": "E25", "type": "location", "descriptor": "outdoor lawn area"}, {"id": "E26", "type": "object", "descriptor": "building with glass windows"}, {"id": "E27", "type": "location", "descriptor": "clear sky"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 172.344, "end_s": 173.291}, "caption": "The scene depicts a modern outdoor space featuring a white building with a glass door facing a green lawn. There is a wooden bench positioned nearby, with potted plants in shades of red visible at the far right. The clear blue sky provides a bright backdrop.", "entities_present": [{"id": "E23", "confidence": 0.9}, {"id": "E26", "confidence": 0.8}], "new_entities": [{"id": "E28", "type": "object", "descriptor": "wooden bench"}, {"id": "E29", "type": "object", "descriptor": "potted plants"}], "relations": [{"type": "near", "subject": "E28|wooden bench", "object": "E29|potted plants", "confidence": 0.85, "evidence": ["Both entities are present in the scene, with the bench adjacent to the plants."], "notes": "The proximity suggests they are part of the same outdoor setup."}, {"type": "faces", "subject": "E26|white building", "object": "E23|outdoor lawn", "confidence": 0.9, "evidence": ["The glass door of the white building directly overlooks the green lawn."], "notes": "The orientation of the building suggests it is designed to interact with the lawn."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 184.291, "end_s": 186.291}, "caption": "The scene displays an outdoor lawn area with a clear blue sky above and a low wall in the distance. In the foreground, there are two potted plants, one red and one in a terracotta color, placed on the green grass. The grassy area is bordered by a gray wall, suggesting a manicured outdoor space.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E20", "confidence": 0.9}], "new_entities": [{"id": "E30", "type": "object", "descriptor": "red potted plant"}, {"id": "E31", "type": "object", "descriptor": "terracotta potted plant"}, {"id": "E32", "type": "location", "descriptor": "green lawn area"}], "relations": [], "on_screen_text": [], "uncertainties": ["The exact role or visibility of the gray wall and the type of surface is unclear."], "confidence": 0.9} -{"window": {"start_s": 192.791, "end_s": 194.793}, "caption": "The scene features a modern outdoor space with a green lawn area bordered by a low gray wall. A wooden bench sits on the lawn, while two potted plants, one red and the other terracotta, are arranged nearby. In the background, a white building with large glass windows is visible. Above, the sky is clear and blue.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E28", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}, {"id": "E26", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 205.295, "end_s": 205.79}, "caption": "The scene shows an outdoor area featuring a wooden bench (E28) with orange slats placed on green grass. Nearby, there are two potted plants, one red (E30) and one terracotta (E31). The background highlights a clear blue sky (E27) above and a low gray wall surrounding the space, creating an open atmosphere.", "entities_present": [{"id": "E28", "confidence": 0.9}, {"id": "E30", "confidence": 0.9}, {"id": "E31", "confidence": 0.9}, {"id": "E27", "confidence": 0.9}, {"id": "E23", "confidence": 0.9}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 0.9} -{"window": {"start_s": 214.793, "end_s": 216.293}, "caption": "The scene features an outdoor space with a clear blue sky visible above. In the foreground, a green lawn area is present, along with two round potted plants: one is light terracotta and the other is dark brown. A wooden bench with orange slats is also seen in the scene. The background includes a blank white wall, contrasting with the vibrant outdoor elements.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E29", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E28", "confidence": 1.0}], "new_entities": [{"id": "E33", "type": "object", "descriptor": "terracotta potted plant"}, {"id": "E34", "type": "object", "descriptor": "dark brown potted plant"}, {"id": "E35", "type": "object", "descriptor": "wooden bench with orange slats"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 226.32, "end_s": 226.81}, "caption": "The scene depicts an outdoor area with a well-maintained green lawn (E23) bordered by a blank white wall on one side. In the foreground, a black watering can (E36) is positioned near the edge of the grassy area. The sky above is a clear blue (E27), with a solid structure visible in the background, likely a building with glass windows (E26).", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [{"id": "E36", "type": "object", "descriptor": "black watering can"}, {"id": "E26", "type": "object", "descriptor": "building with glass windows"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 233.29, "end_s": 234.302}, "caption": "The scene features a flat, green lawn (E23) illuminated by small lights along the ground, leading up to a modern white building (E26) with large glass windows that reflect the interior. A sizeable black tree (E24) stands on the left side against a bright blue sky (E27). The outdoor area appears well-maintained, emphasizing the contrast between the vibrant greenery and the stark wall of the building.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E24", "confidence": 1.0}, {"id": "E26", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [{"id": "E37", "type": "object", "descriptor": "small ground lights"}, {"id": "E38", "type": "location", "descriptor": "lawn area near building"}], "relations": [{"type": "illuminates", "subject": "E37|small ground lights", "object": "E23|lawn area", "confidence": 0.9, "evidence": ["E37 is positioned along the foreground of E23 in the frames."], "notes": "The lights are visible and illuminating the lawn."}, {"type": "located_at", "subject": "E24|tree", "object": "E38|lawn area near building", "confidence": 0.9, "evidence": ["E24 is seen directly on the lawn adjacent to the building (E26)."], "notes": "The tree is firmly placed in the lawn area."}, {"type": "reflected_in", "subject": "E26|building", "object": "E27|blue sky", "confidence": 0.8, "evidence": ["The windows of E26 show a reflection of E27."], "notes": "The clear sky can be seen reflected in the glass."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} -{"window": {"start_s": 287.301, "end_s": 288.298}, "caption": "The scene showcases a view through a large glass window looking into an interior room with dark green walls and wooden flooring. Inside, there is a dining table surrounded by wooden chairs, with a visible bookshelf and lamp enhancing the space's decor. The second frame reveals a painting on the wall, adding visual interest. Outside the window, a green lawn is bordered by a white wall and features small ground lights illuminating the area. A large tree stands nearby under a clear blue sky.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E8", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E12", "confidence": 1.0}, {"id": "E21", "confidence": 1.0}, {"id": "E24", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 296.291, "end_s": 297.291}, "caption": "The scene depicts a small outdoor area enclosed by a white wall on one side. There is a black watering can (E36) placed on the grass alongside a wooden bench (E35) with orange slats, both set against a clear blue sky (E27). The grass is a vibrant green, and the area has a minimalistic aesthetic reminiscent of a small garden space.", "entities_present": [{"id": "E35", "confidence": 1.0}, {"id": "E36", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E23", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 388.291, "end_s": 389.295}, "caption": "The scene depicts an outdoor lawn area (E23) featuring a wooden bench (E35) with orange slats situated against a backdrop of clear blue sky (E27). To the right, there are two red potted plants (E30, E31) on the green grass. The area has minimal furnishings, characterized by an open space that emphasizes the vibrant greenery.", "entities_present": [{"id": "E23", "confidence": 1.0}, {"id": "E27", "confidence": 1.0}, {"id": "E35", "confidence": 1.0}, {"id": "E30", "confidence": 1.0}, {"id": "E31", "confidence": 1.0}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 396.874, "end_s": 397.791}, "caption": "The scene presents a view from an interior room with dark green walls and wooden floors, leading towards a clear sky and outdoor lawn. Visible through a glass window, a green lawn (E23) is illuminated by small ground lights (E37), bordered by a large tree (E24) and accompanied by a few potted plants (E30, E31). The exterior also features a blank white wall and a clear blue sky (E27).", "entities_present": [{"id": "E1", "confidence": 0.8}, {"id": "E2", "confidence": 0.8}, {"id": "E23", "confidence": 0.9}, {"id": "E24", "confidence": 0.9}, {"id": "E27", "confidence": 0.9}, {"id": "E37", "confidence": 0.8}], "new_entities": [], "relations": [], "on_screen_text": [], "uncertainties": ["No new salient entities identified in these frames."], "confidence": 0.9} -{"window": {"start_s": 57.189, "end_s": 60.19}, "caption": "The scene presents a dimly lit interior with a large window showing a serene outdoor view. Through the window, there is a clear blue sky, green grass, and a bench placed outside. In the foreground, a wooden table with red placemats and wooden chairs is visible, along with a dark furniture piece that could be a side table or cabinet. A reflection can be seen in the glass surface of the table.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "wooden table"}, {"id": "E4", "type": "object", "descriptor": "red placemat"}, {"id": "E5", "type": "object", "descriptor": "wooden chairs"}, {"id": "E6", "type": "object", "descriptor": "dark furniture piece"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 43.626, "end_s": 45.628}, "caption": "The frames display an interior view of a room with dark green walls and wooden flooring. A table with dark surface and red placemats is visible in the foreground, while an open window shows a clear sky and a grassy area outside. A bench is positioned in the outside view, emphasizing the peaceful outdoor setting beyond the room's interior.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark surface dining table"}, {"id": "E2", "type": "object", "descriptor": "red placemat"}, {"id": "E3", "type": "location", "descriptor": "outdoor grassy area"}, {"id": "E4", "type": "object", "descriptor": "wooden bench"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 44.633, "end_s": 45.634}, "caption": "A view from inside a room with dark green walls, looking through a glass door that frames an outdoor scene. In the foreground, there is a dark wooden table with red placemats and a set of chairs. Outside, a bench is visible in the sunlight, against a blue sky and some green grass. The room has minimal decor, with a large black television screen mounted on the wall and a mirror hanging nearby.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark wooden table"}, {"id": "E2", "type": "object", "descriptor": "red placemats"}, {"id": "E3", "type": "object", "descriptor": "outdoor bench"}, {"id": "E4", "type": "location", "descriptor": "room with green walls"}, {"id": "E5", "type": "screen", "descriptor": "black television"}, {"id": "E6", "type": "object", "descriptor": "mirror on the wall"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames.", "No significant text is visible."], "confidence": 0.8} -{"window": {"start_s": 41.368, "end_s": 42.37}, "caption": "The frames show a dimly lit room with dark-colored walls. In the first frame, there is a large window providing a view of an outdoor area, which includes a flat lawn and a bench in the distance. Adjacent to the window on the left is a dark TV screen mounted on the wall. The floor is wooden. The second frame shifts perspective to a dining table set with plates and chairs, facing the same window, revealing the lawn and a reflective surface on the wall. The two frames depict an indoor space looking out to the outside.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "window"}, {"id": "E2", "type": "object", "descriptor": "TV screen"}, {"id": "E3", "type": "object", "descriptor": "dining table"}, {"id": "E4", "type": "object", "descriptor": "chairs"}, {"id": "E5", "type": "location", "descriptor": "outdoor lawn"}], "relations": [{"type": "scene_change", "subject": "E1|window", "object": "E5|outdoor lawn", "confidence": 0.9, "evidence": ["The same outdoor view is visible through the window in both frames."], "notes": "Both frames depict the view from the same window."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 0.215, "end_s": 1.215}, "caption": "The scene features a dark green interior room with a visible window showing a clear blue sky and a grassy area outside. A bench can be seen outside, prominently framed by the window. Inside the room, there's a wooden dining table with placemats, and a TV screen is mounted on the wall opposite the window. The lighting is natural, coming from the outdoor sunlight, which adds contrast to the deep green walls.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "dark green wall"}, {"id": "E2", "type": "object", "descriptor": "window"}, {"id": "E3", "type": "object", "descriptor": "bench"}, {"id": "E4", "type": "object", "descriptor": "wooden dining table"}, {"id": "E5", "type": "object", "descriptor": "TV screen"}, {"id": "E6", "type": "object", "descriptor": "placemats"}], "relations": [], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 10.376, "end_s": 11.682}, "caption": "The frames depict two interior settings featuring a dining area and a living space. The first frame shows a dark green wall, a wooden dining table set with red placemats, and a view of an outdoor landscape through a window revealing a clear sky. The second frame presents a similar wall color, a large wall-mounted screen, and a view through a window that includes a park bench and green grass, with the interior focusing on a low, dark wooden table.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "wooden dining table with red placemats"}, {"id": "E2", "type": "object", "descriptor": "dark wooden low table"}, {"id": "E3", "type": "screen", "descriptor": "wall-mounted screen"}, {"id": "E4", "type": "location", "descriptor": "view of outdoor area with park bench in the background"}, {"id": "E5", "type": "location", "descriptor": "view of outdoor area with green grass and clear sky"}], "relations": [], "on_screen_text": [], "uncertainties": ["No people are visible in the frames.", "Specifics about the objects on the tables are unclear."], "confidence": 0.9} -{"window": {"start_s": 11.213, "end_s": 12.754}, "caption": "The scene features a dining area with a dark green wall and a wooden dining table equipped with red placemats. A large window allows a view of an outdoor setting, which is grassy and includes a bench visible through the window. A TV screen is mounted on the wall, adjacent to the window. The room is well-lit, with natural light entering from outside, creating a vibrant atmosphere.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E6", "confidence": 1.0}], "new_entities": [{"id": "E7", "type": "object", "descriptor": "outdoor bench"}, {"id": "E8", "type": "location", "descriptor": "outdoor grassy area"}], "relations": [{"type": "looks_at", "subject": "E2", "object": "E8", "confidence": 0.7, "evidence": ["The window (E2) is facing towards the outdoor area (E8) which is visible in the frame."], "notes": "The window appears to direct the viewer's gaze towards the outdoor space."}], "on_screen_text": [], "uncertainties": ["No people are visible in the frames."], "confidence": 0.9} -{"window": {"start_s": 37.649, "end_s": 39.143}, "caption": "The scene depicts an interior space with a view through a large window. In the window, a clear blue sky and a bench in an outdoor area are visible. Inside, there is a dark-colored cabinet on the left, empty space in front of it, and a dining table with several chairs on the right. The walls are painted dark green, and there is a reflection of the room visible on the window glass.", "entities_present": [], "new_entities": [{"id": "E1", "type": "object", "descriptor": "large window"}, {"id": "E2", "type": "object", "descriptor": "bench"}, {"id": "E3", "type": "object", "descriptor": "dark cabinet"}, {"id": "E4", "type": "object", "descriptor": "dining table"}, {"id": "E5", "type": "object", "descriptor": "chairs"}], "relations": [{"type": "scene_change", "subject": "E1|large window", "object": "E2|bench", "confidence": 0.9, "evidence": ["Both frames display the window looking out towards the bench."], "notes": "The window shows the outdoor view prominently."}], "on_screen_text": [], "uncertainties": [], "confidence": 0.95} -{"window": {"start_s": 48.644, "end_s": 50.16}, "caption": "The scene depicts an interior room with dark green walls featuring a large window (E1) that opens up to a clear blue sky and a view of an outdoor bench (E2) visible through the glass. Inside, a dark cabinet (E3) is located on the left, with a dining table (E4) and several chairs (E5) positioned to the right. The lighting highlights the wooden floor, and there is an empty space in front of the cabinet and behind the table, indicating a spacious design.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E2", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|large window", "object": "E2|bench", "confidence": 0.8, "evidence": ["The window (E1) directly displays the outdoor area, where the bench (E2) is visible."], "notes": "The window is oriented towards the outdoor view."}], "on_screen_text": [], "uncertainties": [], "confidence": 1.0} -{"window": {"start_s": 58.152, "end_s": 59.644}, "caption": "The scene depicts an interior room with dark green walls, featuring a large window (E1) that opens to an outdoor view. Inside, there is a dark cabinet (E3) on the left side and a dining table (E4) with several chairs (E5) on the right. The atmosphere is bright due to natural light coming through the window, illuminating the wooden floor and creating a spacious feeling.", "entities_present": [{"id": "E1", "confidence": 1.0}, {"id": "E4", "confidence": 1.0}, {"id": "E5", "confidence": 1.0}, {"id": "E3", "confidence": 1.0}], "new_entities": [], "relations": [{"type": "looks_at", "subject": "E1|large window", "object": "E2|unknown", "confidence": 0.8, "evidence": ["In both frames, the large window is open, showing an outdoor view."], "notes": "The window directly looks out, suggesting a connection to the outside."}], "on_screen_text": [], "uncertainties": ["Presence of any people is unclear."], "confidence": 1.0} diff --git a/assets/temporal_memory/frames_index.jsonl b/assets/temporal_memory/frames_index.jsonl deleted file mode 100644 index c7b2b7a0ae..0000000000 --- a/assets/temporal_memory/frames_index.jsonl +++ /dev/null @@ -1,50 +0,0 @@ -{"frame_index": 57, "timestamp_s": 70.68190789222717, "timestamp": "01:10.682"} -{"frame_index": 58, "timestamp_s": 72.17690801620483, "timestamp": "01:12.177"} -{"frame_index": 59, "timestamp_s": 72.67690801620483, "timestamp": "01:12.677"} -{"frame_index": 60, "timestamp_s": 74.17690801620483, "timestamp": "01:14.177"} -{"frame_index": 61, "timestamp_s": 74.67590808868408, "timestamp": "01:14.676"} -{"frame_index": 62, "timestamp_s": 76.17990803718567, "timestamp": "01:16.180"} -{"frame_index": 63, "timestamp_s": 76.67690801620483, "timestamp": "01:16.677"} -{"frame_index": 64, "timestamp_s": 77.67790794372559, "timestamp": "01:17.678"} -{"frame_index": 65, "timestamp_s": 78.67690801620483, "timestamp": "01:18.677"} -{"frame_index": 66, "timestamp_s": 79.67690801620483, "timestamp": "01:19.677"} -{"frame_index": 67, "timestamp_s": 80.67590808868408, "timestamp": "01:20.676"} -{"frame_index": 68, "timestamp_s": 81.67690801620483, "timestamp": "01:21.677"} -{"frame_index": 69, "timestamp_s": 82.67590808868408, "timestamp": "01:22.676"} -{"frame_index": 70, "timestamp_s": 84.17690801620483, "timestamp": "01:24.177"} -{"frame_index": 71, "timestamp_s": 84.67590808868408, "timestamp": "01:24.676"} -{"frame_index": 72, "timestamp_s": 85.67690801620483, "timestamp": "01:25.677"} -{"frame_index": 73, "timestamp_s": 87.17590808868408, "timestamp": "01:27.176"} -{"frame_index": 74, "timestamp_s": 88.17690801620483, "timestamp": "01:28.177"} -{"frame_index": 75, "timestamp_s": 88.67690801620483, "timestamp": "01:28.677"} -{"frame_index": 76, "timestamp_s": 89.67590808868408, "timestamp": "01:29.676"} -{"frame_index": 77, "timestamp_s": 90.67590808868408, "timestamp": "01:30.676"} -{"frame_index": 78, "timestamp_s": 91.67690801620483, "timestamp": "01:31.677"} -{"frame_index": 79, "timestamp_s": 92.67890787124634, "timestamp": "01:32.679"} -{"frame_index": 80, "timestamp_s": 93.67690801620483, "timestamp": "01:33.677"} -{"frame_index": 81, "timestamp_s": 94.67590808868408, "timestamp": "01:34.676"} -{"frame_index": 82, "timestamp_s": 95.67590808868408, "timestamp": "01:35.676"} -{"frame_index": 83, "timestamp_s": 96.67890787124634, "timestamp": "01:36.679"} -{"frame_index": 84, "timestamp_s": 97.67690801620483, "timestamp": "01:37.677"} -{"frame_index": 85, "timestamp_s": 98.67690801620483, "timestamp": "01:38.677"} -{"frame_index": 86, "timestamp_s": 100.67690801620483, "timestamp": "01:40.677"} -{"frame_index": 87, "timestamp_s": 101.17590808868408, "timestamp": "01:41.176"} -{"frame_index": 88, "timestamp_s": 102.17990803718567, "timestamp": "01:42.180"} -{"frame_index": 89, "timestamp_s": 104.17590808868408, "timestamp": "01:44.176"} -{"frame_index": 90, "timestamp_s": 104.67690801620483, "timestamp": "01:44.677"} -{"frame_index": 91, "timestamp_s": 105.67690801620483, "timestamp": "01:45.677"} -{"frame_index": 92, "timestamp_s": 106.67590808868408, "timestamp": "01:46.676"} -{"frame_index": 93, "timestamp_s": 107.67690801620483, "timestamp": "01:47.677"} -{"frame_index": 94, "timestamp_s": 109.17590808868408, "timestamp": "01:49.176"} -{"frame_index": 95, "timestamp_s": 110.17690801620483, "timestamp": "01:50.177"} -{"frame_index": 96, "timestamp_s": 111.17890787124634, "timestamp": "01:51.179"} -{"frame_index": 97, "timestamp_s": 112.17690801620483, "timestamp": "01:52.177"} -{"frame_index": 98, "timestamp_s": 113.67690801620483, "timestamp": "01:53.677"} -{"frame_index": 99, "timestamp_s": 114.67590808868408, "timestamp": "01:54.676"} -{"frame_index": 100, "timestamp_s": 115.67590808868408, "timestamp": "01:55.676"} -{"frame_index": 101, "timestamp_s": 117.17690801620483, "timestamp": "01:57.177"} -{"frame_index": 102, "timestamp_s": 118.17690801620483, "timestamp": "01:58.177"} -{"frame_index": 103, "timestamp_s": 119.17590808868408, "timestamp": "01:59.176"} -{"frame_index": 104, "timestamp_s": 120.17590808868408, "timestamp": "02:00.176"} -{"frame_index": 105, "timestamp_s": 121.17590808868408, "timestamp": "02:01.176"} -{"frame_index": 106, "timestamp_s": 122.67690801620483, "timestamp": "02:02.677"} diff --git a/assets/temporal_memory/state.json b/assets/temporal_memory/state.json deleted file mode 100644 index ff9b6ccb03..0000000000 --- a/assets/temporal_memory/state.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "entity_roster": [], - "rolling_summary": "", - "chunk_buffer": [], - "next_summary_at_s": 10.0, - "last_present": [] -} diff --git a/dimos/e2e_tests/test_dimsim_eval.py b/dimos/e2e_tests/test_dimsim_eval.py index dbea3cb28d..c13c6544d9 100644 --- a/dimos/e2e_tests/test_dimsim_eval.py +++ b/dimos/e2e_tests/test_dimsim_eval.py @@ -17,8 +17,8 @@ Uses sim-eval blueprint (includes rerun/browser for manual observation). Run individual evals or all of them: - pytest test_sim_eval_sequential.py -v -s -m slow - pytest test_sim_eval_sequential.py::TestSimEvalSequential::test_go_to_tv -v -s -m slow + pytest dimos/e2e_tests/test_dimsim_eval.py -v -s -m slow + pytest dimos/e2e_tests/test_dimsim_eval.py::TestSimEvalSequential::test_go_to_tv -v -s -m slow """ import json @@ -161,6 +161,7 @@ def sim_eval(): env=env, stdout=log_file or subprocess.DEVNULL, stderr=log_file or subprocess.DEVNULL, + start_new_session=True, ) try: diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py index ec98445679..0fe42d8789 100644 --- a/dimos/e2e_tests/test_dimsim_eval_parallel.py +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -17,7 +17,7 @@ 3 dimos instances + 3 headless browser pages, 1 eval workflow each. Runs all workflows concurrently, cutting wall-clock time to ~1 min. - pytest test_dimsim_eval_parallel.py -v -s -m slow + pytest dimos/e2e_tests/test_dimsim_eval_parallel.py -v -s -m slow """ from concurrent.futures import ThreadPoolExecutor, as_completed @@ -244,6 +244,7 @@ def parallel_env(): env=env0, stdout=log0 or subprocess.DEVNULL, stderr=log0 or subprocess.DEVNULL, + start_new_session=True, ) calls.append(call0) @@ -272,6 +273,7 @@ def parallel_env(): env=env_i, stdout=log_i or subprocess.DEVNULL, stderr=log_i or subprocess.DEVNULL, + start_new_session=True, ) calls.append(call_i) diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index b26b47f3a8..fa9ae8e647 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -17,7 +17,7 @@ Starts dimos sim-nav headless (GPU rendering), then checks that all LCM topics are publishing and that cmd_vel actually moves the robot. - pytest test_sim_nav.py -v -s -m slow + pytest dimos/e2e_tests/test_dimsim_nav.py -v -s """ import math @@ -107,6 +107,7 @@ def sim_nav(): env=env, stdout=log_file or subprocess.DEVNULL, stderr=log_file or subprocess.DEVNULL, + start_new_session=True, ) try: diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 1fa4c9bc81..6054d1acae 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -116,7 +116,7 @@ def _static_base_link(rr: Any) -> list[Any]: }, } -match global_config.viewer_backend: +match global_config.viewer: case "foxglove": from dimos.robot.foxglove_bridge import foxglove_bridge diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 2d1329b08f..6b298f310e 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -37,6 +37,7 @@ from typing import TYPE_CHECKING from dimos import spec +from dimos.core.core import rpc from dimos.core.native_module import NativeModule, NativeModuleConfig from dimos.utils.logging_config import setup_logger @@ -110,6 +111,7 @@ class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): # Control input (consumers publish cmd_vel to LCM, bridge reads it) cmd_vel: In[Twist] + @rpc def start(self) -> None: """Start the DimSim bridge subprocess. @@ -251,6 +253,7 @@ def _maybe_build(self) -> None: capture_output=True, ) dimsim_path = str(dimsim) + self.config.executable = dimsim_path downloaded = True logger.info("dimsim binary installed.") except Exception as exc: diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py index 54d3332d2f..8a1fd9da6a 100644 --- a/dimos/robot/sim/tf_module.py +++ b/dimos/robot/sim/tf_module.py @@ -166,8 +166,20 @@ def stop(self) -> None: @rpc def move(self, twist: Twist, duration: float = 0.0) -> bool: - """Send movement command to the simulator via cmd_vel.""" - self.cmd_vel.publish(twist) + """Send movement command to the simulator via cmd_vel. + + If *duration* > 0, publishes at 20 Hz for that many seconds then sends + a zero-velocity stop command. If 0, publishes once (caller controls). + """ + if duration > 0: + stop = Twist(linear=Vector3(0, 0, 0), angular=Vector3(0, 0, 0)) + deadline = time.monotonic() + duration + while time.monotonic() < deadline: + self.cmd_vel.publish(twist) + time.sleep(0.05) + self.cmd_vel.publish(stop) + else: + self.cmd_vel.publish(twist) return True From 845446534bf895462b6f2f6f86ffcfbb7cbf07d3 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 20:09:28 -0700 Subject: [PATCH 13/19] Fix sim blueprints for dev compatibility: update imports, remove __init__.py files - Replace all removed shorthand aliases with ClassName.blueprint() pattern (CostMapper, VoxelGridMapper, ReplanningAStarPlanner, WavefrontFrontierExplorer, SpatialMemory, WebsocketVisModule, RerunBridgeModule, FoxgloveBridge) - Replace deleted dimos.agents.agent with MCP agent (McpServer + McpClient) - Replace dimos.perception.experimental.temporal_memory shorthand with direct import - Fix spec.Camera/Pointcloud import path (dimos.spec.perception) - Fix CameraInfo import (module vs class) - Fix LCM() constructor (remove autoconf param) - Fix rerun bridge import (RerunBridgeModule + _resolve_viewer_mode) - Remove all __init__.py files (not allowed in dimos) - Fix all_blueprints.py sorting and add dim-sim-bridge/dim-sim-tf modules - Add skipif_in_ci back to test_dimsim_nav (requires dimsim binary) --- dimos/e2e_tests/test_dimsim_nav.py | 2 ++ dimos/robot/all_blueprints.py | 4 ++- dimos/robot/sim/__init__.py | 0 dimos/robot/sim/blueprints/__init__.py | 0 .../robot/sim/blueprints/agentic/__init__.py | 0 .../sim/blueprints/agentic/sim_agentic.py | 24 +++++++------ .../blueprints/agentic/sim_parallel_eval.py | 26 +++++++------- .../blueprints/agentic/sim_temporal_memory.py | 4 +-- dimos/robot/sim/blueprints/basic/__init__.py | 0 dimos/robot/sim/blueprints/basic/sim_basic.py | 36 +++++++++---------- dimos/robot/sim/blueprints/nav/__init__.py | 0 dimos/robot/sim/blueprints/nav/sim_nav.py | 18 +++++----- dimos/robot/sim/blueprints/nav/sim_spatial.py | 4 +-- dimos/robot/sim/bridge.py | 4 +-- dimos/robot/sim/tf_module.py | 2 +- 15 files changed, 66 insertions(+), 58 deletions(-) delete mode 100644 dimos/robot/sim/__init__.py delete mode 100644 dimos/robot/sim/blueprints/__init__.py delete mode 100644 dimos/robot/sim/blueprints/agentic/__init__.py delete mode 100644 dimos/robot/sim/blueprints/basic/__init__.py delete mode 100644 dimos/robot/sim/blueprints/nav/__init__.py diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index fa9ae8e647..3f3e52e36b 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -139,6 +139,8 @@ def spy(sim_nav): s.stop() +@pytest.mark.skipif_in_ci +@pytest.mark.slow class TestSimNav: """Smoke tests for the sim-nav pipeline — sensors, control, and data integrity.""" diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 21864dd31e..c6ad00ef66 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -59,8 +59,8 @@ "sim-agentic": "dimos.robot.sim.blueprints.agentic.sim_agentic:sim_agentic", "sim-basic": "dimos.robot.sim.blueprints.basic.sim_basic:sim_basic", "sim-eval": "dimos.robot.sim.blueprints.agentic.sim_eval:sim_eval", - "sim-parallel-eval": "dimos.robot.sim.blueprints.agentic.sim_parallel_eval:sim_parallel_eval", "sim-nav": "dimos.robot.sim.blueprints.nav.sim_nav:sim_nav", + "sim-parallel-eval": "dimos.robot.sim.blueprints.agentic.sim_parallel_eval:sim_parallel_eval", "sim-spatial": "dimos.robot.sim.blueprints.nav.sim_spatial:sim_spatial", "sim-temporal-memory": "dimos.robot.sim.blueprints.agentic.sim_temporal_memory:sim_temporal_memory", "teleop-phone": "dimos.teleop.phone.blueprints:teleop_phone", @@ -116,6 +116,8 @@ "demo-robot": "dimos.agents.skills.demo_robot", "detection2-d-module": "dimos.perception.detection.module2D", "detection3-d-module": "dimos.perception.detection.module3D", + "dim-sim-bridge": "dimos.robot.sim.bridge", + "dim-sim-tf": "dimos.robot.sim.tf_module", "drone-camera-module": "dimos.robot.drone.camera_module", "drone-connection-module": "dimos.robot.drone.connection_module", "drone-tracking-module": "dimos.robot.drone.drone_tracking_module", diff --git a/dimos/robot/sim/__init__.py b/dimos/robot/sim/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dimos/robot/sim/blueprints/__init__.py b/dimos/robot/sim/blueprints/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dimos/robot/sim/blueprints/agentic/__init__.py b/dimos/robot/sim/blueprints/agentic/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dimos/robot/sim/blueprints/agentic/sim_agentic.py b/dimos/robot/sim/blueprints/agentic/sim_agentic.py index e468267101..4f22174963 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_agentic.py +++ b/dimos/robot/sim/blueprints/agentic/sim_agentic.py @@ -12,24 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""DimSim agentic blueprint — spatial + agent + skills + human input.""" +"""DimSim agentic blueprint — spatial + MCP agent + skills + human input.""" -from dimos.agents.agent import agent -from dimos.agents.skills.navigation import navigation_skill -from dimos.agents.skills.person_follow import person_follow_skill -from dimos.agents.skills.speak_skill import speak_skill -from dimos.agents.web_human_input import web_input +from dimos.agents.mcp.mcp_client import McpClient +from dimos.agents.mcp.mcp_server import McpServer +from dimos.agents.skills.navigation import NavigationSkillContainer +from dimos.agents.skills.person_follow import PersonFollowSkillContainer +from dimos.agents.skills.speak_skill import SpeakSkill +from dimos.agents.web_human_input import WebInput from dimos.core.blueprints import autoconnect from dimos.robot.sim.blueprints.nav.sim_spatial import sim_spatial from dimos.robot.sim.tf_module import _camera_info_static sim_agentic = autoconnect( sim_spatial, - agent(), - navigation_skill(), - person_follow_skill(camera_info=_camera_info_static()), - web_input(), - speak_skill(), + McpServer.blueprint(), + McpClient.blueprint(), + NavigationSkillContainer.blueprint(), + PersonFollowSkillContainer.blueprint(camera_info=_camera_info_static()), + WebInput.blueprint(), + SpeakSkill.blueprint(), ) __all__ = ["sim_agentic"] diff --git a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py index f148e8ada6..c5dd24321a 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py +++ b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py @@ -30,13 +30,15 @@ from dimos.core.blueprints import autoconnect from dimos.core.transport import JpegLcmTransport -from dimos.mapping.costmapper import cost_mapper -from dimos.mapping.voxels import voxel_mapper +from dimos.mapping.costmapper import CostMapper +from dimos.mapping.voxels import VoxelGridMapper from dimos.msgs.sensor_msgs import Image -from dimos.navigation.frontier_exploration import wavefront_frontier_explorer -from dimos.navigation.replanning_a_star.module import replanning_a_star_planner -from dimos.perception.experimental.temporal_memory import temporal_memory -from dimos.perception.spatial_perception import spatial_memory +from dimos.navigation.frontier_exploration.wavefront_frontier_goal_selector import ( + WavefrontFrontierExplorer, +) +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner +from dimos.perception.experimental.temporal_memory.temporal_memory import TemporalMemory +from dimos.perception.spatial_perception import SpatialMemory from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf from dimos.utils.data import DIMOS_PROJECT_ROOT @@ -60,14 +62,14 @@ sim_bridge(), sim_tf(), # sim_nav - voxel_mapper(voxel_size=0.1), - cost_mapper(), - replanning_a_star_planner(), - wavefront_frontier_explorer(), + VoxelGridMapper.blueprint(voxel_size=0.1), + CostMapper.blueprint(), + ReplanningAStarPlanner.blueprint(), + WavefrontFrontierExplorer.blueprint(), # sim_spatial (isolated DB per instance) - spatial_memory(db_path=_db_path, visual_memory_path=_visual_memory_path, new_memory=True), + SpatialMemory.blueprint(db_path=_db_path, visual_memory_path=_visual_memory_path, new_memory=True), # temporal_memory - temporal_memory(), + TemporalMemory.blueprint(), ).global_config(n_workers=8, robot_model="dimsim") __all__ = ["sim_parallel_eval"] diff --git a/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py b/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py index 619d19912f..c4e7388cc9 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py +++ b/dimos/robot/sim/blueprints/agentic/sim_temporal_memory.py @@ -15,12 +15,12 @@ """DimSim temporal memory blueprint — agentic + temporal memory.""" from dimos.core.blueprints import autoconnect -from dimos.perception.experimental.temporal_memory import temporal_memory +from dimos.perception.experimental.temporal_memory.temporal_memory import TemporalMemory from dimos.robot.sim.blueprints.agentic.sim_agentic import sim_agentic sim_temporal_memory = autoconnect( sim_agentic, - temporal_memory(), + TemporalMemory.blueprint(), ) __all__ = ["sim_temporal_memory"] diff --git a/dimos/robot/sim/blueprints/basic/__init__.py b/dimos/robot/sim/blueprints/basic/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 6054d1acae..4e7ece6c70 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -23,7 +23,7 @@ from dimos.protocol.pubsub.impl.lcmpubsub import LCM from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf -from dimos.web.websocket_vis.websocket_vis_module import websocket_vis +from dimos.web.websocket_vis.websocket_vis_module import WebsocketVisModule class _SimLCM(LCM): # type: ignore[misc] @@ -103,7 +103,7 @@ def _static_base_link(rr: Any) -> list[Any]: rerun_config = { - "pubsubs": [_SimLCM(autoconf=True)], + "pubsubs": [_SimLCM()], "visual_override": { "world/camera_info": _convert_camera_info, "world/color_image": _convert_color_image, @@ -116,30 +116,28 @@ def _static_base_link(rr: Any) -> list[Any]: }, } -match global_config.viewer: - case "foxglove": - from dimos.robot.foxglove_bridge import foxglove_bridge +if global_config.viewer == "foxglove": + from dimos.robot.foxglove_bridge import FoxgloveBridge - with_vis = autoconnect( - _transports_base, - foxglove_bridge(shm_channels=["/color_image#sensor_msgs.Image"]), - ) - case "rerun": - from dimos.visualization.rerun.bridge import rerun_bridge - - with_vis = autoconnect(_transports_base, rerun_bridge(**rerun_config)) - case "rerun-web": - from dimos.visualization.rerun.bridge import rerun_bridge + with_vis = autoconnect( + _transports_base, + FoxgloveBridge.blueprint(shm_channels=["/color_image#sensor_msgs.Image"]), + ) +elif global_config.viewer.startswith("rerun"): + from dimos.visualization.rerun.bridge import RerunBridgeModule, _resolve_viewer_mode - with_vis = autoconnect(_transports_base, rerun_bridge(viewer_mode="web", **rerun_config)) - case _: - with_vis = _transports_base + with_vis = autoconnect( + _transports_base, + RerunBridgeModule.blueprint(viewer_mode=_resolve_viewer_mode(), **rerun_config), + ) +else: + with_vis = _transports_base sim_basic = autoconnect( with_vis, sim_bridge(), sim_tf(), - websocket_vis(), + WebsocketVisModule.blueprint(), ).global_config(n_workers=4, robot_model="dimsim") __all__ = ["sim_basic"] diff --git a/dimos/robot/sim/blueprints/nav/__init__.py b/dimos/robot/sim/blueprints/nav/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/dimos/robot/sim/blueprints/nav/sim_nav.py b/dimos/robot/sim/blueprints/nav/sim_nav.py index fa2e829ffe..be0bb92a19 100644 --- a/dimos/robot/sim/blueprints/nav/sim_nav.py +++ b/dimos/robot/sim/blueprints/nav/sim_nav.py @@ -15,21 +15,23 @@ """DimSim navigation blueprint — basic + mapping + planning + exploration.""" from dimos.core.blueprints import autoconnect -from dimos.mapping.costmapper import cost_mapper +from dimos.mapping.costmapper import CostMapper from dimos.mapping.pointclouds.occupancy import ( HeightCostConfig, ) -from dimos.mapping.voxels import voxel_mapper -from dimos.navigation.frontier_exploration import wavefront_frontier_explorer -from dimos.navigation.replanning_a_star.module import replanning_a_star_planner +from dimos.mapping.voxels import VoxelGridMapper +from dimos.navigation.frontier_exploration.wavefront_frontier_goal_selector import ( + WavefrontFrontierExplorer, +) +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner from dimos.robot.sim.blueprints.basic.sim_basic import sim_basic sim_nav = autoconnect( sim_basic, - voxel_mapper(voxel_size=0.1, publish_interval=0.5), - cost_mapper(algo="height_cost", config=HeightCostConfig(can_pass_under=1.5, smoothing=2.0)), - replanning_a_star_planner(), - wavefront_frontier_explorer(), + VoxelGridMapper.blueprint(voxel_size=0.1, publish_interval=0.5), + CostMapper.blueprint(algo="height_cost", config=HeightCostConfig(can_pass_under=1.5, smoothing=2.0)), + ReplanningAStarPlanner.blueprint(), + WavefrontFrontierExplorer.blueprint(), ).global_config(n_workers=6, robot_model="dimsim") __all__ = ["sim_nav"] diff --git a/dimos/robot/sim/blueprints/nav/sim_spatial.py b/dimos/robot/sim/blueprints/nav/sim_spatial.py index 4b252b77db..f2610dbc8d 100644 --- a/dimos/robot/sim/blueprints/nav/sim_spatial.py +++ b/dimos/robot/sim/blueprints/nav/sim_spatial.py @@ -15,12 +15,12 @@ """DimSim spatial blueprint — nav + spatial memory.""" from dimos.core.blueprints import autoconnect -from dimos.perception.spatial_perception import spatial_memory +from dimos.perception.spatial_perception import SpatialMemory from dimos.robot.sim.blueprints.nav.sim_nav import sim_nav sim_spatial = autoconnect( sim_nav, - spatial_memory(), + SpatialMemory.blueprint(), ).global_config(n_workers=8) __all__ = ["sim_spatial"] diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 6b298f310e..71339916a2 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -36,9 +36,9 @@ import shutil from typing import TYPE_CHECKING -from dimos import spec from dimos.core.core import rpc from dimos.core.native_module import NativeModule, NativeModuleConfig +from dimos.spec.perception import Camera, Pointcloud from dimos.utils.logging_config import setup_logger if TYPE_CHECKING: @@ -89,7 +89,7 @@ class DimSimBridgeConfig(NativeModuleConfig): extra_args: list[str] = field(default_factory=list) -class DimSimBridge(NativeModule, spec.Camera, spec.Pointcloud): +class DimSimBridge(NativeModule, Camera, Pointcloud): """NativeModule that manages the DimSim bridge subprocess. The bridge (Deno process) handles Browser-LCM translation and publishes diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py index 8a1fd9da6a..e6f3004cbc 100644 --- a/dimos/robot/sim/tf_module.py +++ b/dimos/robot/sim/tf_module.py @@ -39,7 +39,7 @@ Twist, Vector3, ) -from dimos.msgs.sensor_msgs import CameraInfo +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.utils.logging_config import setup_logger logger = setup_logger() From 43403f04aea34be84fea8a49573027eac260c5be Mon Sep 17 00:00:00 2001 From: Viswa4599 <39920874+Viswa4599@users.noreply.github.com> Date: Sat, 28 Mar 2026 03:10:45 +0000 Subject: [PATCH 14/19] CI code cleanup --- dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py | 4 +++- dimos/robot/sim/blueprints/nav/sim_nav.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py index c5dd24321a..c509458fe4 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py +++ b/dimos/robot/sim/blueprints/agentic/sim_parallel_eval.py @@ -67,7 +67,9 @@ ReplanningAStarPlanner.blueprint(), WavefrontFrontierExplorer.blueprint(), # sim_spatial (isolated DB per instance) - SpatialMemory.blueprint(db_path=_db_path, visual_memory_path=_visual_memory_path, new_memory=True), + SpatialMemory.blueprint( + db_path=_db_path, visual_memory_path=_visual_memory_path, new_memory=True + ), # temporal_memory TemporalMemory.blueprint(), ).global_config(n_workers=8, robot_model="dimsim") diff --git a/dimos/robot/sim/blueprints/nav/sim_nav.py b/dimos/robot/sim/blueprints/nav/sim_nav.py index be0bb92a19..6554fd9534 100644 --- a/dimos/robot/sim/blueprints/nav/sim_nav.py +++ b/dimos/robot/sim/blueprints/nav/sim_nav.py @@ -29,7 +29,9 @@ sim_nav = autoconnect( sim_basic, VoxelGridMapper.blueprint(voxel_size=0.1, publish_interval=0.5), - CostMapper.blueprint(algo="height_cost", config=HeightCostConfig(can_pass_under=1.5, smoothing=2.0)), + CostMapper.blueprint( + algo="height_cost", config=HeightCostConfig(can_pass_under=1.5, smoothing=2.0) + ), ReplanningAStarPlanner.blueprint(), WavefrontFrontierExplorer.blueprint(), ).global_config(n_workers=6, robot_model="dimsim") From 5f24c9c2fb5fff28b4255cdcb86bf4b71c4cb230 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 21:04:58 -0700 Subject: [PATCH 15/19] fix: resolve runtime errors in sim modules after dev merge - Remove `from __future__ import annotations` from bridge.py and tf_module.py (broke Module port creation from type annotations) - Convert DimSimBridgeConfig from @dataclass to pydantic model (parent NativeModuleConfig is pydantic) - Fix all msg imports: use `from dimos.msgs.*.X import X` (module-level imports returned modules, not classes) - Replace `None` with named `_suppress` function in rerun visual_override (pydantic requires callable, lambdas not picklable) - Add missing PatrollingModule to sim_agentic blueprint - Fix os.killpg PermissionError on macOS in test teardown - Add venv bin to PATH in e2e test subprocess env --- dimos/e2e_tests/dimos_cli_call.py | 18 ++++++++++--- dimos/e2e_tests/test_dimsim_eval.py | 5 +++- dimos/e2e_tests/test_dimsim_eval_parallel.py | 5 +++- dimos/e2e_tests/test_dimsim_nav.py | 12 ++++++--- .../sim/blueprints/agentic/sim_agentic.py | 2 ++ dimos/robot/sim/blueprints/basic/sim_basic.py | 8 ++++-- dimos/robot/sim/bridge.py | 27 +++++++++---------- dimos/robot/sim/tf_module.py | 19 ++++++------- 8 files changed, 61 insertions(+), 35 deletions(-) diff --git a/dimos/e2e_tests/dimos_cli_call.py b/dimos/e2e_tests/dimos_cli_call.py index 569dcfa386..620008e28c 100644 --- a/dimos/e2e_tests/dimos_cli_call.py +++ b/dimos/e2e_tests/dimos_cli_call.py @@ -38,6 +38,18 @@ def start(self) -> None: start_new_session=True, ) + def _kill_group(self, sig: int) -> None: + """Send *sig* to the process group, falling back to the process itself. + + On macOS with ``start_new_session=True`` the child runs in its own + session and ``os.killpg`` can raise ``PermissionError``. In that case + we fall back to signalling just the lead process. + """ + try: + os.killpg(self.process.pid, sig) + except PermissionError: + os.kill(self.process.pid, sig) + def stop(self) -> None: if self.process is None: return @@ -45,7 +57,7 @@ def stop(self) -> None: try: # Send SIGTERM to the entire process group so child processes # (e.g. the mujoco viewer subprocess) are also terminated. - os.killpg(self.process.pid, signal.SIGTERM) + self._kill_group(signal.SIGTERM) # Record the time when we sent the kill signal shutdown_start = time.time() @@ -62,7 +74,7 @@ def stop(self) -> None: ) except subprocess.TimeoutExpired: # If we reach here, the process didn't terminate in 30 seconds - os.killpg(self.process.pid, signal.SIGKILL) + self._kill_group(signal.SIGKILL) self.process.wait() # Clean up raise AssertionError( "Process did not shut down within 30 seconds after receiving SIGTERM" @@ -72,7 +84,7 @@ def stop(self) -> None: # Clean up if something goes wrong if self.process.poll() is None: # Process still running try: - os.killpg(self.process.pid, signal.SIGKILL) + self._kill_group(signal.SIGKILL) except ProcessLookupError: pass self.process.wait() diff --git a/dimos/e2e_tests/test_dimsim_eval.py b/dimos/e2e_tests/test_dimsim_eval.py index c13c6544d9..beebf36da6 100644 --- a/dimos/e2e_tests/test_dimsim_eval.py +++ b/dimos/e2e_tests/test_dimsim_eval.py @@ -27,6 +27,7 @@ import signal import socket import subprocess +import sys import time import pytest @@ -153,7 +154,9 @@ def sim_eval(): spy.save_topic("/odom#geometry_msgs.PoseStamped") spy.start() - env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu"} + venv_bin = str(Path(sys.prefix) / "bin") + env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu", + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} call = DimosCliCall() call.demo_args = ["sim-eval"] call.process = subprocess.Popen( diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py index 0fe42d8789..6494ad6b92 100644 --- a/dimos/e2e_tests/test_dimsim_eval_parallel.py +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -27,6 +27,7 @@ import signal import socket import subprocess +import sys import time import pytest @@ -222,7 +223,9 @@ def parallel_env(): calls: list[DimosCliCall] = [] log_files: list = [] - base_env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu"} + venv_bin = str(Path(sys.prefix) / "bin") + base_env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu", + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} log_dir_env = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") log_dir = Path(log_dir_env) if log_dir_env else None if log_dir: diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index 3f3e52e36b..ae14bc8447 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -26,14 +26,18 @@ import signal import socket import subprocess +import sys import time import pytest from dimos.e2e_tests.dimos_cli_call import DimosCliCall from dimos.e2e_tests.lcm_spy import LcmSpy -from dimos.msgs.geometry_msgs import PoseStamped, Twist, Vector3 -from dimos.msgs.sensor_msgs import Image, PointCloud2 +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 BRIDGE_PORT = 8090 @@ -99,7 +103,9 @@ def sim_nav(): log_file = None render = os.environ.get("DIMSIM_RENDER", "cpu") - env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": render} + venv_bin = str(Path(sys.prefix) / "bin") + env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": render, + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} call = DimosCliCall() call.demo_args = ["sim-nav"] call.process = subprocess.Popen( diff --git a/dimos/robot/sim/blueprints/agentic/sim_agentic.py b/dimos/robot/sim/blueprints/agentic/sim_agentic.py index 4f22174963..915b9f5f61 100644 --- a/dimos/robot/sim/blueprints/agentic/sim_agentic.py +++ b/dimos/robot/sim/blueprints/agentic/sim_agentic.py @@ -21,6 +21,7 @@ from dimos.agents.skills.speak_skill import SpeakSkill from dimos.agents.web_human_input import WebInput from dimos.core.blueprints import autoconnect +from dimos.navigation.patrolling.module import PatrollingModule from dimos.robot.sim.blueprints.nav.sim_spatial import sim_spatial from dimos.robot.sim.tf_module import _camera_info_static @@ -30,6 +31,7 @@ McpClient.blueprint(), NavigationSkillContainer.blueprint(), PersonFollowSkillContainer.blueprint(camera_info=_camera_info_static()), + PatrollingModule.blueprint(), WebInput.blueprint(), SpeakSkill.blueprint(), ) diff --git a/dimos/robot/sim/blueprints/basic/sim_basic.py b/dimos/robot/sim/blueprints/basic/sim_basic.py index 4e7ece6c70..9a61da3161 100644 --- a/dimos/robot/sim/blueprints/basic/sim_basic.py +++ b/dimos/robot/sim/blueprints/basic/sim_basic.py @@ -19,7 +19,7 @@ from dimos.core.blueprints import autoconnect from dimos.core.global_config import global_config from dimos.core.transport import JpegLcmTransport -from dimos.msgs.sensor_msgs import Image +from dimos.msgs.sensor_msgs.Image import Image from dimos.protocol.pubsub.impl.lcmpubsub import LCM from dimos.robot.sim.bridge import sim_bridge from dimos.robot.sim.tf_module import sim_tf @@ -92,6 +92,10 @@ def _convert_navigation_costmap(grid: Any) -> Any: ) +def _suppress(_msg: Any) -> None: + return None + + def _static_base_link(rr: Any) -> list[Any]: return [ rr.Boxes3D( @@ -109,7 +113,7 @@ def _static_base_link(rr: Any) -> list[Any]: "world/color_image": _convert_color_image, "world/global_map": _convert_global_map, "world/navigation_costmap": _convert_navigation_costmap, - "world/pointcloud": None, + "world/pointcloud": _suppress, }, "static": { "world/tf/base_link": _static_base_link, diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 71339916a2..df3aff9a9b 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -28,24 +28,24 @@ autoconnect(sim_bridge(), sim_tf(), some_consumer()).build().loop() """ -from __future__ import annotations - -from dataclasses import dataclass, field import os from pathlib import Path import shutil -from typing import TYPE_CHECKING +from typing import Optional + +from pydantic import Field from dimos.core.core import rpc from dimos.core.native_module import NativeModule, NativeModuleConfig +from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 from dimos.spec.perception import Camera, Pointcloud from dimos.utils.logging_config import setup_logger -if TYPE_CHECKING: - from dimos.core.stream import In, Out - from dimos.msgs.geometry_msgs import PoseStamped, Twist - from dimos.msgs.sensor_msgs import CameraInfo, Image, PointCloud2 - logger = setup_logger() _GITHUB_REPO = "Antim-Labs/DimSim" @@ -62,21 +62,20 @@ def _find_deno() -> str: return shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") -def _find_local_cli() -> Path | None: +def _find_local_cli() -> Optional[Path]: """Find local DimSim/dimos-cli/cli.ts for development.""" repo_root = Path(__file__).resolve().parents[4] candidate = repo_root / "DimSim" / "dimos-cli" / "cli.ts" return candidate if candidate.exists() else None -@dataclass(kw_only=True) class DimSimBridgeConfig(NativeModuleConfig): """Configuration for the DimSim bridge subprocess.""" # Resolved in _resolve_paths() — compiled binary or deno (local dev). executable: str = "dimsim" - build_command: str | None = None - cwd: str | None = None + build_command: Optional[str] = None + cwd: Optional[str] = None scene: str = "apt" port: int = 8090 @@ -86,7 +85,7 @@ class DimSimBridgeConfig(NativeModuleConfig): cli_exclude: frozenset[str] = frozenset({"scene", "port", "local"}) # Populated by _resolve_paths() — deno run args + dev subcommand + scene/port. - extra_args: list[str] = field(default_factory=list) + extra_args: list[str] = Field(default_factory=list) class DimSimBridge(NativeModule, Camera, Pointcloud): diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py index e6f3004cbc..3332331b82 100644 --- a/dimos/robot/sim/tf_module.py +++ b/dimos/robot/sim/tf_module.py @@ -23,22 +23,19 @@ DimSimConnection while the NativeModule bridge handles sensor data directly. """ -from __future__ import annotations - import math from threading import Thread import time +from typing import Optional from dimos.core.core import rpc from dimos.core.module import Module from dimos.core.stream import In, Out -from dimos.msgs.geometry_msgs import ( - PoseStamped, - Quaternion, - Transform, - Twist, - Vector3, -) +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Twist import Twist +from dimos.msgs.geometry_msgs.Vector3 import Vector3 from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.utils.logging_config import setup_logger @@ -87,8 +84,8 @@ class DimSimTF(Module): camera_info: Out[CameraInfo] cmd_vel: Out[Twist] - _camera_info_thread: Thread | None = None - _latest_odom: PoseStamped | None = None + _camera_info_thread: Optional[Thread] = None + _latest_odom: Optional[PoseStamped] = None _odom_last_ts: float = 0.0 _odom_count: int = 0 From 9c9d63bae42d838b9b6a42be4d81b7aa54477293 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 21:26:32 -0700 Subject: [PATCH 16/19] fix dimsim binary resolution + add eval creation docs Fix update flow: prefer compiled binary over PATH in _resolve_paths(), always sync self.config.executable after version check, create symlink before running setup. Add eval creation section to simulation.md. Co-Authored-By: Claude Opus 4.6 --- dimos/robot/sim/bridge.py | 28 +++++---- docs/development/simulation.md | 112 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index df3aff9a9b..0c7ee48c5f 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -164,7 +164,9 @@ def _resolve_paths(self) -> None: self.config.cwd = None return - dimsim_path = shutil.which("dimsim") or str(_dimsim_bin()) + # Prefer compiled binary over PATH (PATH may have stale deno-installed version) + dimsim_bin = _dimsim_bin() + dimsim_path = str(dimsim_bin) if dimsim_bin.exists() else shutil.which("dimsim") or str(dimsim_bin) self.config.executable = dimsim_path self.config.extra_args = dev_args self.config.cwd = None @@ -252,7 +254,6 @@ def _maybe_build(self) -> None: capture_output=True, ) dimsim_path = str(dimsim) - self.config.executable = dimsim_path downloaded = True logger.info("dimsim binary installed.") except Exception as exc: @@ -279,20 +280,23 @@ def _maybe_build(self) -> None: "dimsim not found — install Deno and retry, or wait for next release with compiled binaries" ) + # Always update executable to the resolved path (may differ from _resolve_paths) + self.config.executable = dimsim_path + # Symlink to ~/.local/bin so `dimsim` is on PATH for eval authoring + # Must happen BEFORE setup so that `dimsim eval list` etc. use the new binary local_bin = Path.home() / ".local" / "bin" local_bin.mkdir(parents=True, exist_ok=True) symlink = local_bin / "dimsim" - if dimsim_path: - try: - target = Path(dimsim_path).resolve() - if symlink.is_symlink() and symlink.resolve() != target: - symlink.unlink() - if not symlink.exists(): - symlink.symlink_to(target) - logger.info(f"Symlinked dimsim → {symlink}") - except OSError: - pass # no permission + try: + target = Path(dimsim_path).resolve() + if symlink.is_symlink() and symlink.resolve() != target: + symlink.unlink() + if not symlink.exists(): + symlink.symlink_to(target) + logger.info(f"Symlinked dimsim → {symlink}") + except OSError: + pass # no permission # setup/scene have version-aware caching (only downloads if version changed) logger.info("Checking core assets...") diff --git a/docs/development/simulation.md b/docs/development/simulation.md index 2d04a8bf6f..2763c611b0 100644 --- a/docs/development/simulation.md +++ b/docs/development/simulation.md @@ -85,6 +85,118 @@ pytest dimos/e2e_tests/test_dimsim_eval.py::TestSimEvalSequential::test_go_to_tv DIMSIM_EVAL_LOG_DIR=./logs pytest dimos/e2e_tests/test_dimsim_eval.py -v -s -m slow ``` +## Creating Evals + +Evals are JSON workflow files that define a task, starting pose, timeout, and success criteria. They live in `~/.dimsim/evals//.json`. + +### Interactive Wizard + +The fastest way to create an eval: + +```bash +dimsim eval create +``` + +This walks you through picking a scene, rubric type, target object, task prompt, threshold, and timeout. It writes the workflow JSON and prints the command to run it. + +### Manual Creation + +Create a JSON file in `~/.dimsim/evals//.json`: + +```json +{ + "name": "go-to-tv", + "environment": "apt", + "task": "Go to the TV", + "startPose": { "x": 0, "y": 0.5, "z": 3, "yaw": 0 }, + "timeoutSec": 30, + "successCriteria": { + "objectDistance": { + "object": "agent", + "target": "television", + "thresholdM": 2.0 + } + } +} +``` + +**Fields:** + +| Field | Required | Description | +|-------|----------|-------------| +| `name` | Yes | Unique identifier (used in CLI and test selection) | +| `environment` | Yes | Scene name (must be installed via `dimsim scene install`) | +| `task` | Yes | Natural language prompt sent to the agent | +| `startPose` | Yes | Agent spawn position: `x`, `y`, `z`, `yaw` (radians) | +| `timeoutSec` | Yes | Max seconds before the eval is scored | +| `successCriteria` | Yes | One or more rubrics (see below) | + +### Rubric Types + +**`objectDistance`** — Agent must reach a target object within a distance threshold. + +```json +"successCriteria": { + "objectDistance": { + "object": "agent", + "target": "refrigerator", + "thresholdM": 3.0 + } +} +``` + +- `target` is matched by substring against scene object titles/IDs (case-insensitive) +- `thresholdM` is Euclidean distance in meters from agent to the target's bounding box surface +- Use `dimsim list objects --scene ` to see available target objects + +**`llmJudge`** — A VLM judges success from screenshots. + +```json +"successCriteria": { + "llmJudge": { + "prompt": "Did the agent successfully navigate to the kitchen?" + } +} +``` + +**`groundTruth`** — Check spatial ground truth conditions. + +```json +"successCriteria": { + "groundTruth": {} +} +``` + +### Registering in the Manifest + +To include your eval in automated test runs, add it to `~/.dimsim/evals/manifest.json`: + +```json +{ + "version": "1.0", + "environments": [ + { + "name": "apt", + "scene": "apt", + "workflows": ["go-to-tv", "go-to-couch", "your-new-eval"] + } + ] +} +``` + +### Running a Single Eval + +```bash +# Headless (no browser window) +dimsim eval --headless --env apt --workflow go-to-tv + +# Connect to an already-running DimSim server +dimsim eval --connect --env apt --workflow go-to-tv + +# With browser (for debugging) +dimsim dev --eval go-to-tv --scene apt +``` + ## Environment Variables | Variable | Purpose | Default | From cb7b6c2b409510b527eb0ea5e47b87c727c1f3e7 Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 21:28:51 -0700 Subject: [PATCH 17/19] fix dimsim binary resolution, add eval docs, enable dimsim-nav in CI Fix update flow: prefer compiled binary over PATH in _resolve_paths(), always sync self.config.executable after version check, create symlink before running setup. Add eval creation section to simulation.md. Remove skipif_in_ci from test_dimsim_nav only. Co-Authored-By: Claude Opus 4.6 --- dimos/e2e_tests/test_dimsim_nav.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index ae14bc8447..6c48944f44 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -145,7 +145,6 @@ def spy(sim_nav): s.stop() -@pytest.mark.skipif_in_ci @pytest.mark.slow class TestSimNav: """Smoke tests for the sim-nav pipeline — sensors, control, and data integrity.""" From 8f729328bfb59538a5aeff39dbcfbe0dbe78a098 Mon Sep 17 00:00:00 2001 From: Viswa4599 <39920874+Viswa4599@users.noreply.github.com> Date: Sat, 28 Mar 2026 04:34:59 +0000 Subject: [PATCH 18/19] CI code cleanup --- dimos/e2e_tests/test_dimsim_eval.py | 8 ++++++-- dimos/e2e_tests/test_dimsim_eval_parallel.py | 8 ++++++-- dimos/e2e_tests/test_dimsim_nav.py | 8 ++++++-- dimos/robot/sim/bridge.py | 11 ++++++----- dimos/robot/sim/tf_module.py | 5 ++--- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/dimos/e2e_tests/test_dimsim_eval.py b/dimos/e2e_tests/test_dimsim_eval.py index beebf36da6..dd07a983c2 100644 --- a/dimos/e2e_tests/test_dimsim_eval.py +++ b/dimos/e2e_tests/test_dimsim_eval.py @@ -155,8 +155,12 @@ def sim_eval(): spy.start() venv_bin = str(Path(sys.prefix) / "bin") - env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu", - "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} + env = { + **os.environ, + "DIMSIM_HEADLESS": "1", + "DIMSIM_RENDER": "gpu", + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", ""), + } call = DimosCliCall() call.demo_args = ["sim-eval"] call.process = subprocess.Popen( diff --git a/dimos/e2e_tests/test_dimsim_eval_parallel.py b/dimos/e2e_tests/test_dimsim_eval_parallel.py index 6494ad6b92..57d73ff1c2 100644 --- a/dimos/e2e_tests/test_dimsim_eval_parallel.py +++ b/dimos/e2e_tests/test_dimsim_eval_parallel.py @@ -224,8 +224,12 @@ def parallel_env(): calls: list[DimosCliCall] = [] log_files: list = [] venv_bin = str(Path(sys.prefix) / "bin") - base_env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": "gpu", - "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} + base_env = { + **os.environ, + "DIMSIM_HEADLESS": "1", + "DIMSIM_RENDER": "gpu", + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", ""), + } log_dir_env = os.environ.get("DIMSIM_EVAL_LOG_DIR", "") log_dir = Path(log_dir_env) if log_dir_env else None if log_dir: diff --git a/dimos/e2e_tests/test_dimsim_nav.py b/dimos/e2e_tests/test_dimsim_nav.py index 6c48944f44..2ba727f91b 100644 --- a/dimos/e2e_tests/test_dimsim_nav.py +++ b/dimos/e2e_tests/test_dimsim_nav.py @@ -104,8 +104,12 @@ def sim_nav(): render = os.environ.get("DIMSIM_RENDER", "cpu") venv_bin = str(Path(sys.prefix) / "bin") - env = {**os.environ, "DIMSIM_HEADLESS": "1", "DIMSIM_RENDER": render, - "PATH": venv_bin + os.pathsep + os.environ.get("PATH", "")} + env = { + **os.environ, + "DIMSIM_HEADLESS": "1", + "DIMSIM_RENDER": render, + "PATH": venv_bin + os.pathsep + os.environ.get("PATH", ""), + } call = DimosCliCall() call.demo_args = ["sim-nav"] call.process = subprocess.Popen( diff --git a/dimos/robot/sim/bridge.py b/dimos/robot/sim/bridge.py index 0c7ee48c5f..ea594eac2f 100644 --- a/dimos/robot/sim/bridge.py +++ b/dimos/robot/sim/bridge.py @@ -31,7 +31,6 @@ import os from pathlib import Path import shutil -from typing import Optional from pydantic import Field @@ -62,7 +61,7 @@ def _find_deno() -> str: return shutil.which("deno") or str(Path.home() / ".deno" / "bin" / "deno") -def _find_local_cli() -> Optional[Path]: +def _find_local_cli() -> Path | None: """Find local DimSim/dimos-cli/cli.ts for development.""" repo_root = Path(__file__).resolve().parents[4] candidate = repo_root / "DimSim" / "dimos-cli" / "cli.ts" @@ -74,8 +73,8 @@ class DimSimBridgeConfig(NativeModuleConfig): # Resolved in _resolve_paths() — compiled binary or deno (local dev). executable: str = "dimsim" - build_command: Optional[str] = None - cwd: Optional[str] = None + build_command: str | None = None + cwd: str | None = None scene: str = "apt" port: int = 8090 @@ -166,7 +165,9 @@ def _resolve_paths(self) -> None: # Prefer compiled binary over PATH (PATH may have stale deno-installed version) dimsim_bin = _dimsim_bin() - dimsim_path = str(dimsim_bin) if dimsim_bin.exists() else shutil.which("dimsim") or str(dimsim_bin) + dimsim_path = ( + str(dimsim_bin) if dimsim_bin.exists() else shutil.which("dimsim") or str(dimsim_bin) + ) self.config.executable = dimsim_path self.config.extra_args = dev_args self.config.cwd = None diff --git a/dimos/robot/sim/tf_module.py b/dimos/robot/sim/tf_module.py index 3332331b82..a2d531129e 100644 --- a/dimos/robot/sim/tf_module.py +++ b/dimos/robot/sim/tf_module.py @@ -26,7 +26,6 @@ import math from threading import Thread import time -from typing import Optional from dimos.core.core import rpc from dimos.core.module import Module @@ -84,8 +83,8 @@ class DimSimTF(Module): camera_info: Out[CameraInfo] cmd_vel: Out[Twist] - _camera_info_thread: Optional[Thread] = None - _latest_odom: Optional[PoseStamped] = None + _camera_info_thread: Thread | None = None + _latest_odom: PoseStamped | None = None _odom_last_ts: float = 0.0 _odom_count: int = 0 From 09d1296d15f56b064483c9ede248e96bab45747b Mon Sep 17 00:00:00 2001 From: Viswajit Nair Date: Fri, 27 Mar 2026 21:38:47 -0700 Subject: [PATCH 19/19] docs: remove single eval run section from simulation.md Co-Authored-By: Claude Opus 4.6 --- docs/development/simulation.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docs/development/simulation.md b/docs/development/simulation.md index 2763c611b0..8a105e0c00 100644 --- a/docs/development/simulation.md +++ b/docs/development/simulation.md @@ -184,19 +184,6 @@ To include your eval in automated test runs, add it to `~/.dimsim/evals/manifest } ``` -### Running a Single Eval - -```bash -# Headless (no browser window) -dimsim eval --headless --env apt --workflow go-to-tv - -# Connect to an already-running DimSim server -dimsim eval --connect --env apt --workflow go-to-tv - -# With browser (for debugging) -dimsim dev --eval go-to-tv --scene apt -``` - ## Environment Variables | Variable | Purpose | Default |