Quest Teleop PR#1

.gitignore

@@ -63,4 +63,8 @@ yolo11n.pt
 /.mypy_cache*
 
 *mobileclip*
+
 /results
+
+# Teleop SSL certificates
+assets/teleop_certs/

dimos/msgs/geometry_msgs/Pose.py

@@ -222,6 +222,19 @@ def __add__(self, other: Pose | PoseConvertable | LCMTransform | Transform) -> P
 
         return Pose(new_position, new_orientation)
 
+    def __sub__(self, other: Pose) -> Pose:
+        """Compute the delta pose: self - other.
+
+        For position: simple subtraction.
+        For orientation: delta_quat = self.orientation * inverse(other.orientation)
+
+        Returns:
+            A new Pose representing the delta transformation
+        """
+        delta_position = self.position - other.position
+        delta_orientation = self.orientation * other.orientation.inverse()
+        return Pose(delta_position, delta_orientation)
+
     @classmethod
     def from_ros_msg(cls, ros_msg: ROSPose) -> Pose:
         """Create a Pose from a ROS geometry_msgs/Pose message.

dimos/msgs/std_msgs/Float32.py

@@ -0,0 +1,29 @@
+# Copyright 2025-2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Float32 message type."""
+
+from typing import ClassVar
+
+from dimos_lcm.std_msgs import Float32 as LCMFloat32
+
+
+class Float32(LCMFloat32):  # type: ignore[misc]
+    """ROS-compatible Float32 message."""
+
+    msg_name: ClassVar[str] = "std_msgs.Float32"
+
+    def __init__(self, data: float = 0.0) -> None:
+        """Initialize Float32 with data value."""
+        self.data = data

dimos/msgs/std_msgs/__init__.py

@@ -13,8 +13,9 @@
 # limitations under the License.
 
 from .Bool import Bool
+from .Float32 import Float32
 from .Header import Header
 from .Int8 import Int8
 from .Int32 import Int32
 
-__all__ = ["Bool", "Header", "Int8", "Int32"]
+__all__ = ["Bool", "Float32", "Header", "Int8", "Int32"]

dimos/robot/all_blueprints.py

@@ -54,6 +54,8 @@
     "demo-google-maps-skill": "dimos.agents.skills.demo_google_maps_skill:demo_google_maps_skill",
     "demo-object-scene-registration": "dimos.perception.demo_object_scene_registration:demo_object_scene_registration",
     "demo-error-on-name-conflicts": "dimos.robot.unitree_webrtc.demo_error_on_name_conflicts:blueprint",
+    # Teleop blueprints
+    "quest3-teleop": "dimos.teleop.teleop_blueprints:quest3_teleop",
 }
 
 

dimos/teleop/README.md

@@ -0,0 +1,113 @@
+# Teleoperation
+
+VR teleoperation system for controlling robots with Meta Quest controllers.
+
+## Folder Structure
+
+```
+teleop/
+├── __init__.py              # Exports quest3_teleop blueprint
+├── teleop_blueprints.py     # Pre-built system configurations
+├── devices/                 # Teleop input modules
+│   ├── base_teleop_module.py # BaseTeleopModule (calibration, deltas, transforms)
+│   └── vr_teleop_module.py  # VRTeleopModule (LCM inputs)
+└── web/                     # Quest WebXR interface
+    ├── teleop_server.ts     # Deno WebSocket-LCM bridge
+    ├── static/index.html    # WebXR client
+    └── certs/               # SSL certificates
+```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                         Quest 3 Headset                             │
+│  ┌─────────────┐                                                    │
+│  │  WebXR App  │ ──── Controller poses + triggers ────┐             │
+│  └─────────────┘                                      │             │
+└───────────────────────────────────────────────────────│─────────────┘
+                                                        │ WebSocket
+                                                        ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                      Deno Bridge (teleop_server.ts)                 │
+│                   WebSocket ←→ LCM packet forwarding                │
+└───────────────────────────────────────────────────────│─────────────┘
+                                                        │ LCM
+                                                        ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                        VRTeleopModule                               │
+│  ┌──────────────┐    ┌──────────────┐    ┌──────────────┐          │
+│  │  Calibrate   │ →  │ Compute Delta│ →  │  Transform   │          │
+│  │  (X button)  │    │ curr - init  │    │  & Publish   │          │
+│  └──────────────┘    └──────────────┘    └──────────────┘          │
+└───────────────────────────────────────────────────────│─────────────┘
+                                                        │
+                    ┌───────────────────────────────────┴───────────┐
+                    ▼                                               ▼
+           controller_delta_0/1                          controller_delta_2/3
+           (PoseStamped)                                 (TwistStamped)
+           target = init + Δ                             vel = scale(Δ)
+```
+
+## Quick Start
+
+```python
+from dimos.teleop import quest3_teleop
+
+coordinator = quest3_teleop.build()
+coordinator.loop()
+```
+
+Then on Quest 3: Open `https://<your-ip>:8443`, press X to calibrate and start.
+
+## Output Types Configuration
+
+The `output_types` parameter determines how each controller input is mapped:
+
+| Output Type | Indices | Use Case |
+|-------------|---------|----------|
+| `PoseStamped` | 0, 1 | Manipulators, end-effector control |
+| `TwistStamped` | 2, 3 | Locomotion, velocity control |
+
+The system auto-computes active indices from output types:
+- `[PoseStamped, PoseStamped]` → indices `[0, 1]` (dual arm)
+- `[TwistStamped, TwistStamped]` → indices `[2, 3]` (dual locomotion)
+- `[PoseStamped, TwistStamped]` → indices `[0, 2]` (arm + quadruped)
+
+## Custom Setup
+
+```python
+from dimos.teleop.devices import vr_teleop_module
+from dimos.msgs.geometry_msgs import PoseStamped, TwistStamped
+from dimos.core.blueprints import autoconnect
+
+# Dual arm setup (both controllers → PoseStamped)
+dual_arm = autoconnect(
+    vr_teleop_module(
+        output_types=[PoseStamped, PoseStamped],
+        input_labels=["left_arm", "right_arm"],
+    ),
+)
+
+# Arm + Quadruped (left → PoseStamped index 0, right → TwistStamped index 2)
+arm_and_quad = autoconnect(
+    vr_teleop_module(
+        output_types=[PoseStamped, TwistStamped],
+        input_labels=["left_arm", "right_quad"],
+    ),
+)
+
+# With RPC for initial robot pose (arm calibration)
+arm_with_rpc = autoconnect(
+    vr_teleop_module(
+        output_types=[PoseStamped, PoseStamped],
+        input_labels=["left_arm", "right_arm"],
+        robot_pose_rpc_methods=["LeftArm.get_ee_pose", "RightArm.get_ee_pose"],
+    ),
+)
+```
+
+## Submodules
+
+- [devices/](devices/) - Teleop input modules
+- [web/](web/) - Quest WebXR client and Deno server

dimos/teleop/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2025-2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Teleoperation module for dimos."""
+
+from dimos.teleop.teleop_blueprints import quest3_teleop
+
+__all__ = [
+    "quest3_teleop",
+]

dimos/teleop/devices/README.md

@@ -0,0 +1,104 @@
+# Teleop Devices
+
+Input device modules that capture tracking data, compute deltas, and transform to robot commands.
+
+## Modules
+
+### BaseTeleopModule
+
+Abstract base class providing:
+- Multi-controller calibration (capture initial poses)
+- Delta computation (current - initial)
+- Transform to robot commands (PoseStamped or TwistStamped based on `output_types`)
+- Publishing to `controller_delta_*` outputs
+- Optional RPC calls for initial robot pose
+- Rerun visualization
+
+### VRTeleopModule
+
+VR-specific implementation that:
+- Subscribes to LCM Transform messages from Deno bridge
+- Applies coordinate frame transformation (WebXR → Robot)
+- Transforms deltas to robot commands based on output type
+- Runs control loop at 50Hz
+
+## Data Flow
+
+```
+LCM Input (Transform)
+       │
+       ▼
+┌──────────────────┐
+│ Coordinate Frame │  WebXR: X=right, Y=up, Z=back
+│  Transformation  │  Robot: X=forward, Y=left, Z=up
+└──────────────────┘
+       │
+       ▼
+┌──────────────────┐
+│   Calibration    │  On X button: capture initial controller pose
+│                  │  (optional) RPC call for initial robot pose
+└──────────────────┘
+       │
+       ▼
+┌──────────────────┐
+│  Delta Compute   │  delta = current_controller - initial_controller
+└──────────────────┘
+       │
+       ▼
+┌──────────────────┐
+│    Transform     │  PoseStamped: target = initial_robot + delta
+│                  │  TwistStamped: velocity = scale(delta)
+└──────────────────┘
+       │
+       ▼
+   LCM Output (controller_delta_0/1/2/3)
+```
+
+## Configuration
+
+```python
+from dimos.teleop.devices import VRTeleopConfig
+from dimos.msgs.geometry_msgs import PoseStamped, TwistStamped
+
+config = VRTeleopConfig(
+    # Output types determine active indices:
+    # PoseStamped → indices 0,1 | TwistStamped → indices 2,3
+    output_types=[PoseStamped, TwistStamped],
+    input_labels=["left_vr", "right_vr"],
+
+    # Optional RPC methods for initial robot pose (per output)
+    robot_pose_rpc_methods=["ArmDriver.get_ee_pose", None],
+
+    # Visualization
+    visualize_in_rerun=True,
+
+    # Control
+    control_loop_hz=50.0,
+
+    # Transform settings
+    linear_scale=1.0,
+    angular_scale=1.0,
+    max_linear_velocity=0.5,
+    max_angular_velocity=1.0,
+    gripper_threshold=0.5,
+)
+```
+
+## Output Indices
+
+The `output_types` parameter auto-computes active indices:
+
+| Configuration | Active Indices | Outputs |
+|--------------|----------------|---------|
+| `[PoseStamped, PoseStamped]` | `[0, 1]` | Dual arm |
+| `[TwistStamped, TwistStamped]` | `[2, 3]` | Dual locomotion |
+| `[PoseStamped, TwistStamped]` | `[0, 2]` | Arm + quadruped |
+
+## Adding New Devices
+
+1. Inherit from `BaseTeleopModule`
+2. Override `start()` to set up input subscriptions
+3. Transform input to 4x4 pose matrix in robot frame
+4. Call `compute_deltas(poses, trigger_values)` to get delta poses
+5. Use `transform_delta()` from utils to convert to command
+6. Call `publish_command(index, command, aux_command)` to publish

dimos/teleop/devices/__init__.py

@@ -0,0 +1,35 @@
+# Copyright 2025-2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Teleoperation devices."""
+
+from dimos.teleop.devices.base_teleop_module import (
+    BaseTeleopConfig,
+    BaseTeleopModule,
+    TeleopStatusKey,
+)
+from dimos.teleop.devices.vr_teleop_module import (
+    VRTeleopConfig,
+    VRTeleopModule,
+    vr_teleop_module,
+)
+
+__all__ = [
+    "BaseTeleopConfig",
+    "BaseTeleopModule",
+    "TeleopStatusKey",
+    "VRTeleopConfig",
+    "VRTeleopModule",
+    "vr_teleop_module",
+]