-
- );
-};
-
-export const FirstPersonView = () => {
- const [image, setImage] = useState
-
- 俯视视角 (Top-Down)
-
-
- {image ? (
- 
- ) : (
-
-
- 加载中...
-
- )}
-
-
- );
-};
-
-export const ArmControl = () => {
- const [yaw, setYaw] = useState(0);
- const [pitch, setPitch] = useState(0);
- const [roll, setRoll] = useState(0);
- const [state, setState] = useState
-
- 第一人称视角 (First-Person)
-
-
- {image ? (
-
- ) : (
-
-
- 加载中...
-
- )}
-
- {/* 机械臂控制 */}
-
- );
-};
-
-export const CarControl = () => {
- const [velLeft, setVelLeft] = useState(0);
- const [velRight, setVelRight] = useState(0);
-
- const handleCarAction = () => {
- sendMujocoCarAction(mujocoSocket, velLeft, velRight);
- };
-
- return (
-
-
-
- {/* 小车控制 */}
- 机械臂控制
-
-
-
-
- setYaw(parseFloat(e.target.value))}
- className="w-full"
- />
-
-
-
- setPitch(parseFloat(e.target.value))}
- className="w-full"
- />
-
-
-
- setRoll(parseFloat(e.target.value))}
- className="w-full"
- />
-
-
-
-
- 关节状态
-
- {armNames.map((name, i) => (
-
-
- {name}:
-
- pos={armQpos[i]?.toFixed(3) ?? "N/A"}, vel={armQvel[i]?.toFixed(3) ?? "N/A"}
-
-
- ))}
-
-
- );
-};
\ No newline at end of file
diff --git a/ui/src/pages/MujocoPage/ControlPanel.tsx b/ui/src/pages/MujocoPage/ControlPanel.tsx
new file mode 100644
index 0000000..f8da201
--- /dev/null
+++ b/ui/src/pages/MujocoPage/ControlPanel.tsx
@@ -0,0 +1,70 @@
+interface Props {
+ isLoaded: boolean;
+ reset: () => void;
+ showJointOverlay: boolean;
+ setShowJointOverlay: (v: boolean) => void;
+ driveSpeed: number;
+ turnSpeed: number;
+ onDriveSpeedChange: (v: number) => void;
+ onTurnSpeedChange: (v: number) => void;
+}
+
+export function ControlPanel({
+ isLoaded,
+ reset,
+ showJointOverlay,
+ setShowJointOverlay,
+ driveSpeed,
+ turnSpeed,
+ onDriveSpeedChange,
+ onTurnSpeedChange,
+}: Props) {
+ if (!isLoaded) return null;
+
+ return (
+ 小车控制
-
-
-
-
- setVelLeft(parseFloat(e.target.value))}
- className="w-full"
- />
-
-
-
- setVelRight(parseFloat(e.target.value))}
- className="w-full"
- />
-
-
-
+
+ );
+}
diff --git a/ui/src/pages/MujocoPage/MujocoRenderer.tsx b/ui/src/pages/MujocoPage/MujocoRenderer.tsx
new file mode 100644
index 0000000..9017540
--- /dev/null
+++ b/ui/src/pages/MujocoPage/MujocoRenderer.tsx
@@ -0,0 +1,670 @@
+import { useEffect, useRef, useCallback } from "react";
+import * as THREE from "three";
+import type { MainModule, MjModel, MjData } from "@mujoco/mujoco";
+
+const MJ_GEOM_PLANE = 0;
+const MJ_GEOM_SPHERE = 2;
+const MJ_GEOM_CYLINDER = 5;
+const MJ_GEOM_BOX = 6;
+
+const SUBSTEPS = 5;
+
+// Coordinate conversion: MuJoCo (x,y,z) → Three.js (x, z, -y)
+function mjPosToThree(buf: Float64Array, index: number, target: THREE.Vector3): THREE.Vector3 {
+ return target.set(buf[index * 3 + 0], buf[index * 3 + 2], -buf[index * 3 + 1]);
+}
+
+// Quaternion conversion: MuJoCo [w,x,y,z] → Three.js (-x, -z, y, -w)
+function mjQuatToThree(buf: Float64Array, index: number, target: THREE.Quaternion): THREE.Quaternion {
+ return target.set(-buf[index * 4 + 1], -buf[index * 4 + 3], buf[index * 4 + 2], -buf[index * 4 + 0]);
+}
+
+function createGeomMesh(
+ type: number,
+ size: Float64Array,
+ rgba: Float64Array,
+): THREE.Object3D {
+ let mesh: THREE.Object3D;
+
+ const alpha = rgba.length >= 4 ? rgba[3] : 1;
+ const color = new THREE.Color(rgba[0], rgba[1], rgba[2]);
+ const transparent = alpha < 0.99;
+
+ switch (type) {
+ case MJ_GEOM_PLANE: {
+ const geometry = new THREE.PlaneGeometry(20, 20);
+ geometry.rotateX(-Math.PI / 2);
+ const material = new THREE.MeshStandardMaterial({
+ color,
+ side: THREE.DoubleSide,
+ roughness: 0.9,
+ transparent,
+ opacity: alpha,
+ });
+ mesh = new THREE.Mesh(geometry, material);
+ break;
+ }
+ case MJ_GEOM_SPHERE: {
+ const geometry = new THREE.SphereGeometry(size[0], 32, 32);
+ const material = new THREE.MeshStandardMaterial({
+ color,
+ roughness: 0.4,
+ transparent,
+ opacity: alpha,
+ });
+ mesh = new THREE.Mesh(geometry, material);
+ break;
+ }
+ case MJ_GEOM_CYLINDER: {
+ const radius = size[0];
+ const halfHeight = size[1];
+
+ const cylGeom = new THREE.CylinderGeometry(radius, radius, halfHeight * 2, 32);
+
+ const cylMat = new THREE.MeshStandardMaterial({
+ color,
+ roughness: 0.5,
+ metalness: 0.3,
+ transparent,
+ opacity: alpha,
+ });
+ const cylMesh = new THREE.Mesh(cylGeom, cylMat);
+ cylMesh.castShadow = true;
+ cylMesh.receiveShadow = true;
+
+ const group = new THREE.Group();
+ group.add(cylMesh);
+
+ // Cross-spokes only for wheel-like cylinders (radius > halfHeight)
+ if (radius >= halfHeight) {
+ const spokeHalfLen = radius * 0.9;
+ const spokeThick = halfHeight * 0.3;
+ const spokeGeomX = new THREE.BoxGeometry(spokeHalfLen * 2, spokeThick, spokeThick);
+ const spokeGeomZ = new THREE.BoxGeometry(spokeThick, spokeThick, spokeHalfLen * 2);
+ const spokeMat = new THREE.MeshStandardMaterial({
+ color: 0x333333,
+ roughness: 0.5,
+ metalness: 0.4,
+ });
+ const spokeX = new THREE.Mesh(spokeGeomX, spokeMat);
+ const spokeZ = new THREE.Mesh(spokeGeomZ, spokeMat);
+ spokeX.castShadow = true;
+ spokeZ.castShadow = true;
+ group.add(spokeX);
+ group.add(spokeZ);
+ }
+
+ mesh = group;
+ break;
+ }
+ case MJ_GEOM_BOX: {
+ const geometry = new THREE.BoxGeometry(size[0] * 2, size[2] * 2, size[1] * 2);
+ const material = new THREE.MeshStandardMaterial({
+ color,
+ roughness: 0.5,
+ metalness: 0.2,
+ transparent,
+ opacity: alpha,
+ });
+ mesh = new THREE.Mesh(geometry, material);
+ break;
+ }
+ default: {
+ const geometry = new THREE.BoxGeometry(0.1, 0.1, 0.1);
+ const material = new THREE.MeshStandardMaterial({ color: 0xff00ff });
+ mesh = new THREE.Mesh(geometry, material);
+ }
+ }
+
+ mesh.castShadow = true;
+ mesh.receiveShadow = true;
+ return mesh;
+}
+
+function toUint8Array(buf: unknown): Uint8Array | null {
+ if (typeof buf === "string") {
+ return new TextEncoder().encode(buf);
+ }
+ if (buf instanceof ArrayBuffer) {
+ return new Uint8Array(buf);
+ }
+ if (ArrayBuffer.isView(buf)) {
+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+ }
+ return null;
+}
+
+function resolveName(names: unknown, addr: number): string {
+ const buf = toUint8Array(names);
+ if (!buf) return "";
+ let end = addr;
+ while (end < buf.length && buf[end] !== 0) end++;
+ return new TextDecoder().decode(buf.slice(addr, end));
+}
+
+function makeLabelSprite(text: string, color: string, scale: number = 0.4) {
+ const canvas = document.createElement("canvas");
+ canvas.width = 64;
+ canvas.height = 64;
+ const ctx = canvas.getContext("2d")!;
+ ctx.fillStyle = color;
+ ctx.font = "bold 48px Arial";
+ ctx.textAlign = "center";
+ ctx.textBaseline = "middle";
+ ctx.fillText(text, 32, 32);
+
+ const texture = new THREE.CanvasTexture(canvas);
+ texture.minFilter = THREE.LinearFilter;
+ const material = new THREE.SpriteMaterial({ map: texture, depthTest: false });
+ const sprite = new THREE.Sprite(material);
+ sprite.scale.set(scale, scale, 1);
+ return sprite;
+}
+
+function resolveCameraIndices(model: MjModel) {
+ let fpIdx = -1;
+ let tdIdx = -1;
+ const names = model.names;
+ for (let i = 0; i < model.ncam; i++) {
+ const name = resolveName(names, model.name_camadr[i]);
+ if (name === "firstperson") fpIdx = i;
+ else if (name === "topdown") tdIdx = i;
+ }
+ return { fpIdx, tdIdx };
+}
+
+interface JointOverlay {
+ cylinder: THREE.Mesh;
+ sprite: THREE.Sprite;
+ jointIdx: number;
+ name: string;
+ qposAdr: number;
+}
+
+interface Props {
+ mujoco: React.RefObject
+
+
+
+
+
+ Drive Settings
+
+
+ onDriveSpeedChange(parseFloat(e.target.value))}
+ className="w-full h-1.5 bg-slate-700 rounded-lg appearance-none cursor-pointer accent-emerald-500"
+ />
+
+
+
+ onTurnSpeedChange(parseFloat(e.target.value))}
+ className="w-full h-1.5 bg-slate-700 rounded-lg appearance-none cursor-pointer accent-amber-500"
+ />
+
+
+ {!isLoaded && (
+
+ );
+}
diff --git a/ui/src/pages/MujocoPage/carArmXml.ts b/ui/src/pages/MujocoPage/carArmXml.ts
new file mode 100644
index 0000000..3351a20
--- /dev/null
+++ b/ui/src/pages/MujocoPage/carArmXml.ts
@@ -0,0 +1,50 @@
+import { MAZE_XML } from './mazeXml';
+
+export const CAR_ARM_XML = `
+
+ )}
+
+
+ Loading MuJoCo WASM...
+
- {/* 顶部标题栏 */}
- >(new Set());
+ const [showJointOverlay, setShowJointOverlay] = useState(false);
+ const [driveSpeed, setDriveSpeed] = useState(5);
+ const [turnSpeed, setTurnSpeed] = useState(3);
+ const [fps, setFps] = useState(0);
+ const fpsFramesRef = useRef(0);
+ const fpsLastTimeRef = useRef(performance.now());
+
+ const applyDrive = useCallback(() => {
+ let leftVel = 0;
+ let rightVel = 0;
+
+ const k = keysRef.current;
+
+ if (k.has("KeyW") || k.has("ArrowUp")) {
+ leftVel += driveSpeed;
+ rightVel += driveSpeed;
+ }
+ if (k.has("KeyS") || k.has("ArrowDown")) {
+ leftVel -= driveSpeed;
+ rightVel -= driveSpeed;
+ }
+ if (k.has("KeyA") || k.has("ArrowLeft")) {
+ leftVel -= turnSpeed;
+ rightVel += turnSpeed;
+ }
+ if (k.has("KeyD") || k.has("ArrowRight")) {
+ leftVel += turnSpeed;
+ rightVel -= turnSpeed;
+ }
+ if (k.has("Space")) {
+ leftVel = 0;
+ rightVel = 0;
+ }
+
+ setControl("motor_wheel_fl", leftVel);
+ setControl("motor_wheel_rl", leftVel);
+ setControl("motor_wheel_fr", rightVel);
+ setControl("motor_wheel_rr", rightVel);
+ }, [setControl, driveSpeed, turnSpeed]);
+
+ useEffect(() => {
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (DRIVE_KEYS.includes(e.code)) {
+ e.preventDefault();
+ e.stopPropagation();
+ keysRef.current.add(e.code);
+ }
+ };
+ const handleKeyUp = (e: KeyboardEvent) => {
+ keysRef.current.delete(e.code);
+ };
+ // capture:true ensures we intercept before the browser scrolls
+ window.addEventListener("keydown", handleKeyDown, { capture: true });
+ window.addEventListener("keyup", handleKeyUp);
+ return () => {
+ window.removeEventListener("keydown", handleKeyDown, { capture: true });
+ window.removeEventListener("keyup", handleKeyUp);
+ };
+ }, []);
+
+ // Apply drive controls every frame before mj_step, with real FPS
+ const stepWithDrive = useCallback(() => {
+ applyDrive();
+
+ // Real FPS counter
+ fpsFramesRef.current++;
+ const now = performance.now();
+ if (now - fpsLastTimeRef.current >= 1000) {
+ setFps(fpsFramesRef.current);
+ fpsFramesRef.current = 0;
+ fpsLastTimeRef.current = now;
+ }
+
+ step();
+ }, [applyDrive, step]);
+
+ return (
+ ,
+ fpCanvasRef: React.RefObject,
+) {
+ const [isRecording, setIsRecording] = useState(false);
+ const [frameCount, setFrameCount] = useState(0);
+ const [episodeId, setEpisodeId] = useState(1);
+ const [socketStatus, setSocketStatus] = useState("disconnected");
+
+ const socketRef = useRef(null);
+ const recordingRef = useRef(false);
+ const lastCaptureRef = useRef(0);
+ const episodeIdRef = useRef(1);
+ const leftVelRef = useRef(0);
+ const rightVelRef = useRef(0);
+
+ // Connect to backend
+ const connect = useCallback(() => {
+ if (socketRef.current?.connected) return;
+ const s = io("http://localhost:8000", {
+ transports: ["websocket"],
+ namespace: "/sim",
+ });
+ s.on("connect", () => setSocketStatus("connected"));
+ s.on("disconnect", () => setSocketStatus("disconnected"));
+ s.on("connect_error", () => setSocketStatus("error"));
+ s.on("episode_started", (data: { episode_id: number }) => {
+ setEpisodeId(data.episode_id);
+ episodeIdRef.current = data.episode_id;
+ });
+ s.on("collection_count", (data: { count: number }) => {
+ setFrameCount(data.count);
+ });
+ socketRef.current = s;
+ return s;
+ }, []);
+
+ const disconnect = useCallback(() => {
+ socketRef.current?.disconnect();
+ socketRef.current = null;
+ setSocketStatus("disconnected");
+ }, []);
+
+ // Start recording
+ const startRecording = useCallback(() => {
+ const s = connect();
+ recordingRef.current = true;
+ setIsRecording(true);
+ setFrameCount(0);
+ lastCaptureRef.current = 0;
+
+ s.emit("start_episode", {
+ user_id: USER_ID,
+ episode_id: episodeIdRef.current,
+ task_name: "maze_driving",
+ });
+ }, [connect]);
+
+ // Stop recording
+ const stopRecording = useCallback(() => {
+ recordingRef.current = false;
+ setIsRecording(false);
+ const s = socketRef.current;
+ if (s) {
+ s.emit("end_episode", {
+ user_id: USER_ID,
+ episode_id: episodeIdRef.current,
+ });
+ episodeIdRef.current += 1;
+ }
+ }, []);
+
+ // Set current action values (called from drive loop)
+ const setAction = useCallback((left: number, right: number) => {
+ leftVelRef.current = left;
+ rightVelRef.current = right;
+ }, []);
+
+ // Capture frame + send data (called each physics step)
+ const captureFrame = useCallback(() => {
+ if (!recordingRef.current) return;
+
+ const now = performance.now();
+ if (now - lastCaptureRef.current < CAPTURE_INTERVAL_MS) return;
+ lastCaptureRef.current = now;
+
+ // Capture FPV canvas
+ const fpCanvas = fpCanvasRef.current;
+ if (!fpCanvas) return;
+ const image = fpCanvas.toDataURL("image/jpeg", 0.7);
+
+ // Read wheel velocities from MuJoCo data
+ const d = dataRef.current;
+ let velLeft = 0;
+ let velRight = 0;
+ if (d) {
+ // Wheel joints are at indices 0-3 (fl, fr, rl, rr)
+ // Joint velocity = d.qvel[joint_dof_adr]
+ const getJointVel = (name: string) => {
+ // Simplified: use the first 4 hinge joints
+ // wheel_fl=0, wheel_fr=1, wheel_rl=2, wheel_rr=3
+ const idx = ["wheel_fl", "wheel_fr", "wheel_rl", "wheel_rr"].indexOf(name);
+ if (idx >= 0 && d.qvel.length > idx) return d.qvel[idx];
+ return 0;
+ };
+ velLeft = (getJointVel("wheel_fl") + getJointVel("wheel_rl")) / 2;
+ velRight = (getJointVel("wheel_fr") + getJointVel("wheel_rr")) / 2;
+ }
+
+ const action: [number, number, number] = [
+ leftVelRef.current,
+ rightVelRef.current,
+ 0, // gripper (not used for car)
+ ];
+
+ socketRef.current?.emit("collect_data", {
+ user_id: USER_ID,
+ image: image.replace(/^data:image\/jpeg;base64,/, ""),
+ dataset_name: "maze",
+ timestamp: now,
+ state: { vel_left: velLeft, vel_right: velRight },
+ action,
+ });
+ }, [dataRef, fpCanvasRef]);
+
+ return {
+ isRecording,
+ frameCount,
+ episodeId,
+ socketStatus,
+ connect,
+ disconnect,
+ startRecording,
+ stopRecording,
+ setAction,
+ captureFrame,
+ };
+}
diff --git a/ui/src/pages/MujocoPage/useMujoco.ts b/ui/src/pages/MujocoPage/useMujoco.ts
new file mode 100644
index 0000000..a9b9eee
--- /dev/null
+++ b/ui/src/pages/MujocoPage/useMujoco.ts
@@ -0,0 +1,93 @@
+import { useEffect, useRef, useState, useCallback } from "react";
+import type { MainModule, MjModel, MjData } from "@mujoco/mujoco";
+import wasmUrl from "@mujoco/mujoco/mujoco.wasm?url";
+import { CAR_ARM_XML } from "./carArmXml";
+
+export function useMujoco() {
+ const [isLoaded, setIsLoaded] = useState(false);
+ const mujocoRef = useRef(null);
+ const modelRef = useRef(null);
+ const dataRef = useRef(null);
+
+ useEffect(() => {
+ let cancelled = false;
+
+ async function init() {
+ const { default: loadMujoco } = await import("@mujoco/mujoco");
+ const mujoco = await loadMujoco({
+ locateFile: (path: string) => {
+ if (path.endsWith(".wasm")) return wasmUrl;
+ return path;
+ },
+ });
+ if (cancelled) return;
+
+ mujocoRef.current = mujoco;
+
+ const model = (mujoco.MjModel as unknown as { from_xml_string: (xml: string) => MjModel }).from_xml_string(CAR_ARM_XML);
+ const data = new mujoco.MjData(model);
+
+ modelRef.current = model;
+ dataRef.current = data;
+ setIsLoaded(true);
+ }
+
+ init();
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+
+ const step = useCallback(() => {
+ const m = mujocoRef.current;
+ const model = modelRef.current;
+ const data = dataRef.current;
+ if (!m || !model || !data) return;
+ m.mj_step(model, data);
+ }, []);
+
+ const setControl = useCallback((name: string, value: number) => {
+ const model = modelRef.current;
+ const data = dataRef.current;
+ if (!model || !data) return;
+ const names = model.names;
+ let buf: Uint8Array;
+ if (typeof names === "string") {
+ buf = new TextEncoder().encode(names);
+ } else if (names instanceof ArrayBuffer) {
+ buf = new Uint8Array(names);
+ } else if (ArrayBuffer.isView(names)) {
+ buf = new Uint8Array(names.buffer, names.byteOffset, names.byteLength);
+ } else {
+ return;
+ }
+ const decoder = new TextDecoder();
+ for (let i = 0; i < model.nu; i++) {
+ const addr = model.name_actuatoradr[i];
+ let end = addr;
+ while (end < buf.length && buf[end] !== 0) end++;
+ if (decoder.decode(buf.slice(addr, end)) === name) {
+ data.ctrl[i] = value;
+ return;
+ }
+ }
+ }, []);
+
+ const reset = useCallback(() => {
+ const m = mujocoRef.current;
+ const model = modelRef.current;
+ const data = dataRef.current;
+ if (!m || !model || !data) return;
+ m.mj_resetData(model, data);
+ }, []);
+
+ return {
+ isLoaded,
+ mujoco: mujocoRef,
+ model: modelRef,
+ data: dataRef,
+ step,
+ setControl,
+ reset,
+ };
+}
-
-
- {/* 主内容区 */}
-
-
-
- MJC
-
-
-
- MuJoCo 小车+机械臂
-3D Physics Simulation
-
-
-
-
- {connected ? 'Connected' : 'Disconnected'}
-
-
- {/* 中间 - 俯视视角 */}
-
+ const { isLoaded, mujoco, model, data, step, setControl, reset } =
+ useMujoco();
+ const keysRef = useRef
-
-
- {/* 右侧 - 第一人称视角 */}
-
-
-
+
+ );
+}
diff --git a/ui/src/pages/MujocoPage/mazeXml.ts b/ui/src/pages/MujocoPage/mazeXml.ts
new file mode 100644
index 0000000..76435e0
--- /dev/null
+++ b/ui/src/pages/MujocoPage/mazeXml.ts
@@ -0,0 +1,33 @@
+export const MAZE_XML = `
+
+
+
+
+
+
+ MJC
+
+
+
+ + MuJoCo WASM + Three.js +
+Browser-native simulation
+
+
+ FPS: {fps || "--"}
+
+
+
+
+
+ {isLoaded ? "Running" : "Loading..."}
+
+
+
+
+
+
+
+
+
- );
-}
\ No newline at end of file
+ + WASD / Arrows: drive · Space: brake +