{
- e.preventDefault();
- const el = getViewElement();
- if (!el) return;
- const rect = el.getBoundingClientRect();
- const x = (e.clientX - rect.left) / rect.width;
- const y = (e.clientY - rect.top) / rect.height;
-
- if (e.altKey) {
- // Multi-touch mode: begin gesture
- multiTouchActiveRef.current = true;
- multiTouchShiftRef.current = e.shiftKey;
- const fingers = { x1: x, y1: y, x2: 1.0 - x, y2: 1.0 - y };
- // For pan mode, lock the offset between fingers
- panOffsetRef.current = { dx: 1.0 - x - x, dy: 1.0 - y - y };
- setFingerIndicators(fingers);
- sendMultiTouch({ type: "begin", ...fingers });
- return;
- }
+ {relayMode && !useNativeRelayVideo && (
+
{
+ const el = e.currentTarget;
+ if (el.naturalWidth > 0 && el.naturalHeight > 0) {
+ setScreenSize((prev) =>
+ prev && prev.width === el.naturalWidth && prev.height === el.naturalHeight
+ ? prev
+ : { width: el.naturalWidth, height: el.naturalHeight },
+ );
+ }
+ }}
+ style={{
+ position: "absolute",
+ inset: 0,
+ width: "100%",
+ height: "100%",
+ cursor: FINGER_CURSOR,
+ display: "block",
+ userSelect: "none",
+ WebkitUserSelect: "none" as any,
+ touchAction: "none",
+ ...imageStyle,
+ }}
+ />
+ )}
+ {/* Interactive overlay — captures all pointer events */}
+
{
+ e.preventDefault();
+ const el = getViewElement();
+ if (!el) return;
+ const rect = el.getBoundingClientRect();
+ const x = (e.clientX - rect.left) / rect.width;
+ const y = (e.clientY - rect.top) / rect.height;
- showTouchIndicator(x, y);
- if (y > 0.88) {
- edgeGestureRef.current = true;
- sendTouch({ type: "begin", x, y, edge: EDGE_BOTTOM });
- } else {
- edgeGestureRef.current = false;
- handleTouch("begin", e);
- }
- }}
- onMouseMove={(e) => {
- const el = getViewElement();
- if (!el) return;
- const rect = el.getBoundingClientRect();
- const x = (e.clientX - rect.left) / rect.width;
- const y = (e.clientY - rect.top) / rect.height;
- lastMousePosRef.current = { x, y };
-
- // Alt-hover preview (no buttons pressed)
- if (e.buttons === 0) {
if (e.altKey) {
- setFingerIndicators({
- x1: x,
- y1: y,
- x2: 1.0 - x,
- y2: 1.0 - y,
- });
+ // Multi-touch mode: begin gesture
+ multiTouchActiveRef.current = true;
+ multiTouchShiftRef.current = e.shiftKey;
+ const fingers = { x1: x, y1: y, x2: 1.0 - x, y2: 1.0 - y };
+ // For pan mode, lock the offset between fingers
+ panOffsetRef.current = { dx: 1.0 - x - x, dy: 1.0 - y - y };
+ setFingerIndicators(fingers);
+ sendMultiTouch({ type: "begin", ...fingers });
+ return;
}
- return;
- }
- if (multiTouchActiveRef.current) {
- let fingers;
- if (multiTouchShiftRef.current) {
- // Pan: both fingers translate together, maintaining fixed spacing
- const off = panOffsetRef.current;
- fingers = { x1: x, y1: y, x2: x + off.dx, y2: y + off.dy };
+ showTouchIndicator(x, y);
+ if (y > 0.88) {
+ edgeGestureRef.current = true;
+ sendTouch({ type: "begin", x, y, edge: EDGE_BOTTOM });
} else {
- // Pinch: fingers mirror around screen center (0.5, 0.5)
- fingers = { x1: x, y1: y, x2: 1.0 - x, y2: 1.0 - y };
+ edgeGestureRef.current = false;
+ handleTouch("begin", e);
}
- setFingerIndicators(fingers);
- sendMultiTouch({ type: "move", ...fingers });
- return;
- }
-
- moveTouchIndicator(x, y);
- if (edgeGestureRef.current) {
- sendTouch({ type: "move", x, y, edge: EDGE_BOTTOM });
- } else {
- handleTouch("move", e);
- }
- }}
- onMouseUp={(e) => {
- if (multiTouchActiveRef.current) {
+ }}
+ onMouseMove={(e) => {
const el = getViewElement();
- if (el) {
- const rect = el.getBoundingClientRect();
- const x = (e.clientX - rect.left) / rect.width;
- const y = (e.clientY - rect.top) / rect.height;
- if (multiTouchShiftRef.current) {
- const off = panOffsetRef.current;
- sendMultiTouch({
- type: "end",
- x1: x,
- y1: y,
- x2: x + off.dx,
- y2: y + off.dy,
- });
- } else {
- sendMultiTouch({
- type: "end",
+ if (!el) return;
+ const rect = el.getBoundingClientRect();
+ const x = (e.clientX - rect.left) / rect.width;
+ const y = (e.clientY - rect.top) / rect.height;
+ lastMousePosRef.current = { x, y };
+
+ // Alt-hover preview (no buttons pressed)
+ if (e.buttons === 0) {
+ if (e.altKey) {
+ setFingerIndicators({
x1: x,
y1: y,
x2: 1.0 - x,
y2: 1.0 - y,
});
}
+ return;
}
- multiTouchActiveRef.current = false;
- // Keep showing preview if alt is still held
- if (!e.altKey) setFingerIndicators(null);
- return;
- }
- hideTouchIndicator();
- if (edgeGestureRef.current) {
- const el = getViewElement();
- if (el) {
- const rect = el.getBoundingClientRect();
- const x = (e.clientX - rect.left) / rect.width;
- const y = (e.clientY - rect.top) / rect.height;
- sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
+ if (multiTouchActiveRef.current) {
+ let fingers;
+ if (multiTouchShiftRef.current) {
+ // Pan: both fingers translate together, maintaining fixed spacing
+ const off = panOffsetRef.current;
+ fingers = { x1: x, y1: y, x2: x + off.dx, y2: y + off.dy };
+ } else {
+ // Pinch: fingers mirror around screen center (0.5, 0.5)
+ fingers = { x1: x, y1: y, x2: 1.0 - x, y2: 1.0 - y };
+ }
+ setFingerIndicators(fingers);
+ sendMultiTouch({ type: "move", ...fingers });
+ return;
}
- edgeGestureRef.current = false;
- return;
- }
- handleTouch("end", e);
- }}
- onMouseLeave={(e) => {
- if (multiTouchActiveRef.current) {
- if (fingerIndicators) {
- sendMultiTouch({ type: "end", ...fingerIndicators });
+
+ moveTouchIndicator(x, y);
+ if (edgeGestureRef.current) {
+ sendTouch({ type: "move", x, y, edge: EDGE_BOTTOM });
+ } else {
+ handleTouch("move", e);
+ }
+ }}
+ onMouseUp={(e) => {
+ if (multiTouchActiveRef.current) {
+ const el = getViewElement();
+ if (el) {
+ const rect = el.getBoundingClientRect();
+ const x = (e.clientX - rect.left) / rect.width;
+ const y = (e.clientY - rect.top) / rect.height;
+ if (multiTouchShiftRef.current) {
+ const off = panOffsetRef.current;
+ sendMultiTouch({
+ type: "end",
+ x1: x,
+ y1: y,
+ x2: x + off.dx,
+ y2: y + off.dy,
+ });
+ } else {
+ sendMultiTouch({
+ type: "end",
+ x1: x,
+ y1: y,
+ x2: 1.0 - x,
+ y2: 1.0 - y,
+ });
+ }
+ }
+ multiTouchActiveRef.current = false;
+ // Keep showing preview if alt is still held
+ if (!e.altKey) setFingerIndicators(null);
+ return;
}
- multiTouchActiveRef.current = false;
- setFingerIndicators(null);
- return;
- }
- hideTouchIndicator();
- if (edgeGestureRef.current) {
- const el = getViewElement();
- if (el) {
- const rect = el.getBoundingClientRect();
- const x = (e.clientX - rect.left) / rect.width;
- const y = (e.clientY - rect.top) / rect.height;
- sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
+ hideTouchIndicator();
+ if (edgeGestureRef.current) {
+ const el = getViewElement();
+ if (el) {
+ const rect = el.getBoundingClientRect();
+ const x = (e.clientX - rect.left) / rect.width;
+ const y = (e.clientY - rect.top) / rect.height;
+ sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
+ }
+ edgeGestureRef.current = false;
+ return;
}
- edgeGestureRef.current = false;
- return;
- }
- if (e.buttons > 0) handleTouch("end", e);
- setFingerIndicators(null);
- }}
- onTouchStart={(e) => {
- e.preventDefault();
- const el = getViewElement();
- if (!el) return;
- const rect = el.getBoundingClientRect();
-
- if (e.touches.length >= 2) {
- // Two fingers down — start multi-touch
+ handleTouch("end", e);
+ }}
+ onMouseLeave={(e) => {
+ if (multiTouchActiveRef.current) {
+ if (fingerIndicators) {
+ sendMultiTouch({ type: "end", ...fingerIndicators });
+ }
+ multiTouchActiveRef.current = false;
+ setFingerIndicators(null);
+ return;
+ }
+
hideTouchIndicator();
- const t1 = e.touches[0];
- const t2 = e.touches[1];
- const fingers = {
- x1: (t1.clientX - rect.left) / rect.width,
- y1: (t1.clientY - rect.top) / rect.height,
- x2: (t2.clientX - rect.left) / rect.width,
- y2: (t2.clientY - rect.top) / rect.height,
- };
- // If a single-touch gesture was already in progress, end it first
- if (!realMultiTouchRef.current && !edgeGestureRef.current) {
- sendTouch({ type: "end", x: fingers.x1, y: fingers.y1 });
+ if (edgeGestureRef.current) {
+ const el = getViewElement();
+ if (el) {
+ const rect = el.getBoundingClientRect();
+ const x = (e.clientX - rect.left) / rect.width;
+ const y = (e.clientY - rect.top) / rect.height;
+ sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
+ }
+ edgeGestureRef.current = false;
+ return;
}
- realMultiTouchRef.current = true;
- multiTouchActiveRef.current = true;
- edgeGestureRef.current = false;
- setFingerIndicators(fingers);
- sendMultiTouch({ type: "begin", ...fingers });
- return;
- }
+ if (e.buttons > 0) handleTouch("end", e);
+ setFingerIndicators(null);
+ }}
+ onTouchStart={(e) => {
+ e.preventDefault();
+ const el = getViewElement();
+ if (!el) return;
+ const rect = el.getBoundingClientRect();
- const touch = e.touches[0];
- if (!touch) return;
- const x = (touch.clientX - rect.left) / rect.width;
- const y = (touch.clientY - rect.top) / rect.height;
- showTouchIndicator(x, y);
- if (y > 0.88) {
- edgeGestureRef.current = true;
- sendTouch({ type: "begin", x, y, edge: EDGE_BOTTOM });
- } else {
- edgeGestureRef.current = false;
- sendTouch({ type: "begin", x, y });
- }
- }}
- onTouchMove={(e) => {
- e.preventDefault();
- const el = getViewElement();
- if (!el) return;
- const rect = el.getBoundingClientRect();
-
- if (realMultiTouchRef.current && e.touches.length >= 2) {
- const t1 = e.touches[0];
- const t2 = e.touches[1];
- const fingers = {
- x1: (t1.clientX - rect.left) / rect.width,
- y1: (t1.clientY - rect.top) / rect.height,
- x2: (t2.clientX - rect.left) / rect.width,
- y2: (t2.clientY - rect.top) / rect.height,
- };
- setFingerIndicators(fingers);
- sendMultiTouch({ type: "move", ...fingers });
- return;
- }
+ if (e.touches.length >= 2) {
+ // Two fingers down — start multi-touch
+ hideTouchIndicator();
+ const t1 = e.touches[0];
+ const t2 = e.touches[1];
+ const fingers = {
+ x1: (t1.clientX - rect.left) / rect.width,
+ y1: (t1.clientY - rect.top) / rect.height,
+ x2: (t2.clientX - rect.left) / rect.width,
+ y2: (t2.clientY - rect.top) / rect.height,
+ };
+ // If a single-touch gesture was already in progress, end it first
+ if (!realMultiTouchRef.current && !edgeGestureRef.current) {
+ sendTouch({ type: "end", x: fingers.x1, y: fingers.y1 });
+ }
+ realMultiTouchRef.current = true;
+ multiTouchActiveRef.current = true;
+ edgeGestureRef.current = false;
+ setFingerIndicators(fingers);
+ sendMultiTouch({ type: "begin", ...fingers });
+ return;
+ }
- const touch = e.touches[0];
- if (!touch) return;
- const x = (touch.clientX - rect.left) / rect.width;
- const y = (touch.clientY - rect.top) / rect.height;
- moveTouchIndicator(x, y);
- if (edgeGestureRef.current) {
- sendTouch({ type: "move", x, y, edge: EDGE_BOTTOM });
- } else {
- sendTouch({ type: "move", x, y });
- }
- }}
- onTouchEnd={(e) => {
- e.preventDefault();
- const el = getViewElement();
- if (!el) return;
- const rect = el.getBoundingClientRect();
-
- if (realMultiTouchRef.current) {
- // End multi-touch when all fingers lift (touches.length is remaining fingers)
- if (e.touches.length < 2) {
- const t1 = e.changedTouches[0];
- // Use last known indicator positions as fallback for the second finger
- const last = fingerIndicators;
- if (t1 && last) {
- sendMultiTouch({
- type: "end",
- x1: (t1.clientX - rect.left) / rect.width,
- y1: (t1.clientY - rect.top) / rect.height,
- x2: last.x2,
- y2: last.y2,
- });
- } else if (last) {
- sendMultiTouch({ type: "end", ...last });
+ const touch = e.touches[0];
+ if (!touch) return;
+ const x = (touch.clientX - rect.left) / rect.width;
+ const y = (touch.clientY - rect.top) / rect.height;
+ showTouchIndicator(x, y);
+ if (y > 0.88) {
+ edgeGestureRef.current = true;
+ sendTouch({ type: "begin", x, y, edge: EDGE_BOTTOM });
+ } else {
+ edgeGestureRef.current = false;
+ sendTouch({ type: "begin", x, y });
+ }
+ }}
+ onTouchMove={(e) => {
+ e.preventDefault();
+ const el = getViewElement();
+ if (!el) return;
+ const rect = el.getBoundingClientRect();
+
+ if (realMultiTouchRef.current && e.touches.length >= 2) {
+ const t1 = e.touches[0];
+ const t2 = e.touches[1];
+ const fingers = {
+ x1: (t1.clientX - rect.left) / rect.width,
+ y1: (t1.clientY - rect.top) / rect.height,
+ x2: (t2.clientX - rect.left) / rect.width,
+ y2: (t2.clientY - rect.top) / rect.height,
+ };
+ setFingerIndicators(fingers);
+ sendMultiTouch({ type: "move", ...fingers });
+ return;
+ }
+
+ const touch = e.touches[0];
+ if (!touch) return;
+ const x = (touch.clientX - rect.left) / rect.width;
+ const y = (touch.clientY - rect.top) / rect.height;
+ moveTouchIndicator(x, y);
+ if (edgeGestureRef.current) {
+ sendTouch({ type: "move", x, y, edge: EDGE_BOTTOM });
+ } else {
+ sendTouch({ type: "move", x, y });
+ }
+ }}
+ onTouchEnd={(e) => {
+ e.preventDefault();
+ const el = getViewElement();
+ if (!el) return;
+ const rect = el.getBoundingClientRect();
+
+ if (realMultiTouchRef.current) {
+ // End multi-touch when all fingers lift (touches.length is remaining fingers)
+ if (e.touches.length < 2) {
+ const t1 = e.changedTouches[0];
+ // Use last known indicator positions as fallback for the second finger
+ const last = fingerIndicators;
+ if (t1 && last) {
+ sendMultiTouch({
+ type: "end",
+ x1: (t1.clientX - rect.left) / rect.width,
+ y1: (t1.clientY - rect.top) / rect.height,
+ x2: last.x2,
+ y2: last.y2,
+ });
+ } else if (last) {
+ sendMultiTouch({ type: "end", ...last });
+ }
+ realMultiTouchRef.current = false;
+ multiTouchActiveRef.current = false;
+ setFingerIndicators(null);
}
- realMultiTouchRef.current = false;
- multiTouchActiveRef.current = false;
- setFingerIndicators(null);
+ return;
}
- return;
- }
- const touch = e.changedTouches[0];
- if (!touch) return;
- const x = (touch.clientX - rect.left) / rect.width;
- const y = (touch.clientY - rect.top) / rect.height;
- hideTouchIndicator();
- if (edgeGestureRef.current) {
- sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
- edgeGestureRef.current = false;
- } else {
- sendTouch({ type: "end", x, y });
- }
- }}
- />
- {/* Single-touch indicator (hidden by default, shown via ref) */}
-
- {/* Multi-touch finger indicators */}
- {fingerIndicators && (
- <>
-
-
- >
- )}
- {!connected && !error && (
-
- Connecting...
-
- )}
- {error && (
-
-
- {error}
-
-
- )}
- {showSlowOverlay && (
-
-
- Slow connection
-
-
- )}
+ const touch = e.changedTouches[0];
+ if (!touch) return;
+ const x = (touch.clientX - rect.left) / rect.width;
+ const y = (touch.clientY - rect.top) / rect.height;
+ hideTouchIndicator();
+ if (edgeGestureRef.current) {
+ sendTouch({ type: "end", x, y, edge: EDGE_BOTTOM });
+ edgeGestureRef.current = false;
+ } else {
+ sendTouch({ type: "end", x, y });
+ }
+ }}
+ />
+ {/* Single-touch indicator (hidden by default, shown via ref) */}
+
+ {/* Multi-touch finger indicators */}
+ {fingerIndicators && (
+ <>
+
+
+ >
+ )}
+ {!connected && !error && (
+
+ Connecting...
+
+ )}
+ {error && (
+
+
+ {error}
+
+
+ )}
+ {showSlowOverlay && (
+
+
+ Slow connection
+
+
+ )}