diff --git a/demo/app.css b/demo/app.css index aa00deb6..e499c955 100644 --- a/demo/app.css +++ b/demo/app.css @@ -165,6 +165,16 @@ input[type='text'][placeholder='Chat Mode (Functional | Conversational)']:focus background-color: black; } +.camera-preview { + width: 100%; + aspect-ratio: 4 / 3; + border-radius: 8px; + background-color: #000; + border: 1px solid #444; + object-fit: cover; + transform: scaleX(-1); +} + .animated { border: 2px solid #ffcc00; } diff --git a/demo/app.tsx b/demo/app.tsx index 4d10b3ac..b99035c6 100644 --- a/demo/app.tsx +++ b/demo/app.tsx @@ -20,7 +20,15 @@ export function App() { const [audioInputDevices, setAudioInputDevices] = useState([]); const [selectedAudioDeviceId, setSelectedAudioDeviceId] = useState(''); + const [isCameraOn, setIsCameraOn] = useState(false); + const [cameraStream, setCameraStream] = useState(undefined); + const [videoInputDevices, setVideoInputDevices] = useState([]); + const [selectedVideoDeviceId, setSelectedVideoDeviceId] = useState(''); + const cameraStreamRef = useRef(undefined); + const hasSetDefaultVideoDevice = useRef(false); + const videoRef = useRef(null); + const cameraPreviewRef = useRef(null); const { srcObject, @@ -36,6 +44,10 @@ export function App() { unpublishMicrophoneStream, microphoneEnabled, isMicrophonePublished, + publishCameraStream, + unpublishCameraStream, + isCameraPublished, + cameraEnabled, } = useAgentManager({ debug, agentId, @@ -55,6 +67,68 @@ export function App() { } }, []); + const cleanupCameraStream = useCallback(() => { + if (cameraStreamRef.current) { + cameraStreamRef.current.getTracks().forEach(track => track.stop()); + cameraStreamRef.current = undefined; + setCameraStream(undefined); + hasSetDefaultVideoDevice.current = false; + } + }, []); + + const handleDisconnect = useCallback(async () => { + try { + await disconnect(); + } finally { + cleanupCameraStream(); + setIsCameraOn(false); + } + }, [disconnect, cleanupCameraStream]); + + const updateVideoDevices = useCallback(async () => { + try { + const tempStream = await navigator.mediaDevices.getUserMedia({ video: true }); + tempStream.getTracks().forEach(track => track.stop()); + const devices = await navigator.mediaDevices.enumerateDevices(); + const videoInputs = devices.filter(device => device.kind === 'videoinput'); + setVideoInputDevices(videoInputs); + if (videoInputs.length > 0 && !hasSetDefaultVideoDevice.current) { + hasSetDefaultVideoDevice.current = true; + setSelectedVideoDeviceId(videoInputs[0].deviceId); + } + } catch (error) { + console.error('Failed to enumerate video devices:', error); + } + }, []); + + const handleCameraToggle = useCallback( + async (enabled: boolean) => { + if (enabled) { + try { + const videoConstraints: MediaStreamConstraints['video'] = selectedVideoDeviceId + ? { deviceId: { exact: selectedVideoDeviceId } } + : true; + const stream = await navigator.mediaDevices.getUserMedia({ video: videoConstraints }); + cameraStreamRef.current = stream; + setCameraStream(stream); + } catch (error) { + console.error('Failed to get camera access:', error); + alert('Failed to access camera. Please check permissions.'); + return; + } + } else { + try { + await unpublishCameraStream(); + } catch (error) { + console.error('Failed to unpublish camera stream:', error); + } + cleanupCameraStream(); + } + setIsCameraOn(enabled); + }, + [selectedVideoDeviceId, unpublishCameraStream, cleanupCameraStream] + ); + const updateAudioDevices = useCallback(async () => { try { await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -163,6 +237,36 @@ export function App() { } }, [enableMicrophone, updateAudioDevices]); + useEffect(() => { + return cleanupCameraStream; + }, [cleanupCameraStream]); + + useEffect(() => { + if (cameraPreviewRef.current) { + cameraPreviewRef.current.srcObject = cameraStream ?? null; + } + }, [cameraStream]); + + useEffect(() => { + if (isCameraOn) { + updateVideoDevices(); + } + }, [isCameraOn, updateVideoDevices]); + + useEffect(() => { + if ( + connectionState === ConnectionState.Connected && + isCameraOn && + publishCameraStream && + cameraStreamRef.current && + !isCameraPublished + ) { + publishCameraStream(cameraStreamRef.current).catch(error => { + console.error('Failed to publish camera stream:', error); + }); + } + }, [connectionState, isCameraOn, publishCameraStream, isCameraPublished]); + useEffect(() => { if (srcObject && videoRef.current) { videoRef.current.srcObject = srcObject; @@ -225,7 +329,7 @@ export function App() { Interrupt - @@ -261,6 +365,18 @@ export function App() { Microphone )} + + {cameraEnabled && ( + + )} {microphoneEnabled && enableMicrophone && audioInputDevices.length > 0 && (
@@ -280,6 +396,33 @@ export function App() {
)} + {isCameraOn && videoInputDevices.length > 0 && ( +
+ +
+ )} + {isCameraOn && ( +