diff --git a/gemini-demo/.dockerignore b/gemini-demo/.dockerignore new file mode 100644 index 0000000..c637362 --- /dev/null +++ b/gemini-demo/.dockerignore @@ -0,0 +1,7 @@ +node_modules/ +dist/ +.env +*.log +.DS_Store +backend/yarn.lock +web/yarn.lock diff --git a/gemini-demo/.env.example b/gemini-demo/.env.example new file mode 100644 index 0000000..ab6f20d --- /dev/null +++ b/gemini-demo/.env.example @@ -0,0 +1,4 @@ +FISHJAM_ID= +FISHJAM_MANAGEMENT_TOKEN= +GEMINI_API_KEY= +VITE_FISHJAM_ID= diff --git a/gemini-demo/.gitignore b/gemini-demo/.gitignore new file mode 100644 index 0000000..fb61065 --- /dev/null +++ b/gemini-demo/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +.yarn +dist/ +.env +*.log +.DS_Store diff --git a/gemini-demo/.yarnrc.yml b/gemini-demo/.yarnrc.yml new file mode 100644 index 0000000..3186f3f --- /dev/null +++ b/gemini-demo/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/gemini-demo/README.md b/gemini-demo/README.md new file mode 100644 index 0000000..dfe6ef8 --- /dev/null +++ b/gemini-demo/README.md @@ -0,0 +1,57 @@ +# Gemini Demo + +A minimal example of a video call with a Gemini Live AI agent using Fishjam Cloud. + +## What it does + +- Create a room and join a video call +- Spawn a Gemini Live voice agent with a custom system prompt +- The agent joins the call, listens to participants, and responds with voice +- Supports Google Search for real-time information + +## Setup + +1. Copy `.env.example` to `.env` and fill in your credentials: + +``` +cp .env.example .env +``` + +2. Install dependencies: + +``` +cd backend && npm install +cd ../web && npm install +``` + +3. Start the backend: + +``` +cd backend && npm run start +``` + +4. Start the frontend (in another terminal): + +``` +cd web && npm run start +``` + +5. Open http://localhost:5173 + +## Architecture + +``` +backend/src/main.ts - Fastify + tRPC server, Fishjam SDK, Gemini Live API +web/src/App.tsx - React frontend with Fishjam React Client +web/src/trpc.ts - tRPC client setup +``` + +### Audio flow + +``` +Peer audio (16kHz) → Fishjam Agent → Gemini Live API + ↓ + Fishjam Agent Track ← Gemini response (24kHz) + ↓ + All peers hear the agent +``` diff --git a/gemini-demo/backend/.yarnrc.yml b/gemini-demo/backend/.yarnrc.yml new file mode 100644 index 0000000..3186f3f --- /dev/null +++ b/gemini-demo/backend/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/gemini-demo/backend/Dockerfile b/gemini-demo/backend/Dockerfile new file mode 100644 index 0000000..1fb24dd --- /dev/null +++ b/gemini-demo/backend/Dockerfile @@ -0,0 +1,37 @@ +FROM node:24-alpine AS builder + +WORKDIR /app + +RUN apk add --no-cache python3 make g++ +RUN corepack enable + +COPY package.json yarn.lock .yarnrc.yml ./ +COPY backend ./backend +COPY web ./web + +RUN yarn install --immutable + +RUN yarn workspace gemini-demo-backend build + +FROM node:24-alpine AS runner + +WORKDIR /app + +RUN apk add --no-cache dumb-init python3 make g++ +RUN corepack enable + +COPY package.json yarn.lock .yarnrc.yml ./ +COPY backend ./backend +COPY web ./web + +RUN yarn install --immutable + +COPY --from=builder /app/backend/dist ./backend/dist + +RUN addgroup -g 1001 -S nodejs && adduser -S nodejs -u 1001 +USER nodejs + +EXPOSE 8000 + +ENTRYPOINT ["dumb-init", "--"] +CMD ["node", "backend/dist/main.js"] diff --git a/gemini-demo/backend/package.json b/gemini-demo/backend/package.json new file mode 100644 index 0000000..61213e5 --- /dev/null +++ b/gemini-demo/backend/package.json @@ -0,0 +1,36 @@ +{ + "name": "gemini-demo-backend", + "private": true, + "version": "0.0.0", + "type": "module", + "exports": { + ".": { + "types": "./src/main.ts" + } + }, + "types": "./src/main.ts", + "dependencies": { + "@fastify/cors": "^11.1.0", + "@fishjam-cloud/js-server-sdk": "^0.25.4", + "@google/genai": "^1.44.0", + "@trpc/server": "^11.6.0", + "dotenv": "^17.2.3", + "fastify": "^5.6.1", + "pino-pretty": "^13.1.1", + "ws": "^8.18.0", + "zod": "^4.1.11" + }, + "devDependencies": { + "@tsconfig/node24": "^24.0.1", + "@types/node": "^24.5.2", + "@types/ws": "^8.18.1", + "tsx": "^4.20.6", + "typescript": "^5.9.2" + }, + "scripts": { + "start": "tsx watch src/main.ts", + "build": "tsc -p tsconfig.json", + "typecheck": "tsc --noEmit" + }, + "packageManager": "yarn@4.12.0" +} diff --git a/gemini-demo/backend/src/agents.ts b/gemini-demo/backend/src/agents.ts new file mode 100644 index 0000000..0735330 --- /dev/null +++ b/gemini-demo/backend/src/agents.ts @@ -0,0 +1,96 @@ +import { LiveServerMessage, Modality } from "@google/genai"; +import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini"; +import { type TrackId, type RoomId } from "@fishjam-cloud/js-server-sdk"; + +import { fishjam, genai } from "./clients.js"; + +export const createAgent = async ( + roomId: RoomId, + systemInstruction: string, +) => { + const { agent: fishjamAgent } = await fishjam.createAgent(roomId, { + output: FishjamGemini.geminiInputAudioSettings, + }); + + const agentTrack = fishjamAgent.createTrack( + FishjamGemini.geminiOutputAudioSettings, + ); + + let cleanup: CallableFunction | undefined; + + const session = await genai.live.connect({ + model: "gemini-3.1-flash-live-preview", + config: { + responseModalities: [Modality.AUDIO], + systemInstruction, + tools: [ + { googleSearch: {} }, + { + functionDeclarations: [ + { + name: "disconnect", + description: `Disconnect yourself from the room. + Use this when the user asks you to disconnect.`, + }, + ], + }, + ], + }, + callbacks: { + onmessage: async (message: LiveServerMessage) => { + if (message.data) { + const audio = Buffer.from(message.data, "base64"); + fishjamAgent.sendData(agentTrack.id, audio); + } + + if (message.serverContent?.interrupted) { + fishjamAgent.interruptTrack(agentTrack.id); + } + + message.toolCall?.functionCalls?.forEach((call) => { + if (call.name === "disconnect") { + cleanup?.(); + } + }); + }, + }, + }); + + const room = await fishjam.getRoom(roomId); + + const interval = setInterval(async () => { + const humanPeerVideoTrack = room.peers + .find(({ type }) => type === "webrtc") + ?.tracks.find(({ type }) => type === "video"); + + if (!humanPeerVideoTrack?.id) return; + + const image = await fishjamAgent.captureImage( + humanPeerVideoTrack.id as TrackId, + ); + + session.sendRealtimeInput({ + video: { + data: Buffer.from(image.data).toString("base64"), + mimeType: image.contentType, + }, + }); + }, 1000); + + fishjamAgent.on("trackData", ({ data }) => { + session.sendRealtimeInput({ + audio: { + data: Buffer.from(data).toString("base64"), + mimeType: FishjamGemini.inputMimeType, + }, + }); + }); + + cleanup = () => { + clearInterval(interval); + session.close(); + fishjamAgent.deleteTrack(agentTrack.id); + fishjamAgent.removeAllListeners("trackData"); + fishjamAgent.disconnect(); + }; +}; diff --git a/gemini-demo/backend/src/clients.ts b/gemini-demo/backend/src/clients.ts new file mode 100644 index 0000000..8a2dfd8 --- /dev/null +++ b/gemini-demo/backend/src/clients.ts @@ -0,0 +1,13 @@ +import { FishjamClient } from "@fishjam-cloud/js-server-sdk"; +import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini"; + +import config from "./config.js"; + +export const fishjam = new FishjamClient({ + fishjamId: config.FISHJAM_ID, + managementToken: config.FISHJAM_MANAGEMENT_TOKEN, +}); + +export const genai = FishjamGemini.createClient({ + apiKey: config.GEMINI_API_KEY, +}); diff --git a/gemini-demo/backend/src/config.ts b/gemini-demo/backend/src/config.ts new file mode 100644 index 0000000..a451773 --- /dev/null +++ b/gemini-demo/backend/src/config.ts @@ -0,0 +1,13 @@ +import z from "zod"; +import dotenv from "dotenv"; + +dotenv.config({ path: "../.env", quiet: true }); + +export default z + .object({ + PORT: z.coerce.number().int().default(8000), + FISHJAM_ID: z.string(), + FISHJAM_MANAGEMENT_TOKEN: z.string(), + GEMINI_API_KEY: z.string(), + }) + .parse(process.env); diff --git a/gemini-demo/backend/src/main.ts b/gemini-demo/backend/src/main.ts new file mode 100644 index 0000000..09c1732 --- /dev/null +++ b/gemini-demo/backend/src/main.ts @@ -0,0 +1,40 @@ +import cors from "@fastify/cors"; + +import { + type FastifyTRPCPluginOptions, + fastifyTRPCPlugin, +} from "@trpc/server/adapters/fastify"; +import { applyWSSHandler } from "@trpc/server/adapters/ws"; +import Fastify from "fastify"; +import { WebSocketServer } from "ws"; +import { type AppRouter, appRouter } from "./router.js"; +export type { AppRouter } from "./router.js"; +import config from "./config.js"; + +const fastify = Fastify({ + logger: { transport: { target: "pino-pretty" } }, +}); + +await fastify.register(cors, { origin: true, credentials: true }); + +fastify.register(fastifyTRPCPlugin, { + prefix: "/api/v1", + trpcOptions: { + router: appRouter, + onError({ path, error }) { + fastify.log.error("tRPC error on %s: %O", path, error); + }, + } satisfies FastifyTRPCPluginOptions["trpcOptions"], +}); + +await fastify.ready(); +await fastify.listen({ port: config.PORT, host: "0.0.0.0" }); + +const wss = new WebSocketServer({ + server: fastify.server, + path: "/api/v1", +}); + +applyWSSHandler({ wss, router: appRouter }); + +fastify.log.info(`Server running on port ${config.PORT}`); diff --git a/gemini-demo/backend/src/peers.ts b/gemini-demo/backend/src/peers.ts new file mode 100644 index 0000000..6afc018 --- /dev/null +++ b/gemini-demo/backend/src/peers.ts @@ -0,0 +1,26 @@ +import type { RoomId } from "@fishjam-cloud/js-server-sdk"; +import { fishjam } from "./clients.js"; + +const roomNameToId = new Map(); + +export const getPeerToken = async (roomName: string, peerName: string) => { + const roomId = roomNameToId.get(roomName); + + const room = await (roomId ? fishjam.getRoom(roomId) : fishjam.createRoom()); + + roomNameToId.set(roomName, room.id); + + const isNameTaken = room.peers.some( + (p) => (p.metadata as { name?: string } | null)?.name === peerName, + ); + + if (isNameTaken) { + throw new Error("Peer name is already taken"); + } + + const { peer, peerToken } = await fishjam.createPeer(room.id, { + metadata: { name: peerName }, + }); + + return { roomId: room.id, peer, peerToken }; +}; diff --git a/gemini-demo/backend/src/router.ts b/gemini-demo/backend/src/router.ts new file mode 100644 index 0000000..16bf5b0 --- /dev/null +++ b/gemini-demo/backend/src/router.ts @@ -0,0 +1,22 @@ +import { initTRPC } from "@trpc/server"; +import z from "zod"; +import { getPeerToken } from "./peers.js"; +import { createAgent } from "./agents.js"; +import type { RoomId } from "@fishjam-cloud/js-server-sdk"; + +const t = initTRPC.create(); + +export const appRouter = t.router({ + getPeerToken: t.procedure + .input(z.object({ roomName: z.string().min(1), peerName: z.string() })) + .mutation(async ({ input }) => + getPeerToken(input.roomName, input.peerName), + ), + createAgent: t.procedure + .input(z.object({ roomId: z.string(), systemPrompt: z.string().min(1) })) + .mutation(async ({ input }) => + createAgent(input.roomId as RoomId, input.systemPrompt), + ), +}); + +export type AppRouter = typeof appRouter; diff --git a/gemini-demo/backend/tsconfig.json b/gemini-demo/backend/tsconfig.json new file mode 100644 index 0000000..398df1c --- /dev/null +++ b/gemini-demo/backend/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@tsconfig/node24/tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "module": "NodeNext", + "moduleResolution": "nodenext", + "declaration": false + } +} diff --git a/gemini-demo/docker-compose.yml b/gemini-demo/docker-compose.yml new file mode 100644 index 0000000..87afd34 --- /dev/null +++ b/gemini-demo/docker-compose.yml @@ -0,0 +1,35 @@ +services: + backend: + build: + context: . + dockerfile: backend/Dockerfile + expose: + - "8000" + environment: + - NODE_ENV=production + - PORT=8000 + - FISHJAM_ID=${FISHJAM_ID} + - FISHJAM_MANAGEMENT_TOKEN=${FISHJAM_MANAGEMENT_TOKEN} + - GEMINI_API_KEY=${GEMINI_API_KEY} + restart: unless-stopped + + web: + build: + context: . + dockerfile: web/Dockerfile + args: + - VITE_FISHJAM_ID=${FISHJAM_ID} + restart: unless-stopped + depends_on: + - backend + + nginx: + image: nginx:alpine + ports: + - "5001:80" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - backend + - web + restart: unless-stopped diff --git a/gemini-demo/nginx.conf b/gemini-demo/nginx.conf new file mode 100644 index 0000000..6f18ec6 --- /dev/null +++ b/gemini-demo/nginx.conf @@ -0,0 +1,77 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + gzip on; + gzip_vary on; + gzip_types text/plain text/css text/xml text/javascript + application/x-javascript application/xml+rss + application/javascript application/json; + + upstream backend { + server backend:8000; + } + + upstream web { + server web:3000; + } + + server { + listen 80; + server_name _; + client_max_body_size 100M; + + # Backend API and WebSocket + location /api { + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + + # Frontend + location / { + proxy_pass http://web; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + } + + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + } +} diff --git a/gemini-demo/package.json b/gemini-demo/package.json new file mode 100644 index 0000000..d892aba --- /dev/null +++ b/gemini-demo/package.json @@ -0,0 +1,19 @@ +{ + "name": "gemini-demo", + "packageManager": "yarn@4.12.0", + "private": true, + "workspaces": [ + "backend", + "web" + ], + "devDependencies": { + "typescript": "^5.9.2" + }, + "scripts": { + "build": "yarn workspaces foreach -A -p run build", + "typecheck": "yarn workspaces foreach -A -p run typecheck" + }, + "engines": { + "node": ">= 24.0.0" + } +} diff --git a/gemini-demo/web/Dockerfile b/gemini-demo/web/Dockerfile new file mode 100644 index 0000000..68083af --- /dev/null +++ b/gemini-demo/web/Dockerfile @@ -0,0 +1,33 @@ +FROM node:24-alpine AS builder + +WORKDIR /app + +RUN apk add --no-cache python3 make g++ +RUN corepack enable + +COPY package.json yarn.lock .yarnrc.yml ./ +COPY backend ./backend +COPY web ./web + +ARG VITE_FISHJAM_ID +ENV VITE_FISHJAM_ID=$VITE_FISHJAM_ID + +RUN yarn install --immutable + +RUN ./node_modules/.bin/vite build web + +FROM node:24-alpine + +WORKDIR /app + +RUN apk add --no-cache dumb-init + +COPY --from=builder /app/web/dist ./web/dist + +RUN addgroup -g 1001 -S nodejs && adduser -S nodejs -u 1001 +USER nodejs + +EXPOSE 3000 + +ENTRYPOINT ["dumb-init", "--"] +CMD ["npx", "serve", "-s", "web/dist", "-l", "tcp://0.0.0.0:3000"] diff --git a/gemini-demo/web/gemini.png b/gemini-demo/web/gemini.png new file mode 100644 index 0000000..223f3c6 Binary files /dev/null and b/gemini-demo/web/gemini.png differ diff --git a/gemini-demo/web/index.html b/gemini-demo/web/index.html new file mode 100644 index 0000000..d7ad052 --- /dev/null +++ b/gemini-demo/web/index.html @@ -0,0 +1,16 @@ + + + + + + Gemini Demo + + + +
+ + + diff --git a/gemini-demo/web/package.json b/gemini-demo/web/package.json new file mode 100644 index 0000000..72bf95a --- /dev/null +++ b/gemini-demo/web/package.json @@ -0,0 +1,26 @@ +{ + "name": "gemini-demo-web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "start": "vite", + "build": "tsc -b && vite build", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@fishjam-cloud/react-client": "^0.22.0", + "@trpc/client": "^11.6.0", + "@trpc/server": "^11.6.0", + "gemini-demo-backend": "workspace:*", + "react": "^19.1.1", + "react-dom": "^19.1.1" + }, + "devDependencies": { + "@types/react": "^19.1.13", + "@types/react-dom": "^19.1.9", + "@vitejs/plugin-react": "^5.0.3", + "typescript": "^5.9.2", + "vite": "^7.1.7" + } +} diff --git a/gemini-demo/web/src/App.tsx b/gemini-demo/web/src/App.tsx new file mode 100644 index 0000000..09813b8 --- /dev/null +++ b/gemini-demo/web/src/App.tsx @@ -0,0 +1,37 @@ +import { useConnection } from "@fishjam-cloud/react-client"; +import { useCallback, useState } from "react"; +import { Lobby } from "./components/Lobby"; +import { CallView } from "./components/CallView"; + +type View = "lobby" | "call"; + +export default function App() { + const [view, setView] = useState("lobby"); + const [roomId, setRoomId] = useState(null); + const [roomName, setRoomName] = useState(""); + + const { leaveRoom } = useConnection(); + + const handleLeave = useCallback(() => { + leaveRoom(); + setView("lobby"); + setRoomId(null); + setRoomName(""); + }, [leaveRoom]); + + if (view === "lobby") { + return ( + { + setView("call"); + setRoomId(roomId); + setRoomName(roomName); + }} + /> + ); + } + + return ( + + ); +} diff --git a/gemini-demo/web/src/components/AgentTile.tsx b/gemini-demo/web/src/components/AgentTile.tsx new file mode 100644 index 0000000..dc3f224 --- /dev/null +++ b/gemini-demo/web/src/components/AgentTile.tsx @@ -0,0 +1,70 @@ +import type { PeerWithTracks } from "@fishjam-cloud/react-client"; +import geminiLogo from "/gemini.png"; +import { useEffect, useRef } from "react"; + +export function AgentTile({ + agentPeer, +}: { + agentPeer: PeerWithTracks | undefined; +}) { + const agentAudioRef = useRef(null); + + useEffect(() => { + if (!agentAudioRef.current) return; + agentAudioRef.current.srcObject = agentPeer?.tracks[0]?.stream ?? null; + }, [agentPeer?.tracks[0]?.stream]); + + if (!agentPeer) return null; + return ( +
+ Gemini + +
+ ); +} + +const styles: Record = { + tile: { + position: "relative", + background: "#eee", + border: "1px solid #ddd", + borderRadius: 10, + overflow: "hidden", + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + height: 560, + }, + logo: { + width: 64, + height: 64, + }, + name: { + position: "absolute", + bottom: 8, + left: 10, + fontSize: 13, + color: "#fff", + background: "rgba(0,0,0,0.5)", + padding: "2px 8px", + borderRadius: 4, + }, +}; diff --git a/gemini-demo/web/src/components/CallView.tsx b/gemini-demo/web/src/components/CallView.tsx new file mode 100644 index 0000000..92b4958 --- /dev/null +++ b/gemini-demo/web/src/components/CallView.tsx @@ -0,0 +1,129 @@ +import { + useCamera, + useMicrophone, + usePeers, +} from "@fishjam-cloud/react-client"; +import { useMemo, useState } from "react"; +import { trpc } from "../trpc"; +import { PeerTile } from "./PeerTile"; +import { AgentTile } from "./AgentTile"; +import { Toolbar } from "./Toolbar"; +import { SystemPromptModal } from "./SystemPromptModal"; + +interface CallViewProps { + roomId: string; + roomName: string; + onLeave: () => void; +} + +export function CallView({ roomId, roomName, onLeave }: CallViewProps) { + const [showPromptModal, setShowPromptModal] = useState(false); + + const { isCameraOn, toggleCamera, cameraStream } = useCamera(); + const { isMicrophoneMuted, toggleMicrophoneMute } = useMicrophone(); + + const { remotePeers } = usePeers<{ name: string }>(); + + const agentPeer = useMemo( + () => remotePeers.find((p) => !p.metadata), + [remotePeers], + ); + const humanPeers = useMemo( + () => remotePeers.filter((p) => p.metadata), + [remotePeers], + ); + + const handleStartAgent = async (systemPrompt: string) => { + setShowPromptModal(false); + await trpc.createAgent.mutate({ roomId, systemPrompt }); + }; + + const handleLeave = async () => { + onLeave(); + }; + + return ( +
+ setShowPromptModal(true)} + onLeave={handleLeave} + /> + +
+ + + +
+ } + /> + + {humanPeers.map((peer) => ( + + ))} + + +
+ + {showPromptModal && ( + setShowPromptModal(false)} + /> + )} + + ); +} + +const styles: Record = { + container: { + display: "flex", + flexDirection: "column", + height: "100vh", + fontFamily: + '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', + background: "#fff", + }, + peerGrid: { + flex: 1, + display: "grid", + gridTemplateColumns: "repeat(auto-fill, minmax(560px, 1fr))", + gap: 32, + padding: 32, + overflow: "hidden", + alignContent: "center", + maxWidth: 1300, + width: "100%", + margin: "0 auto", + }, + controls: { + position: "absolute", + bottom: 8, + right: 10, + display: "flex", + gap: 6, + }, + controlBtn: { + padding: "4px 10px", + fontSize: 12, + background: "rgba(0,0,0,0.5)", + color: "#fff", + border: "none", + borderRadius: 4, + cursor: "pointer", + }, +}; diff --git a/gemini-demo/web/src/components/Lobby.tsx b/gemini-demo/web/src/components/Lobby.tsx new file mode 100644 index 0000000..38fd763 --- /dev/null +++ b/gemini-demo/web/src/components/Lobby.tsx @@ -0,0 +1,119 @@ +import { useState, type FC } from "react"; +import { trpc } from "../trpc"; +import { + useConnection, + useInitializeDevices, +} from "@fishjam-cloud/react-client"; + +interface LobbyProps { + onJoined: (roomId: string, roomName: string) => void; +} + +export const Lobby: FC = (props) => { + const [roomName, setRoomName] = useState(""); + const [userName, setUserName] = useState(""); + const [loading, setLoading] = useState(null); + const { joinRoom } = useConnection(); + const { initializeDevices } = useInitializeDevices(); + + const canJoin = roomName.trim() && userName.trim() && !loading; + + const handleSubmit = async () => { + setLoading("Joining..."); + try { + const trimmedPeerName = userName.trim(); + const trimmedRoomName = roomName.trim(); + + const { roomId, peerToken } = await trpc.getPeerToken.mutate({ + roomName: trimmedRoomName, + peerName: trimmedPeerName, + }); + + props.onJoined(roomId, roomName); + await initializeDevices(); + await joinRoom({ peerToken, peerMetadata: { name: trimmedPeerName } }); + } finally { + setLoading(null); + } + }; + + return ( +
+

Gemini x Fishjam Demo

+

Videoconference with a Gemini Live AI agent

+ +
+ setRoomName(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && handleSubmit()} + autoFocus + /> + setUserName(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && handleSubmit()} + /> + +
+
+ ); +}; + +const styles: Record = { + container: { + display: "flex", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + minHeight: "100vh", + fontFamily: + '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', + background: "#fff", + color: "#111", + }, + title: { + fontSize: 28, + fontWeight: 600, + marginBottom: 4, + }, + subtitle: { + fontSize: 14, + color: "#666", + marginBottom: 32, + }, + card: { + display: "flex", + flexDirection: "column", + gap: 12, + width: 320, + alignItems: "stretch", + }, + input: { + padding: "10px 12px", + fontSize: 14, + border: "1px solid #ddd", + borderRadius: 6, + outline: "none", + }, + button: { + padding: "10px 20px", + fontSize: 14, + fontWeight: 500, + background: "#111", + color: "#fff", + border: "none", + borderRadius: 6, + cursor: "pointer", + }, +}; diff --git a/gemini-demo/web/src/components/PeerTile.tsx b/gemini-demo/web/src/components/PeerTile.tsx new file mode 100644 index 0000000..6a93c15 --- /dev/null +++ b/gemini-demo/web/src/components/PeerTile.tsx @@ -0,0 +1,74 @@ +import { useEffect, useRef } from "react"; + +export function PeerTile({ + name, + stream, + audioStream, + controls, +}: { + name: string; + stream?: MediaStream | null; + audioStream?: MediaStream | null; + controls?: React.ReactNode; +}) { + const videoRef = useRef(null); + const audioRef = useRef(null); + + useEffect(() => { + if (videoRef.current) { + videoRef.current.srcObject = stream ?? null; + } + }, [stream]); + + useEffect(() => { + if (audioRef.current) { + audioRef.current.srcObject = audioStream ?? null; + } + }, [audioStream]); + + return ( +
+ {stream ? ( +
+ ); +} + +const styles: Record = { + peerTile: { + position: "relative", + background: "#f5f5f5", + borderRadius: 10, + overflow: "hidden", + display: "flex", + alignItems: "center", + justifyContent: "center", + height: 560, + }, + video: { + width: "100%", + height: "100%", + objectFit: "cover", + }, + noVideo: { + fontSize: 18, + color: "#999", + }, + peerName: { + position: "absolute", + bottom: 8, + left: 10, + fontSize: 13, + color: "#fff", + background: "rgba(0,0,0,0.5)", + padding: "2px 8px", + borderRadius: 4, + }, +}; diff --git a/gemini-demo/web/src/components/SystemPromptModal.tsx b/gemini-demo/web/src/components/SystemPromptModal.tsx new file mode 100644 index 0000000..80f3d1f --- /dev/null +++ b/gemini-demo/web/src/components/SystemPromptModal.tsx @@ -0,0 +1,104 @@ +import { useState } from "react"; + +const DEFAULT_SYSTEM_PROMPT = `You are a helpful voice assistant in a video call. +Keep your responses concise and conversational. +You can use Google Search to look up current information when asked.`; + +export function SystemPromptModal({ + onStart, + onClose, +}: { + onStart: (prompt: string) => void; + onClose: () => void; +}) { + const [prompt, setPrompt] = useState(DEFAULT_SYSTEM_PROMPT); + + return ( +
+
e.stopPropagation()}> +

Agent System Prompt

+