Add gemini demo

gemini-demo/.dockerignore

@@ -0,0 +1,7 @@
+node_modules/
+dist/
+.env
+*.log
+.DS_Store
+backend/yarn.lock
+web/yarn.lock

gemini-demo/.env.example

@@ -0,0 +1,4 @@
+FISHJAM_ID=
+FISHJAM_MANAGEMENT_TOKEN=
+GEMINI_API_KEY=
+VITE_FISHJAM_ID=

gemini-demo/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+.yarn
+dist/
+.env
+*.log
+.DS_Store

gemini-demo/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

gemini-demo/README.md

@@ -0,0 +1,57 @@
+# Gemini Demo
+
+A minimal example of a video call with a Gemini Live AI agent using Fishjam Cloud.
+
+## What it does
+
+- Create a room and join a video call
+- Spawn a Gemini Live voice agent with a custom system prompt
+- The agent joins the call, listens to participants, and responds with voice
+- Supports Google Search for real-time information
+
+## Setup
+
+1. Copy `.env.example` to `.env` and fill in your credentials:
+
+```
+cp .env.example .env
+```
+
+2. Install dependencies:
+
+```
+cd backend && npm install
+cd ../web && npm install
+```
+
+3. Start the backend:
+
+```
+cd backend && npm run start
+```
+
+4. Start the frontend (in another terminal):
+
+```
+cd web && npm run start
+```
+
+5. Open http://localhost:5173
+
+## Architecture
+
+```
+backend/src/main.ts   - Fastify + tRPC server, Fishjam SDK, Gemini Live API
+web/src/App.tsx        - React frontend with Fishjam React Client
+web/src/trpc.ts        - tRPC client setup
+```
+
+### Audio flow
+
+```
+Peer audio (16kHz) → Fishjam Agent → Gemini Live API
+                                          ↓
+                    Fishjam Agent Track ← Gemini response (24kHz)
+                          ↓
+                    All peers hear the agent
+```

gemini-demo/backend/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

gemini-demo/backend/Dockerfile

@@ -0,0 +1,37 @@
+FROM node:24-alpine AS builder
+
+WORKDIR /app
+
+RUN apk add --no-cache python3 make g++
+RUN corepack enable
+
+COPY package.json yarn.lock .yarnrc.yml ./
+COPY backend ./backend
+COPY web ./web
+
+RUN yarn install --immutable
+
+RUN yarn workspace gemini-demo-backend build
+
+FROM node:24-alpine AS runner
+
+WORKDIR /app
+
+RUN apk add --no-cache dumb-init python3 make g++
+RUN corepack enable
+
+COPY package.json yarn.lock .yarnrc.yml ./
+COPY backend ./backend
+COPY web ./web
+
+RUN yarn install --immutable
+
+COPY --from=builder /app/backend/dist ./backend/dist
+
+RUN addgroup -g 1001 -S nodejs && adduser -S nodejs -u 1001
+USER nodejs
+
+EXPOSE 8000
+
+ENTRYPOINT ["dumb-init", "--"]
+CMD ["node", "backend/dist/main.js"]

gemini-demo/backend/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "gemini-demo-backend",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/main.ts"
+    }
+  },
+  "types": "./src/main.ts",
+  "dependencies": {
+    "@fastify/cors": "^11.1.0",
+    "@fishjam-cloud/js-server-sdk": "^0.25.4",
+    "@google/genai": "^1.44.0",
+    "@trpc/server": "^11.6.0",
+    "dotenv": "^17.2.3",
+    "fastify": "^5.6.1",
+    "pino-pretty": "^13.1.1",
+    "ws": "^8.18.0",
+    "zod": "^4.1.11"
+  },
+  "devDependencies": {
+    "@tsconfig/node24": "^24.0.1",
+    "@types/node": "^24.5.2",
+    "@types/ws": "^8.18.1",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.2"
+  },
+  "scripts": {
+    "start": "tsx watch src/main.ts",
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit"
+  },
+  "packageManager": "yarn@4.12.0"
+}

gemini-demo/backend/src/agents.ts

@@ -0,0 +1,96 @@
+import { LiveServerMessage, Modality } from "@google/genai";
+import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini";
+import { type TrackId, type RoomId } from "@fishjam-cloud/js-server-sdk";
+
+import { fishjam, genai } from "./clients.js";
+
+export const createAgent = async (
+  roomId: RoomId,
+  systemInstruction: string,
+) => {
+  const { agent: fishjamAgent } = await fishjam.createAgent(roomId, {
+    output: FishjamGemini.geminiInputAudioSettings,
+  });
+
+  const agentTrack = fishjamAgent.createTrack(
+    FishjamGemini.geminiOutputAudioSettings,
+  );
+
+  let cleanup: CallableFunction | undefined;
+
+  const session = await genai.live.connect({
+    model: "gemini-3.1-flash-live-preview",
+    config: {
+      responseModalities: [Modality.AUDIO],
+      systemInstruction,
+      tools: [
+        { googleSearch: {} },
+        {
+          functionDeclarations: [
+            {
+              name: "disconnect",
+              description: `Disconnect yourself from the room. 
+                            Use this when the user asks you to disconnect.`,
+            },
+          ],
+        },
+      ],
+    },
+    callbacks: {
+      onmessage: async (message: LiveServerMessage) => {
+        if (message.data) {
+          const audio = Buffer.from(message.data, "base64");
+          fishjamAgent.sendData(agentTrack.id, audio);
+        }
+
+        if (message.serverContent?.interrupted) {
+          fishjamAgent.interruptTrack(agentTrack.id);
+        }
+
+        message.toolCall?.functionCalls?.forEach((call) => {
+          if (call.name === "disconnect") {
+            cleanup?.();
+          }
+        });
+      },
+    },
+  });
+
+  const room = await fishjam.getRoom(roomId);
+
+  const interval = setInterval(async () => {
+    const humanPeerVideoTrack = room.peers
+      .find(({ type }) => type === "webrtc")
+      ?.tracks.find(({ type }) => type === "video");
+
+    if (!humanPeerVideoTrack?.id) return;
+
+    const image = await fishjamAgent.captureImage(
+      humanPeerVideoTrack.id as TrackId,
+    );
+
+    session.sendRealtimeInput({
+      video: {
+        data: Buffer.from(image.data).toString("base64"),
+        mimeType: image.contentType,
+      },
+    });
+  }, 1000);
+
+  fishjamAgent.on("trackData", ({ data }) => {
+    session.sendRealtimeInput({
+      audio: {
+        data: Buffer.from(data).toString("base64"),
+        mimeType: FishjamGemini.inputMimeType,
+      },
+    });
+  });
+
+  cleanup = () => {
+    clearInterval(interval);
+    session.close();
+    fishjamAgent.deleteTrack(agentTrack.id);
+    fishjamAgent.removeAllListeners("trackData");
+    fishjamAgent.disconnect();
+  };
+};

gemini-demo/backend/src/clients.ts

@@ -0,0 +1,13 @@
+import { FishjamClient } from "@fishjam-cloud/js-server-sdk";
+import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini";
+
+import config from "./config.js";
+
+export const fishjam = new FishjamClient({
+  fishjamId: config.FISHJAM_ID,
+  managementToken: config.FISHJAM_MANAGEMENT_TOKEN,
+});
+
+export const genai = FishjamGemini.createClient({
+  apiKey: config.GEMINI_API_KEY,
+});

gemini-demo/backend/src/config.ts

@@ -0,0 +1,13 @@
+import z from "zod";
+import dotenv from "dotenv";
+
+dotenv.config({ path: "../.env", quiet: true });
+
+export default z
+  .object({
+    PORT: z.coerce.number().int().default(8000),
+    FISHJAM_ID: z.string(),
+    FISHJAM_MANAGEMENT_TOKEN: z.string(),
+    GEMINI_API_KEY: z.string(),
+  })
+  .parse(process.env);

gemini-demo/backend/src/main.ts

@@ -0,0 +1,40 @@
+import cors from "@fastify/cors";
+
+import {
+  type FastifyTRPCPluginOptions,
+  fastifyTRPCPlugin,
+} from "@trpc/server/adapters/fastify";
+import { applyWSSHandler } from "@trpc/server/adapters/ws";
+import Fastify from "fastify";
+import { WebSocketServer } from "ws";
+import { type AppRouter, appRouter } from "./router.js";
+export type { AppRouter } from "./router.js";
+import config from "./config.js";
+
+const fastify = Fastify({
+  logger: { transport: { target: "pino-pretty" } },
+});
+
+await fastify.register(cors, { origin: true, credentials: true });
+
+fastify.register(fastifyTRPCPlugin, {
+  prefix: "/api/v1",
+  trpcOptions: {
+    router: appRouter,
+    onError({ path, error }) {
+      fastify.log.error("tRPC error on %s: %O", path, error);
+    },
+  } satisfies FastifyTRPCPluginOptions<AppRouter>["trpcOptions"],
+});
+
+await fastify.ready();
+await fastify.listen({ port: config.PORT, host: "0.0.0.0" });
+
+const wss = new WebSocketServer({
+  server: fastify.server,
+  path: "/api/v1",
+});
+
+applyWSSHandler({ wss, router: appRouter });
+
+fastify.log.info(`Server running on port ${config.PORT}`);

gemini-demo/backend/src/peers.ts

@@ -0,0 +1,26 @@
+import type { RoomId } from "@fishjam-cloud/js-server-sdk";
+import { fishjam } from "./clients.js";
+
+const roomNameToId = new Map<string, RoomId>();
+
+export const getPeerToken = async (roomName: string, peerName: string) => {
+  const roomId = roomNameToId.get(roomName);
+
+  const room = await (roomId ? fishjam.getRoom(roomId) : fishjam.createRoom());
+
+  roomNameToId.set(roomName, room.id);
+
+  const isNameTaken = room.peers.some(
+    (p) => (p.metadata as { name?: string } | null)?.name === peerName,
+  );
+
+  if (isNameTaken) {
+    throw new Error("Peer name is already taken");
+  }
+
+  const { peer, peerToken } = await fishjam.createPeer(room.id, {
+    metadata: { name: peerName },
+  });
+
+  return { roomId: room.id, peer, peerToken };
+};

gemini-demo/backend/src/router.ts

@@ -0,0 +1,22 @@
+import { initTRPC } from "@trpc/server";
+import z from "zod";
+import { getPeerToken } from "./peers.js";
+import { createAgent } from "./agents.js";
+import type { RoomId } from "@fishjam-cloud/js-server-sdk";
+
+const t = initTRPC.create();
+
+export const appRouter = t.router({
+  getPeerToken: t.procedure
+    .input(z.object({ roomName: z.string().min(1), peerName: z.string() }))
+    .mutation(async ({ input }) =>
+      getPeerToken(input.roomName, input.peerName),
+    ),
+  createAgent: t.procedure
+    .input(z.object({ roomId: z.string(), systemPrompt: z.string().min(1) }))
+    .mutation(async ({ input }) =>
+      createAgent(input.roomId as RoomId, input.systemPrompt),
+    ),
+});
+
+export type AppRouter = typeof appRouter;

gemini-demo/backend/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "@tsconfig/node24/tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "module": "NodeNext",
+    "moduleResolution": "nodenext",
+    "declaration": false
+  }
+}