From 6b8b87a82b1f73c0b92541ae7b401e8bcfef315c Mon Sep 17 00:00:00 2001 From: Yehoav Rabinovich Date: Mon, 26 Jan 2026 14:29:29 +0200 Subject: [PATCH 1/4] fix(realtime): increase connection timeout in WebRTCManager to 5 minutes Updated the connection timeout from 60 seconds to 300 seconds to improve reliability during connection attempts. --- packages/sdk/src/realtime/webrtc-manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/src/realtime/webrtc-manager.ts b/packages/sdk/src/realtime/webrtc-manager.ts index 026269d..5be7616 100644 --- a/packages/sdk/src/realtime/webrtc-manager.ts +++ b/packages/sdk/src/realtime/webrtc-manager.ts @@ -52,7 +52,7 @@ export class WebRTCManager { async connect(localStream: MediaStream): Promise { return pRetry( async () => { - await this.connection.connect(this.config.webrtcUrl, localStream, 60000, this.config.integration); + await this.connection.connect(this.config.webrtcUrl, localStream, 300000, this.config.integration); return true; }, { From be92f338a816468ca7ea5db05d4595989ab89ecc Mon Sep 17 00:00:00 2001 From: Yehoav Rabinovich Date: Mon, 26 Jan 2026 14:43:07 +0200 Subject: [PATCH 2/4] feat(realtime): add status and queue position events to WebRTC connection Introduced new event types for status and queue position in the RealTimeClient and WebRTCConnection. Updated message types to handle incoming status and queue position messages, enhancing real-time feedback capabilities. --- packages/sdk/src/realtime/client.ts | 11 +++++++++++ packages/sdk/src/realtime/types.ts | 16 +++++++++++++++- packages/sdk/src/realtime/webrtc-connection.ts | 14 ++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index f7220ce..3e9df76 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -56,6 +56,8 @@ export type RealTimeClientConnectOptions = z.infer { initialPrompt, }); + // Wire up queue status events + const wsEmitter = webrtcManager.getWebsocketMessageEmitter(); + wsEmitter.on("status", (msg) => { + eventEmitter.emit("status", msg.status); + }); + wsEmitter.on("queuePosition", (msg) => { + eventEmitter.emit("queuePosition", { position: msg.position, queueSize: msg.queue_size }); + }); + await webrtcManager.connect(inputStream); const methods = realtimeMethods(webrtcManager); diff --git a/packages/sdk/src/realtime/types.ts b/packages/sdk/src/realtime/types.ts index bd07c2c..2df6a18 100644 --- a/packages/sdk/src/realtime/types.ts +++ b/packages/sdk/src/realtime/types.ts @@ -60,6 +60,18 @@ export type SetImageAckMessage = { error: null | string; }; +// Queue message types +export type StatusMessage = { + type: "status"; + status: string; +}; + +export type QueuePositionMessage = { + type: "queue_position"; + position: number; + queue_size: number; +}; + // Incoming message types (from server) export type IncomingWebRTCMessage = | ReadyMessage @@ -69,7 +81,9 @@ export type IncomingWebRTCMessage = | IceRestartMessage | PromptAckMessage | ErrorMessage - | SetImageAckMessage; + | SetImageAckMessage + | StatusMessage + | QueuePositionMessage; // Outgoing message types (to server) export type OutgoingWebRTCMessage = diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index 107923a..99fc73b 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -4,7 +4,9 @@ import type { IncomingWebRTCMessage, OutgoingWebRTCMessage, PromptAckMessage, + QueuePositionMessage, SetImageAckMessage, + StatusMessage, TurnConfig, } from "./types"; @@ -28,6 +30,8 @@ export type ConnectionState = "connecting" | "connected" | "disconnected"; type WsMessageEvents = { promptAck: PromptAckMessage; setImageAck: SetImageAckMessage; + status: StatusMessage; + queuePosition: QueuePositionMessage; }; export class WebRTCConnection { @@ -122,6 +126,16 @@ export class WebRTCConnection { return; } + if (msg.type === "status") { + this.websocketMessagesEmitter.emit("status", msg); + return; + } + + if (msg.type === "queue_position") { + this.websocketMessagesEmitter.emit("queuePosition", msg); + return; + } + // All other messages require peer connection if (!this.pc) return; From a81e638f6af180ad5c0cb6998793d24a60bdb85c Mon Sep 17 00:00:00 2001 From: Yehoav Rabinovich Date: Mon, 26 Jan 2026 14:59:27 +0200 Subject: [PATCH 3/4] feat(realtime): add optional onStatus and onQueuePosition callbacks to RealTimeClient Enhanced the RealTimeClient by introducing optional callback functions for handling status updates and queue position events. This allows users to respond to real-time changes more effectively. --- packages/sdk/src/realtime/client.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 3e9df76..cf0748d 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -42,6 +42,9 @@ const avatarOptionsSchema = z.object({ }); export type AvatarOptions = z.infer; +type OnStatusFn = (status: string) => void; +type OnQueuePositionFn = (data: { position: number; queueSize: number }) => void; + const realTimeClientConnectOptionsSchema = z.object({ model: modelDefinitionSchema, onRemoteStream: z.custom((val) => typeof val === "function", { @@ -50,6 +53,16 @@ const realTimeClientConnectOptionsSchema = z.object({ initialState: realTimeClientInitialStateSchema.optional(), customizeOffer: createAsyncFunctionSchema(z.function()).optional(), avatar: avatarOptionsSchema.optional(), + onStatus: z + .custom((val) => typeof val === "function", { + message: "onStatus must be a function", + }) + .optional(), + onQueuePosition: z + .custom((val) => typeof val === "function", { + message: "onQueuePosition must be a function", + }) + .optional(), }); export type RealTimeClientConnectOptions = z.infer; @@ -148,13 +161,16 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialPrompt, }); - // Wire up queue status events + // Wire up queue status events (called before connect so we don't miss early messages) const wsEmitter = webrtcManager.getWebsocketMessageEmitter(); wsEmitter.on("status", (msg) => { eventEmitter.emit("status", msg.status); + options.onStatus?.(msg.status); }); wsEmitter.on("queuePosition", (msg) => { - eventEmitter.emit("queuePosition", { position: msg.position, queueSize: msg.queue_size }); + const data = { position: msg.position, queueSize: msg.queue_size }; + eventEmitter.emit("queuePosition", data); + options.onQueuePosition?.(data); }); await webrtcManager.connect(inputStream); From 7cc1d1ba41aa107930c2dc260c6a9f59f1ca5874 Mon Sep 17 00:00:00 2001 From: Dan Shemesh Date: Tue, 27 Jan 2026 00:30:14 +0200 Subject: [PATCH 4/4] add an optional timeout parameter to setImage (#72) > [!NOTE] > Adds configurable timeout support for sending reference images in realtime flows. > > - Extends `setImage` (client and manager) and `setImageBase64` (connection) to accept `options.timeout` with default `15000ms` > - Uses `options.timeout ?? AVATAR_SETUP_TIMEOUT_MS` for `set_image` ack waiting; preserves existing behavior when not provided > - Adds unit tests verifying custom and default timeout behavior; updates test imports to use `vi` timers > > Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit 014310a6f8aae84e54f128e631d6abb2f57d0532. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot). --- packages/sdk/src/realtime/client.ts | 10 ++- .../sdk/src/realtime/webrtc-connection.ts | 7 ++- packages/sdk/src/realtime/webrtc-manager.ts | 5 +- packages/sdk/tests/unit.test.ts | 63 ++++++++++++++++++- 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index cf0748d..6755cba 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -81,7 +81,10 @@ export type RealTimeClient = { on: (event: K, listener: (data: Events[K]) => void) => void; off: (event: K, listener: (data: Events[K]) => void) => void; sessionId: string; - setImage: (image: Blob | File | string | null, options?: { prompt?: string; enhance?: boolean }) => Promise; + setImage: ( + image: Blob | File | string | null, + options?: { prompt?: string; enhance?: boolean; timeout?: number }, + ) => Promise; // live_avatar audio method (only available when model is live_avatar and no stream is provided) playAudio?: (audio: Blob | File | ArrayBuffer) => Promise; }; @@ -194,7 +197,10 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { on: eventEmitter.on, off: eventEmitter.off, sessionId, - setImage: async (image: Blob | File | string | null, options?: { prompt?: string; enhance?: boolean }) => { + setImage: async ( + image: Blob | File | string | null, + options?: { prompt?: string; enhance?: boolean; timeout?: number }, + ) => { if (image === null) { return webrtcManager.setImage(null, options); } diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index 99fc73b..0f22379 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -195,12 +195,15 @@ export class WebRTCConnection { * Pass null to clear the reference image or use a placeholder. * Optionally include a prompt to send with the image. */ - async setImageBase64(imageBase64: string | null, options?: { prompt?: string; enhance?: boolean }): Promise { + async setImageBase64( + imageBase64: string | null, + options?: { prompt?: string; enhance?: boolean; timeout?: number }, + ): Promise { return new Promise((resolve, reject) => { const timeoutId = setTimeout(() => { this.websocketMessagesEmitter.off("setImageAck", listener); reject(new Error("Image send timed out")); - }, AVATAR_SETUP_TIMEOUT_MS); + }, options?.timeout ?? AVATAR_SETUP_TIMEOUT_MS); const listener = (msg: SetImageAckMessage) => { clearTimeout(timeoutId); diff --git a/packages/sdk/src/realtime/webrtc-manager.ts b/packages/sdk/src/realtime/webrtc-manager.ts index 5be7616..894a4b9 100644 --- a/packages/sdk/src/realtime/webrtc-manager.ts +++ b/packages/sdk/src/realtime/webrtc-manager.ts @@ -92,7 +92,10 @@ export class WebRTCManager { return this.connection.websocketMessagesEmitter; } - setImage(imageBase64: string | null, options?: { prompt?: string; enhance?: boolean }): Promise { + setImage( + imageBase64: string | null, + options?: { prompt?: string; enhance?: boolean; timeout?: number }, + ): Promise { return this.connection.setImageBase64(imageBase64, options); } } diff --git a/packages/sdk/tests/unit.test.ts b/packages/sdk/tests/unit.test.ts index c79fd3b..a35e431 100644 --- a/packages/sdk/tests/unit.test.ts +++ b/packages/sdk/tests/unit.test.ts @@ -1,6 +1,6 @@ import { HttpResponse, http } from "msw"; import { setupServer } from "msw/node"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { createDecartClient, models } from "../src/index.js"; const MOCK_RESPONSE_DATA = new Uint8Array([0x00, 0x01, 0x02]).buffer; @@ -943,6 +943,67 @@ describe("Lucy 14b realtime", () => { }); }); +describe("WebRTCConnection", () => { + describe("setImageBase64 timeout", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("uses custom timeout when provided", async () => { + const { WebRTCConnection } = await import("../src/realtime/webrtc-connection.js"); + const connection = new WebRTCConnection(); + + const customTimeout = 5000; + let rejected = false; + let rejectionError: Error | null = null; + + const promise = connection.setImageBase64("base64data", { timeout: customTimeout }).catch((err) => { + rejected = true; + rejectionError = err; + }); + + // Advance time to just before the custom timeout - should not have rejected yet + await vi.advanceTimersByTimeAsync(customTimeout - 1); + expect(rejected).toBe(false); + + // Advance past the custom timeout - now it should reject + await vi.advanceTimersByTimeAsync(2); + await promise; + + expect(rejected).toBe(true); + expect(rejectionError?.message).toBe("Image send timed out"); + }); + + it("uses default timeout (15000ms) when not provided", async () => { + const { WebRTCConnection } = await import("../src/realtime/webrtc-connection.js"); + const connection = new WebRTCConnection(); + + let rejected = false; + let rejectionError: Error | null = null; + + const promise = connection.setImageBase64("base64data").catch((err) => { + rejected = true; + rejectionError = err; + }); + + // Advance to just before the default timeout (15000ms) - should not reject yet + await vi.advanceTimersByTimeAsync(14999); + expect(rejected).toBe(false); + + // Now advance past the default timeout + await vi.advanceTimersByTimeAsync(2); + await promise; + + expect(rejected).toBe(true); + expect(rejectionError?.message).toBe("Image send timed out"); + }); + }); +}); + describe("live_avatar Model", () => { describe("Model Definition", () => { it("has correct model name", () => {