feat(realtime): add generating lifecycle state and event buffering

examples/sdk-core/realtime/connection-events.ts

@@ -39,6 +39,12 @@ async function main() {
       case "connected":
         console.log("Connected! Streaming active.");
         break;
+      case "generating":
+        console.log("Generation started! Frames incoming.");
+        break;
+      case "reconnecting":
+        console.log("Connection lost, reconnecting...");
+        break;
       case "disconnected":
         console.log("Disconnected from server.");
         break;

packages/sdk/src/index.ts

@@ -25,7 +25,7 @@ export type {
   RealTimeClientInitialState,
 } from "./realtime/client";
 export type { SetInput } from "./realtime/methods";
-export type { ConnectionState } from "./realtime/webrtc-connection";
+export type { ConnectionState } from "./realtime/types";
 export {
   type ImageModelDefinition,
   type ImageModels,

packages/sdk/src/realtime/client.ts

@@ -6,6 +6,7 @@ import { modelStateSchema } from "../shared/types";
 import { createWebrtcError, type DecartSDKError } from "../utils/errors";
 import { AudioStreamManager } from "./audio-stream-manager";
 import { realtimeMethods, type SetInput } from "./methods";
+import type { ConnectionState } from "./types";
 import { WebRTCManager } from "./webrtc-manager";
 
 async function blobToBase64(blob: Blob): Promise<string> {
@@ -90,15 +91,15 @@ const realTimeClientConnectOptionsSchema = z.object({
 export type RealTimeClientConnectOptions = z.infer<typeof realTimeClientConnectOptionsSchema>;
 
 export type Events = {
-  connectionChange: "connected" | "connecting" | "disconnected" | "reconnecting";
+  connectionChange: ConnectionState;
   error: DecartSDKError;
 };
 
 export type RealTimeClient = {
   set: (input: SetInput) => Promise<void>;
   setPrompt: (prompt: string, { enhance }?: { enhance?: boolean }) => Promise<void>;
   isConnected: () => boolean;
-  getConnectionState: () => "connected" | "connecting" | "disconnected" | "reconnecting";
+  getConnectionState: () => ConnectionState;
   disconnect: () => void;
   on: <K extends keyof Events>(event: K, listener: (data: Events[K]) => void) => void;
   off: <K extends keyof Events>(event: K, listener: (data: Events[K]) => void) => void;
@@ -167,16 +168,38 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
           : undefined;
 
       const url = `${baseUrl}${options.model.urlPath}`;
+
+      const eventBuffer: Array<{ event: keyof Events; data: Events[keyof Events] }> = [];
+      let buffering = true;
+
+      const emitOrBuffer = <K extends keyof Events>(event: K, data: Events[K]) => {
+        if (buffering) {
+          eventBuffer.push({ event, data: data as Events[keyof Events] });
+        } else {
+          eventEmitter.emit(event, data);
+        }
+      };
+
+      const flushBufferedEvents = () => {
+        setTimeout(() => {
+          buffering = false;
+          for (const { event, data } of eventBuffer) {
+            (eventEmitter.emit as (type: keyof Events, data: Events[keyof Events]) => void)(event, data);
+          }
+          eventBuffer.length = 0;
+        }, 0);
+      };
+
       webrtcManager = new WebRTCManager({
         webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}`,
         integration,
         onRemoteStream,
         onConnectionStateChange: (state) => {
-          eventEmitter.emit("connectionChange", state);
+          emitOrBuffer("connectionChange", state);
         },
         onError: (error) => {
           console.error("WebRTC error:", error);
-          eventEmitter.emit("error", createWebrtcError(error));
+          emitOrBuffer("error", createWebrtcError(error));
         },
         customizeOffer: options.customizeOffer as ((offer: RTCSessionDescriptionInit) => Promise<void>) | undefined,
         vp8MinBitrate: 300,
@@ -203,6 +226,8 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         isConnected: () => manager.isConnected(),
         getConnectionState: () => manager.getConnectionState(),
         disconnect: () => {
+          buffering = false;
+          eventBuffer.length = 0;
           manager.cleanup();
           audioStreamManager?.cleanup();
         },
@@ -227,6 +252,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
         client.playAudio = (audio: Blob | File | ArrayBuffer) => manager.playAudio(audio);
       }
 
+      flushBufferedEvents();
       return client;
     } catch (error) {
       webrtcManager?.cleanup();

packages/sdk/src/realtime/methods.ts

@@ -28,7 +28,7 @@ export const realtimeMethods = (
 ) => {
   const assertConnected = () => {
     const state = webrtcManager.getConnectionState();
-    if (state !== "connected") {
+    if (state !== "connected" && state !== "generating") {
       throw new Error(`Cannot send message: connection is ${state}`);
     }
   };

packages/sdk/src/realtime/types.ts

@@ -60,6 +60,12 @@ export type SetImageAckMessage = {
   error: null | string;
 };
 
+export type GenerationStartedMessage = {
+  type: "generation_started";
+};
+
+export type ConnectionState = "connecting" | "connected" | "generating" | "disconnected" | "reconnecting";
+
 // Incoming message types (from server)
 export type IncomingWebRTCMessage =
   | ReadyMessage
@@ -69,7 +75,8 @@ export type IncomingWebRTCMessage =
   | IceRestartMessage
   | PromptAckMessage
   | ErrorMessage
-  | SetImageAckMessage;
+  | SetImageAckMessage
+  | GenerationStartedMessage;
 
 // Outgoing message types (to server)
 export type OutgoingWebRTCMessage =

packages/sdk/src/realtime/webrtc-connection.ts

@@ -1,6 +1,7 @@
 import mitt from "mitt";
 import { buildUserAgent } from "../utils/user-agent";
 import type {
+  ConnectionState,
   IncomingWebRTCMessage,
   OutgoingWebRTCMessage,
   PromptAckMessage,
@@ -23,8 +24,6 @@ interface ConnectionCallbacks {
   initialPrompt?: { text: string; enhance?: boolean };
 }
 
-export type ConnectionState = "connecting" | "connected" | "disconnected" | "reconnecting";
-
 type WsMessageEvents = {
   promptAck: PromptAckMessage;
   setImageAck: SetImageAckMessage;
@@ -107,7 +106,7 @@ export class WebRTCConnection {
       await Promise.race([
         new Promise<void>((resolve, reject) => {
           const checkConnection = setInterval(() => {
-            if (this.state === "connected") {
+            if (this.state === "connected" || this.state === "generating") {
               clearInterval(checkConnection);
               resolve();
             } else if (this.state === "disconnected") {
@@ -151,6 +150,11 @@ export class WebRTCConnection {
         return;
       }
 
+      if (msg.type === "generation_started") {
+        this.setState("generating");
+        return;
+      }
+
       // All other messages require peer connection
       if (!this.pc) return;
 
@@ -340,9 +344,11 @@ export class WebRTCConnection {
     this.pc.onconnectionstatechange = () => {
       if (!this.pc) return;
       const s = this.pc.connectionState;
-      this.setState(
-        s === "connected" ? "connected" : ["connecting", "new"].includes(s) ? "connecting" : "disconnected",
-      );
+      const nextState =
+        s === "connected" ? "connected" : ["connecting", "new"].includes(s) ? "connecting" : "disconnected";
+      // Keep "generating" sticky unless the connection is actually lost.
+      if (this.state === "generating" && nextState !== "disconnected") return;
+      this.setState(nextState);
     };
 
     this.pc.oniceconnectionstatechange = () => {

packages/sdk/src/realtime/webrtc-manager.ts

@@ -1,6 +1,6 @@
 import pRetry, { AbortError } from "p-retry";
-import type { OutgoingMessage } from "./types";
-import { type ConnectionState, WebRTCConnection } from "./webrtc-connection";
+import type { ConnectionState, OutgoingMessage } from "./types";
+import { WebRTCConnection } from "./webrtc-connection";
 
 export interface WebRTCConfig {
   webrtcUrl: string;
@@ -62,7 +62,7 @@ export class WebRTCManager {
   private emitState(state: ConnectionState): void {
     if (this.managerState !== state) {
       this.managerState = state;
-      if (state === "connected") this.hasConnected = true;
+      if (state === "connected" || state === "generating") this.hasConnected = true;
       this.config.onConnectionStateChange?.(state);
     }
   }
@@ -75,12 +75,10 @@ export class WebRTCManager {
 
     // During reconnection, intercept state changes from the connection layer
     if (this.isReconnecting) {
-      if (state === "connected") {
-        // Reconnection succeeded
+      if (state === "connected" || state === "generating") {
         this.isReconnecting = false;
-        this.emitState("connected");
+        this.emitState(state);
       }
-      // Swallow other states during reconnection (connecting, disconnected)
       return;
     }
 
@@ -197,7 +195,7 @@ export class WebRTCManager {
   }
 
   isConnected(): boolean {
-    return this.managerState === "connected";
+    return this.managerState === "connected" || this.managerState === "generating";
   }
 
   getConnectionState(): ConnectionState {