From 8123a027082315c16ebe55ce1b0cdff4bc2e95c5 Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Tue, 20 Jan 2026 13:53:43 +0200 Subject: [PATCH 1/6] Handle loader and interrupt --- .../agent-manager/connect-to-manager.ts | 3 ++ src/services/agent-manager/index.ts | 46 +++++++++++-------- .../streaming-manager/livekit-manager.ts | 25 +++++++++- src/types/stream/stream.ts | 3 ++ 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index d2e0a9ee..a2c6e4af 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -16,6 +16,7 @@ import { ConnectionState, CreateSessionV2Options, CreateStreamOptions, + Interrupt, StreamEvents, StreamType, StreamingState, @@ -165,6 +166,8 @@ type ConnectToManagerOptions = AgentManagerOptions & { onVideoIdChange?: (videoId: string | null) => void; /** Internal callback for livekit-manager data channel events */ onMessage?: ChatProgressCallback; + /** Internal callback for when interrupt is detected by streaming manager */ + onInterruptDetected?: (interrupt: Interrupt) => void; }; chatId?: string; }; diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index cb4b72f9..b19d5349 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -97,6 +97,26 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt videoId = newVideoId; }; + const interrupt = ({ type }: Interrupt) => { + const lastMessage = items.messages[items.messages.length - 1]; + + analytics.track('agent-video-interrupt', { + type: type || 'click', + video_duration_to_interrupt: interruptTimestampTracker.get(true), + message_duration_to_interrupt: latencyTimestampTracker.get(true), + }); + + lastMessage.interrupted = true; + options.callbacks.onNewMessage?.([...items.messages], 'answer'); + + if (isStreamsV2) { + sendInterruptV2(items.streamingManager! as StreamingManager); + } else { + validateInterrupt(items.streamingManager, items.streamingManager?.streamType, videoId); + sendInterrupt(items.streamingManager!, videoId!); + } + }; + const loadedTimestamp = Date.now(); defer(() => { analytics.track('agent-sdk', { event: 'loaded', ...getAnalyticsInfo(agentEntity) }, loadedTimestamp); @@ -129,7 +149,12 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt { ...options, mode, - callbacks: { ...options.callbacks, onVideoIdChange: updateVideoId, onMessage }, + callbacks: { + ...options.callbacks, + onVideoIdChange: updateVideoId, + onMessage, + onInterruptDetected: interrupt, + }, }, agentsApi, analytics, @@ -506,23 +531,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, }); }, - async interrupt({ type }: Interrupt) { - const lastMessage = items.messages[items.messages.length - 1]; - - analytics.track('agent-video-interrupt', { - type: type || 'click', - video_duration_to_interrupt: interruptTimestampTracker.get(true), - message_duration_to_interrupt: latencyTimestampTracker.get(true), - }); - - lastMessage.interrupted = true; - options.callbacks.onNewMessage?.([...items.messages], 'answer'); - if (isStreamsV2) { - sendInterruptV2(items.streamingManager! as StreamingManager); - } else { - validateInterrupt(items.streamingManager, items.streamingManager?.streamType, videoId); - sendInterrupt(items.streamingManager!, videoId!); - } - }, + interrupt, }; } diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 7057388f..92645d73 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -102,6 +102,7 @@ export async function createLiveKitStreamingManager | null = null; const TRACK_SUBSCRIPTION_TIMEOUT_MS = 20000; + let currentActivityState: AgentActivityState = AgentActivityState.Idle; const streamApi = createStreamApiV2(auth, baseURL || didApiUrl, agentId, callbacks.onError); let sessionId: string | undefined; @@ -216,11 +217,22 @@ export async function createLiveKitStreamingManager speaker.isLocal); + const isRemoteParticipantSpeaking = activeSpeakers.find(speaker => !speaker.isLocal); + + if (isLocalParticipantSpeaking) { + callbacks.onAgentActivityStateChange?.(AgentActivityState.Idle); + + if (currentActivityState !== AgentActivityState.Idle) { + callbacks.onInterruptDetected?.({ type: 'audio' }); + currentActivityState = AgentActivityState.Idle; + } + } else if (isRemoteParticipantSpeaking) { + currentActivityState = AgentActivityState.Talking; callbacks.onAgentActivityStateChange?.(AgentActivityState.Talking); } else { callbacks.onAgentActivityStateChange?.(AgentActivityState.Idle); + currentActivityState = AgentActivityState.Idle; } } @@ -310,6 +322,11 @@ export async function createLiveKitStreamingManager { + callbacks.onAgentActivityStateChange?.(AgentActivityState.Loading); + }); } } catch (e) { log('Failed to parse data channel message:', e); @@ -481,6 +498,10 @@ export async function createLiveKitStreamingManager void; onStreamCreated?: (stream: { stream_id: string; session_id: string; agent_id: string }) => void; onStreamReady?: () => void; + onInterruptDetected?: (interrupt: Interrupt) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; From f0cfca73cebe8a0ec5a0fe2e4969caed298e0419 Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Tue, 20 Jan 2026 14:21:02 +0200 Subject: [PATCH 2/6] interrupt shouldnt be async --- src/services/agent-manager/index.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/agent-manager/index.test.ts b/src/services/agent-manager/index.test.ts index 14614e56..3a4ab324 100644 --- a/src/services/agent-manager/index.test.ts +++ b/src/services/agent-manager/index.test.ts @@ -518,7 +518,7 @@ describe('createAgentManager', () => { // Add a message to interrupt await manager.chat('Hello'); - await manager.interrupt({ type: 'click' }); + manager.interrupt({ type: 'click' }); expect(validateInterrupt).toHaveBeenCalledWith(mockStreamingManager, StreamType.Legacy, null); expect(sendInterrupt).toHaveBeenCalledWith(mockStreamingManager, null); @@ -538,7 +538,7 @@ describe('createAgentManager', () => { throw new Error('Interrupt validation failed'); }); - await expect(manager.interrupt({ type: 'click' })).rejects.toThrow('Interrupt validation failed'); + expect(() => manager.interrupt({ type: 'click' })).toThrow('Interrupt validation failed'); // Verify validateInterrupt was called expect(validateInterrupt).toHaveBeenCalledWith(mockStreamingManager, StreamType.Legacy, null); From ace32e9efa3dd8fc03741c4154e958ae53f766ff Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Tue, 20 Jan 2026 14:33:00 +0200 Subject: [PATCH 3/6] fix comment --- src/services/streaming-manager/livekit-manager.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 92645d73..4cf7e5bc 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -498,10 +498,7 @@ export async function createLiveKitStreamingManager Date: Tue, 20 Jan 2026 14:39:50 +0200 Subject: [PATCH 4/6] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b7da3eac..5155bd4e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.27", + "version": "1.1.28", "type": "module", "description": "d-id client sdk", "repository": { From 885d421f9b2ce3dc4131d138d43b7279999c163f Mon Sep 17 00:00:00 2001 From: ReutAtias3 <141619499+ReutAtias3@users.noreply.github.com> Date: Tue, 20 Jan 2026 20:29:54 +0200 Subject: [PATCH 5/6] set inturrupt on first transcription event (#280) * set inturrupt on first transcription event * format --- .../streaming-manager/livekit-manager.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 4cf7e5bc..bd167c55 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -27,6 +27,7 @@ import type { RoomEvent, SubscriptionError, Track, + TranscriptionSegment, } from 'livekit-client'; async function importLiveKit(): Promise<{ @@ -142,11 +143,18 @@ export async function createLiveKitStreamingManager speaker.isLocal); const isRemoteParticipantSpeaking = activeSpeakers.find(speaker => !speaker.isLocal); - if (isLocalParticipantSpeaking) { - callbacks.onAgentActivityStateChange?.(AgentActivityState.Idle); - - if (currentActivityState !== AgentActivityState.Idle) { - callbacks.onInterruptDetected?.({ type: 'audio' }); - currentActivityState = AgentActivityState.Idle; - } - } else if (isRemoteParticipantSpeaking) { + if (isRemoteParticipantSpeaking) { currentActivityState = AgentActivityState.Talking; callbacks.onAgentActivityStateChange?.(AgentActivityState.Talking); } else { From 16db2a389fa232b62d16a2ce4cf7760ec6fbdcd0 Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Tue, 20 Jan 2026 20:30:19 +0200 Subject: [PATCH 6/6] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5155bd4e..17847dd7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.28", + "version": "1.1.29", "type": "module", "description": "d-id client sdk", "repository": {