From 4651370a2d7e351cfa715a7d2d32bcafccf15958 Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Sun, 25 Jan 2026 13:26:42 +0200 Subject: [PATCH 1/9] Implemented sentiment tracking and optional debug metadata callback --- .../streaming-manager/livekit-manager.ts | 25 ++++++++++++++++--- src/types/entities/agents/manager.ts | 11 ++++++++ src/types/stream/stream.ts | 3 ++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index b744dc85..30aa8b3d 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -308,9 +308,28 @@ export async function createLiveKitStreamingManager void; export type VideoStateChangeCallback = (state: StreamingState, data: any) => void; +export type DebugMetadata = + { + type: string; + [key: string]: any; + }; + interface ManagerCallbacks { /** * Optional callback will be triggered each time the RTC connection changes state @@ -74,6 +80,11 @@ interface ManagerCallbacks { * @param messages - array of messages */ onNewMessage?(messages: Message[], type: 'answer' | 'partial' | 'user'): void; + /** + * Optional callback function that will be triggered with debug metadata when debug mode is enabled + * @param metadata - debug metadata object containing type and data + */ + onDebugMetadata?(metadata: DebugMetadata): void; /** * Optional callback function that will be triggered each time new chat is created * @param chatId - id of the new chat diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 3dec24b9..0ab14838 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -2,7 +2,7 @@ import { Analytics } from '@sdk/services/analytics/mixpanel'; import { VideoRTCStatsReport } from '@sdk/services/streaming-manager/stats/report'; import { Auth } from '../auth'; import { Interrupt } from '../entities'; -import { ChatProgressCallback } from '../entities/agents/manager'; +import { ChatProgressCallback, DebugMetadata } from '../entities/agents/manager'; import { CreateClipStreamRequest, CreateTalkStreamRequest, SendClipStreamPayload, SendTalkStreamPayload } from './api'; import { ICreateStreamRequestResponse, IceCandidate, SendStreamPayloadResponse, Status } from './rtc'; @@ -68,6 +68,7 @@ export interface ManagerCallbacks { onStreamCreated?: (stream: { stream_id: string; session_id: string; agent_id: string }) => void; onStreamReady?: () => void; onInterruptDetected?: (interrupt: Interrupt) => void; + onDebugMetadata?: (metadata: DebugMetadata) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; From 85527252473ff8c770dfb5f22d5281d8a0bee1bb Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Sun, 25 Jan 2026 13:31:09 +0200 Subject: [PATCH 2/9] prettier --- src/services/streaming-manager/livekit-manager.ts | 5 +++-- src/types/entities/agents/manager.ts | 9 ++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 30aa8b3d..fb5414c5 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -308,7 +308,6 @@ export async function createLiveKitStreamingManager void; export type VideoStateChangeCallback = (state: StreamingState, data: any) => void; -export type DebugMetadata = - { - type: string; - [key: string]: any; - }; +export type DebugMetadata = { + type: string; + [key: string]: any; +}; interface ManagerCallbacks { /** From f08afc2cab6ed87212336233847135f140ce39ab Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Sun, 25 Jan 2026 19:52:08 +0200 Subject: [PATCH 3/9] use onmessage --- src/services/socket-manager/message-queue.ts | 25 ++++++++++++++++- .../streaming-manager/livekit-manager.ts | 28 ++++++------------- src/types/entities/agents/chat.ts | 4 +++ src/types/entities/agents/manager.ts | 10 ------- src/types/stream/stream.ts | 3 +- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index d2d26571..40ab2467 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -110,10 +110,12 @@ export function createMessageEventQueue( ) { let chatEventQueue: ChatEventQueue = {}; const clearQueue = () => (chatEventQueue = {}); + let lastMessageType: 'answer' | 'partial' | 'user' = 'answer'; const onNewMessage: AgentManagerOptions['callbacks']['onNewMessage'] = (messages, event) => { if (event === 'user') { clearQueue(); } + lastMessageType = event; options.callbacks.onNewMessage?.(messages, event); }; @@ -145,7 +147,28 @@ export function createMessageEventQueue( if (event === SEvent.StreamVideoCreated) { analytics.linkTrack('agent-video', props, SEvent.StreamVideoCreated, ['start']); - } else if (completedEvents.includes(event)) { + + // Attach sentiment to the last assistant message if present + if (data.sentiment) { + const lastMessage = items.messages[items.messages.length - 1]; + if (lastMessage?.role === 'assistant') { + lastMessage.sentiment = data.sentiment; + onNewMessage?.([...items.messages], lastMessageType); + } + } + } + + else if (event === SEvent.StreamVideoDone) { + if (data.sentiment === null) { + const lastMessage = items.messages[items.messages.length - 1]; + if (lastMessage?.role === 'assistant') { + delete lastMessage.sentiment; + onNewMessage?.([...items.messages], lastMessageType); + } + } + } + + if (completedEvents.includes(event)) { // Stream video event const streamEvent = event.split('/')[1]; diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index fb5414c5..93b9e96c 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -308,29 +308,17 @@ export async function createLiveKitStreamingManager void; export type VideoStateChangeCallback = (state: StreamingState, data: any) => void; -export type DebugMetadata = { - type: string; - [key: string]: any; -}; - interface ManagerCallbacks { /** * Optional callback will be triggered each time the RTC connection changes state @@ -79,11 +74,6 @@ interface ManagerCallbacks { * @param messages - array of messages */ onNewMessage?(messages: Message[], type: 'answer' | 'partial' | 'user'): void; - /** - * Optional callback function that will be triggered with debug metadata when debug mode is enabled - * @param metadata - debug metadata object containing type and data - */ - onDebugMetadata?(metadata: DebugMetadata): void; /** * Optional callback function that will be triggered each time new chat is created * @param chatId - id of the new chat diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 0ab14838..3dec24b9 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -2,7 +2,7 @@ import { Analytics } from '@sdk/services/analytics/mixpanel'; import { VideoRTCStatsReport } from '@sdk/services/streaming-manager/stats/report'; import { Auth } from '../auth'; import { Interrupt } from '../entities'; -import { ChatProgressCallback, DebugMetadata } from '../entities/agents/manager'; +import { ChatProgressCallback } from '../entities/agents/manager'; import { CreateClipStreamRequest, CreateTalkStreamRequest, SendClipStreamPayload, SendTalkStreamPayload } from './api'; import { ICreateStreamRequestResponse, IceCandidate, SendStreamPayloadResponse, Status } from './rtc'; @@ -68,7 +68,6 @@ export interface ManagerCallbacks { onStreamCreated?: (stream: { stream_id: string; session_id: string; agent_id: string }) => void; onStreamReady?: () => void; onInterruptDetected?: (interrupt: Interrupt) => void; - onDebugMetadata?: (metadata: DebugMetadata) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; From 01c06c1fd5685455b8957866d61dea8bcb89991f Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Sun, 25 Jan 2026 20:05:47 +0200 Subject: [PATCH 4/9] use onmessage --- src/services/socket-manager/message-queue.ts | 7 +++++-- src/services/streaming-manager/livekit-manager.ts | 14 +++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index 40ab2467..533e9ea9 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -152,7 +152,8 @@ export function createMessageEventQueue( if (data.sentiment) { const lastMessage = items.messages[items.messages.length - 1]; if (lastMessage?.role === 'assistant') { - lastMessage.sentiment = data.sentiment; + const updatedMessage = { ...lastMessage, sentiment: data.sentiment }; + items.messages[items.messages.length - 1] = updatedMessage; onNewMessage?.([...items.messages], lastMessageType); } } @@ -162,7 +163,9 @@ export function createMessageEventQueue( if (data.sentiment === null) { const lastMessage = items.messages[items.messages.length - 1]; if (lastMessage?.role === 'assistant') { - delete lastMessage.sentiment; + + const updatedMessage = { ...lastMessage, sentiment: data.sentiment }; + items.messages[items.messages.length - 1] = updatedMessage; onNewMessage?.([...items.messages], lastMessageType); } } diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 93b9e96c..ab200df1 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -311,11 +311,15 @@ export async function createLiveKitStreamingManager Date: Sun, 25 Jan 2026 21:06:56 +0200 Subject: [PATCH 5/9] chore: format code with prettier --- src/services/socket-manager/message-queue.ts | 9 +++------ src/services/streaming-manager/livekit-manager.ts | 6 +++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index 533e9ea9..d4b537ab 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -147,7 +147,7 @@ export function createMessageEventQueue( if (event === SEvent.StreamVideoCreated) { analytics.linkTrack('agent-video', props, SEvent.StreamVideoCreated, ['start']); - + // Attach sentiment to the last assistant message if present if (data.sentiment) { const lastMessage = items.messages[items.messages.length - 1]; @@ -157,20 +157,17 @@ export function createMessageEventQueue( onNewMessage?.([...items.messages], lastMessageType); } } - } - - else if (event === SEvent.StreamVideoDone) { + } else if (event === SEvent.StreamVideoDone) { if (data.sentiment === null) { const lastMessage = items.messages[items.messages.length - 1]; if (lastMessage?.role === 'assistant') { - const updatedMessage = { ...lastMessage, sentiment: data.sentiment }; items.messages[items.messages.length - 1] = updatedMessage; onNewMessage?.([...items.messages], lastMessageType); } } } - + if (completedEvents.includes(event)) { // Stream video event const streamEvent = event.split('/')[1]; diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 409448ef..dab72a09 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -320,14 +320,14 @@ export async function createLiveKitStreamingManager Date: Mon, 26 Jan 2026 11:40:53 +0200 Subject: [PATCH 6/9] pr --- src/services/socket-manager/message-queue.ts | 9 --------- src/services/streaming-manager/livekit-manager.ts | 14 ++++++++------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index d4b537ab..1e5160a9 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -157,15 +157,6 @@ export function createMessageEventQueue( onNewMessage?.([...items.messages], lastMessageType); } } - } else if (event === SEvent.StreamVideoDone) { - if (data.sentiment === null) { - const lastMessage = items.messages[items.messages.length - 1]; - if (lastMessage?.role === 'assistant') { - const updatedMessage = { ...lastMessage, sentiment: data.sentiment }; - items.messages[items.messages.length - 1] = updatedMessage; - onNewMessage?.([...items.messages], lastMessageType); - } - } } if (completedEvents.includes(event)) { diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index dab72a09..21d45c26 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -69,6 +69,11 @@ export enum DataChannelTopic { Interrupt = 'did.interrupt', } +interface VideoMessageData { + sentiment?: { id: string; name: string } | null; + [role: string]: unknown; +} + export function handleInitError( error: unknown, log: (message?: any, ...optionalParams: any[]) => void, @@ -321,17 +326,14 @@ export async function createLiveKitStreamingManager Date: Mon, 26 Jan 2026 11:46:06 +0200 Subject: [PATCH 7/9] Fx --- src/services/streaming-manager/livekit-manager.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 21d45c26..cfab2423 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -4,6 +4,7 @@ import { ConnectivityState, CreateSessionV2Options, CreateStreamOptions, + Message, PayloadType, StreamEvents, StreamingManagerOptions, @@ -69,10 +70,7 @@ export enum DataChannelTopic { Interrupt = 'did.interrupt', } -interface VideoMessageData { - sentiment?: { id: string; name: string } | null; - [role: string]: unknown; -} +type VideoMessageData = Pick export function handleInitError( error: unknown, From e0c80a41647e0874c9e003ec59d811902592bd9e Mon Sep 17 00:00:00 2001 From: ReutAtias Date: Mon, 26 Jan 2026 11:50:15 +0200 Subject: [PATCH 8/9] push --- src/services/streaming-manager/livekit-manager.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index cfab2423..61c12ff0 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -70,7 +70,7 @@ export enum DataChannelTopic { Interrupt = 'did.interrupt', } -type VideoMessageData = Pick +type VideoMessageData = Pick; export function handleInitError( error: unknown, @@ -327,11 +327,10 @@ export async function createLiveKitStreamingManager Date: Mon, 26 Jan 2026 11:57:11 +0200 Subject: [PATCH 9/9] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de330d97..6409435e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.33", + "version": "1.1.34", "type": "module", "description": "d-id client sdk", "repository": {