From 32947eb118aa11b1c3c8e3caf6e56a59d5f5cea4 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Tue, 20 May 2025 11:08:14 +0300 Subject: [PATCH 01/35] feature: stream type analytics enrich (#145) --- src/services/streaming-manager/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 0bf841f0..4c8772c8 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -119,7 +119,7 @@ function handleStreamState({ export async function createStreamingManager( agentId: string, agent: T, - { debug = false, callbacks, auth, baseURL = didApiUrl }: StreamingManagerOptions + { debug = false, callbacks, auth, baseURL = didApiUrl, analytics }: StreamingManagerOptions ) { _debug = debug; let srcObject: MediaStream | null = null; @@ -143,6 +143,11 @@ export async function createStreamingManager( } const streamType = fluent ? StreamType.Fluent : StreamType.Legacy; + + analytics.enrich({ + 'stream-type': streamType, + }); + const warmup = agent.stream_warmup && !fluent; const getIsConnected = () => isConnected; From ce55d336a3858282f519195ced737393a0b516ff Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Wed, 21 May 2025 12:06:59 +0300 Subject: [PATCH 02/35] bugfix: speak when textual (#144) * bugfix: speak when textual * cr fixes * cr fixes --- src/services/agent-manager/index.ts | 13 ++++++++++++- src/utils/chat.ts | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 src/utils/chat.ts diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 65149b07..4bfe49c0 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -16,6 +16,7 @@ import { CONNECTION_RETRY_TIMEOUT_MS } from '$/config/consts'; import { didApiUrl, didSocketApiUrl, mixpanelKey } from '$/config/environment'; import { ChatCreationFailed, ValidationError } from '$/errors'; import { getRandom } from '$/utils'; +import { isTextualChat } from '$/utils/chat'; import { createAgentsApi } from '../../api/agents'; import { getAnalyticsInfo } from '../../utils/analytics'; import { retryOperation } from '../../utils/retry-operation'; @@ -364,7 +365,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return agentsApi.deleteRating(agentEntity.id, items.chat.id, id); }, - speak(payload: string | SupportedStreamScript) { + async speak(payload: string | SupportedStreamScript) { if (!items.streamingManager) { throw new Error('Please connect to the agent first'); } @@ -413,6 +414,16 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt options.callbacks.onNewMessage?.([...items.messages], 'answer'); } + const isTextual = isTextualChat(items.chatMode); + + // If the current chat is textual, we shouldn't activate the TTS. + if (items.chat && isTextual) { + return { + duration: 0, + status: 'success', + }; + } + return items.streamingManager.speak({ script, metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, diff --git a/src/utils/chat.ts b/src/utils/chat.ts new file mode 100644 index 00000000..a9460e2a --- /dev/null +++ b/src/utils/chat.ts @@ -0,0 +1,4 @@ +import { ChatMode } from '$/types'; + +export const isTextualChat = (chatMode: ChatMode) => + [ChatMode.TextOnly, ChatMode.Playground, ChatMode.Maintenance].includes(chatMode); From b988a88f38dc1093470a356bc6427aacb566f3ae Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Wed, 21 May 2025 12:32:49 +0300 Subject: [PATCH 03/35] bump version (#146) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 61567e77..b7dae4a1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.5", + "version": "1.1.0-beta.6", "type": "module", "description": "d-id client sdk", "repository": { From c74bc4bf1238d2061d28aa2acdccb94efd9ef819 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Mon, 9 Jun 2025 15:59:17 +0300 Subject: [PATCH 04/35] add rtt to report + add freeze to low_conn indicator --- src/services/streaming-manager/index.ts | 2 +- src/services/streaming-manager/stats/poll.ts | 5 +++-- src/services/streaming-manager/stats/report.ts | 18 +++++++++++++++++- src/types/stream/stream.ts | 2 ++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 4c8772c8..c341d56d 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -172,7 +172,7 @@ export async function createStreamingManager( report, streamType, }), - state => callbacks.onConnectivityStateChange?.(connectivityState), + state => callbacks.onConnectivityStateChange?.(state), warmup ); diff --git a/src/services/streaming-manager/stats/poll.ts b/src/services/streaming-manager/stats/poll.ts index 9cdb7416..db756480 100644 --- a/src/services/streaming-manager/stats/poll.ts +++ b/src/services/streaming-manager/stats/poll.ts @@ -74,7 +74,7 @@ export function pollStats( : avgJitterDelayInInterval > HIGH_JITTER_TRESHOLD && currFreezeCount > 1 ? ConnectivityState.Weak : prevLowConnState; - + console.log(currFreezeCount, currLowConnState, avgJitterDelayInInterval) if (currLowConnState !== prevLowConnState) { onConnectivityStateChange?.(currLowConnState); prevLowConnState = currLowConnState; @@ -103,7 +103,8 @@ export function pollStats( if (!getIsConnected()) { onConnected(); } - + console.log("statsReport", statsReport) + prevFreezeCount = freezeCount isStreaming = false; } } diff --git a/src/services/streaming-manager/stats/report.ts b/src/services/streaming-manager/stats/report.ts index abdc309e..be643bb7 100644 --- a/src/services/streaming-manager/stats/report.ts +++ b/src/services/streaming-manager/stats/report.ts @@ -1,10 +1,14 @@ import { AnalyticsRTCStatsReport, SlimRTCStatsReport } from '$/types'; import { average } from '$/utils/analytics'; +import { stat } from 'fs'; export interface VideoRTCStatsReport { webRTCStats: { anomalies: AnalyticsRTCStatsReport[]; aggregateReport: AnalyticsRTCStatsReport; + minRtt:number; + maxRtt:number; + avgRtt:number; minJitterDelayInInterval: number; maxJitterDelayInInterval: number; avgJitterDelayInInterval: number; @@ -71,13 +75,18 @@ function extractAnomalies(stats: AnalyticsRTCStatsReport[]): AnalyticsRTCStatsRe export function formatStats(stats: RTCStatsReport): SlimRTCStatsReport { let codec = ''; + let currRtt: number = 0; for (const report of stats.values()) { if (report && report.type === 'codec' && report.mimeType.startsWith('video')) { codec = report.mimeType.split('/')[1]; } + if(report && report.type === 'candidate-pair'){ + currRtt = report.currentRoundTripTime; + } if (report && report.type === 'inbound-rtp' && report.kind === 'video') { return { codec, + rtt: currRtt, timestamp: report.timestamp, bytesReceived: report.bytesReceived, packetsReceived: report.packetsReceived, @@ -109,6 +118,7 @@ export function createVideoStatsReport( if (!previousStats) { return { timestamp: report.timestamp, + rtt: report.rtt, duration: 0, bytesReceived: report.bytesReceived, bitrate: (report.bytesReceived * 8) / (interval / 1000), @@ -129,6 +139,7 @@ export function createVideoStatsReport( return { timestamp: report.timestamp, duration: 0, + rtt: report.rtt, bytesReceived: report.bytesReceived - previousStats.bytesReceived, bitrate: ((report.bytesReceived - previousStats.bytesReceived) * 8) / (interval / 1000), packetsReceived: report.packetsReceived - previousStats.packetsReceived, @@ -150,6 +161,7 @@ export function createVideoStatsReport( return { timestamp: report.timestamp, duration: (interval * index) / 1000, + rtt: report.rtt, bytesReceived: report.bytesReceived - stats[index - 1].bytesReceived, bitrate: ((report.bytesReceived - stats[index - 1].bytesReceived) * 8) / (interval / 1000), packetsReceived: report.packetsReceived - stats[index - 1].packetsReceived, @@ -169,11 +181,15 @@ export function createVideoStatsReport( }); const anomalies = extractAnomalies(differentialReport); const lowFpsCount = anomalies.reduce((acc, report) => acc + (report.causes!.includes('low fps') ? 1 : 0), 0); - const avgJittersSamples = differentialReport.map(stat => stat.avgJitterDelayInInterval); + const avgJittersSamples = differentialReport.filter(stat => !!stat.avgJitterDelayInInterval).map(stat => stat.avgJitterDelayInInterval); + const avgRttSamples = differentialReport.filter(stat => !!stat.rtt).map(stat => stat.rtt); return { webRTCStats: { anomalies: anomalies, + minRtt: Math.min(...avgRttSamples), + avgRtt: average(avgRttSamples), + maxRtt: Math.max(...avgRttSamples), aggregateReport: createAggregateReport(stats[0], stats[stats.length - 1], lowFpsCount), minJitterDelayInInterval: Math.min(...avgJittersSamples), maxJitterDelayInInterval: Math.max(...avgJittersSamples), diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 7a169c51..00ab27f3 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -109,6 +109,7 @@ export interface StreamingManagerOptions { export interface SlimRTCStatsReport { index: number; codec: string; + rtt: number; duration?: number; bitrate?: number; timestamp: any; @@ -130,6 +131,7 @@ export interface SlimRTCStatsReport { export interface AnalyticsRTCStatsReport { timestamp?: number; + rtt: number; duration: number; bytesReceived: number; bitrate: number; From a1b64e5b4dc05dce5eb9718864e594240e373d83 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Mon, 9 Jun 2025 17:21:05 +0300 Subject: [PATCH 05/35] fix build --- src/types/stream/stream.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 00ab27f3..9753b102 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -131,7 +131,6 @@ export interface SlimRTCStatsReport { export interface AnalyticsRTCStatsReport { timestamp?: number; - rtt: number; duration: number; bytesReceived: number; bitrate: number; From af6add35b8b0b4c3ad98a1ee69cd32f39a07fdac Mon Sep 17 00:00:00 2001 From: benyael15 Date: Tue, 10 Jun 2025 11:40:39 +0300 Subject: [PATCH 06/35] fix CR --- src/services/streaming-manager/stats/poll.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/streaming-manager/stats/poll.ts b/src/services/streaming-manager/stats/poll.ts index db756480..011118da 100644 --- a/src/services/streaming-manager/stats/poll.ts +++ b/src/services/streaming-manager/stats/poll.ts @@ -74,7 +74,7 @@ export function pollStats( : avgJitterDelayInInterval > HIGH_JITTER_TRESHOLD && currFreezeCount > 1 ? ConnectivityState.Weak : prevLowConnState; - console.log(currFreezeCount, currLowConnState, avgJitterDelayInInterval) + if (currLowConnState !== prevLowConnState) { onConnectivityStateChange?.(currLowConnState); prevLowConnState = currLowConnState; @@ -103,7 +103,7 @@ export function pollStats( if (!getIsConnected()) { onConnected(); } - console.log("statsReport", statsReport) + prevFreezeCount = freezeCount isStreaming = false; } From 84a9fe2a2518135d671d0180709924635a8a4e67 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Tue, 10 Jun 2025 11:45:54 +0300 Subject: [PATCH 07/35] fix CR2 --- src/services/streaming-manager/index.ts | 1 - src/services/streaming-manager/stats/report.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index c341d56d..00533853 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -127,7 +127,6 @@ export async function createStreamingManager( let isDatachannelOpen = false; let dataChannelSignal: StreamingState = StreamingState.Stop; let statsSignal: StreamingState = StreamingState.Stop; - let connectivityState: ConnectivityState = ConnectivityState.Unknown; const { startConnection, sendStreamRequest, close, createStream, addIceCandidate } = agent.videoType === VideoType.Clip diff --git a/src/services/streaming-manager/stats/report.ts b/src/services/streaming-manager/stats/report.ts index be643bb7..a6a75f84 100644 --- a/src/services/streaming-manager/stats/report.ts +++ b/src/services/streaming-manager/stats/report.ts @@ -1,6 +1,5 @@ import { AnalyticsRTCStatsReport, SlimRTCStatsReport } from '$/types'; import { average } from '$/utils/analytics'; -import { stat } from 'fs'; export interface VideoRTCStatsReport { webRTCStats: { From 5f138af0ad3e97a40a7125ba4f3deae67c3da2a7 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Tue, 10 Jun 2025 11:54:07 +0300 Subject: [PATCH 08/35] format --- src/services/streaming-manager/index.ts | 2 +- src/services/streaming-manager/stats/poll.ts | 8 ++++---- src/services/streaming-manager/stats/report.ts | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 00533853..877a464d 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -283,7 +283,7 @@ export async function createStreamingManager( try { if (state === ConnectionState.Connected) { - await close(streamIdFromServer, session_id).catch(_ => {}); + await close(streamIdFromServer, session_id).catch(_ => { }); } } catch (e) { log('Error on close stream connection', e); diff --git a/src/services/streaming-manager/stats/poll.ts b/src/services/streaming-manager/stats/poll.ts index 011118da..ab907ec9 100644 --- a/src/services/streaming-manager/stats/poll.ts +++ b/src/services/streaming-manager/stats/poll.ts @@ -72,9 +72,9 @@ export function pollStats( avgJitterDelayInInterval < LOW_JITTER_TRESHOLD ? ConnectivityState.Strong : avgJitterDelayInInterval > HIGH_JITTER_TRESHOLD && currFreezeCount > 1 - ? ConnectivityState.Weak - : prevLowConnState; - + ? ConnectivityState.Weak + : prevLowConnState; + if (currLowConnState !== prevLowConnState) { onConnectivityStateChange?.(currLowConnState); prevLowConnState = currLowConnState; @@ -103,7 +103,7 @@ export function pollStats( if (!getIsConnected()) { onConnected(); } - + prevFreezeCount = freezeCount isStreaming = false; } diff --git a/src/services/streaming-manager/stats/report.ts b/src/services/streaming-manager/stats/report.ts index a6a75f84..220ac820 100644 --- a/src/services/streaming-manager/stats/report.ts +++ b/src/services/streaming-manager/stats/report.ts @@ -5,9 +5,9 @@ export interface VideoRTCStatsReport { webRTCStats: { anomalies: AnalyticsRTCStatsReport[]; aggregateReport: AnalyticsRTCStatsReport; - minRtt:number; - maxRtt:number; - avgRtt:number; + minRtt: number; + maxRtt: number; + avgRtt: number; minJitterDelayInInterval: number; maxJitterDelayInInterval: number; avgJitterDelayInInterval: number; @@ -79,7 +79,7 @@ export function formatStats(stats: RTCStatsReport): SlimRTCStatsReport { if (report && report.type === 'codec' && report.mimeType.startsWith('video')) { codec = report.mimeType.split('/')[1]; } - if(report && report.type === 'candidate-pair'){ + if (report && report.type === 'candidate-pair') { currRtt = report.currentRoundTripTime; } if (report && report.type === 'inbound-rtp' && report.kind === 'video') { From 5485668a084490d8d8ceb4a60ca566780fe8c280 Mon Sep 17 00:00:00 2001 From: Dor Eitan Date: Tue, 10 Jun 2025 13:38:05 +0300 Subject: [PATCH 09/35] bugfix: mixpanel webrtc stats in fluent --- .../agent-manager/connect-to-manager.ts | 123 ++++++++++++++---- src/services/streaming-manager/index.ts | 2 - 2 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index e28c7759..51aba050 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -31,35 +31,99 @@ function getAgentStreamArgs(agent: Agent, options?: AgentManagerOptions): Create }; } -function trackStreamingStateAnalytics( +function trackVideoStateChangeAnalytics( state: StreamingState, agent: Agent, statsReport: any, analytics: Analytics, streamType: StreamType ) { - if (timestampTracker.get() > 0) { - if (state === StreamingState.Start) { - analytics.linkTrack( - 'agent-video', - { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, - 'start', - [StreamEvents.StreamVideoCreated] - ); - } else if (state === StreamingState.Stop) { - analytics.linkTrack( - 'agent-video', - { - event: 'stop', - is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, - background: agent.presenter.type === 'clip' && agent.presenter.background, - 'stream-type': streamType, - ...statsReport, - }, - 'done', - [StreamEvents.StreamVideoDone] - ); - } + if (streamType === StreamType.Fluent) { + trackVideoStreamAnalytics(state, agent, statsReport, analytics, streamType); + } else { + trackLegacyVideoAnalytics(state, agent, statsReport, analytics, streamType); + } +} + +function trackVideoStreamAnalytics( + state: StreamingState, + agent: Agent, + statsReport: any, + analytics: Analytics, + streamType: StreamType +) { + if (state === StreamingState.Start) { + analytics.track('stream-session', { event: 'start', 'stream-type': streamType }); + } else if (state === StreamingState.Stop) { + analytics.track('stream-session', { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + ...statsReport, + }); + } +} + +function trackAgentActivityAnalytics( + state: StreamingState, + agent: Agent, + analytics: Analytics, + streamType: StreamType +) { + if (timestampTracker.get() <= 0) return; + + if (state === StreamingState.Start) { + analytics.linkTrack( + 'agent-video', + { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, + 'start', + [StreamEvents.StreamVideoCreated] + ); + } else if (state === StreamingState.Stop) { + analytics.linkTrack( + 'agent-video', + { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + }, + 'done', + [StreamEvents.StreamVideoDone] + ); + } +} + +function trackLegacyVideoAnalytics( + state: StreamingState, + agent: Agent, + statsReport: any, + analytics: Analytics, + streamType: StreamType +) { + if (timestampTracker.get() <= 0) return; + + if (state === StreamingState.Start) { + analytics.linkTrack( + 'agent-video', + { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, + 'start', + [StreamEvents.StreamVideoCreated] + ); + } else if (state === StreamingState.Stop) { + analytics.linkTrack( + 'agent-video', + { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + ...statsReport, + }, + 'done', + [StreamEvents.StreamVideoDone] + ); } } @@ -86,14 +150,21 @@ function connectToManager( }, onVideoStateChange: (state: StreamingState, statsReport?: any) => { options.callbacks.onVideoStateChange?.(state); - trackStreamingStateAnalytics(state, agent, statsReport, analytics, streamingManager.streamType); + + trackVideoStateChangeAnalytics( + state, + agent, + statsReport, + analytics, + streamingManager.streamType + ); }, onAgentActivityStateChange: (state: AgentActivityState) => { options.callbacks.onAgentActivityStateChange?.(state); - trackStreamingStateAnalytics( + + trackAgentActivityAnalytics( state === AgentActivityState.Talking ? StreamingState.Start : StreamingState.Stop, agent, - undefined, analytics, streamingManager.streamType ); diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 4c8772c8..998e6380 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -270,7 +270,6 @@ export async function createStreamingManager( if (peerConnection) { if (state === ConnectionState.New) { // Connection already closed - callbacks.onVideoStateChange?.(StreamingState.Stop); clearInterval(videoStatsInterval); return; } @@ -290,7 +289,6 @@ export async function createStreamingManager( log('Error on close stream connection', e); } - callbacks.onVideoStateChange?.(StreamingState.Stop); callbacks.onAgentActivityStateChange?.(AgentActivityState.Idle); clearInterval(videoStatsInterval); } From 034c51fa80264cee1e163edfa50b3aa10afade5a Mon Sep 17 00:00:00 2001 From: benyael15 Date: Wed, 11 Jun 2025 15:10:51 +0300 Subject: [PATCH 10/35] window.crypto --- src/auth/get-auth-header.ts | 5 +++-- src/services/analytics/mixpanel.ts | 3 ++- src/utils/index.ts | 7 ++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/auth/get-auth-header.ts b/src/auth/get-auth-header.ts index 1fdfd48b..bb6bf268 100644 --- a/src/auth/get-auth-header.ts +++ b/src/auth/get-auth-header.ts @@ -6,8 +6,9 @@ export function getExternalId() { let key = window.localStorage.getItem('did_external_key_id'); if (!key) { - key = Math.random().toString(16).slice(2); - window.localStorage.setItem('did_external_key_id', key); + let newKey = getRandom() + window.localStorage.setItem('did_external_key_id', newKey); + key = newKey } return key; diff --git a/src/services/analytics/mixpanel.ts b/src/services/analytics/mixpanel.ts index b06fc5de..f049603b 100644 --- a/src/services/analytics/mixpanel.ts +++ b/src/services/analytics/mixpanel.ts @@ -1,6 +1,7 @@ import { getExternalId } from '$/auth/get-auth-header'; import { Agent } from '$/types'; import { getAgentType } from '$/utils/agent'; +import { getRandom } from '$/utils'; export interface AnalyticsOptions { token: string; @@ -65,7 +66,7 @@ export function initializeAnalytics(config: AnalyticsOptions): Analytics { ...analyticProps, additionalProperties: {}, isEnabled: config.isEnabled ?? true, - getRandom: () => Math.random().toString(16).slice(2), + getRandom, enrich(properties: Record) { const props = {}; diff --git a/src/utils/index.ts b/src/utils/index.ts index a8bbcfc6..411f0443 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,2 +1,7 @@ export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); -export const getRandom = () => Math.random().toString(16).slice(2); +export const getRandom = (length: number = 16) => { + const arr = new Uint8Array(length); + window.crypto.getRandomValues(arr); + return Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join(''); +} + From 71e64305f95dc7955ac1279a6e01823a0e840b11 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Wed, 11 Jun 2025 15:48:53 +0300 Subject: [PATCH 11/35] Update index.ts --- src/utils/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/index.ts b/src/utils/index.ts index 411f0443..3bc1dc9a 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -2,6 +2,6 @@ export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, export const getRandom = (length: number = 16) => { const arr = new Uint8Array(length); window.crypto.getRandomValues(arr); - return Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join(''); + return Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join('').slice(0,13); } From f90d48aad1d34d42f695d11161989621f980d7f3 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Wed, 11 Jun 2025 16:12:19 +0300 Subject: [PATCH 12/35] - --- package.json | 2 +- yarn.lock | 38 +++++++++++--------------------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index b7dae4a1..e1c4d03f 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "devDependencies": { "@preact/preset-vite": "^2.8.1", "@trivago/prettier-plugin-sort-imports": "^4.3.0", - "@types/node": "^20.11.24", + "@types/node": "^24.0.0", "commander": "^11.1.0", "glob": "^10.3.10", "preact": "^10.19.6", diff --git a/yarn.lock b/yarn.lock index 056a5ad0..4ad15f6c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -657,12 +657,12 @@ resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== -"@types/node@^20.11.24": - version "20.11.25" - resolved "https://registry.npmjs.org/@types/node/-/node-20.11.25.tgz#0f50d62f274e54dd7a49f7704cc16bfbcccaf49f" - integrity sha512-TBHyJxk2b7HceLVGFcpAUjsa5zIdsPWlR6XHfyGzd0SFu+/NFgQgMAl96MSDZgQDvJAvV6BKsFOrt6zIL09JDw== +"@types/node@^24.0.0": + version "24.0.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-24.0.0.tgz#14a278ce74dd33993f2c4e5dd614760728c0fba8" + integrity sha512-yZQa2zm87aRVcqDyH5+4Hv9KYgSdgwX1rFnGvpbzMaC7YAljmhBET93TPiTd3ObwTL+gSpIzPKg5BqVxdCvxKg== dependencies: - undici-types "~5.26.4" + undici-types "~7.8.0" "@volar/language-core@1.11.1", "@volar/language-core@~1.11.1": version "1.11.1" @@ -1438,16 +1438,7 @@ string-argv@~0.3.1: resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -1465,14 +1456,7 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -1528,10 +1512,10 @@ typescript@^5.3.3: resolved "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz#0ae9cebcfae970718474fe0da2c090cad6577372" integrity sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ== -undici-types@~5.26.4: - version "5.26.5" - resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" - integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici-types@~7.8.0: + version "7.8.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.8.0.tgz#de00b85b710c54122e44fbfd911f8d70174cd294" + integrity sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw== universalify@^0.1.0: version "0.1.2" From b4f0d8c500a3b406da164b3b32d604aacbc2939a Mon Sep 17 00:00:00 2001 From: benyael15 Date: Wed, 11 Jun 2025 16:17:46 +0300 Subject: [PATCH 13/35] Update app.tsx --- demo/app.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/app.tsx b/demo/app.tsx index cc0f99e0..2e6480f3 100644 --- a/demo/app.tsx +++ b/demo/app.tsx @@ -8,7 +8,7 @@ import { useAgentManager } from './hooks/useAgentManager'; export function App() { const [warmup, setWarmup] = useState(true); const [text, setText] = useState( - 'oded bobobobo sagi mamamama . bla raga ode ovem. lol cha cha cha cha cha . bobobobo. cha cha cha cha. bobobobo cha cha cha cha bobobobo. ssssssss cha cha cha cha cha bobobobo . cha cha cha cha bobobobo . cha cha cha cha. bobobobo ssssssss' + 'Ben bobobobo sagi mamamama . bla raga ode ovem. lol cha cha cha cha cha . bobobobo. cha cha cha cha. bobobobo cha cha cha cha bobobobo. ssssssss cha cha cha cha cha bobobobo . cha cha cha cha bobobobo . cha cha cha cha. bobobobo ssssssss' ); const [mode, setMode] = useState(ChatMode.Functional); const [sessionTimeout, setSessionTimeout] = useState(); From 3e443598f02427badd91765a77e98256275a9108 Mon Sep 17 00:00:00 2001 From: benyael15 Date: Wed, 11 Jun 2025 16:21:03 +0300 Subject: [PATCH 14/35] - --- package.json | 2 +- yarn.lock | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index e1c4d03f..fb90e252 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "devDependencies": { "@preact/preset-vite": "^2.8.1", "@trivago/prettier-plugin-sort-imports": "^4.3.0", - "@types/node": "^24.0.0", + "@types/node": "^22.15.0", "commander": "^11.1.0", "glob": "^10.3.10", "preact": "^10.19.6", diff --git a/yarn.lock b/yarn.lock index 4ad15f6c..4cfabeb9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -657,12 +657,12 @@ resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== -"@types/node@^24.0.0": - version "24.0.0" - resolved "https://registry.yarnpkg.com/@types/node/-/node-24.0.0.tgz#14a278ce74dd33993f2c4e5dd614760728c0fba8" - integrity sha512-yZQa2zm87aRVcqDyH5+4Hv9KYgSdgwX1rFnGvpbzMaC7YAljmhBET93TPiTd3ObwTL+gSpIzPKg5BqVxdCvxKg== +"@types/node@^22.15.0": + version "22.15.31" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.15.31.tgz#454f11e2061150135c8353d7f3b3b1823fca9f3f" + integrity sha512-jnVe5ULKl6tijxUhvQeNbQG/84fHfg+yMak02cT8QVhBx/F05rAVxCGBYYTh2EKz22D6JF5ktXuNwdx7b9iEGw== dependencies: - undici-types "~7.8.0" + undici-types "~6.21.0" "@volar/language-core@1.11.1", "@volar/language-core@~1.11.1": version "1.11.1" @@ -1439,6 +1439,7 @@ string-argv@~0.3.1: integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== "string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: + name string-width-cjs version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -1457,6 +1458,7 @@ string-width@^5.0.1, string-width@^5.1.2: strip-ansi "^7.0.1" "strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: + name strip-ansi-cjs version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -1512,10 +1514,10 @@ typescript@^5.3.3: resolved "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz#0ae9cebcfae970718474fe0da2c090cad6577372" integrity sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ== -undici-types@~7.8.0: - version "7.8.0" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.8.0.tgz#de00b85b710c54122e44fbfd911f8d70174cd294" - integrity sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw== +undici-types@~6.21.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" + integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== universalify@^0.1.0: version "0.1.2" From b33606f8b34a701ca11c0e9a027db32ae70e8f5a Mon Sep 17 00:00:00 2001 From: Ofek Simhi <158498125+osimhi213@users.noreply.github.com> Date: Sun, 29 Jun 2025 12:01:05 +0300 Subject: [PATCH 15/35] Feature/enrich mixpanel with stream metadata (#153) * enrich mixpanel events with stream metadata * fix datachannel signal * add mixpanel event of agent-chat with status ready * CR * CR * CR --- src/services/analytics/mixpanel.ts | 14 +------ src/services/streaming-manager/index.ts | 55 +++++++++++++++++++------ src/types/stream/stream.ts | 5 --- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/services/analytics/mixpanel.ts b/src/services/analytics/mixpanel.ts index f049603b..10cddb4d 100644 --- a/src/services/analytics/mixpanel.ts +++ b/src/services/analytics/mixpanel.ts @@ -67,19 +67,7 @@ export function initializeAnalytics(config: AnalyticsOptions): Analytics { additionalProperties: {}, isEnabled: config.isEnabled ?? true, getRandom, - enrich(properties: Record) { - const props = {}; - - if (properties && typeof properties !== 'object') { - throw new Error('properties must be a flat json object'); - } - - for (let prop in properties) { - if (typeof properties[prop] === 'string' || typeof properties[prop] === 'number') { - props[prop] = properties[prop]; - } - } - + enrich(props: Record) { this.additionalProperties = { ...this.additionalProperties, ...props }; }, async track(event: string, props?: Record) { diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index c6bb042a..b68a12f9 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -4,14 +4,13 @@ import { AgentActivityState, ConnectionState, CreateStreamOptions, - DataChannelSignalMap, PayloadType, + StreamEvents, StreamType, StreamingManagerOptions, StreamingState, VideoType, } from '$/types/index'; -import { ConnectivityState } from '$/types/stream/stream'; import { pollStats } from './stats/poll'; import { VideoRTCStatsReport } from './stats/report'; @@ -23,6 +22,9 @@ const actualRTCPC = ( (window as any).mozRTCPeerConnection ).bind(window); +type DataChannelPayload = string | Record; +type DataChannelMessageHandler = (subject: S, payload?: DataChannelPayload) => void + function mapConnectionState(state: RTCIceConnectionState): ConnectionState { switch (state) { case 'connected': @@ -44,6 +46,18 @@ function mapConnectionState(state: RTCIceConnectionState): ConnectionState { } } +function parseDataChannelMessage(message: string): { subject: StreamEvents, data: DataChannelPayload } { + const [subject, rawData = ''] = message.split(/:(.+)/); + try { + const data = JSON.parse(rawData); + log('parsed data channel message', { subject, data }); + return { subject: subject as StreamEvents, data }; + } catch (e) { + log('Failed to parse data channel message, returning data as string', { subject, rawData, error: e }); + return { subject: subject as StreamEvents, data: rawData }; + } +} + function handleLegacyStreamState({ statsSignal, dataChannelSignal, @@ -205,18 +219,33 @@ export async function createStreamingManager( } }; - pcDataChannel.onmessage = (event: MessageEvent) => { - if (event.data in DataChannelSignalMap) { - dataChannelSignal = DataChannelSignalMap[event.data]; + function handleStreamVideoEvent(subject: StreamEvents.StreamStarted | StreamEvents.StreamDone) { + dataChannelSignal = subject === StreamEvents.StreamStarted ? StreamingState.Start : StreamingState.Stop; - handleStreamState({ - statsSignal: streamType === StreamType.Legacy ? statsSignal : undefined, - dataChannelSignal, - onVideoStateChange: callbacks.onVideoStateChange, - onAgentActivityStateChange: callbacks.onAgentActivityStateChange, - streamType, - }); - } + handleStreamState({ + statsSignal: streamType === StreamType.Legacy ? statsSignal : undefined, + dataChannelSignal, + onVideoStateChange: callbacks.onVideoStateChange, + onAgentActivityStateChange: callbacks.onAgentActivityStateChange, + streamType, + }); + } + + function handleStreamReadyEvent(_subject: StreamEvents.StreamReady, payload?: DataChannelPayload) { + const streamMetadata = typeof payload === 'string' ? payload : payload?.metadata; + streamMetadata && analytics.enrich({ streamMetadata }); + analytics.track('agent-chat', { event: 'ready' }); + } + + const dataChannelHandlers = { + [StreamEvents.StreamStarted]: handleStreamVideoEvent, + [StreamEvents.StreamDone]: handleStreamVideoEvent, + [StreamEvents.StreamReady]: handleStreamReadyEvent, + } satisfies Partial<{ [K in StreamEvents]: DataChannelMessageHandler }> + + pcDataChannel.onmessage = (event: MessageEvent) => { + const { subject, data } = parseDataChannelMessage(event.data); + dataChannelHandlers[subject]?.(subject, data); }; peerConnection.oniceconnectionstatechange = () => { diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 9753b102..87b9a1a2 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -23,11 +23,6 @@ export enum AgentActivityState { Talking = 'TALKING', } -export const DataChannelSignalMap: Record = { - 'stream/started': StreamingState.Start, - 'stream/done': StreamingState.Stop, -}; - export enum StreamEvents { ChatAnswer = 'chat/answer', ChatPartial = 'chat/partial', From d4593642205ac506caf50f24e9592dbbba26778e Mon Sep 17 00:00:00 2001 From: Ofek Simhi Date: Sun, 29 Jun 2025 12:12:34 +0300 Subject: [PATCH 16/35] chore: publish @d-id/client-sdk:v1.1.0-beta.12 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fb90e252..eadf079e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.6", + "version": "1.1.0-beta.12", "type": "module", "description": "d-id client sdk", "repository": { From 06342122830dd1524476e066054ad2ef091fcf3e Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:38:12 +0300 Subject: [PATCH 17/35] Feature: interrupt (#152) * feature: interrupt * feature: interrupt * feature: interrupt * make it more robust --- demo/app.tsx | 33 ++++++++------- demo/hooks/useAgentManager.ts | 12 ++++++ src/services/agent-manager/index.ts | 42 +++++++++++++++++++- src/services/interrupt/index.ts | 39 ++++++++++++++++++ src/services/socket-manager/message-queue.ts | 2 +- src/services/streaming-manager/index.ts | 31 ++++++++++++++- src/types/entities/agents/chat.ts | 3 ++ src/types/entities/agents/manager.ts | 12 ++++++ src/types/index.ts | 2 +- src/types/stream/rtc.ts | 1 + src/types/stream/stream.ts | 7 ++++ 11 files changed, 165 insertions(+), 19 deletions(-) create mode 100644 src/services/interrupt/index.ts diff --git a/demo/app.tsx b/demo/app.tsx index 2e6480f3..8fbe657f 100644 --- a/demo/app.tsx +++ b/demo/app.tsx @@ -17,20 +17,21 @@ export function App() { const videoRef = useRef(null); - const { srcObject, connectionState, messages, isSpeaking, connect, disconnect, speak, chat } = useAgentManager({ - agentId, - baseURL: didApiUrl, - wsURL: didSocketApiUrl, - mode, - enableAnalytics: false, - auth: { type: 'key', clientKey }, - streamOptions: { - streamWarmup: warmup, - sessionTimeout, - compatibilityMode, - fluent, - }, - }); + const { srcObject, connectionState, messages, isSpeaking, connect, disconnect, speak, chat, interrupt } = + useAgentManager({ + agentId, + baseURL: didApiUrl, + wsURL: didSocketApiUrl, + mode, + enableAnalytics: false, + auth: { type: 'key', clientKey }, + streamOptions: { + streamWarmup: warmup, + sessionTimeout, + compatibilityMode, + fluent, + }, + }); async function onClick() { if (connectionState === ConnectionState.New || connectionState === ConnectionState.Fail) { @@ -81,6 +82,10 @@ export function App() { Send to Chat + + diff --git a/demo/hooks/useAgentManager.ts b/demo/hooks/useAgentManager.ts index 2c966bf3..a84e8a64 100644 --- a/demo/hooks/useAgentManager.ts +++ b/demo/hooks/useAgentManager.ts @@ -150,6 +150,17 @@ export function useAgentManager(props: UseAgentManagerOptions) { [agentManager, connectionState] ); + const interrupt = useCallback(async () => { + if (!agentManager || connectionState !== ConnectionState.Connected) return; + + try { + await agentManager.interrupt(); + } catch (e) { + console.error('Error interrupting:', e); + throw e; + } + }, [agentManager, connectionState]); + return { connectionState, messages, @@ -159,5 +170,6 @@ export function useAgentManager(props: UseAgentManagerOptions) { disconnect, speak, chat, + interrupt, }; } diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 4bfe49c0..790197cd 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -24,6 +24,7 @@ import { initializeAnalytics } from '../analytics/mixpanel'; import { timestampTracker } from '../analytics/timestamp-tracker'; import { createChat, getRequestHeaders } from '../chat'; import { getInitialMessages } from '../chat/intial-messages'; +import { sendInterrupt, validateInterrupt } from '../interrupt'; import { SocketManager, createSocketManager } from '../socket-manager'; import { createMessageEventQueue } from '../socket-manager/message-queue'; import { StreamingManager } from '../streaming-manager'; @@ -51,12 +52,16 @@ export interface AgentManagerItems { */ export async function createAgentManager(agent: string, options: AgentManagerOptions): Promise { let firstConnection = true; + let queuedInterrupt = false; const mxKey = options.mixpanelKey || mixpanelKey; const wsURL = options.wsURL || didSocketApiUrl; const baseURL = options.baseURL || didApiUrl; - const items: AgentManagerItems = { messages: [], chatMode: options.mode || ChatMode.Functional }; + const items: AgentManagerItems = { + messages: [], + chatMode: options.mode || ChatMode.Functional, + }; const agentsApi = createAgentsApi(options.auth, baseURL, options.callbacks.onError); const agentEntity = await agentsApi.getById(agent); const analytics = initializeAnalytics({ @@ -80,6 +85,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt timestampTracker.reset(); + queuedInterrupt = false; + if (newChat && !firstConnection) { delete items.chat; @@ -128,6 +135,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt items.socketManager?.disconnect(); await items.streamingManager?.disconnect(); + queuedInterrupt = false; + delete items.streamingManager; delete items.socketManager; @@ -150,6 +159,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return { agent: agentEntity, getStreamType: () => items.streamingManager?.streamType, + getIsInterruptEnabled: () => items.streamingManager?.interruptEnabled ?? false, starterMessages: agentEntity.knowledge?.starter_message || [], getSTTToken: () => agentsApi.getSTTToken(agentEntity.id), changeMode, @@ -287,8 +297,15 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt created_at: new Date().toISOString(), context: response.context, matches: response.matches, + videoId: response.videoId, }); + if (queuedInterrupt && response.videoId && items.streamingManager) { + queuedInterrupt = false; + items.messages[items.messages.length - 1].interrupted = true; + await sendInterrupt(items.streamingManager, response.videoId); + } + analytics.track('agent-message-send', { event: 'success', mode: items.chatMode, @@ -307,6 +324,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return response; } catch (e) { + queuedInterrupt = false; + if (items.messages[items.messages.length - 1]?.role === 'assistant') { items.messages.pop(); } @@ -429,6 +448,27 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, }); }, + async interrupt() { + const lastMessage = items.messages[items.messages.length - 1]; + const chatRequestPending = lastMessage?.role === 'user'; + + validateInterrupt( + items.streamingManager, + items.chat, + items.streamingManager?.streamType, + chatRequestPending, + !!lastMessage?.videoId + ); + + if (chatRequestPending) { + queuedInterrupt = true; + return; + } + + lastMessage.interrupted = true; + options.callbacks.onNewMessage?.([...items.messages], 'answer'); + sendInterrupt(items.streamingManager!, lastMessage?.videoId!); + }, }; } diff --git a/src/services/interrupt/index.ts b/src/services/interrupt/index.ts new file mode 100644 index 00000000..ec267887 --- /dev/null +++ b/src/services/interrupt/index.ts @@ -0,0 +1,39 @@ +import { Chat, CreateStreamOptions, StreamEvents, StreamInterruptPayload, StreamType } from '$/types'; +import { StreamingManager } from '../streaming-manager'; + +export function validateInterrupt( + streamingManager: StreamingManager | undefined, + chat: Chat | undefined, + streamType: StreamType | undefined, + hasChatPending: boolean, + hasVideoId: boolean +): void { + if (!streamingManager || !chat) { + throw new Error('Please connect to the agent first'); + } + + if (!streamingManager.interruptEnabled) { + throw new Error('Interrupt is not enabled for this stream'); + } + + if (streamType !== StreamType.Fluent) { + throw new Error('Interrupt only available for Fluent streams'); + } + + if (!hasChatPending && !hasVideoId) { + throw new Error('No active video to interrupt'); + } +} + +export async function sendInterrupt( + streamingManager: StreamingManager, + videoId: string +): Promise { + const payload: StreamInterruptPayload = { + type: StreamEvents.StreamInterrupt, + videoId, + timestamp: Date.now(), + }; + + streamingManager.sendDataChannelMessage(JSON.stringify(payload)); +} diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index d4e97e65..6400e03e 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -87,7 +87,7 @@ export function createMessageEventQueue( } else if (completedEvents.includes(event)) { // Stream video event const streamEvent = event.split('/')[1]; - + if (failedEvents.includes(event)) { // Dont depend on video state change if stream failed analytics.track('agent-video', { ...props, event: streamEvent }); diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index b68a12f9..6e99d230 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -147,7 +147,14 @@ export async function createStreamingManager( ? createClipApi(auth, baseURL, agentId, callbacks.onError) : createTalkApi(auth, baseURL, agentId, callbacks.onError); - const { id: streamIdFromServer, offer, ice_servers, session_id, fluent } = await createStream(agent); + const { + id: streamIdFromServer, + offer, + ice_servers, + session_id, + fluent, + interrupt_enabled: interruptEnabled, + } = await createStream(agent); const peerConnection = new actualRTCPC({ iceServers: ice_servers }); const pcDataChannel = peerConnection.createDataChannel('JanusDataChannel'); @@ -311,7 +318,7 @@ export async function createStreamingManager( try { if (state === ConnectionState.Connected) { - await close(streamIdFromServer, session_id).catch(_ => { }); + await close(streamIdFromServer, session_id).catch(_ => {}); } } catch (e) { log('Error on close stream connection', e); @@ -321,6 +328,25 @@ export async function createStreamingManager( clearInterval(videoStatsInterval); } }, + /** + * Method to send data channel messages to the server + */ + sendDataChannelMessage(payload: string) { + if (!isConnected || pcDataChannel.readyState !== 'open') { + log('Data channel is not ready for sending messages'); + callbacks.onError?.(new Error('Data channel is not ready for sending messages'), { + streamId: streamIdFromServer, + }); + return; + } + + try { + pcDataChannel.send(payload); + } catch (e: any) { + log('Error sending data channel message', e); + callbacks.onError?.(e, { streamId: streamIdFromServer }); + } + }, /** * Session identifier information, should be returned in the body of all streaming requests */ @@ -331,6 +357,7 @@ export async function createStreamingManager( streamId: streamIdFromServer, streamType, + interruptEnabled, }; } diff --git a/src/types/entities/agents/chat.ts b/src/types/entities/agents/chat.ts index f2b0c287..33d21e88 100644 --- a/src/types/entities/agents/chat.ts +++ b/src/types/entities/agents/chat.ts @@ -31,6 +31,8 @@ export interface Message { created_at?: string; matches?: ChatResponse['matches']; context?: string; + videoId?: string; + interrupted?: boolean; } export interface ChatPayload { @@ -63,6 +65,7 @@ export interface ChatResponse { matches?: IRetrivalMetadata[]; chatMode?: ChatMode; context?: string; + videoId?: string; } export interface Chat { diff --git a/src/types/entities/agents/manager.ts b/src/types/entities/agents/manager.ts index 4142a168..30d2f3d2 100644 --- a/src/types/entities/agents/manager.ts +++ b/src/types/entities/agents/manager.ts @@ -166,6 +166,12 @@ export interface AgentManager { * Get the current stream type of the agent */ getStreamType: () => StreamType | undefined; + + /** + * Get if the stream supports interrupt + */ + getIsInterruptEnabled: () => boolean; + /** * Array of starter messages that will be sent to the agent when the chat starts */ @@ -220,4 +226,10 @@ export interface AgentManager { * @param properties flat json object with properties that will be added to analytics events fired from the sdk */ enrichAnalytics: (properties: Record) => void; + + /** + * Method to interrupt the current video stream + * Only available for Fluent streams and when there's an active video to interrupt + */ + interrupt: () => void; } diff --git a/src/types/index.ts b/src/types/index.ts index 37e40862..a84963f8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,6 @@ -export * from './stream-script'; export * from './auth'; export * from './entities'; export * from './stream'; +export * from './stream-script'; export * from './voice/stt'; export * from './voice/tts'; diff --git a/src/types/stream/rtc.ts b/src/types/stream/rtc.ts index 879ef2cf..95ba5a89 100644 --- a/src/types/stream/rtc.ts +++ b/src/types/stream/rtc.ts @@ -38,6 +38,7 @@ export interface ICreateStreamRequestResponse extends StickyRequest { offer: any; ice_servers: IceServer[]; fluent?: boolean; + interrupt_enabled?: boolean; } export interface IceCandidate { diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 87b9a1a2..4be7ba1d 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -31,6 +31,7 @@ export enum StreamEvents { StreamFailed = 'stream/error', StreamReady = 'stream/ready', StreamCreated = 'stream/created', + StreamInterrupt = 'stream/interrupt', StreamVideoCreated = 'stream-video/started', StreamVideoDone = 'stream-video/done', StreamVideoError = 'stream-video/error', @@ -143,3 +144,9 @@ export interface AnalyticsRTCStatsReport { lowFpsCount?: number; causes?: string[]; } + +export interface StreamInterruptPayload { + type: StreamEvents.StreamInterrupt; + videoId: string; + timestamp: number; +} From 021f1f51f69aa37b0909cf9ba7b925808a1f4c18 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:12:28 +0300 Subject: [PATCH 18/35] chore: publish sdk v1.1.0-beta.13 (#154) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eadf079e..f927c027 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.12", + "version": "1.1.0-beta.13", "type": "module", "description": "d-id client sdk", "repository": { From 1f65d5b957c04b5804a8e5bbe30b1791f810a0a1 Mon Sep 17 00:00:00 2001 From: Dariusz Date: Wed, 2 Jul 2025 16:58:31 +0200 Subject: [PATCH 19/35] fixed greetings in playground --- src/services/agent-manager/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 4bfe49c0..705d3202 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -366,10 +366,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return agentsApi.deleteRating(agentEntity.id, items.chat.id, id); }, async speak(payload: string | SupportedStreamScript) { - if (!items.streamingManager) { - throw new Error('Please connect to the agent first'); - } - function getScript(): StreamScript { if (typeof payload === 'string') { if (!agentEntity.presenter.voice) { @@ -404,7 +400,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt analytics.track('agent-speak', script); timestampTracker.update(); - if (items.chat?.id && script.type === 'text') { + if (items.messages && script.type === 'text') { items.messages.push({ id: getRandom(), role: 'assistant', @@ -417,13 +413,17 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const isTextual = isTextualChat(items.chatMode); // If the current chat is textual, we shouldn't activate the TTS. - if (items.chat && isTextual) { + if (isTextual) { return { duration: 0, status: 'success', }; } + if (!items.streamingManager) { + throw new Error('Please connect to the agent first'); + } + return items.streamingManager.speak({ script, metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, From 108973e707ad0f073ce60a7beb26661505a25297 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Wed, 2 Jul 2025 18:06:26 +0300 Subject: [PATCH 20/35] bump version 1.1.0-beta.14 (#157) * bump version 1.1.0-beta.15 * bump version 1.1.0-beta.14 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f927c027..566cdfdb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.13", + "version": "1.1.0-beta.14", "type": "module", "description": "d-id client sdk", "repository": { @@ -45,4 +45,4 @@ "vite": "^5.1.4", "vite-plugin-dts": "^3.7.3" } -} \ No newline at end of file +} From a7056a1d6f98c84974a0917ff690b2d91ab2e7e6 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Thu, 3 Jul 2025 11:59:09 +0300 Subject: [PATCH 21/35] feature: interrupt type mixpanel event (#155) * feature: interrupt type mixpanel event * extract type --- demo/hooks/useAgentManager.ts | 2 +- .../agent-manager/connect-to-manager.ts | 18 ++++++++---- src/services/agent-manager/index.ts | 29 ++++++++++++++----- src/services/analytics/timestamp-tracker.ts | 3 +- src/types/entities/agents/chat.ts | 4 +++ src/types/entities/agents/manager.ts | 4 +-- 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/demo/hooks/useAgentManager.ts b/demo/hooks/useAgentManager.ts index a84e8a64..84806a83 100644 --- a/demo/hooks/useAgentManager.ts +++ b/demo/hooks/useAgentManager.ts @@ -154,7 +154,7 @@ export function useAgentManager(props: UseAgentManagerOptions) { if (!agentManager || connectionState !== ConnectionState.Connected) return; try { - await agentManager.interrupt(); + agentManager.interrupt({ type: 'click' }); } catch (e) { console.error('Error interrupting:', e); throw e; diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index 51aba050..caf304e3 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -15,7 +15,7 @@ import { mapVideoType, } from '$/types'; import { Analytics } from '../analytics/mixpanel'; -import { timestampTracker } from '../analytics/timestamp-tracker'; +import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; import { createChat } from '../chat'; function getAgentStreamArgs(agent: Agent, options?: AgentManagerOptions): CreateStreamOptions { @@ -71,12 +71,12 @@ function trackAgentActivityAnalytics( analytics: Analytics, streamType: StreamType ) { - if (timestampTracker.get() <= 0) return; + if (latencyTimestampTracker.get() <= 0) return; if (state === StreamingState.Start) { analytics.linkTrack( 'agent-video', - { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, + { event: 'start', latency: latencyTimestampTracker.get(true), 'stream-type': streamType }, 'start', [StreamEvents.StreamVideoCreated] ); @@ -102,12 +102,12 @@ function trackLegacyVideoAnalytics( analytics: Analytics, streamType: StreamType ) { - if (timestampTracker.get() <= 0) return; + if (latencyTimestampTracker.get() <= 0) return; if (state === StreamingState.Start) { analytics.linkTrack( 'agent-video', - { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, + { event: 'start', latency: latencyTimestampTracker.get(true), 'stream-type': streamType }, 'start', [StreamEvents.StreamVideoCreated] ); @@ -132,7 +132,7 @@ function connectToManager( options: AgentManagerOptions, analytics: Analytics ): Promise> { - timestampTracker.reset(); + latencyTimestampTracker.reset(); return new Promise(async (resolve, reject) => { try { @@ -162,6 +162,12 @@ function connectToManager( onAgentActivityStateChange: (state: AgentActivityState) => { options.callbacks.onAgentActivityStateChange?.(state); + if (state === AgentActivityState.Talking) { + interruptTimestampTracker.update(); + } else { + interruptTimestampTracker.reset(); + } + trackAgentActivityAnalytics( state === AgentActivityState.Talking ? StreamingState.Start : StreamingState.Stop, agent, diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 5791c7c2..2063a620 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -7,6 +7,7 @@ import { ChatMode, ConnectionState, CreateStreamOptions, + Interrupt, Message, StreamScript, SupportedStreamScript, @@ -21,7 +22,7 @@ import { createAgentsApi } from '../../api/agents'; import { getAnalyticsInfo } from '../../utils/analytics'; import { retryOperation } from '../../utils/retry-operation'; import { initializeAnalytics } from '../analytics/mixpanel'; -import { timestampTracker } from '../analytics/timestamp-tracker'; +import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; import { createChat, getRequestHeaders } from '../chat'; import { getInitialMessages } from '../chat/intial-messages'; import { sendInterrupt, validateInterrupt } from '../interrupt'; @@ -83,7 +84,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt async function connect(newChat: boolean) { options.callbacks.onConnectionStateChange?.(ConnectionState.Connecting); - timestampTracker.reset(); + latencyTimestampTracker.reset(); queuedInterrupt = false; @@ -282,7 +283,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt id: getRandom(), role: 'user', content: userMessage, - created_at: new Date(timestampTracker.update()).toISOString(), + created_at: new Date(latencyTimestampTracker.update()).toISOString(), }); options.callbacks.onNewMessage?.([...items.messages], 'user'); @@ -316,7 +317,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt options.callbacks.onNewMessage?.([...items.messages], 'answer'); analytics.track('agent-message-received', { - latency: timestampTracker.get(true), + latency: latencyTimestampTracker.get(true), mode: items.chatMode, messages: items.messages.length, }); @@ -417,14 +418,14 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const script = getScript(); analytics.track('agent-speak', script); - timestampTracker.update(); + latencyTimestampTracker.update(); if (items.messages && script.type === 'text') { items.messages.push({ id: getRandom(), role: 'assistant', content: script.input, - created_at: new Date(timestampTracker.get(true)).toISOString(), + created_at: new Date(latencyTimestampTracker.get(true)).toISOString(), }); options.callbacks.onNewMessage?.([...items.messages], 'answer'); } @@ -448,7 +449,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, }); }, - async interrupt() { + async interrupt({ type }: Interrupt) { const lastMessage = items.messages[items.messages.length - 1]; const chatRequestPending = lastMessage?.role === 'user'; @@ -460,6 +461,18 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt !!lastMessage?.videoId ); + analytics.track('agent-video-interrupt', { + type: type || 'click', + stream_id: items.streamingManager?.streamId, + agent_id: agentEntity.id, + owner_id: agentEntity.owner_id, + video_duration_to_interrupt: interruptTimestampTracker.get(true), + message_duration_to_interrupt: latencyTimestampTracker.get(true), + chat_id: items.chat?.id, + mode: items.chatMode, + queued_interrupt: chatRequestPending, + }); + if (chatRequestPending) { queuedInterrupt = true; return; @@ -467,7 +480,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt lastMessage.interrupted = true; options.callbacks.onNewMessage?.([...items.messages], 'answer'); - sendInterrupt(items.streamingManager!, lastMessage?.videoId!); + sendInterrupt(items.streamingManager!, lastMessage.videoId!); }, }; } diff --git a/src/services/analytics/timestamp-tracker.ts b/src/services/analytics/timestamp-tracker.ts index 5d282d0d..c3b217f5 100644 --- a/src/services/analytics/timestamp-tracker.ts +++ b/src/services/analytics/timestamp-tracker.ts @@ -8,4 +8,5 @@ function createTimestampTracker() { }; } -export const timestampTracker = createTimestampTracker(); +export const latencyTimestampTracker = createTimestampTracker(); +export const interruptTimestampTracker = createTimestampTracker(); diff --git a/src/types/entities/agents/chat.ts b/src/types/entities/agents/chat.ts index 33d21e88..0fb7c17c 100644 --- a/src/types/entities/agents/chat.ts +++ b/src/types/entities/agents/chat.ts @@ -79,3 +79,7 @@ export interface Chat { agent_id__modified_at: string; chat_mode?: ChatMode; } + +export interface Interrupt { + type: 'text' | 'audio' | 'click'; +} diff --git a/src/types/entities/agents/manager.ts b/src/types/entities/agents/manager.ts index 30d2f3d2..20405470 100644 --- a/src/types/entities/agents/manager.ts +++ b/src/types/entities/agents/manager.ts @@ -12,7 +12,7 @@ import { } from '$/types/stream'; import { SupportedStreamScript } from '$/types/stream-script'; import { Agent } from './agent'; -import { ChatMode, ChatResponse, Message, RatingEntity } from './chat'; +import { ChatMode, ChatResponse, Interrupt, Message, RatingEntity } from './chat'; /** * Types of events provided in Chat Progress Callback @@ -231,5 +231,5 @@ export interface AgentManager { * Method to interrupt the current video stream * Only available for Fluent streams and when there's an active video to interrupt */ - interrupt: () => void; + interrupt: (interrupt: Interrupt) => void; } From 450bceb5178200f8d1204fb2b8aa12d9c50ef6dd Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Thu, 3 Jul 2025 14:26:01 +0300 Subject: [PATCH 22/35] bump version 1.1.0-beta.15 (#158) --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 566cdfdb..6a3df430 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.14", + "version": "1.1.0-beta.15", "type": "module", "description": "d-id client sdk", "repository": { @@ -45,4 +45,4 @@ "vite": "^5.1.4", "vite-plugin-dts": "^3.7.3" } -} +} \ No newline at end of file From a6df5ed9e2f7e061e1de2510e3f890fc0125e14f Mon Sep 17 00:00:00 2001 From: Dariusz Date: Wed, 16 Jul 2025 17:46:47 +0200 Subject: [PATCH 23/35] catched limit error --- demo/hooks/useAgentManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/demo/hooks/useAgentManager.ts b/demo/hooks/useAgentManager.ts index 84806a83..82e04223 100644 --- a/demo/hooks/useAgentManager.ts +++ b/demo/hooks/useAgentManager.ts @@ -142,6 +142,10 @@ export function useAgentManager(props: UseAgentManagerOptions) { try { await agentManager.chat(userMessage.trim()); } catch (e) { + if (e instanceof Error && e.message?.includes('User stream has reached pending requests limit')) { + console.log('User stream has reached pending requests limit'); + return; + } setConnectionState(ConnectionState.Fail); throw e; From cc15f01abbb854fcbab70d33d675727169e19d3a Mon Sep 17 00:00:00 2001 From: Dariusz Date: Sun, 20 Jul 2025 13:24:32 +0200 Subject: [PATCH 24/35] rename `interruptEnabled` func --- src/services/agent-manager/index.ts | 2 +- src/services/interrupt/index.ts | 2 +- src/services/streaming-manager/index.ts | 4 ++-- src/types/entities/agents/manager.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 2063a620..01eeb8b1 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -160,7 +160,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return { agent: agentEntity, getStreamType: () => items.streamingManager?.streamType, - getIsInterruptEnabled: () => items.streamingManager?.interruptEnabled ?? false, + getIsInterruptAvailable: () => items.streamingManager?.interruptAvailable ?? false, starterMessages: agentEntity.knowledge?.starter_message || [], getSTTToken: () => agentsApi.getSTTToken(agentEntity.id), changeMode, diff --git a/src/services/interrupt/index.ts b/src/services/interrupt/index.ts index ec267887..2aaeac91 100644 --- a/src/services/interrupt/index.ts +++ b/src/services/interrupt/index.ts @@ -12,7 +12,7 @@ export function validateInterrupt( throw new Error('Please connect to the agent first'); } - if (!streamingManager.interruptEnabled) { + if (!streamingManager.interruptAvailable) { throw new Error('Interrupt is not enabled for this stream'); } diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 6e99d230..8e734203 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -153,7 +153,7 @@ export async function createStreamingManager( ice_servers, session_id, fluent, - interrupt_enabled: interruptEnabled, + interrupt_enabled: interruptAvailable, } = await createStream(agent); const peerConnection = new actualRTCPC({ iceServers: ice_servers }); const pcDataChannel = peerConnection.createDataChannel('JanusDataChannel'); @@ -357,7 +357,7 @@ export async function createStreamingManager( streamId: streamIdFromServer, streamType, - interruptEnabled, + interruptAvailable, }; } diff --git a/src/types/entities/agents/manager.ts b/src/types/entities/agents/manager.ts index 20405470..39730d69 100644 --- a/src/types/entities/agents/manager.ts +++ b/src/types/entities/agents/manager.ts @@ -170,7 +170,7 @@ export interface AgentManager { /** * Get if the stream supports interrupt */ - getIsInterruptEnabled: () => boolean; + getIsInterruptAvailable: () => boolean; /** * Array of starter messages that will be sent to the agent when the chat starts From bbd9b0a759be59c7c5498f566c4e267ec1a711e2 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:50:15 +0300 Subject: [PATCH 25/35] chore: bump 1.1.0-beta.20 (#163) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6a3df430..19d820ce 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.15", + "version": "1.1.0-beta.20", "type": "module", "description": "d-id client sdk", "repository": { From 583f3cd4c61c1fa45c98976443c454c75d9f5b43 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Mon, 21 Jul 2025 19:52:22 +0300 Subject: [PATCH 26/35] feature: interrupt speak (#164) * feature: interrupt speak * feature: interrupt speak --- src/services/agent-manager/index.ts | 38 ++++++++++++++++++++++++----- src/types/stream/rtc.ts | 1 + 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 01eeb8b1..82f1d78e 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -54,6 +54,7 @@ export interface AgentManagerItems { export async function createAgentManager(agent: string, options: AgentManagerOptions): Promise { let firstConnection = true; let queuedInterrupt = false; + let speakPending = false; const mxKey = options.mixpanelKey || mixpanelKey; const wsURL = options.wsURL || didSocketApiUrl; @@ -87,6 +88,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt latencyTimestampTracker.reset(); queuedInterrupt = false; + speakPending = false; if (newChat && !firstConnection) { delete items.chat; @@ -137,6 +139,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt await items.streamingManager?.disconnect(); queuedInterrupt = false; + speakPending = false; delete items.streamingManager; delete items.socketManager; @@ -436,6 +439,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt if (isTextual) { return { duration: 0, + video_id: '', status: 'success', }; } @@ -444,20 +448,42 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt throw new Error('Please connect to the agent first'); } - return items.streamingManager.speak({ - script, - metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, - }); + speakPending = true; + + try { + const response = await items.streamingManager.speak({ + script, + metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, + }); + + speakPending = false; + + items.messages[items.messages.length - 1].videoId = response.video_id; + + if (queuedInterrupt && response.video_id && items.streamingManager) { + queuedInterrupt = false; + items.messages[items.messages.length - 1].interrupted = true; + await sendInterrupt(items.streamingManager, response.video_id); + } + + options.callbacks.onNewMessage?.([...items.messages], 'answer'); + + return response; + } finally { + speakPending = false; + } }, async interrupt({ type }: Interrupt) { const lastMessage = items.messages[items.messages.length - 1]; const chatRequestPending = lastMessage?.role === 'user'; + const isStreamRequestPending = chatRequestPending || speakPending; + validateInterrupt( items.streamingManager, items.chat, items.streamingManager?.streamType, - chatRequestPending, + isStreamRequestPending, !!lastMessage?.videoId ); @@ -473,7 +499,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt queued_interrupt: chatRequestPending, }); - if (chatRequestPending) { + if (isStreamRequestPending) { queuedInterrupt = true; return; } diff --git a/src/types/stream/rtc.ts b/src/types/stream/rtc.ts index 95ba5a89..e74db024 100644 --- a/src/types/stream/rtc.ts +++ b/src/types/stream/rtc.ts @@ -68,4 +68,5 @@ export interface Status { export interface SendStreamPayloadResponse extends Status, StickyRequest { duration: number; + video_id: string; } From 0de3964f0972156b2b8fce133c3f7eaf5a9a418a Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:17:34 +0300 Subject: [PATCH 27/35] bugfix: allow to interrupt speak without chat open (#165) --- src/services/agent-manager/index.ts | 1 - src/services/interrupt/index.ts | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 82f1d78e..1914ddb6 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -481,7 +481,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt validateInterrupt( items.streamingManager, - items.chat, items.streamingManager?.streamType, isStreamRequestPending, !!lastMessage?.videoId diff --git a/src/services/interrupt/index.ts b/src/services/interrupt/index.ts index 2aaeac91..115aa176 100644 --- a/src/services/interrupt/index.ts +++ b/src/services/interrupt/index.ts @@ -1,14 +1,13 @@ -import { Chat, CreateStreamOptions, StreamEvents, StreamInterruptPayload, StreamType } from '$/types'; +import { CreateStreamOptions, StreamEvents, StreamInterruptPayload, StreamType } from '$/types'; import { StreamingManager } from '../streaming-manager'; export function validateInterrupt( streamingManager: StreamingManager | undefined, - chat: Chat | undefined, streamType: StreamType | undefined, - hasChatPending: boolean, + hasStreamRequestPending: boolean, hasVideoId: boolean ): void { - if (!streamingManager || !chat) { + if (!streamingManager) { throw new Error('Please connect to the agent first'); } @@ -20,7 +19,7 @@ export function validateInterrupt( throw new Error('Interrupt only available for Fluent streams'); } - if (!hasChatPending && !hasVideoId) { + if (!hasStreamRequestPending && !hasVideoId) { throw new Error('No active video to interrupt'); } } From 83c7da11b07ccc8f05aecf014348fa6176ed162a Mon Sep 17 00:00:00 2001 From: ReutAtias3 <141619499+ReutAtias3@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:38:28 +0300 Subject: [PATCH 28/35] Merge pull request #166 from de-id/refactor/report-event-on-start Report load event --- src/services/agent-manager/index.ts | 11 ++++++---- src/services/analytics/mixpanel.ts | 31 +++++------------------------ src/utils/analytics.ts | 19 ++++++++++++++++++ 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 1914ddb6..42ad18d2 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -19,7 +19,7 @@ import { ChatCreationFailed, ValidationError } from '$/errors'; import { getRandom } from '$/utils'; import { isTextualChat } from '$/utils/chat'; import { createAgentsApi } from '../../api/agents'; -import { getAnalyticsInfo } from '../../utils/analytics'; +import { getAgentInfo, getAnalyticsInfo } from '../../utils/analytics'; import { retryOperation } from '../../utils/retry-operation'; import { initializeAnalytics } from '../analytics/mixpanel'; import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; @@ -64,14 +64,17 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt messages: [], chatMode: options.mode || ChatMode.Functional, }; - const agentsApi = createAgentsApi(options.auth, baseURL, options.callbacks.onError); - const agentEntity = await agentsApi.getById(agent); const analytics = initializeAnalytics({ token: mxKey, - agent: agentEntity, + agentId: agent, isEnabled: options.enableAnalitics, distinctId: options.distinctId, }); + analytics.track('agent-sdk', { event: 'init' }); + const agentsApi = createAgentsApi(options.auth, baseURL, options.callbacks.onError); + + const agentEntity = await agentsApi.getById(agent); + analytics.enrich(getAgentInfo(agentEntity)); const { onMessage, clearQueue } = createMessageEventQueue(analytics, items, options, agentEntity, () => items.socketManager?.disconnect() ); diff --git a/src/services/analytics/mixpanel.ts b/src/services/analytics/mixpanel.ts index 10cddb4d..c2a993a1 100644 --- a/src/services/analytics/mixpanel.ts +++ b/src/services/analytics/mixpanel.ts @@ -1,11 +1,9 @@ import { getExternalId } from '$/auth/get-auth-header'; -import { Agent } from '$/types'; -import { getAgentType } from '$/utils/agent'; import { getRandom } from '$/utils'; export interface AnalyticsOptions { token: string; - agent: Agent; + agentId: string; isEnabled?: boolean; distinctId?: string; } @@ -16,7 +14,7 @@ export interface Analytics { isEnabled: boolean; chatId?: string; agentId: string; - owner_id: string; + owner_id?: string; getRandom(): string; track(event: string, props?: Record): Promise; linkTrack(mixpanelEvent: string, props: Record, event: string, dependencies: string[]): any; @@ -40,30 +38,11 @@ const mixpanelUrl = 'https://api-js.mixpanel.com/track/?verbose=1&ip=1'; export function initializeAnalytics(config: AnalyticsOptions): Analytics { const source = window?.hasOwnProperty('DID_AGENTS_API') ? 'agents-ui' : 'agents-sdk'; - const presenter = config.agent.presenter; - const promptCustomization = config.agent.llm?.prompt_customization; - const analyticProps = { + return { token: config.token || 'testKey', distinct_id: config.distinctId || getExternalId(), - agentId: config.agent.id, - agentType: getAgentType(presenter), - owner_id: config.agent.owner_id ?? '', - promptVersion: config.agent.llm?.prompt_version, - behavior: { - role: promptCustomization?.role, - personality: promptCustomization?.personality, - instructions: config.agent.llm?.instructions, - }, - temperature: config.agent.llm?.temperature, - knowledgeSource: promptCustomization?.knowledge_source, - starterQuestionsCount: config.agent.knowledge?.starter_message?.length, - topicsToAvoid: promptCustomization?.topics_to_avoid, - maxResponseLength: promptCustomization?.max_response_length, - }; - - return { - ...analyticProps, + agentId: config.agentId, additionalProperties: {}, isEnabled: config.isEnabled ?? true, getRandom, @@ -90,7 +69,7 @@ export function initializeAnalytics(config: AnalyticsOptions): Analytics { properties: { ...this.additionalProperties, ...sendProps, - ...analyticProps, + agentId: this.agentId, source, time: Date.now(), $insert_id: this.getRandom(), diff --git a/src/utils/analytics.ts b/src/utils/analytics.ts index acd30637..d90a4449 100644 --- a/src/utils/analytics.ts +++ b/src/utils/analytics.ts @@ -33,6 +33,25 @@ export function getAnalyticsInfo(agent: Agent) { }; } +export function getAgentInfo(agent: Agent) { + const promptCustomization = agent.llm?.prompt_customization; + + return { + agentType: getAgentType(agent.presenter), + owner_id: agent.owner_id ?? '', + promptVersion: agent.llm?.prompt_version, + behavior: { + role: promptCustomization?.role, + personality: promptCustomization?.personality, + instructions: agent.llm?.instructions, + }, + temperature: agent.llm?.temperature, + knowledgeSource: promptCustomization?.knowledge_source, + starterQuestionsCount: agent.knowledge?.starter_message?.length, + topicsToAvoid: promptCustomization?.topics_to_avoid, + maxResponseLength: promptCustomization?.max_response_length, + }; +} export const sumFunc = (numbers: number[]) => numbers.reduce((total, aNumber) => total + aNumber, 0); export const average = (numbers: number[]) => sumFunc(numbers) / numbers.length; From 61e9bb01e62c780323cd2e09f7db5e693c25395f Mon Sep 17 00:00:00 2001 From: dariusz-did Date: Mon, 4 Aug 2025 16:28:59 +0200 Subject: [PATCH 29/35] video id from data channel (#170) * videoId from DC * typo --- .../agent-manager/connect-to-manager.ts | 10 ++++++-- src/services/agent-manager/index.ts | 15 ++++++++++-- src/services/streaming-manager/index.ts | 24 +++++++++++++++---- src/types/stream/stream.ts | 1 + 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index caf304e3..f7a7fd04 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -127,9 +127,15 @@ function trackLegacyVideoAnalytics( } } +type ConnectToManagerOptions = AgentManagerOptions & { + callbacks: AgentManagerOptions['callbacks'] & { + onVideoIdChange?: (videoId: string | null) => void; + }; +}; + function connectToManager( agent: Agent, - options: AgentManagerOptions, + options: ConnectToManagerOptions, analytics: Analytics ): Promise> { latencyTimestampTracker.reset(); @@ -185,7 +191,7 @@ function connectToManager( export async function initializeStreamAndChat( agent: Agent, - options: AgentManagerOptions, + options: ConnectToManagerOptions, agentsApi: AgentsAPI, analytics: Analytics, chat?: Chat diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 42ad18d2..7859b6f2 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -55,6 +55,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt let firstConnection = true; let queuedInterrupt = false; let speakPending = false; + let videoId: string | null = null; const mxKey = options.mixpanelKey || mixpanelKey; const wsURL = options.wsURL || didSocketApiUrl; @@ -83,6 +84,10 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt options.callbacks.onNewMessage?.([...items.messages], 'answer'); + const updateVideoId = (newVideoId: string | null) => { + videoId = newVideoId; + }; + analytics.track('agent-sdk', { event: 'loaded', ...getAnalyticsInfo(agentEntity) }); async function connect(newChat: boolean) { @@ -106,7 +111,13 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const initPromise = retryOperation( () => { - return initializeStreamAndChat(agentEntity, options, agentsApi, analytics, items.chat); + return initializeStreamAndChat( + agentEntity, + { ...options, callbacks: { ...options.callbacks, onVideoIdChange: updateVideoId } }, + agentsApi, + analytics, + items.chat + ); }, { limit: 3, @@ -486,7 +497,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt items.streamingManager, items.streamingManager?.streamType, isStreamRequestPending, - !!lastMessage?.videoId + !!videoId ); analytics.track('agent-video-interrupt', { diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 8e734203..969aa44f 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -23,7 +23,7 @@ const actualRTCPC = ( ).bind(window); type DataChannelPayload = string | Record; -type DataChannelMessageHandler = (subject: S, payload?: DataChannelPayload) => void +type DataChannelMessageHandler = (subject: S, payload?: DataChannelPayload) => void; function mapConnectionState(state: RTCIceConnectionState): ConnectionState { switch (state) { @@ -46,7 +46,7 @@ function mapConnectionState(state: RTCIceConnectionState): ConnectionState { } } -function parseDataChannelMessage(message: string): { subject: StreamEvents, data: DataChannelPayload } { +function parseDataChannelMessage(message: string): { subject: StreamEvents; data: DataChannelPayload } { const [subject, rawData = ''] = message.split(/:(.+)/); try { const data = JSON.parse(rawData); @@ -226,7 +226,23 @@ export async function createStreamingManager( } }; - function handleStreamVideoEvent(subject: StreamEvents.StreamStarted | StreamEvents.StreamDone) { + const handleStreamVideoIdChange = (videoId: string | null) => { + callbacks.onVideoIdChange?.(videoId); + }; + + function handleStreamVideoEvent( + subject: StreamEvents.StreamStarted | StreamEvents.StreamDone, + payload?: DataChannelPayload + ) { + if (subject === StreamEvents.StreamStarted && typeof payload === 'object' && 'metadata' in payload) { + const metadata = payload.metadata as { videoId: string }; + handleStreamVideoIdChange(metadata.videoId); + } + + if (subject === StreamEvents.StreamDone) { + handleStreamVideoIdChange(null); + } + dataChannelSignal = subject === StreamEvents.StreamStarted ? StreamingState.Start : StreamingState.Stop; handleStreamState({ @@ -248,7 +264,7 @@ export async function createStreamingManager( [StreamEvents.StreamStarted]: handleStreamVideoEvent, [StreamEvents.StreamDone]: handleStreamVideoEvent, [StreamEvents.StreamReady]: handleStreamReadyEvent, - } satisfies Partial<{ [K in StreamEvents]: DataChannelMessageHandler }> + } satisfies Partial<{ [K in StreamEvents]: DataChannelMessageHandler }>; pcDataChannel.onmessage = (event: MessageEvent) => { const { subject, data } = parseDataChannelMessage(event.data); diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 4be7ba1d..a75470ca 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -61,6 +61,7 @@ export interface ManagerCallbacks { onError?: (error: Error, errorData: object) => void; onConnectivityStateChange?: (state: ConnectivityState) => void; onAgentActivityStateChange?: (state: AgentActivityState) => void; + onVideoIdChange?: (videoId: string | null) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; From d649e7418a4badd8f31d9a70601f8619a7fad416 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:27:58 +0300 Subject: [PATCH 30/35] Feature: sdk ci cd (#169) * add workflows to prod and staging * fix: update workflow names to reflect correct package identifiers * feature: add E2E test dispatch after publishing to staging * change org * wip * feat: enhance workflows with version bumping and environment testing options * feat: add DEPLOYMENT documentation for staging and production workflows * feat: implement production deployment workflows with E2E validation * feat: add SDK environment management and E2E testing workflows * feat: enhance deployment workflows with version management and E2E testing updates * enhance ci/cd workflow * change runner to medium * fix yarn * fix yarn * fix yarn * change agents-ui to prod branch, added manual trigger * add manual workflow * open pr to agents ui * increase timeout * Refactor E2E workflows: remove manual branch input, add concurrency settings --------- Co-authored-by: Daniel Abitbul --- .github/workflows/manual-e2e.yml | 106 +++++++++++++ .github/workflows/pr-main-e2e.yml | 101 +++++++++++++ .github/workflows/pr-prod-e2e.yml | 71 +++++++++ .github/workflows/publish-on-merge.yml | 196 +++++++++++++++++++++++++ 4 files changed, 474 insertions(+) create mode 100644 .github/workflows/manual-e2e.yml create mode 100644 .github/workflows/pr-main-e2e.yml create mode 100644 .github/workflows/pr-prod-e2e.yml create mode 100644 .github/workflows/publish-on-merge.yml diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml new file mode 100644 index 00000000..ab14a93b --- /dev/null +++ b/.github/workflows/manual-e2e.yml @@ -0,0 +1,106 @@ +name: Manual E2E Validation + +on: + workflow_dispatch: + inputs: + sdk_branch: + description: 'SDK branch to test' + required: true + default: 'main' + type: string + ui_branch: + description: 'agents-ui branch to test' + required: true + default: 'staging' + type: string + +jobs: + e2e-validation: + runs-on: ${{ github.event.inputs.ui_branch == 'prod' && 'ubuntu-latest' || 'aws-medium' }} + timeout-minutes: 30 + environment: + name: ${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + env: + ENV: ${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + + steps: + - name: Checkout SDK branch + uses: actions/checkout@v4 + with: + path: agents-sdk + ref: ${{ github.event.inputs.sdk_branch }} + + - name: Setup Node.js for SDK + uses: actions/setup-node@v4 + with: + node-version: 20 + cache-dependency-path: agents-sdk/yarn.lock + + - name: Install Yarn + run: npm install -g yarn + + - name: Install SDK dependencies + working-directory: agents-sdk + run: yarn install --frozen-lockfile + + - name: Build SDK + working-directory: agents-sdk + run: yarn build + + - name: Pack SDK for testing + working-directory: agents-sdk + run: | + npm pack + echo "SDK_PACKAGE=$(ls *.tgz)" >> $GITHUB_ENV + + - name: Checkout agents-ui branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: ${{ github.event.inputs.ui_branch }} + path: agents-ui + token: ${{ secrets.DEVOPS_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js for agents-ui + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + sed "s/\$NPM_AUTH_TOKEN/${{ secrets.NPM_TOKEN }}/g" .npmrc.template > .npmrc + fi + + - name: Install local SDK build in agents-ui + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add file:../agents-sdk/${{ env.SDK_PACKAGE }} + yarn install --frozen-lockfile + + - name: Install Playwright Chrome + working-directory: agents-ui + run: yarn playwright install chrome + + - name: Run E2E tests + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + ASSERT_CHAT_RESTART: 'false' + run: yarn test:${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-manual-${{ github.event.inputs.sdk_branch }}-${{ github.event.inputs.ui_branch }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/pr-main-e2e.yml b/.github/workflows/pr-main-e2e.yml new file mode 100644 index 00000000..d3f572a4 --- /dev/null +++ b/.github/workflows/pr-main-e2e.yml @@ -0,0 +1,101 @@ +name: agents-ui prod with local sdk build e2e validation + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + e2e-validation: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: prod + env: + ENV: prod + + steps: + - name: Checkout SDK branch + uses: actions/checkout@v4 + with: + path: agents-sdk + ref: ${{ github.head_ref || github.ref_name }} + + - name: Setup Node.js for SDK + uses: actions/setup-node@v4 + with: + node-version: 20 + cache-dependency-path: agents-sdk/yarn.lock + + - name: Install Yarn + run: npm install -g yarn + + - name: Install SDK dependencies + working-directory: agents-sdk + run: yarn install --frozen-lockfile + + - name: Build SDK + working-directory: agents-sdk + run: yarn build + + - name: Pack SDK for testing + working-directory: agents-sdk + run: | + npm pack + echo "SDK_PACKAGE=$(ls *.tgz)" >> $GITHUB_ENV + + - name: Checkout agents-ui production branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: prod + path: agents-ui + token: ${{ secrets.DEVOPS_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js for agents-ui + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + sed "s/\$NPM_AUTH_TOKEN/${{ secrets.NPM_TOKEN }}/g" .npmrc.template > .npmrc + fi + + - name: Install local SDK build in agents-ui + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add file:../agents-sdk/${{ env.SDK_PACKAGE }} + yarn install --frozen-lockfile + + - name: Install Playwright Chrome + working-directory: agents-ui + run: yarn playwright install chrome + + - name: Run E2E tests against production + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + ASSERT_CHAT_RESTART: 'false' + run: yarn test:prod + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-main-pr-${{ github.event.number }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/pr-prod-e2e.yml b/.github/workflows/pr-prod-e2e.yml new file mode 100644 index 00000000..53144fd8 --- /dev/null +++ b/.github/workflows/pr-prod-e2e.yml @@ -0,0 +1,71 @@ +name: agents-ui prod with staging sdk build e2e validation + +on: + pull_request: + branches: [prod] + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + e2e-validation: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: prod + env: + ENV: prod + + steps: + - name: Checkout agents-ui production branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: prod + path: agents-ui + fetch-depth: 0 + lfs: true + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + envsubst < .npmrc.template > .npmrc + fi + env: + NPM_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Install staging SDK version + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add @d-id/client-sdk@staging + npm install -g yarn && yarn + + - name: Run E2E tests against production environment + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + run: yarn test:prod + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-prod-pr-${{ github.event.number }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/publish-on-merge.yml b/.github/workflows/publish-on-merge.yml new file mode 100644 index 00000000..334b9fa8 --- /dev/null +++ b/.github/workflows/publish-on-merge.yml @@ -0,0 +1,196 @@ +name: Auto Publish SDK + +on: + push: + branches: [main, prod] + workflow_dispatch: + inputs: + dry_run: + description: 'Run in dry-run mode (no actual publishing)' + required: false + default: false + type: boolean + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: 'https://registry.npmjs.org' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Build package + run: yarn build + + - name: Determine version and tag + id: version + run: | + if [ "${{ github.ref_name }}" = "main" ]; then + # For main branch - staging versions with run number + # Use jq to safely read JSON (standard tool) + BASE_VERSION=$(jq -r '.version' package.json) + if [ "$BASE_VERSION" = "null" ]; then + echo "Error: Could not read version from package.json" + exit 1 + fi + STAGING_VERSION="${BASE_VERSION}-staging.${{ github.run_number }}" + echo "version=$STAGING_VERSION" >> $GITHUB_OUTPUT + echo "tag=staging" >> $GITHUB_OUTPUT + echo "description=Staging release from main branch" >> $GITHUB_OUTPUT + echo "should_sync=false" >> $GITHUB_OUTPUT + else + # For prod branch - production versions + # Use npm version command (official npm way to bump versions) + NEW_VERSION=$(npm version patch --no-git-tag-version --silent) + NEW_VERSION=${NEW_VERSION#v} # Remove 'v' prefix if present + echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "tag=latest" >> $GITHUB_OUTPUT + echo "description=Production release" >> $GITHUB_OUTPUT + echo "should_sync=true" >> $GITHUB_OUTPUT + fi + + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + if [ "${{ github.event.inputs.dry_run }}" = "true" ]; then + echo "🔍 DRY RUN MODE: Would publish version ${{ steps.version.outputs.version }} with tag ${{ steps.version.outputs.tag }}" + echo "📦 Package would be published to: https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}" + echo "🏷️ NPM tag would be: ${{ steps.version.outputs.tag }}" + echo "✅ Dry run completed successfully - no actual publishing occurred" + else + echo "🚀 Publishing version ${{ steps.version.outputs.version }} with tag ${{ steps.version.outputs.tag }}" + npm publish --access public --tag ${{ steps.version.outputs.tag }} + echo "✅ Successfully published to NPM" + fi + + - name: Create Git tag for production + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag "v${{ steps.version.outputs.version }}" + git push origin "v${{ steps.version.outputs.version }}" + + - name: Commit version bump (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: bump version to ${{ steps.version.outputs.version }} [skip ci]" || echo "No changes to commit" + git push origin prod + + - name: Sync version back to main (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + # Fetch latest main + git fetch origin main + git checkout main + git pull origin main + + jq --arg version "${{ steps.version.outputs.version }}" '.version = $version' package.json > package.json.tmp + + if ! jq empty package.json.tmp 2>/dev/null; then + echo "Error: Generated invalid JSON" + rm -f package.json.tmp + exit 1 + fi + + mv package.json.tmp package.json + + if git diff --quiet package.json; then + echo "No version changes needed" + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: sync version ${{ steps.version.outputs.version }} from prod [skip ci]" + git push origin main + fi + + - name: Create GitHub Release (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.version.outputs.version }} + release_name: Release v${{ steps.version.outputs.version }} + body: | + ## Release v${{ steps.version.outputs.version }} + + **Published to NPM:** [@d-id/client-sdk@${{ steps.version.outputs.version }}](https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}) + + ### Installation + ```bash + npm install @d-id/client-sdk@${{ steps.version.outputs.version }} + ``` + + ${{ steps.version.outputs.description }} + draft: false + prerelease: false + + - name: Checkout agents-ui repository + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + token: ${{ secrets.GITHUB_TOKEN }} + path: agents-ui + + - name: Update SDK version and create PR + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + cd agents-ui + + jq --arg version "${{ steps.version.outputs.version }}" '.dependencies."@d-id/client-sdk" = $version' package.json > package.json.tmp + mv package.json.tmp package.json + + if git diff --quiet package.json; then + echo "No version changes needed in agents-ui" + exit 0 + fi + + git checkout -b "chore/bump-sdk-version-${{ steps.version.outputs.version }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: bump @d-id/client-sdk to v${{ steps.version.outputs.version }}" + git push origin "chore/bump-sdk-version-${{ steps.version.outputs.version }}" + + gh pr create \ + --repo de-id/agents-ui \ + --title "chore: bump @d-id/client-sdk to v${{ steps.version.outputs.version }}" \ + --body "## SDK Version Update + + This PR updates the @d-id/client-sdk dependency to version ${{ steps.version.outputs.version }}. + + ### Changes + - Updated @d-id/client-sdk from previous version to v${{ steps.version.outputs.version }} + + ### Related + - SDK Release: [v${{ steps.version.outputs.version }}](https://github.com/d-id/agents-sdk/releases/tag/v${{ steps.version.outputs.version }}) + - NPM Package: [@d-id/client-sdk@${{ steps.version.outputs.version }}](https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}) + + ### Next Steps + - [ ] Review the changes + - [ ] Run tests to ensure compatibility + - [ ] Merge when ready" \ + --base main \ + --head "chore/bump-sdk-version-${{ steps.version.outputs.version }}" \ + --label "dependencies" \ + --label "automated" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5a3840e66717ea255347ded6e080d9248fd5f6af Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:57:04 +0300 Subject: [PATCH 31/35] chore: bump version to 1.1.1 and update workflow names for clarity (#175) --- .github/workflows/pr-main-e2e.yml | 2 +- .github/workflows/pr-prod-e2e.yml | 2 +- .github/workflows/publish-on-merge.yml | 18 ++++++++++++++++-- package.json | 4 ++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-main-e2e.yml b/.github/workflows/pr-main-e2e.yml index d3f572a4..5867c17f 100644 --- a/.github/workflows/pr-main-e2e.yml +++ b/.github/workflows/pr-main-e2e.yml @@ -1,4 +1,4 @@ -name: agents-ui prod with local sdk build e2e validation +name: UI prod e2e with local sdk build on: pull_request: diff --git a/.github/workflows/pr-prod-e2e.yml b/.github/workflows/pr-prod-e2e.yml index 53144fd8..2ca04f3c 100644 --- a/.github/workflows/pr-prod-e2e.yml +++ b/.github/workflows/pr-prod-e2e.yml @@ -1,4 +1,4 @@ -name: agents-ui prod with staging sdk build e2e validation +name: UI prod e2e with staging sdk build on: pull_request: diff --git a/.github/workflows/publish-on-merge.yml b/.github/workflows/publish-on-merge.yml index 334b9fa8..a60990d3 100644 --- a/.github/workflows/publish-on-merge.yml +++ b/.github/workflows/publish-on-merge.yml @@ -38,13 +38,13 @@ jobs: run: | if [ "${{ github.ref_name }}" = "main" ]; then # For main branch - staging versions with run number - # Use jq to safely read JSON (standard tool) BASE_VERSION=$(jq -r '.version' package.json) if [ "$BASE_VERSION" = "null" ]; then echo "Error: Could not read version from package.json" exit 1 fi - STAGING_VERSION="${BASE_VERSION}-staging.${{ github.run_number }}" + CLEAN_VERSION=$(echo "$BASE_VERSION" | sed 's/-.*$//') + STAGING_VERSION="${CLEAN_VERSION}-staging.${{ github.run_number }}" echo "version=$STAGING_VERSION" >> $GITHUB_OUTPUT echo "tag=staging" >> $GITHUB_OUTPUT echo "description=Staging release from main branch" >> $GITHUB_OUTPUT @@ -60,6 +60,20 @@ jobs: echo "should_sync=true" >> $GITHUB_OUTPUT fi + - name: Update package.json version + run: | + jq --arg version "${{ steps.version.outputs.version }}" '.version = $version' package.json > package.json.tmp + + if ! jq empty package.json.tmp 2>/dev/null; then + echo "Error: Generated invalid JSON" + rm -f package.json.tmp + exit 1 + fi + + mv package.json.tmp package.json + + echo "Updated package.json version to: $(jq -r '.version' package.json)" + - name: Publish to NPM env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/package.json b/package.json index 19d820ce..6f4f26bc 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.20", + "version": "1.1.1", "type": "module", "description": "d-id client sdk", "repository": { @@ -45,4 +45,4 @@ "vite": "^5.1.4", "vite-plugin-dts": "^3.7.3" } -} \ No newline at end of file +} From 082d89fff8d80cad3ce1ef7d4f104976a179580c Mon Sep 17 00:00:00 2001 From: dariusz-did Date: Tue, 5 Aug 2025 10:02:17 +0200 Subject: [PATCH 32/35] removed queued interrupt logic (#174) --- src/services/agent-manager/index.ts | 63 +++-------------------------- src/services/interrupt/index.ts | 5 +-- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 7859b6f2..6c82a1d7 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -53,8 +53,6 @@ export interface AgentManagerItems { */ export async function createAgentManager(agent: string, options: AgentManagerOptions): Promise { let firstConnection = true; - let queuedInterrupt = false; - let speakPending = false; let videoId: string | null = null; const mxKey = options.mixpanelKey || mixpanelKey; @@ -95,9 +93,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt latencyTimestampTracker.reset(); - queuedInterrupt = false; - speakPending = false; - if (newChat && !firstConnection) { delete items.chat; @@ -152,9 +147,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt items.socketManager?.disconnect(); await items.streamingManager?.disconnect(); - queuedInterrupt = false; - speakPending = false; - delete items.streamingManager; delete items.socketManager; @@ -318,12 +310,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt videoId: response.videoId, }); - if (queuedInterrupt && response.videoId && items.streamingManager) { - queuedInterrupt = false; - items.messages[items.messages.length - 1].interrupted = true; - await sendInterrupt(items.streamingManager, response.videoId); - } - analytics.track('agent-message-send', { event: 'success', mode: items.chatMode, @@ -342,8 +328,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return response; } catch (e) { - queuedInterrupt = false; - if (items.messages[items.messages.length - 1]?.role === 'assistant') { items.messages.pop(); } @@ -462,43 +446,14 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt throw new Error('Please connect to the agent first'); } - speakPending = true; - - try { - const response = await items.streamingManager.speak({ - script, - metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, - }); - - speakPending = false; - - items.messages[items.messages.length - 1].videoId = response.video_id; - - if (queuedInterrupt && response.video_id && items.streamingManager) { - queuedInterrupt = false; - items.messages[items.messages.length - 1].interrupted = true; - await sendInterrupt(items.streamingManager, response.video_id); - } - - options.callbacks.onNewMessage?.([...items.messages], 'answer'); - - return response; - } finally { - speakPending = false; - } + return items.streamingManager.speak({ + script, + metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, + }); }, async interrupt({ type }: Interrupt) { + validateInterrupt(items.streamingManager, items.streamingManager?.streamType, videoId); const lastMessage = items.messages[items.messages.length - 1]; - const chatRequestPending = lastMessage?.role === 'user'; - - const isStreamRequestPending = chatRequestPending || speakPending; - - validateInterrupt( - items.streamingManager, - items.streamingManager?.streamType, - isStreamRequestPending, - !!videoId - ); analytics.track('agent-video-interrupt', { type: type || 'click', @@ -509,17 +464,11 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt message_duration_to_interrupt: latencyTimestampTracker.get(true), chat_id: items.chat?.id, mode: items.chatMode, - queued_interrupt: chatRequestPending, }); - if (isStreamRequestPending) { - queuedInterrupt = true; - return; - } - lastMessage.interrupted = true; options.callbacks.onNewMessage?.([...items.messages], 'answer'); - sendInterrupt(items.streamingManager!, lastMessage.videoId!); + sendInterrupt(items.streamingManager!, videoId!); }, }; } diff --git a/src/services/interrupt/index.ts b/src/services/interrupt/index.ts index 115aa176..95d196c3 100644 --- a/src/services/interrupt/index.ts +++ b/src/services/interrupt/index.ts @@ -4,8 +4,7 @@ import { StreamingManager } from '../streaming-manager'; export function validateInterrupt( streamingManager: StreamingManager | undefined, streamType: StreamType | undefined, - hasStreamRequestPending: boolean, - hasVideoId: boolean + videoId: string | null ): void { if (!streamingManager) { throw new Error('Please connect to the agent first'); @@ -19,7 +18,7 @@ export function validateInterrupt( throw new Error('Interrupt only available for Fluent streams'); } - if (!hasStreamRequestPending && !hasVideoId) { + if (!videoId) { throw new Error('No active video to interrupt'); } } From 9238e46bd92a1fc46ba07ac0f1d1098a1067d117 Mon Sep 17 00:00:00 2001 From: benyael15 <32225598+benyael15@users.noreply.github.com> Date: Tue, 5 Aug 2025 14:05:30 +0300 Subject: [PATCH 33/35] Feature/outer control mode (#173) * add outer control mode * exclude text creation from new mode * add onStreamCreated callback * rename chat mode * put back websockets * getting video id form socket (#168) * adjustments before merge --------- Co-authored-by: Niv Zelber Co-authored-by: Dor Eitan Co-authored-by: dariusz-did --- src/services/agent-manager/index.ts | 8 ++++---- src/services/chat/index.ts | 6 ++++-- src/services/streaming-manager/index.ts | 1 + src/types/entities/agents/chat.ts | 1 + src/types/entities/agents/manager.ts | 9 ++++++--- src/types/stream/stream.ts | 1 + src/utils/chat.ts | 3 +++ 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 6c82a1d7..944fa2c2 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -17,7 +17,7 @@ import { CONNECTION_RETRY_TIMEOUT_MS } from '$/config/consts'; import { didApiUrl, didSocketApiUrl, mixpanelKey } from '$/config/environment'; import { ChatCreationFailed, ValidationError } from '$/errors'; import { getRandom } from '$/utils'; -import { isTextualChat } from '$/utils/chat'; +import { isChatModeWithoutChat, isTextualChat } from '$/utils/chat'; import { createAgentsApi } from '../../api/agents'; import { getAgentInfo, getAnalyticsInfo } from '../../utils/analytics'; import { retryOperation } from '../../utils/retry-operation'; @@ -74,6 +74,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const agentEntity = await agentsApi.getById(agent); analytics.enrich(getAgentInfo(agentEntity)); + const { onMessage, clearQueue } = createMessageEventQueue(analytics, items, options, agentEntity, () => items.socketManager?.disconnect() ); @@ -207,8 +208,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }, async chat(userMessage: string) { const validateChatRequest = () => { - if (options.mode === ChatMode.DirectPlayback) { - throw new ValidationError('Direct playback is enabled, chat is disabled'); + if (isChatModeWithoutChat(options.mode)) { + throw new ValidationError(`${options.mode} is enabled, chat is disabled`); } else if (userMessage.length >= 800) { throw new ValidationError('Message cannot be more than 800 characters'); } else if (userMessage.length === 0) { @@ -307,7 +308,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt created_at: new Date().toISOString(), context: response.context, matches: response.matches, - videoId: response.videoId, }); analytics.track('agent-message-send', { diff --git a/src/services/chat/index.ts b/src/services/chat/index.ts index c4992d23..d0bce303 100644 --- a/src/services/chat/index.ts +++ b/src/services/chat/index.ts @@ -1,5 +1,7 @@ import { PLAYGROUND_HEADER } from '$/config/consts'; -import { Agent, AgentsAPI, Chat, ChatMode } from '$/types'; +import type { Agent, AgentsAPI, Chat } from '$/types'; +import { ChatMode } from '$/types'; +import { isChatModeWithoutChat } from '$/utils/chat'; import { Analytics } from '../analytics/mixpanel'; export function getRequestHeaders(chatMode?: ChatMode): Record> { @@ -15,7 +17,7 @@ export async function createChat( chat?: Chat ) { try { - if (!chat && chatMode !== ChatMode.DirectPlayback) { + if (!chat && !isChatModeWithoutChat(chatMode)) { chat = await agentsApi.newChat(agent.id, { persist }, getRequestHeaders(chatMode)); analytics.track('agent-chat', { diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 969aa44f..478a3769 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -155,6 +155,7 @@ export async function createStreamingManager( fluent, interrupt_enabled: interruptAvailable, } = await createStream(agent); + callbacks.onStreamCreated?.({ stream_id: streamIdFromServer, session_id: session_id as string, agent_id: agentId }); const peerConnection = new actualRTCPC({ iceServers: ice_servers }); const pcDataChannel = peerConnection.createDataChannel('JanusDataChannel'); diff --git a/src/types/entities/agents/chat.ts b/src/types/entities/agents/chat.ts index 0fb7c17c..d08bbf04 100644 --- a/src/types/entities/agents/chat.ts +++ b/src/types/entities/agents/chat.ts @@ -57,6 +57,7 @@ export enum ChatMode { Maintenance = 'Maintenance', Playground = 'Playground', DirectPlayback = 'DirectPlayback', + Off = 'Off', } export interface ChatResponse { diff --git a/src/types/entities/agents/manager.ts b/src/types/entities/agents/manager.ts index 39730d69..b4204175 100644 --- a/src/types/entities/agents/manager.ts +++ b/src/types/entities/agents/manager.ts @@ -11,6 +11,7 @@ import { StreamingState, } from '$/types/stream'; import { SupportedStreamScript } from '$/types/stream-script'; +import type { ManagerCallbacks as StreamManagerCallbacks } from '../../stream/stream'; import { Agent } from './agent'; import { ChatMode, ChatResponse, Interrupt, Message, RatingEntity } from './chat'; @@ -74,13 +75,11 @@ interface ManagerCallbacks { * @param chatId - id of the new chat */ onNewChat?(chatId: string): void; - /** * Optional callback function that will be triggered each time the chat mode changes * @param mode - ChatMode */ onModeChange?(mode: ChatMode): void; - /** * Optional callback function that will be triggered each time the user internet connectivity state change by realtime estimated bitrate * @param state - ConnectivityState @@ -90,12 +89,16 @@ interface ManagerCallbacks { * Optional callback function that will be triggered on fetch request errors */ onError?: (error: Error, errorData?: object) => void; - /** * Optional callback function that will be triggered each time the agent activity state changes * @param state - AgentActivityState */ onAgentActivityStateChange?(state: AgentActivityState): void; + /** + * Optional callback function that will be triggered each time a new stream is created + * @param stream - object containing stream_id, session_id and agent_id + */ + onStreamCreated?: StreamManagerCallbacks['onStreamCreated']; } interface StreamOptions { diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index a75470ca..e135f13b 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -62,6 +62,7 @@ export interface ManagerCallbacks { onConnectivityStateChange?: (state: ConnectivityState) => void; onAgentActivityStateChange?: (state: AgentActivityState) => void; onVideoIdChange?: (videoId: string | null) => void; + onStreamCreated?: (stream: { stream_id: string; session_id: string; agent_id: string }) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; diff --git a/src/utils/chat.ts b/src/utils/chat.ts index a9460e2a..cb722025 100644 --- a/src/utils/chat.ts +++ b/src/utils/chat.ts @@ -2,3 +2,6 @@ import { ChatMode } from '$/types'; export const isTextualChat = (chatMode: ChatMode) => [ChatMode.TextOnly, ChatMode.Playground, ChatMode.Maintenance].includes(chatMode); + +export const isChatModeWithoutChat = (chatMode: ChatMode | undefined) => + chatMode && [ChatMode.DirectPlayback, ChatMode.Off].includes(chatMode); From 1ae25d6f10adbcc9d27b4300c363b13c86940320 Mon Sep 17 00:00:00 2001 From: dariusz-did Date: Tue, 5 Aug 2025 16:44:30 +0200 Subject: [PATCH 34/35] interrupt type extended (#176) --- src/types/entities/agents/chat.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/entities/agents/chat.ts b/src/types/entities/agents/chat.ts index d08bbf04..c626e636 100644 --- a/src/types/entities/agents/chat.ts +++ b/src/types/entities/agents/chat.ts @@ -82,5 +82,5 @@ export interface Chat { } export interface Interrupt { - type: 'text' | 'audio' | 'click'; + type: 'text' | 'audio' | 'click' | 'manual'; } From 3b8a8787589ab73e17d2f895df6defe7bc720443 Mon Sep 17 00:00:00 2001 From: dor-eitan <164745144+dor-eitan@users.noreply.github.com> Date: Thu, 7 Aug 2025 10:15:09 +0300 Subject: [PATCH 35/35] Update GitHub Actions workflow to use DEVOPS_TOKEN instead of GITHUB_TOKEN for authentication (#178) --- .github/workflows/pr-prod-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-prod-e2e.yml b/.github/workflows/pr-prod-e2e.yml index 2ca04f3c..885c9848 100644 --- a/.github/workflows/pr-prod-e2e.yml +++ b/.github/workflows/pr-prod-e2e.yml @@ -27,7 +27,7 @@ jobs: path: agents-ui fetch-depth: 0 lfs: true - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.DEVOPS_TOKEN }} - name: Set github environment variables uses: rlespinasse/github-slug-action@v4