diff --git a/package.json b/package.json index 96dcf7be..7b52a26a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.13", + "version": "1.1.14", "type": "module", "description": "d-id client sdk", "repository": { diff --git a/src/services/agent-manager/connect-to-manager.test.ts b/src/services/agent-manager/connect-to-manager.test.ts index c9be3eff..f4afdb5c 100644 --- a/src/services/agent-manager/connect-to-manager.test.ts +++ b/src/services/agent-manager/connect-to-manager.test.ts @@ -6,12 +6,13 @@ import { ConnectionState, Providers, StreamEvents, - StreamType, StreamingState, + StreamType, + TransportProvider, } from '../../types'; import { Analytics } from '../analytics/mixpanel'; import { createChat } from '../chat'; -import { createStreamingManager } from '../streaming-manager'; +import { createStreamingManager, StreamApiVersion } from '../streaming-manager'; import { initializeStreamAndChat } from './connect-to-manager'; // Mock dependencies @@ -116,7 +117,8 @@ describe('connect-to-manager', () => { mockAgentsApi = { getById: jest.fn().mockResolvedValue(mockAgent), chat: jest.fn(), createRating: jest.fn() }; // Setup mocks - (createStreamingManager as jest.Mock).mockImplementation((agentId, streamArgs, options) => { + (createStreamingManager as jest.Mock).mockReset(); + (createStreamingManager as jest.Mock).mockImplementation((agent, streamOptions, options) => { // Immediately trigger the connection state change to Connected setTimeout(() => { if (options.callbacks.onConnectionStateChange) { @@ -128,6 +130,7 @@ describe('connect-to-manager', () => { return Promise.resolve(mockStreamingManager); }); + (createChat as jest.Mock).mockReset(); (createChat as jest.Mock).mockResolvedValue({ chat: mockChat, chatMode: ChatMode.Functional }); }); @@ -137,9 +140,18 @@ describe('connect-to-manager', () => { expect(result.streamingManager).toBe(mockStreamingManager); expect(result.chat).toBe(mockChat); + expect(createChat).toHaveBeenCalledWith( + mockAgent, + mockAgentsApi, + mockAnalytics, + ChatMode.Functional, + true, + undefined + ); expect(createStreamingManager).toHaveBeenCalledWith( mockAgent, { + version: StreamApiVersion.V1, output_resolution: 1080, session_timeout: 30000, stream_warmup: true, @@ -155,14 +167,6 @@ describe('connect-to-manager', () => { }), }) ); - expect(createChat).toHaveBeenCalledWith( - mockAgent, - mockAgentsApi, - mockAnalytics, - ChatMode.Functional, - true, - undefined - ); }); it('should initialize with existing chat', async () => { @@ -233,7 +237,7 @@ describe('connect-to-manager', () => { onVideoStateChange = jest.fn(); onAgentActivityStateChange = jest.fn(); - (createStreamingManager as jest.Mock).mockImplementation((agentId, streamArgs, options) => { + (createStreamingManager as jest.Mock).mockImplementation((agent, streamOptions, options) => { onConnectionStateChange = options.callbacks.onConnectionStateChange; onVideoStateChange = options.callbacks.onVideoStateChange; onAgentActivityStateChange = options.callbacks.onAgentActivityStateChange; @@ -380,13 +384,16 @@ describe('connect-to-manager', () => { expect(createStreamingManager).toHaveBeenCalledWith( mockAgent, { + version: StreamApiVersion.V1, output_resolution: 720, session_timeout: 60000, stream_warmup: false, compatibility_mode: 'on', fluent: true, }, - expect.any(Object) + expect.not.objectContaining({ + chatId: expect.anything(), + }) ); }); @@ -398,13 +405,40 @@ describe('connect-to-manager', () => { expect(createStreamingManager).toHaveBeenCalledWith( mockAgent, { + version: StreamApiVersion.V1, output_resolution: undefined, session_timeout: undefined, stream_warmup: undefined, compatibility_mode: undefined, fluent: undefined, }, - expect.any(Object) + expect.not.objectContaining({ + chatId: expect.anything(), + }) + ); + }); + + it('should include analytics data when provided', async () => { + const optionsWithAnalytics = { + ...mockOptions, + distinctId: 'analytics-user', + mixpanelAdditionalProperties: { plan: 'scale' }, + }; + + await initializeStreamAndChat(mockAgent, optionsWithAnalytics, mockAgentsApi, mockAnalytics); + + expect(createStreamingManager).toHaveBeenCalledWith( + mockAgent, + expect.objectContaining({ + version: StreamApiVersion.V1, + end_user_data: { + distinct_id: 'analytics-user', + plan: 'scale', + }, + }), + expect.not.objectContaining({ + chatId: expect.anything(), + }) ); }); }); @@ -458,16 +492,14 @@ describe('connect-to-manager', () => { (createStreamingManager as jest.Mock).mockRejectedValueOnce(streamError); (createChat as jest.Mock).mockRejectedValueOnce(chatError); - // Should reject with the first error encountered - await expect( - initializeStreamAndChat(mockAgent, mockOptions, mockAgentsApi, mockAnalytics) - ).rejects.toThrow(); + await expect(initializeStreamAndChat(mockAgent, mockOptions, mockAgentsApi, mockAnalytics)).rejects.toThrow( + 'Chat failed' + ); }); }); - describe('Concurrent Operations', () => { - it('should handle streaming manager and chat creation concurrently', async () => { - // Simplified test to avoid timing issues + describe('Sequential Operations', () => { + it('should handle streaming manager and chat creation sequentially', async () => { const result = await initializeStreamAndChat(mockAgent, mockOptions, mockAgentsApi, mockAnalytics); expect(result.streamingManager).toBeDefined(); @@ -476,4 +508,46 @@ describe('connect-to-manager', () => { expect(createChat).toHaveBeenCalled(); }); }); + + describe('Streams V2 Support', () => { + it('should use CreateStreamV2Options for expressive agents', async () => { + const expressiveAgent = { + ...mockAgent, + presenter: { + type: 'expressive' as const, + voice: { type: Providers.Microsoft, voice_id: 'voice-123' }, + }, + }; + + await initializeStreamAndChat(expressiveAgent, mockOptions, mockAgentsApi, mockAnalytics); + + expect(createStreamingManager).toHaveBeenCalledWith( + expressiveAgent, + { + version: StreamApiVersion.V2, + transport_provider: TransportProvider.Livekit, + chat_id: 'chat-123', + }, + expect.objectContaining({ + chatId: 'chat-123', + }) + ); + }); + + it('should use CreateStreamOptions for non-expressive agents', async () => { + await initializeStreamAndChat(mockAgent, mockOptions, mockAgentsApi, mockAnalytics); + + expect(createStreamingManager).toHaveBeenCalledWith( + mockAgent, + expect.objectContaining({ + version: StreamApiVersion.V1, + output_resolution: 1080, + session_timeout: 30000, + }), + expect.not.objectContaining({ + chatId: expect.anything(), + }) + ); + }); + }); }); diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index 7f3fc6cd..161d99a6 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -1,5 +1,10 @@ import { ChatModeDowngraded } from '$/errors'; -import { StreamingManager, createStreamingManager } from '$/services/streaming-manager'; +import { + ExtendedStreamOptions, + StreamApiVersion, + StreamingManager, + createStreamingManager, +} from '$/services/streaming-manager'; import { Agent, AgentActivityState, @@ -9,24 +14,52 @@ import { ChatMode, ConnectionState, CreateStreamOptions, + CreateStreamV2Options, StreamEvents, StreamType, StreamingState, + TransportProvider, } from '$/types'; +import { isStreamsV2Agent } from '$/utils/agent'; import { Analytics } from '../analytics/mixpanel'; import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; import { createChat } from '../chat'; -function getAgentStreamArgs(options?: AgentManagerOptions): CreateStreamOptions { +function getAgentStreamV2Options(options?: ConnectToManagerOptions): CreateStreamV2Options { + return { + transport_provider: TransportProvider.Livekit, + chat_id: options?.chatId, + }; +} + +function getAgentStreamV1Options(options?: ConnectToManagerOptions): CreateStreamOptions { const { streamOptions } = options ?? {}; - return { + const endUserData = + options?.distinctId || options?.mixpanelAdditionalProperties?.plan !== undefined + ? { + ...(options?.distinctId ? { distinct_id: options.distinctId } : {}), + ...(options?.mixpanelAdditionalProperties?.plan !== undefined + ? { plan: options.mixpanelAdditionalProperties?.plan } + : {}), + } + : undefined; + + const streamArgs = { output_resolution: streamOptions?.outputResolution, session_timeout: streamOptions?.sessionTimeout, stream_warmup: streamOptions?.streamWarmup, compatibility_mode: streamOptions?.compatibilityMode, fluent: streamOptions?.fluent, }; + + return { ...streamArgs, ...(endUserData && { end_user_data: endUserData }) }; +} + +function getAgentStreamOptions(agent: Agent, options?: ConnectToManagerOptions): ExtendedStreamOptions { + return isStreamsV2Agent(agent.presenter.type) + ? { version: StreamApiVersion.V2, ...getAgentStreamV2Options(options) } + : { version: StreamApiVersion.V1, ...getAgentStreamV1Options(options) }; } function trackVideoStateChangeAnalytics( @@ -129,19 +162,20 @@ type ConnectToManagerOptions = AgentManagerOptions & { callbacks: AgentManagerOptions['callbacks'] & { onVideoIdChange?: (videoId: string | null) => void; }; + chatId?: string; }; function connectToManager( agent: Agent, options: ConnectToManagerOptions, analytics: Analytics -): Promise> { +): Promise> { latencyTimestampTracker.reset(); return new Promise(async (resolve, reject) => { try { - let streamingManager: StreamingManager; - streamingManager = await createStreamingManager(agent, getAgentStreamArgs(options), { + let streamingManager: StreamingManager; + streamingManager = await createStreamingManager(agent, getAgentStreamOptions(agent, options), { ...options, analytics, callbacks: { @@ -194,12 +228,39 @@ export async function initializeStreamAndChat( agentsApi: AgentsAPI, analytics: Analytics, chat?: Chat -): Promise<{ chat?: Chat; streamingManager?: StreamingManager }> { - const createChatPromise = createChat(agent, agentsApi, analytics, options.mode, options.persistentChat, chat); - const connectToManagerPromise = connectToManager(agent, options, analytics); - - const [chatResult, streamingManager] = await Promise.all([createChatPromise, connectToManagerPromise]); +): Promise<{ chat?: Chat; streamingManager?: StreamingManager }> { + const resolveStreamAndChat = async () => { + if (isStreamsV2Agent(agent.presenter.type)) { + const chatResult = await createChat( + agent, + agentsApi, + analytics, + options.mode, + options.persistentChat, + chat + ); + const streamingManager = await connectToManager( + agent, + { ...options, chatId: chatResult.chat?.id }, + analytics + ); + return { chatResult, streamingManager }; + } else { + const createChatPromise = createChat( + agent, + agentsApi, + analytics, + options.mode, + options.persistentChat, + chat + ); + const connectToManagerPromise = connectToManager(agent, options, analytics); + const [chatResult, streamingManager] = await Promise.all([createChatPromise, connectToManagerPromise]); + return { chatResult, streamingManager }; + } + }; + const { chatResult, streamingManager } = await resolveStreamAndChat(); const { chat: newChat, chatMode } = chatResult; if (chatMode && chatMode !== options.mode) { diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index e7131185..e6bd23b2 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -424,7 +424,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt id: getRandom(), role: 'assistant', content: script.input, - created_at: new Date(latencyTimestampTracker.get(true)).toISOString(), + created_at: new Date().toISOString(), }); options.callbacks.onNewMessage?.([...items.messages], 'answer'); } diff --git a/src/services/streaming-manager/common.ts b/src/services/streaming-manager/common.ts index 250b2211..25f51e7a 100644 --- a/src/services/streaming-manager/common.ts +++ b/src/services/streaming-manager/common.ts @@ -1,4 +1,4 @@ -import { CreateStreamOptions, PayloadType, StreamType } from '$/types'; +import { CreateStreamOptions, CreateStreamV2Options, PayloadType, StreamType } from '$/types'; export const createStreamingLogger = (debug: boolean, prefix: string) => (message: string, extra?: any) => debug && console.log(`[${prefix}] ${message}`, extra ?? ''); @@ -7,7 +7,7 @@ export const createStreamingLogger = (debug: boolean, prefix: string) => (messag * Shared type for all streaming managers (LiveKit, WebRTC, etc.) * This type represents the return value of any streaming manager implementation */ -export type StreamingManager = { +export type StreamingManager = { /** * Method to send request to server to get clip or talk depending on payload * @param payload The payload to send to the streaming service diff --git a/src/services/streaming-manager/factory.test.ts b/src/services/streaming-manager/factory.test.ts index 52ad3ada..7982ba2b 100644 --- a/src/services/streaming-manager/factory.test.ts +++ b/src/services/streaming-manager/factory.test.ts @@ -1,6 +1,12 @@ import { AgentFactory, StreamingManagerOptionsFactory } from '../../test-utils/factories'; -import { CreateStreamOptions, Providers, StreamingManagerOptions } from '../../types'; -import { createStreamingManager } from './factory'; +import { + CreateStreamOptions, + CreateStreamV2Options, + Providers, + StreamingManagerOptions, + TransportProvider, +} from '../../types'; +import { createStreamingManager, StreamApiVersion } from './factory'; const mockCreateWebRTCStreamingManager = jest.fn(); jest.mock('./webrtc-manager', () => ({ @@ -38,7 +44,7 @@ describe('createStreamingManager', () => { }, }); - await createStreamingManager(agent, mockStreamOptions, mockOptions); + await createStreamingManager(agent, { version: StreamApiVersion.V1, ...mockStreamOptions }, mockOptions); expect(mockCreateWebRTCStreamingManager).toHaveBeenCalledWith(agent.id, mockStreamOptions, mockOptions); expect(mockCreateLiveKitStreamingManager).not.toHaveBeenCalled(); @@ -57,26 +63,31 @@ describe('createStreamingManager', () => { }, }); - await createStreamingManager(agent, mockStreamOptions, mockOptions); + await createStreamingManager(agent, { version: StreamApiVersion.V1, ...mockStreamOptions }, mockOptions); expect(mockCreateWebRTCStreamingManager).toHaveBeenCalledWith(agent.id, mockStreamOptions, mockOptions); expect(mockCreateLiveKitStreamingManager).not.toHaveBeenCalled(); }); - // it('calls to createLiveKitStreamingManager when agent presenter type is expressive', async () => { - // const agent = AgentFactory.build({ - // presenter: { - // type: 'expressive', - // voice: { - // type: Providers.Microsoft, - // voice_id: 'voice-123', - // }, - // }, - // }); - - // await createStreamingManager(agent, mockStreamOptions, mockOptions); - - // expect(mockCreateLiveKitStreamingManager).toHaveBeenCalledWith(agent.id, mockStreamOptions, mockOptions); - // expect(mockCreateWebRTCStreamingManager).not.toHaveBeenCalled(); - // }); + it('calls to createLiveKitStreamingManager when agent presenter type is expressive', async () => { + const agent = AgentFactory.build({ + presenter: { + type: 'expressive', + voice: { + type: Providers.Microsoft, + voice_id: 'voice-123', + }, + }, + }); + + const v2StreamOptions: CreateStreamV2Options = { + transport_provider: TransportProvider.Livekit, + chat_id: 'chat-123', + }; + + await createStreamingManager(agent, { version: StreamApiVersion.V2, ...v2StreamOptions }, mockOptions); + + expect(mockCreateLiveKitStreamingManager).toHaveBeenCalledWith(agent.id, v2StreamOptions, mockOptions); + expect(mockCreateWebRTCStreamingManager).not.toHaveBeenCalled(); + }); }); diff --git a/src/services/streaming-manager/factory.ts b/src/services/streaming-manager/factory.ts index a6087253..7f73383d 100644 --- a/src/services/streaming-manager/factory.ts +++ b/src/services/streaming-manager/factory.ts @@ -1,21 +1,42 @@ -import { Agent, CreateStreamOptions, StreamingManagerOptions, VideoType } from '$/types'; +import { Agent, CreateStreamOptions, CreateStreamV2Options, StreamingManagerOptions, TransportProvider } from '$/types'; +import { StreamingManager } from './common'; import { createWebRTCStreamingManager } from './webrtc-manager'; -const isLiveKitAgent = (agent: Agent): boolean => agent.presenter.type === VideoType.Expressive; +export enum StreamApiVersion { + V1 = 'v1', + V2 = 'v2', +} + +export type ExtendedStreamOptions = + | ({ version: StreamApiVersion.V1 } & CreateStreamOptions) + | ({ version: StreamApiVersion.V2 } & CreateStreamV2Options); -export async function createStreamingManager( +export async function createStreamingManager( agent: Agent, - streamOptions: T, + streamOptions: ExtendedStreamOptions, options: StreamingManagerOptions -) { +): Promise> { const agentId = agent.id; - if (isLiveKitAgent(agent)) { - // Lazy import the LiveKit manager only when needed - // const { createLiveKitStreamingManager } = await import('./livekit-manager'); - // return createLiveKitStreamingManager(agentId, streamOptions, options); - return {} as any; - } else { - return createWebRTCStreamingManager(agentId, streamOptions, options); + switch (streamOptions.version) { + case StreamApiVersion.V1: { + const { version, ...createStreamOptions } = streamOptions; + return createWebRTCStreamingManager(agentId, createStreamOptions, options); + } + + case StreamApiVersion.V2: { + const { version, ...createStreamOptions } = streamOptions; + + switch (createStreamOptions.transport_provider) { + case TransportProvider.Livekit: + const { createLiveKitStreamingManager } = await import('./livekit-manager'); + return createLiveKitStreamingManager(agentId, createStreamOptions, options); + default: + throw new Error(`Unsupported transport provider: ${createStreamOptions.transport_provider}`); + } + } + + default: + throw new Error(`Invalid stream version: ${(streamOptions as any).version}`); } } diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index b4933788..0b375850 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -1,5 +1,6 @@ -export { createStreamingManager } from './factory'; +export { StreamApiVersion as StreamApiVersion, createStreamingManager } from './factory'; +export type { ExtendedStreamOptions } from './factory'; export type { StreamingManager } from './common'; -// export type { LiveKitStreamingManager } from './livekit-manager'; +export type { LiveKitStreamingManager } from './livekit-manager'; export * from './webrtc-manager'; diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 4c53ca34..86d8c594 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -3,12 +3,13 @@ import { ConnectionState, ConnectivityState, CreateStreamOptions, + CreateStreamV2Options, PayloadType, StreamEvents, StreamingManagerOptions, StreamingState, StreamType, - Transport, + TransportProvider, } from '$/types'; import { createStreamApiV2 } from '../../api/streams/streamsApiV2'; import { didApiUrl } from '../../config/environment'; @@ -38,7 +39,7 @@ async function importLiveKit(): Promise<{ } } -export async function createLiveKitStreamingManager( +export async function createLiveKitStreamingManager( agentId: string, agent: T, options: StreamingManagerOptions @@ -160,7 +161,8 @@ export async function createLiveKitStreamingManager presenter.type === 'clip' && presenter.presenter_id.startsWith('v2_') ? 'clip_v2' : presenter.type; + +export const isStreamsV2Agent = (type: AgentType): boolean => type === VideoType.Expressive; diff --git a/tsconfig.json b/tsconfig.json index b9079c95..5636db0c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,5 +46,6 @@ } }, "include": ["src", "demo"], + "exclude": ["node_modules", "**/*.test.ts"], "references": [{ "path": "./tsconfig.node.json" }] }