diff --git a/package.json b/package.json index f6b8de21..7e3f4db0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.56", + "version": "1.1.57", "type": "module", "description": "d-id client sdk", "repository": { diff --git a/src/services/agent-manager/index.test.ts b/src/services/agent-manager/index.test.ts index f6a7e584..e5b914c0 100644 --- a/src/services/agent-manager/index.test.ts +++ b/src/services/agent-manager/index.test.ts @@ -511,6 +511,7 @@ describe('createAgentManager', () => { describe('interrupt', () => { beforeEach(async () => { + mockStreamingManager.interruptAvailable = true; await manager.connect(); }); @@ -742,12 +743,10 @@ describe('createAgentManager', () => { expect(mockUnpublish).toHaveBeenCalled(); }); - it('should throw error when unpublishMicrophoneStream is not available', async () => { + it('should no-op when unpublishMicrophoneStream is not available', async () => { mockStreamingManager.unpublishMicrophoneStream = undefined; - await expect(manager.unpublishMicrophoneStream?.()).rejects.toThrow( - 'unpublishMicrophoneStream is not available for this streaming manager' - ); + await expect(manager.unpublishMicrophoneStream?.()).resolves.toBeUndefined(); }); }); @@ -795,12 +794,10 @@ describe('createAgentManager', () => { expect(mockUnpublish).toHaveBeenCalled(); }); - it('should throw error when unpublishCameraStream is not available', async () => { + it('should no-op when unpublishCameraStream is not available', async () => { mockStreamingManager.unpublishCameraStream = undefined; - await expect(manager.unpublishCameraStream?.()).rejects.toThrow( - 'unpublishCameraStream is not available for this streaming manager' - ); + await expect(manager.unpublishCameraStream?.()).resolves.toBeUndefined(); }); }); @@ -835,4 +832,75 @@ describe('createAgentManager', () => { }); }); }); + + describe('registerClientTool', () => { + let manager: AgentManager; + + beforeEach(async () => { + mockStreamingManager.registerRpcMethod = jest.fn(); + mockStreamingManager.unregisterRpcMethod = jest.fn(); + manager = await createAgentManager('agent-123', mockOptions); + }); + + it('should register tool and call registerRpcMethod after connect', async () => { + const handler = jest.fn().mockResolvedValue('result'); + + await manager.connect(); + manager.registerClientTool('testTool', handler); + + expect(mockStreamingManager.registerRpcMethod).toHaveBeenCalledWith('testTool', expect.any(Function)); + }); + + it('should buffer tool registration before connect and flush on connect', async () => { + const handler = jest.fn().mockResolvedValue('result'); + + manager.registerClientTool('testTool', handler); + expect(mockStreamingManager.registerRpcMethod).not.toHaveBeenCalled(); + + await manager.connect(); + expect(mockStreamingManager.registerRpcMethod).toHaveBeenCalledWith('testTool', expect.any(Function)); + }); + + it('should not call registerRpcMethod twice for same tool name', async () => { + const handler1 = jest.fn().mockResolvedValue('result1'); + const handler2 = jest.fn().mockResolvedValue('result2'); + + await manager.connect(); + manager.registerClientTool('testTool', handler1); + manager.registerClientTool('testTool', handler2); + + expect(mockStreamingManager.registerRpcMethod).toHaveBeenCalledTimes(1); + }); + + it('should unregister tool from map and room', async () => { + const handler = jest.fn().mockResolvedValue('result'); + + await manager.connect(); + manager.registerClientTool('testTool', handler); + manager.unregisterClientTool('testTool'); + + expect(mockStreamingManager.unregisterRpcMethod).toHaveBeenCalledWith('testTool'); + }); + + it('should invoke latest handler when RPC is called after re-registration', async () => { + const handler1 = jest.fn().mockResolvedValue('result1'); + const handler2 = jest.fn().mockResolvedValue('result2'); + + await manager.connect(); + manager.registerClientTool('testTool', handler1); + + // Get the RPC handler that was registered + const rpcHandler = mockStreamingManager.registerRpcMethod.mock.calls[0][1]; + + // Re-register with new handler (same name — only updates Map) + manager.registerClientTool('testTool', handler2); + + // Invoke the RPC handler — should use handler2 from Map + const result = await rpcHandler({ payload: '{"key": "val"}' }); + + expect(handler1).not.toHaveBeenCalled(); + expect(handler2).toHaveBeenCalledWith({ key: 'val' }); + expect(result).toBe('result2'); + }); + }); }); diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index c8f43615..0b719f37 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -5,6 +5,7 @@ import { Chat, ChatMode, ChatResponse, + ClientToolHandler, ConnectionState, CreateSessionV2Options, CreateStreamOptions, @@ -104,6 +105,9 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }; const interrupt = ({ type }: Interrupt) => { + if (!items.streamingManager?.interruptAvailable) { + return; + } if (!items.streamingManager?.isInterruptible) return; const lastMessage = items.messages[items.messages.length - 1]; @@ -125,6 +129,43 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt } }; + const clientToolHandlers = new Map(); + + function createRpcHandler(toolName: string) { + return async (data: { payload: string }): Promise => { + const handler = clientToolHandlers.get(toolName); + if (!handler) { + throw new Error(`No handler registered for client tool: ${toolName}`); + } + try { + const args = JSON.parse(data.payload); + return await handler(args); + } catch (error) { + throw new Error(`Client tool "${toolName}" failed: ${(error as Error).message}`); + } + }; + } + + function flushClientToolsToRoom() { + for (const [name] of clientToolHandlers) { + items.streamingManager?.unregisterRpcMethod?.(name); + items.streamingManager?.registerRpcMethod?.(name, createRpcHandler(name)); + } + } + + function registerClientTool(name: string, handler: ClientToolHandler): void { + const isNew = !clientToolHandlers.has(name); + clientToolHandlers.set(name, handler); + if (isNew) { + items.streamingManager?.registerRpcMethod?.(name, createRpcHandler(name)); + } + } + + function unregisterClientTool(name: string): void { + clientToolHandlers.delete(name); + items.streamingManager?.unregisterRpcMethod?.(name); + } + const loadedTimestamp = Date.now(); defer(() => { analytics.track('agent-sdk', { event: 'loaded', ...getAnalyticsInfo(agentEntity) }, loadedTimestamp); @@ -194,6 +235,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt items.socketManager = socketManager; items.chat = chat; + flushClientToolsToRoom(); + firstConnection = false; analytics.enrich({ @@ -232,7 +275,6 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt agent: agentEntity, getStreamType: () => items.streamingManager?.streamType, getIsInterruptAvailable: () => items.streamingManager?.interruptAvailable ?? false, - getIsTriggersAvailable: () => items.streamingManager?.triggersAvailable ?? false, starterMessages: agentEntity.knowledge?.starter_message || [], getSTTToken: () => agentsApi.getSTTToken(agentEntity.id), changeMode, @@ -286,7 +328,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }, async unpublishMicrophoneStream() { if (!items.streamingManager?.unpublishMicrophoneStream) { - throw new Error('unpublishMicrophoneStream is not available for this streaming manager'); + return; } return items.streamingManager.unpublishMicrophoneStream(); }, @@ -298,7 +340,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }, async unpublishCameraStream() { if (!items.streamingManager?.unpublishCameraStream) { - throw new Error('unpublishCameraStream is not available for this streaming manager'); + return; } return items.streamingManager.unpublishCameraStream(); }, @@ -552,5 +594,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }); }, interrupt, + registerClientTool, + unregisterClientTool, }; } diff --git a/src/services/streaming-manager/common.ts b/src/services/streaming-manager/common.ts index 63295562..a8c4a432 100644 --- a/src/services/streaming-manager/common.ts +++ b/src/services/streaming-manager/common.ts @@ -87,7 +87,15 @@ export type StreamingManager Promise): void; + + /** + * Unregister a previously registered RPC method. + * supported only for livekit manager */ - triggersAvailable: boolean; + unregisterRpcMethod?(method: string): void; }; diff --git a/src/services/streaming-manager/livekit-manager.ts b/src/services/streaming-manager/livekit-manager.ts index 0f45a7fc..e714d376 100644 --- a/src/services/streaming-manager/livekit-manager.ts +++ b/src/services/streaming-manager/livekit-manager.ts @@ -129,6 +129,7 @@ export async function createLiveKitStreamingManager Promise) { + room?.registerRpcMethod(method, handler); + }, + unregisterRpcMethod(method: string) { + room?.unregisterRpcMethod(method); + }, + sessionId, streamId: sessionId, streamType, - interruptAvailable: true, + interruptAvailable: interruptEnabled, isInterruptible: currentInterruptible, - triggersAvailable: false, }; } diff --git a/src/services/streaming-manager/webrtc-manager.ts b/src/services/streaming-manager/webrtc-manager.ts index d311efd9..6b33e808 100644 --- a/src/services/streaming-manager/webrtc-manager.ts +++ b/src/services/streaming-manager/webrtc-manager.ts @@ -170,7 +170,6 @@ export async function createWebRTCStreamingManager boolean; - /** - * Get if the stream supports triggers - */ - getIsTriggersAvailable: () => boolean; - /** * Array of starter messages that will be sent to the agent when the chat starts */ @@ -284,4 +280,18 @@ export interface AgentManager { * Only available for Fluent streams and when there's an active video to interrupt */ interrupt: (interrupt: Interrupt) => void; + + /** + * Register a handler for a client tool. When the agent's LLM calls this tool, + * the handler executes on the client and returns the result to the LLM. + * @param name - Tool name (must match the tool name defined in the agent config) + * @param handler - Async function receiving args, must return a JSON string (max 15KiB) + */ + registerClientTool: (name: string, handler: ClientToolHandler) => void; + + /** + * Remove a previously registered client tool handler. + * @param name - Tool name to unregister + */ + unregisterClientTool: (name: string) => void; } diff --git a/src/types/stream/rtc.ts b/src/types/stream/rtc.ts index 0cf603bb..e74db024 100644 --- a/src/types/stream/rtc.ts +++ b/src/types/stream/rtc.ts @@ -39,7 +39,6 @@ export interface ICreateStreamRequestResponse extends StickyRequest { ice_servers: IceServer[]; fluent?: boolean; interrupt_enabled?: boolean; - triggers_enabled?: boolean; } export interface IceCandidate { diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 9fef29de..55f3c5df 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -194,6 +194,8 @@ export interface StreamInterruptPayload { timestamp: number; } +export type ClientToolHandler = (args: Record) => Promise; + export interface ToolCallingPayload { execution_id: string; tool_name: string; diff --git a/src/types/stream/streams-v2.ts b/src/types/stream/streams-v2.ts index accc95f7..aad7e0ac 100644 --- a/src/types/stream/streams-v2.ts +++ b/src/types/stream/streams-v2.ts @@ -13,4 +13,5 @@ export interface CreateSessionV2Response { id: string; session_url: string; session_token: string; + interrupt_enabled: boolean; } diff --git a/src/utils/analytics.ts b/src/utils/analytics.ts index 657e8963..ea29dc26 100644 --- a/src/utils/analytics.ts +++ b/src/utils/analytics.ts @@ -54,7 +54,7 @@ export function getAgentInfo(agent: Agent) { maxResponseLength: promptCustomization?.max_response_length, agentId: agent.id, access: agent.access, - name: agent.preview_name, + agentName: agent.preview_name, ...(agent.access === 'public' ? { from: 'agent-template' } : {}), }; }