diff --git a/apps/sim/lib/copilot/request/handlers/handlers.test.ts b/apps/sim/lib/copilot/request/handlers/handlers.test.ts index c3f92fe3c06..7c69cbf96c4 100644 --- a/apps/sim/lib/copilot/request/handlers/handlers.test.ts +++ b/apps/sim/lib/copilot/request/handlers/handlers.test.ts @@ -408,6 +408,7 @@ describe('sse-handlers tool lifecycle', () => { const updated = context.toolCalls.get('tool-1') expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.success) + expect(updated?.agentId).toBe('main') // Display titles are derived client-side from the tool name (+args), not the // stream; read with no path resolves to the static "Reading file". expect(updated?.displayTitle).toBe('Reading file') @@ -889,9 +890,11 @@ describe('sse-handlers tool lifecycle', () => { expect.any(Object) ) expect(context.toolCalls.get('sub-tool-1')?.params).toEqual({ name: 'Example Workflow' }) + expect(context.toolCalls.get('sub-tool-1')?.agentId).toBe('workflow') expect(context.subAgentToolCalls['parent-1']?.[0]?.params).toEqual({ name: 'Example Workflow', }) + expect(context.subAgentToolCalls['parent-1']?.[0]?.agentId).toBe('workflow') }) it('routes subagent text using the event scope parent tool call id', async () => { @@ -973,6 +976,40 @@ describe('sse-handlers tool lifecycle', () => { await sleep(0) expect(context.subAgentToolCalls['parent-1']?.[0]?.id).toBe('sub-tool-scope-1') + expect(context.toolCalls.get('sub-tool-scope-1')?.agentId).toBe('deploy') + }) + + it('retains the first agent attribution on replayed partial tool calls', async () => { + context.toolCalls.set('replayed-read', { + id: 'replayed-read', + name: 'read', + status: 'executing', + }) + + const replayPartial = (agentId: string) => + subAgentHandlers.tool( + { + type: MothershipStreamV1EventType.tool, + scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId }, + payload: { + toolCallId: 'replayed-read', + toolName: 'read', + executor: MothershipStreamV1ToolExecutor.go, + mode: MothershipStreamV1ToolMode.sync, + phase: MothershipStreamV1ToolPhase.call, + status: 'generating', + partial: true, + }, + } satisfies StreamEvent, + context, + execContext, + { interactive: false, timeout: 1000 } + ) + + await replayPartial('workflow') + await replayPartial('deploy') + + expect(context.toolCalls.get('replayed-read')?.agentId).toBe('workflow') }) it('pairs compaction lifecycle events within each scoped subagent lane', async () => { diff --git a/apps/sim/lib/copilot/request/handlers/tool.ts b/apps/sim/lib/copilot/request/handlers/tool.ts index 6634a625308..aeabc123d71 100644 --- a/apps/sim/lib/copilot/request/handlers/tool.ts +++ b/apps/sim/lib/copilot/request/handlers/tool.ts @@ -285,6 +285,7 @@ export async function handleToolEvent( ): Promise { const isSubagent = scope === 'subagent' const parentToolCallId = isSubagent ? getScopedParentToolCallId(event, context) : undefined + const agentId = event.scope?.agentId ?? 'main' if (isSubagent && !parentToolCallId) return @@ -332,6 +333,7 @@ export async function handleToolEvent( options, parentToolCallId, scope, + agentId, getScopedSpanIdentity(event) ) } @@ -409,6 +411,7 @@ async function handleCallPhase( options: OrchestratorOptions, parentToolCallId: string | undefined, scope: ToolScope, + agentId: string, spanIdentity: { spanId?: string; parentSpanId?: string } ): Promise { const { toolCallId, toolName } = data @@ -416,6 +419,7 @@ async function handleCallPhase( const isGenerating = data.status === TOOL_CALL_STATUS.generating const isPartial = data.partial === true || isGenerating const existing = context.toolCalls.get(toolCallId) + if (existing) existing.agentId ??= agentId const isSubagent = scope === 'subagent' const ui = getToolCallUI(data) @@ -459,12 +463,13 @@ async function handleCallPhase( toolName, args, parentToolCallId!, + agentId, ui, spanIdentity, !isPartial ) } else { - registerMainToolCall(context, toolCallId, toolName, args, existing, ui, !isPartial) + registerMainToolCall(context, toolCallId, toolName, args, existing, agentId, ui, !isPartial) } if (isPartial) return @@ -554,6 +559,7 @@ function registerSubagentToolCall( toolName: string, args: Record | undefined, parentToolCallId: string, + agentId: string, ui: { title?: string; phaseLabel?: string; hidden?: boolean }, spanIdentity: { spanId?: string; parentSpanId?: string }, finalized: boolean @@ -574,6 +580,7 @@ function registerSubagentToolCall( id: toolCallId, name: toolName, status: 'pending', + agentId, params: args, startTime: Date.now(), } @@ -594,6 +601,7 @@ function registerSubagentToolCall( const subagentToolCalls = context.subAgentToolCalls[parentToolCallId] const existingSubagentToolCall = subagentToolCalls.find((tc) => tc.id === toolCallId) if (existingSubagentToolCall) { + existingSubagentToolCall.agentId ??= agentId if (!rebindResolvedIntegrationCall(existingSubagentToolCall, toolName, args)) { updateToolCallFromFrame(existingSubagentToolCall, toolName, args, finalized) } @@ -609,6 +617,7 @@ function registerMainToolCall( toolName: string, args: Record | undefined, existing: ToolCallState | undefined, + agentId: string, ui: { title?: string; phaseLabel?: string; hidden?: boolean }, finalized: boolean ): void { @@ -633,6 +642,7 @@ function registerMainToolCall( id: toolCallId, name: toolName, status: 'pending', + agentId, params: args, startTime: Date.now(), } diff --git a/apps/sim/lib/copilot/request/metrics.test.ts b/apps/sim/lib/copilot/request/metrics.test.ts new file mode 100644 index 00000000000..77e00075061 --- /dev/null +++ b/apps/sim/lib/copilot/request/metrics.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { toolCallsAdd, toolDurationRecord } = vi.hoisted(() => ({ + toolCallsAdd: vi.fn(), + toolDurationRecord: vi.fn(), +})) + +vi.mock('@opentelemetry/api', () => ({ + metrics: { + getMeter: vi.fn(() => ({ + createCounter: vi.fn(() => ({ add: toolCallsAdd })), + createHistogram: vi.fn((name: string) => ({ + record: name === 'copilot.tool.duration' ? toolDurationRecord : vi.fn(), + })), + })), + }, +})) + +import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' +import { normalizeToolAgentId, recordSimToolMetric } from '@/lib/copilot/request/metrics' + +describe('recordSimToolMetric', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(['main', 'workflow'])( + 'attributes call counts and duration to the registered %s agent', + (agentId) => { + recordSimToolMetric('read', agentId, 'success', 125) + + const baseAttributes = { + [TraceAttr.ToolName]: 'read', + [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.ToolOutcome]: 'success', + } + expect(toolCallsAdd).toHaveBeenCalledWith(1, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: agentId, + }) + expect(toolDurationRecord).toHaveBeenCalledWith(125, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: agentId, + }) + } + ) + + it('collapses unknown agent IDs to the bounded fallback', () => { + recordSimToolMetric('read', 'tenant-defined-agent', 'success', 125) + + const baseAttributes = { + [TraceAttr.ToolName]: 'read', + [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.ToolOutcome]: 'success', + } + expect(toolCallsAdd).toHaveBeenCalledWith(1, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: 'other', + }) + expect(toolDurationRecord).toHaveBeenCalledWith(125, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: 'other', + }) + }) + + it.each([ + { agentId: 'main', expected: 'main' }, + { agentId: 'workflow', expected: 'workflow' }, + { agentId: 'tenant-defined-agent', expected: 'other' }, + { agentId: '', expected: 'other' }, + ])('normalizes $agentId to $expected for every telemetry signal', ({ agentId, expected }) => { + expect(normalizeToolAgentId(agentId)).toBe(expected) + }) +}) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index 31f0e7996f6..d3bfb804382 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -5,9 +5,9 @@ // contracts/metrics_v1.go) so the Go∪Sim union is queryable as one series set // — e.g. `copilot.tool.duration` split by `tool.executor` (go|client|sim). // -// Bounded cardinality only: tool.name is capped to the shared tool catalog -// (else "other"); vfs phase / file-read outcome are bounded sets. NEVER a -// user/chat/request id (those explode Prometheus series). +// Bounded cardinality only: tool.name and gen_ai.agent.name are capped to the +// shared catalogs (else "other"); vfs phase / file-read outcome are bounded +// sets. NEVER a user/chat/request id (those explode Prometheus series). import { type Counter, type Histogram, metrics } from '@opentelemetry/api' import { Metric } from '@/lib/copilot/generated/metrics-v1' import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1' @@ -68,15 +68,30 @@ function cappedToolName(name: string): string { return TOOL_CATALOG[name] ? name : 'other' } +const REGISTERED_AGENT_IDS = new Set([ + 'main', + ...Object.values(TOOL_CATALOG).flatMap(({ subagentId }) => (subagentId ? [subagentId] : [])), +]) + +export function normalizeToolAgentId(agentId: string): string { + return REGISTERED_AGENT_IDS.has(agentId) ? agentId : 'other' +} + // recordSimToolMetric emits copilot.tool.calls (+1) and copilot.tool.duration // for one server-side Sim tool dispatch (executor=sim). outcome is the bounded // tool outcome (success/error/…). Pure telemetry. -export function recordSimToolMetric(name: string, outcome: string, durationMs: number): void { +export function recordSimToolMetric( + name: string, + agentId: string, + outcome: string, + durationMs: number +): void { const { toolDuration, toolCalls } = instruments() const attrs = { [TraceAttr.ToolName]: cappedToolName(name), [TraceAttr.ToolExecutor]: 'sim', [TraceAttr.ToolOutcome]: outcome, + [TraceAttr.GenAiAgentName]: normalizeToolAgentId(agentId), } toolCalls.add(1, attrs) if (durationMs >= 0) toolDuration.record(durationMs, attrs) diff --git a/apps/sim/lib/copilot/request/otel.ts b/apps/sim/lib/copilot/request/otel.ts index e93f9f53d5a..3804eb5ebe3 100644 --- a/apps/sim/lib/copilot/request/otel.ts +++ b/apps/sim/lib/copilot/request/otel.ts @@ -22,6 +22,7 @@ import { import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1' import { contextFromRequestHeaders } from '@/lib/copilot/request/go/propagation' +import { normalizeToolAgentId } from '@/lib/copilot/request/metrics' import { isExplicitStopReason } from '@/lib/copilot/request/session/abort-reason' // OTel GenAI content-capture env var (spec: @@ -284,6 +285,7 @@ export async function withCopilotToolSpan( input: { toolName: string toolCallId: string + agentName: string runId?: string chatId?: string argsBytes?: number @@ -299,6 +301,7 @@ export async function withCopilotToolSpan( [TraceAttr.ToolName]: input.toolName, [TraceAttr.ToolCallId]: input.toolCallId, [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.GenAiAgentName]: normalizeToolAgentId(input.agentName), ...(input.runId ? { [TraceAttr.RunId]: input.runId } : {}), ...(input.chatId ? { [TraceAttr.ChatId]: input.chatId } : {}), ...(typeof input.argsBytes === 'number' diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index 8d60e889a05..b2ca84938ee 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -1,13 +1,37 @@ import '@sim/testing/mocks/executor' -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { recordSimToolMetric, setAttribute, withCopilotToolSpan } = vi.hoisted(() => { + const setAttribute = vi.fn() + return { + recordSimToolMetric: vi.fn(), + setAttribute, + withCopilotToolSpan: vi.fn( + (_input: unknown, fn: (span: { setAttribute: typeof setAttribute }) => Promise) => + fn({ setAttribute }) + ), + } +}) + +vi.mock('@/lib/copilot/request/metrics', () => ({ + recordSimToolMetric, +})) + +vi.mock('@/lib/copilot/request/otel', () => ({ + withCopilotToolSpan, +})) + import { TOOL_WATCHDOG_DEFAULT_MS, TOOL_WATCHDOG_LONG_RUNNING_MS } from '@/lib/copilot/constants' +import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1' +import { createStreamingContext } from '@/lib/copilot/request/context/request-context' import { buildToolExecutionContext, + executeToolAndReport, pendingToolWaitBudgetMs, toolWatchdogTimeoutMs, } from '@/lib/copilot/request/tools/executor' -import type { ExecutionContext } from '@/lib/copilot/request/types' +import type { ExecutionContext, ToolCallState } from '@/lib/copilot/request/types' describe('toolWatchdogTimeoutMs', () => { it('gives request-scoped MCP tools the long-running watchdog', () => { @@ -58,3 +82,74 @@ describe('buildToolExecutionContext', () => { }) }) }) + +describe('executeToolAndReport metrics', () => { + const executionContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('forwards the stored agent on normal completion', async () => { + const toolCall: ToolCallState = { + id: 'call-1', + name: 'read', + status: MothershipStreamV1ToolOutcome.success, + result: { success: true, output: 'done' }, + agentId: 'workflow', + endTime: Date.now(), + } + const context = createStreamingContext({ + toolCalls: new Map([[toolCall.id, toolCall]]), + }) + + await executeToolAndReport(toolCall.id, context, executionContext) + + expect(recordSimToolMetric).toHaveBeenCalledWith( + 'read', + 'workflow', + MothershipStreamV1ToolOutcome.success, + expect.any(Number) + ) + expect(withCopilotToolSpan).toHaveBeenCalledWith( + expect.objectContaining({ agentName: 'workflow' }), + expect.any(Function) + ) + }) + + it.each([ + { agentId: 'workflow', expectedAgentId: 'workflow' }, + { agentId: undefined, expectedAgentId: 'main' }, + ])( + 'forwards $expectedAgentId when an unexpected error occurs', + async ({ agentId, expectedAgentId }) => { + const toolCall: ToolCallState = { + id: 'call-2', + name: 'read', + status: MothershipStreamV1ToolOutcome.error, + agentId, + endTime: Date.now(), + } + const context = createStreamingContext({ + toolCalls: new Map([[toolCall.id, toolCall]]), + }) + + await expect(executeToolAndReport(toolCall.id, context, executionContext)).rejects.toThrow( + 'missing a canonical error' + ) + expect(recordSimToolMetric).toHaveBeenCalledWith( + 'read', + expectedAgentId, + MothershipStreamV1ToolOutcome.error, + expect.any(Number) + ) + expect(withCopilotToolSpan).toHaveBeenCalledWith( + expect.objectContaining({ agentName: expectedAgentId }), + expect.any(Function) + ) + } + ) +}) diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 10f0f84402c..388f8356663 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -408,6 +408,7 @@ export async function executeToolAndReport( { toolName: toolCall.name, toolCallId: toolCall.id, + agentName: toolCall.agentId ?? 'main', runId: context.runId, chatId: execContext.chatId, argsBytes: argsPayload?.length, @@ -428,7 +429,12 @@ export async function executeToolAndReport( } // Durable Grafana signal for "which Sim tool is slowest" (executor=sim); // pairs with the Go executor-boundary metric (U15) as one series set. - recordSimToolMetric(toolCall.name, completion.status, durationMs) + recordSimToolMetric( + toolCall.name, + toolCall.agentId ?? 'main', + completion.status, + durationMs + ) return completion } catch (err) { // executeToolAndReportInner threw (infra/unexpected error, not a normal @@ -437,7 +443,12 @@ export async function executeToolAndReport( const durationMs = Date.now() - startedAt otelSpan.setAttribute(TraceAttr.ToolOutcome, 'error') otelSpan.setAttribute(TraceAttr.ToolDurationMs, durationMs) - recordSimToolMetric(toolCall.name, 'error', durationMs) + recordSimToolMetric( + toolCall.name, + toolCall.agentId ?? 'main', + MothershipStreamV1ToolOutcome.error, + durationMs + ) throw err } } diff --git a/apps/sim/lib/copilot/request/types.ts b/apps/sim/lib/copilot/request/types.ts index bf4908896db..ed76e5cd505 100644 --- a/apps/sim/lib/copilot/request/types.ts +++ b/apps/sim/lib/copilot/request/types.ts @@ -25,6 +25,8 @@ export interface ToolCallState { id: string name: string status: ToolCallStatus + /** Bounded registry ID of the agent that invoked this tool. */ + agentId?: string displayTitle?: string /** Model-authored activity text for a gateway-resolved integration call. */ integrationDescription?: string