From 29fc7966c28794d570be688b6312e6d975b8ed34 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:33:02 -0700 Subject: [PATCH 1/6] feat(copilot): attribute Sim tool calls to agents --- .../copilot/request/handlers/handlers.test.ts | 37 +++++++++ apps/sim/lib/copilot/request/handlers/tool.ts | 12 ++- apps/sim/lib/copilot/request/metrics.test.ts | 41 ++++++++++ apps/sim/lib/copilot/request/metrics.ts | 16 +++- .../copilot/request/tools/executor.test.ts | 80 ++++++++++++++++++- .../sim/lib/copilot/request/tools/executor.ts | 14 +++- apps/sim/lib/copilot/request/types.ts | 2 + 7 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 apps/sim/lib/copilot/request/metrics.test.ts diff --git a/apps/sim/lib/copilot/request/handlers/handlers.test.ts b/apps/sim/lib/copilot/request/handlers/handlers.test.ts index c3f92fe3c06..7c69cbf96c4 100644 --- a/apps/sim/lib/copilot/request/handlers/handlers.test.ts +++ b/apps/sim/lib/copilot/request/handlers/handlers.test.ts @@ -408,6 +408,7 @@ describe('sse-handlers tool lifecycle', () => { const updated = context.toolCalls.get('tool-1') expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.success) + expect(updated?.agentId).toBe('main') // Display titles are derived client-side from the tool name (+args), not the // stream; read with no path resolves to the static "Reading file". expect(updated?.displayTitle).toBe('Reading file') @@ -889,9 +890,11 @@ describe('sse-handlers tool lifecycle', () => { expect.any(Object) ) expect(context.toolCalls.get('sub-tool-1')?.params).toEqual({ name: 'Example Workflow' }) + expect(context.toolCalls.get('sub-tool-1')?.agentId).toBe('workflow') expect(context.subAgentToolCalls['parent-1']?.[0]?.params).toEqual({ name: 'Example Workflow', }) + expect(context.subAgentToolCalls['parent-1']?.[0]?.agentId).toBe('workflow') }) it('routes subagent text using the event scope parent tool call id', async () => { @@ -973,6 +976,40 @@ describe('sse-handlers tool lifecycle', () => { await sleep(0) expect(context.subAgentToolCalls['parent-1']?.[0]?.id).toBe('sub-tool-scope-1') + expect(context.toolCalls.get('sub-tool-scope-1')?.agentId).toBe('deploy') + }) + + it('retains the first agent attribution on replayed partial tool calls', async () => { + context.toolCalls.set('replayed-read', { + id: 'replayed-read', + name: 'read', + status: 'executing', + }) + + const replayPartial = (agentId: string) => + subAgentHandlers.tool( + { + type: MothershipStreamV1EventType.tool, + scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId }, + payload: { + toolCallId: 'replayed-read', + toolName: 'read', + executor: MothershipStreamV1ToolExecutor.go, + mode: MothershipStreamV1ToolMode.sync, + phase: MothershipStreamV1ToolPhase.call, + status: 'generating', + partial: true, + }, + } satisfies StreamEvent, + context, + execContext, + { interactive: false, timeout: 1000 } + ) + + await replayPartial('workflow') + await replayPartial('deploy') + + expect(context.toolCalls.get('replayed-read')?.agentId).toBe('workflow') }) it('pairs compaction lifecycle events within each scoped subagent lane', async () => { diff --git a/apps/sim/lib/copilot/request/handlers/tool.ts b/apps/sim/lib/copilot/request/handlers/tool.ts index 6634a625308..aeabc123d71 100644 --- a/apps/sim/lib/copilot/request/handlers/tool.ts +++ b/apps/sim/lib/copilot/request/handlers/tool.ts @@ -285,6 +285,7 @@ export async function handleToolEvent( ): Promise { const isSubagent = scope === 'subagent' const parentToolCallId = isSubagent ? getScopedParentToolCallId(event, context) : undefined + const agentId = event.scope?.agentId ?? 'main' if (isSubagent && !parentToolCallId) return @@ -332,6 +333,7 @@ export async function handleToolEvent( options, parentToolCallId, scope, + agentId, getScopedSpanIdentity(event) ) } @@ -409,6 +411,7 @@ async function handleCallPhase( options: OrchestratorOptions, parentToolCallId: string | undefined, scope: ToolScope, + agentId: string, spanIdentity: { spanId?: string; parentSpanId?: string } ): Promise { const { toolCallId, toolName } = data @@ -416,6 +419,7 @@ async function handleCallPhase( const isGenerating = data.status === TOOL_CALL_STATUS.generating const isPartial = data.partial === true || isGenerating const existing = context.toolCalls.get(toolCallId) + if (existing) existing.agentId ??= agentId const isSubagent = scope === 'subagent' const ui = getToolCallUI(data) @@ -459,12 +463,13 @@ async function handleCallPhase( toolName, args, parentToolCallId!, + agentId, ui, spanIdentity, !isPartial ) } else { - registerMainToolCall(context, toolCallId, toolName, args, existing, ui, !isPartial) + registerMainToolCall(context, toolCallId, toolName, args, existing, agentId, ui, !isPartial) } if (isPartial) return @@ -554,6 +559,7 @@ function registerSubagentToolCall( toolName: string, args: Record | undefined, parentToolCallId: string, + agentId: string, ui: { title?: string; phaseLabel?: string; hidden?: boolean }, spanIdentity: { spanId?: string; parentSpanId?: string }, finalized: boolean @@ -574,6 +580,7 @@ function registerSubagentToolCall( id: toolCallId, name: toolName, status: 'pending', + agentId, params: args, startTime: Date.now(), } @@ -594,6 +601,7 @@ function registerSubagentToolCall( const subagentToolCalls = context.subAgentToolCalls[parentToolCallId] const existingSubagentToolCall = subagentToolCalls.find((tc) => tc.id === toolCallId) if (existingSubagentToolCall) { + existingSubagentToolCall.agentId ??= agentId if (!rebindResolvedIntegrationCall(existingSubagentToolCall, toolName, args)) { updateToolCallFromFrame(existingSubagentToolCall, toolName, args, finalized) } @@ -609,6 +617,7 @@ function registerMainToolCall( toolName: string, args: Record | undefined, existing: ToolCallState | undefined, + agentId: string, ui: { title?: string; phaseLabel?: string; hidden?: boolean }, finalized: boolean ): void { @@ -633,6 +642,7 @@ function registerMainToolCall( id: toolCallId, name: toolName, status: 'pending', + agentId, params: args, startTime: Date.now(), } diff --git a/apps/sim/lib/copilot/request/metrics.test.ts b/apps/sim/lib/copilot/request/metrics.test.ts new file mode 100644 index 00000000000..e5d72313075 --- /dev/null +++ b/apps/sim/lib/copilot/request/metrics.test.ts @@ -0,0 +1,41 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it, vi } from 'vitest' + +const { toolCallsAdd, toolDurationRecord } = vi.hoisted(() => ({ + toolCallsAdd: vi.fn(), + toolDurationRecord: vi.fn(), +})) + +vi.mock('@opentelemetry/api', () => ({ + metrics: { + getMeter: vi.fn(() => ({ + createCounter: vi.fn(() => ({ add: toolCallsAdd })), + createHistogram: vi.fn((name: string) => ({ + record: name === 'copilot.tool.duration' ? toolDurationRecord : vi.fn(), + })), + })), + }, +})) + +import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' +import { recordSimToolMetric } from '@/lib/copilot/request/metrics' + +describe('recordSimToolMetric', () => { + it('attributes call counts to the agent without adding it to duration', () => { + recordSimToolMetric('read', 'workflow', 'success', 125) + + const baseAttributes = { + [TraceAttr.ToolName]: 'read', + [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.ToolOutcome]: 'success', + } + expect(toolCallsAdd).toHaveBeenCalledWith(1, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: 'workflow', + }) + expect(toolDurationRecord).toHaveBeenCalledWith(125, baseAttributes) + }) +}) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index 31f0e7996f6..a9537638879 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -71,15 +71,23 @@ function cappedToolName(name: string): string { // recordSimToolMetric emits copilot.tool.calls (+1) and copilot.tool.duration // for one server-side Sim tool dispatch (executor=sim). outcome is the bounded // tool outcome (success/error/…). Pure telemetry. -export function recordSimToolMetric(name: string, outcome: string, durationMs: number): void { +export function recordSimToolMetric( + name: string, + agentId: string, + outcome: string, + durationMs: number +): void { const { toolDuration, toolCalls } = instruments() - const attrs = { + const baseAttrs = { [TraceAttr.ToolName]: cappedToolName(name), [TraceAttr.ToolExecutor]: 'sim', [TraceAttr.ToolOutcome]: outcome, } - toolCalls.add(1, attrs) - if (durationMs >= 0) toolDuration.record(durationMs, attrs) + toolCalls.add(1, { + ...baseAttrs, + [TraceAttr.GenAiAgentName]: agentId, + }) + if (durationMs >= 0) toolDuration.record(durationMs, baseAttrs) } // recordVfsMaterialize records VFS materialization time. Call once per phase diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index 8d60e889a05..3f3c9216896 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -1,13 +1,33 @@ import '@sim/testing/mocks/executor' -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { recordSimToolMetric, setAttribute } = vi.hoisted(() => ({ + recordSimToolMetric: vi.fn(), + setAttribute: vi.fn(), +})) + +vi.mock('@/lib/copilot/request/metrics', () => ({ + recordSimToolMetric, +})) + +vi.mock('@/lib/copilot/request/otel', () => ({ + withCopilotToolSpan: ( + _input: unknown, + fn: (span: { setAttribute: typeof setAttribute }) => Promise + ) => fn({ setAttribute }), +})) + import { TOOL_WATCHDOG_DEFAULT_MS, TOOL_WATCHDOG_LONG_RUNNING_MS } from '@/lib/copilot/constants' +import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1' +import { createStreamingContext } from '@/lib/copilot/request/context/request-context' import { buildToolExecutionContext, + executeToolAndReport, pendingToolWaitBudgetMs, toolWatchdogTimeoutMs, } from '@/lib/copilot/request/tools/executor' -import type { ExecutionContext } from '@/lib/copilot/request/types' +import type { ExecutionContext, ToolCallState } from '@/lib/copilot/request/types' describe('toolWatchdogTimeoutMs', () => { it('gives request-scoped MCP tools the long-running watchdog', () => { @@ -58,3 +78,59 @@ describe('buildToolExecutionContext', () => { }) }) }) + +describe('executeToolAndReport metrics', () => { + const executionContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('forwards the stored agent on normal completion', async () => { + const toolCall: ToolCallState = { + id: 'call-1', + name: 'read', + status: MothershipStreamV1ToolOutcome.success, + result: { success: true, output: 'done' }, + agentId: 'workflow', + endTime: Date.now(), + } + const context = createStreamingContext({ + toolCalls: new Map([[toolCall.id, toolCall]]), + }) + + await executeToolAndReport(toolCall.id, context, executionContext) + + expect(recordSimToolMetric).toHaveBeenCalledWith( + 'read', + 'workflow', + MothershipStreamV1ToolOutcome.success, + expect.any(Number) + ) + }) + + it('falls back to main when forwarding an unexpected throw', async () => { + const toolCall: ToolCallState = { + id: 'call-2', + name: 'read', + status: MothershipStreamV1ToolOutcome.error, + endTime: Date.now(), + } + const context = createStreamingContext({ + toolCalls: new Map([[toolCall.id, toolCall]]), + }) + + await expect(executeToolAndReport(toolCall.id, context, executionContext)).rejects.toThrow( + 'missing a canonical error' + ) + expect(recordSimToolMetric).toHaveBeenCalledWith( + 'read', + 'main', + MothershipStreamV1ToolOutcome.error, + expect.any(Number) + ) + }) +}) diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 10f0f84402c..9524fefdd05 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -428,7 +428,12 @@ export async function executeToolAndReport( } // Durable Grafana signal for "which Sim tool is slowest" (executor=sim); // pairs with the Go executor-boundary metric (U15) as one series set. - recordSimToolMetric(toolCall.name, completion.status, durationMs) + recordSimToolMetric( + toolCall.name, + toolCall.agentId ?? 'main', + completion.status, + durationMs + ) return completion } catch (err) { // executeToolAndReportInner threw (infra/unexpected error, not a normal @@ -437,7 +442,12 @@ export async function executeToolAndReport( const durationMs = Date.now() - startedAt otelSpan.setAttribute(TraceAttr.ToolOutcome, 'error') otelSpan.setAttribute(TraceAttr.ToolDurationMs, durationMs) - recordSimToolMetric(toolCall.name, 'error', durationMs) + recordSimToolMetric( + toolCall.name, + toolCall.agentId ?? 'main', + MothershipStreamV1ToolOutcome.error, + durationMs + ) throw err } } diff --git a/apps/sim/lib/copilot/request/types.ts b/apps/sim/lib/copilot/request/types.ts index bf4908896db..ed76e5cd505 100644 --- a/apps/sim/lib/copilot/request/types.ts +++ b/apps/sim/lib/copilot/request/types.ts @@ -25,6 +25,8 @@ export interface ToolCallState { id: string name: string status: ToolCallStatus + /** Bounded registry ID of the agent that invoked this tool. */ + agentId?: string displayTitle?: string /** Model-authored activity text for a gateway-resolved integration call. */ integrationDescription?: string From f8e61448a94a58385c4bdedb667455fa6a5d2e4f Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:43:35 -0700 Subject: [PATCH 2/6] fix(review): bound agent metric labels --- apps/sim/lib/copilot/request/metrics.test.ts | 30 ++++++++++-- apps/sim/lib/copilot/request/metrics.ts | 11 ++++- .../copilot/request/tools/executor.test.ts | 47 +++++++++++-------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/apps/sim/lib/copilot/request/metrics.test.ts b/apps/sim/lib/copilot/request/metrics.test.ts index e5d72313075..8a1bad619ed 100644 --- a/apps/sim/lib/copilot/request/metrics.test.ts +++ b/apps/sim/lib/copilot/request/metrics.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' const { toolCallsAdd, toolDurationRecord } = vi.hoisted(() => ({ toolCallsAdd: vi.fn(), @@ -24,8 +24,30 @@ import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { recordSimToolMetric } from '@/lib/copilot/request/metrics' describe('recordSimToolMetric', () => { - it('attributes call counts to the agent without adding it to duration', () => { - recordSimToolMetric('read', 'workflow', 'success', 125) + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(['main', 'workflow'])( + 'attributes call counts to the registered %s agent without adding it to duration', + (agentId) => { + recordSimToolMetric('read', agentId, 'success', 125) + + const baseAttributes = { + [TraceAttr.ToolName]: 'read', + [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.ToolOutcome]: 'success', + } + expect(toolCallsAdd).toHaveBeenCalledWith(1, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: agentId, + }) + expect(toolDurationRecord).toHaveBeenCalledWith(125, baseAttributes) + } + ) + + it('collapses unknown agent IDs to the bounded fallback', () => { + recordSimToolMetric('read', 'tenant-defined-agent', 'success', 125) const baseAttributes = { [TraceAttr.ToolName]: 'read', @@ -34,7 +56,7 @@ describe('recordSimToolMetric', () => { } expect(toolCallsAdd).toHaveBeenCalledWith(1, { ...baseAttributes, - [TraceAttr.GenAiAgentName]: 'workflow', + [TraceAttr.GenAiAgentName]: 'other', }) expect(toolDurationRecord).toHaveBeenCalledWith(125, baseAttributes) }) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index a9537638879..a2a66c73da8 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -68,6 +68,15 @@ function cappedToolName(name: string): string { return TOOL_CATALOG[name] ? name : 'other' } +const REGISTERED_AGENT_IDS = new Set([ + 'main', + ...Object.values(TOOL_CATALOG).flatMap(({ subagentId }) => (subagentId ? [subagentId] : [])), +]) + +function cappedAgentId(agentId: string): string { + return REGISTERED_AGENT_IDS.has(agentId) ? agentId : 'other' +} + // recordSimToolMetric emits copilot.tool.calls (+1) and copilot.tool.duration // for one server-side Sim tool dispatch (executor=sim). outcome is the bounded // tool outcome (success/error/…). Pure telemetry. @@ -85,7 +94,7 @@ export function recordSimToolMetric( } toolCalls.add(1, { ...baseAttrs, - [TraceAttr.GenAiAgentName]: agentId, + [TraceAttr.GenAiAgentName]: cappedAgentId(agentId), }) if (durationMs >= 0) toolDuration.record(durationMs, baseAttrs) } diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index 3f3c9216896..5e1d564d176 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -112,25 +112,32 @@ describe('executeToolAndReport metrics', () => { ) }) - it('falls back to main when forwarding an unexpected throw', async () => { - const toolCall: ToolCallState = { - id: 'call-2', - name: 'read', - status: MothershipStreamV1ToolOutcome.error, - endTime: Date.now(), + it.each([ + { agentId: 'workflow', expectedAgentId: 'workflow' }, + { agentId: undefined, expectedAgentId: 'main' }, + ])( + 'forwards $expectedAgentId when an unexpected error occurs', + async ({ agentId, expectedAgentId }) => { + const toolCall: ToolCallState = { + id: 'call-2', + name: 'read', + status: MothershipStreamV1ToolOutcome.error, + agentId, + endTime: Date.now(), + } + const context = createStreamingContext({ + toolCalls: new Map([[toolCall.id, toolCall]]), + }) + + await expect(executeToolAndReport(toolCall.id, context, executionContext)).rejects.toThrow( + 'missing a canonical error' + ) + expect(recordSimToolMetric).toHaveBeenCalledWith( + 'read', + expectedAgentId, + MothershipStreamV1ToolOutcome.error, + expect.any(Number) + ) } - const context = createStreamingContext({ - toolCalls: new Map([[toolCall.id, toolCall]]), - }) - - await expect(executeToolAndReport(toolCall.id, context, executionContext)).rejects.toThrow( - 'missing a canonical error' - ) - expect(recordSimToolMetric).toHaveBeenCalledWith( - 'read', - 'main', - MothershipStreamV1ToolOutcome.error, - expect.any(Number) - ) - }) + ) }) From 8f6a08da923fe078fb4f4bf0afb9f23122f38ed2 Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:06:31 -0700 Subject: [PATCH 3/6] feat(copilot): attribute tool latency to agents --- apps/sim/lib/copilot/request/metrics.test.ts | 12 +++++++++--- apps/sim/lib/copilot/request/metrics.ts | 12 +++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/apps/sim/lib/copilot/request/metrics.test.ts b/apps/sim/lib/copilot/request/metrics.test.ts index 8a1bad619ed..bbc4e87e5a8 100644 --- a/apps/sim/lib/copilot/request/metrics.test.ts +++ b/apps/sim/lib/copilot/request/metrics.test.ts @@ -29,7 +29,7 @@ describe('recordSimToolMetric', () => { }) it.each(['main', 'workflow'])( - 'attributes call counts to the registered %s agent without adding it to duration', + 'attributes call counts and duration to the registered %s agent', (agentId) => { recordSimToolMetric('read', agentId, 'success', 125) @@ -42,7 +42,10 @@ describe('recordSimToolMetric', () => { ...baseAttributes, [TraceAttr.GenAiAgentName]: agentId, }) - expect(toolDurationRecord).toHaveBeenCalledWith(125, baseAttributes) + expect(toolDurationRecord).toHaveBeenCalledWith(125, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: agentId, + }) } ) @@ -58,6 +61,9 @@ describe('recordSimToolMetric', () => { ...baseAttributes, [TraceAttr.GenAiAgentName]: 'other', }) - expect(toolDurationRecord).toHaveBeenCalledWith(125, baseAttributes) + expect(toolDurationRecord).toHaveBeenCalledWith(125, { + ...baseAttributes, + [TraceAttr.GenAiAgentName]: 'other', + }) }) }) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index a2a66c73da8..9084544d0f5 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -5,9 +5,9 @@ // contracts/metrics_v1.go) so the Go∪Sim union is queryable as one series set // — e.g. `copilot.tool.duration` split by `tool.executor` (go|client|sim). // -// Bounded cardinality only: tool.name is capped to the shared tool catalog -// (else "other"); vfs phase / file-read outcome are bounded sets. NEVER a -// user/chat/request id (those explode Prometheus series). +// Bounded cardinality only: tool.name and gen_ai.agent.name are capped to the +// shared catalogs (else "other"); vfs phase / file-read outcome are bounded +// sets. NEVER a user/chat/request id (those explode Prometheus series). import { type Counter, type Histogram, metrics } from '@opentelemetry/api' import { Metric } from '@/lib/copilot/generated/metrics-v1' import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1' @@ -91,11 +91,9 @@ export function recordSimToolMetric( [TraceAttr.ToolName]: cappedToolName(name), [TraceAttr.ToolExecutor]: 'sim', [TraceAttr.ToolOutcome]: outcome, - } - toolCalls.add(1, { - ...baseAttrs, [TraceAttr.GenAiAgentName]: cappedAgentId(agentId), - }) + } + toolCalls.add(1, baseAttrs) if (durationMs >= 0) toolDuration.record(durationMs, baseAttrs) } From 7903265d8651268d373906b4fdad2b34f990f41c Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:55:12 -0700 Subject: [PATCH 4/6] feat(copilot): attribute tool spans to agents --- apps/sim/lib/copilot/request/otel.ts | 2 ++ .../copilot/request/tools/executor.test.ts | 28 +++++++++++++------ .../sim/lib/copilot/request/tools/executor.ts | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/copilot/request/otel.ts b/apps/sim/lib/copilot/request/otel.ts index e93f9f53d5a..d575807dd34 100644 --- a/apps/sim/lib/copilot/request/otel.ts +++ b/apps/sim/lib/copilot/request/otel.ts @@ -284,6 +284,7 @@ export async function withCopilotToolSpan( input: { toolName: string toolCallId: string + agentName: string runId?: string chatId?: string argsBytes?: number @@ -299,6 +300,7 @@ export async function withCopilotToolSpan( [TraceAttr.ToolName]: input.toolName, [TraceAttr.ToolCallId]: input.toolCallId, [TraceAttr.ToolExecutor]: 'sim', + [TraceAttr.GenAiAgentName]: input.agentName, ...(input.runId ? { [TraceAttr.RunId]: input.runId } : {}), ...(input.chatId ? { [TraceAttr.ChatId]: input.chatId } : {}), ...(typeof input.argsBytes === 'number' diff --git a/apps/sim/lib/copilot/request/tools/executor.test.ts b/apps/sim/lib/copilot/request/tools/executor.test.ts index 5e1d564d176..b2ca84938ee 100644 --- a/apps/sim/lib/copilot/request/tools/executor.test.ts +++ b/apps/sim/lib/copilot/request/tools/executor.test.ts @@ -2,20 +2,24 @@ import '@sim/testing/mocks/executor' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { recordSimToolMetric, setAttribute } = vi.hoisted(() => ({ - recordSimToolMetric: vi.fn(), - setAttribute: vi.fn(), -})) +const { recordSimToolMetric, setAttribute, withCopilotToolSpan } = vi.hoisted(() => { + const setAttribute = vi.fn() + return { + recordSimToolMetric: vi.fn(), + setAttribute, + withCopilotToolSpan: vi.fn( + (_input: unknown, fn: (span: { setAttribute: typeof setAttribute }) => Promise) => + fn({ setAttribute }) + ), + } +}) vi.mock('@/lib/copilot/request/metrics', () => ({ recordSimToolMetric, })) vi.mock('@/lib/copilot/request/otel', () => ({ - withCopilotToolSpan: ( - _input: unknown, - fn: (span: { setAttribute: typeof setAttribute }) => Promise - ) => fn({ setAttribute }), + withCopilotToolSpan, })) import { TOOL_WATCHDOG_DEFAULT_MS, TOOL_WATCHDOG_LONG_RUNNING_MS } from '@/lib/copilot/constants' @@ -110,6 +114,10 @@ describe('executeToolAndReport metrics', () => { MothershipStreamV1ToolOutcome.success, expect.any(Number) ) + expect(withCopilotToolSpan).toHaveBeenCalledWith( + expect.objectContaining({ agentName: 'workflow' }), + expect.any(Function) + ) }) it.each([ @@ -138,6 +146,10 @@ describe('executeToolAndReport metrics', () => { MothershipStreamV1ToolOutcome.error, expect.any(Number) ) + expect(withCopilotToolSpan).toHaveBeenCalledWith( + expect.objectContaining({ agentName: expectedAgentId }), + expect.any(Function) + ) } ) }) diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 9524fefdd05..388f8356663 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -408,6 +408,7 @@ export async function executeToolAndReport( { toolName: toolCall.name, toolCallId: toolCall.id, + agentName: toolCall.agentId ?? 'main', runId: context.runId, chatId: execContext.chatId, argsBytes: argsPayload?.length, From f4703a966a5aa4afdb188643b53d9afc3fdef28a Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:43:26 -0700 Subject: [PATCH 5/6] refactor(copilot): simplify tool attribute naming --- apps/sim/lib/copilot/request/metrics.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index 9084544d0f5..8c987c95ca2 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -87,14 +87,14 @@ export function recordSimToolMetric( durationMs: number ): void { const { toolDuration, toolCalls } = instruments() - const baseAttrs = { + const attrs = { [TraceAttr.ToolName]: cappedToolName(name), [TraceAttr.ToolExecutor]: 'sim', [TraceAttr.ToolOutcome]: outcome, [TraceAttr.GenAiAgentName]: cappedAgentId(agentId), } - toolCalls.add(1, baseAttrs) - if (durationMs >= 0) toolDuration.record(durationMs, baseAttrs) + toolCalls.add(1, attrs) + if (durationMs >= 0) toolDuration.record(durationMs, attrs) } // recordVfsMaterialize records VFS materialization time. Call once per phase From 560fa267c38779c68a9f8cc4ac47d0da5f14b02e Mon Sep 17 00:00:00 2001 From: Justin Blumencranz <96924014+j15z@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:29:34 -0700 Subject: [PATCH 6/6] fix(copilot): bound tool span agent labels --- apps/sim/lib/copilot/request/metrics.test.ts | 11 ++++++++++- apps/sim/lib/copilot/request/metrics.ts | 4 ++-- apps/sim/lib/copilot/request/otel.ts | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/copilot/request/metrics.test.ts b/apps/sim/lib/copilot/request/metrics.test.ts index bbc4e87e5a8..77e00075061 100644 --- a/apps/sim/lib/copilot/request/metrics.test.ts +++ b/apps/sim/lib/copilot/request/metrics.test.ts @@ -21,7 +21,7 @@ vi.mock('@opentelemetry/api', () => ({ })) import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' -import { recordSimToolMetric } from '@/lib/copilot/request/metrics' +import { normalizeToolAgentId, recordSimToolMetric } from '@/lib/copilot/request/metrics' describe('recordSimToolMetric', () => { beforeEach(() => { @@ -66,4 +66,13 @@ describe('recordSimToolMetric', () => { [TraceAttr.GenAiAgentName]: 'other', }) }) + + it.each([ + { agentId: 'main', expected: 'main' }, + { agentId: 'workflow', expected: 'workflow' }, + { agentId: 'tenant-defined-agent', expected: 'other' }, + { agentId: '', expected: 'other' }, + ])('normalizes $agentId to $expected for every telemetry signal', ({ agentId, expected }) => { + expect(normalizeToolAgentId(agentId)).toBe(expected) + }) }) diff --git a/apps/sim/lib/copilot/request/metrics.ts b/apps/sim/lib/copilot/request/metrics.ts index 8c987c95ca2..d3bfb804382 100644 --- a/apps/sim/lib/copilot/request/metrics.ts +++ b/apps/sim/lib/copilot/request/metrics.ts @@ -73,7 +73,7 @@ const REGISTERED_AGENT_IDS = new Set([ ...Object.values(TOOL_CATALOG).flatMap(({ subagentId }) => (subagentId ? [subagentId] : [])), ]) -function cappedAgentId(agentId: string): string { +export function normalizeToolAgentId(agentId: string): string { return REGISTERED_AGENT_IDS.has(agentId) ? agentId : 'other' } @@ -91,7 +91,7 @@ export function recordSimToolMetric( [TraceAttr.ToolName]: cappedToolName(name), [TraceAttr.ToolExecutor]: 'sim', [TraceAttr.ToolOutcome]: outcome, - [TraceAttr.GenAiAgentName]: cappedAgentId(agentId), + [TraceAttr.GenAiAgentName]: normalizeToolAgentId(agentId), } toolCalls.add(1, attrs) if (durationMs >= 0) toolDuration.record(durationMs, attrs) diff --git a/apps/sim/lib/copilot/request/otel.ts b/apps/sim/lib/copilot/request/otel.ts index d575807dd34..3804eb5ebe3 100644 --- a/apps/sim/lib/copilot/request/otel.ts +++ b/apps/sim/lib/copilot/request/otel.ts @@ -22,6 +22,7 @@ import { import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1' import { contextFromRequestHeaders } from '@/lib/copilot/request/go/propagation' +import { normalizeToolAgentId } from '@/lib/copilot/request/metrics' import { isExplicitStopReason } from '@/lib/copilot/request/session/abort-reason' // OTel GenAI content-capture env var (spec: @@ -300,7 +301,7 @@ export async function withCopilotToolSpan( [TraceAttr.ToolName]: input.toolName, [TraceAttr.ToolCallId]: input.toolCallId, [TraceAttr.ToolExecutor]: 'sim', - [TraceAttr.GenAiAgentName]: input.agentName, + [TraceAttr.GenAiAgentName]: normalizeToolAgentId(input.agentName), ...(input.runId ? { [TraceAttr.RunId]: input.runId } : {}), ...(input.chatId ? { [TraceAttr.ChatId]: input.chatId } : {}), ...(typeof input.argsBytes === 'number'