From 54567d03d898b65ed8637dedf023fcd80f555225 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 4 May 2026 15:27:21 -0700 Subject: [PATCH 1/2] fix(openrouter): Capture reasoning --- .../openrouter-instrumentation/assertions.ts | 48 ++++++++++++++ .../openrouter-instrumentation/constants.mjs | 10 ++- .../scenario.impl.mjs | 31 +++++++++ .../plugins/openrouter-agent-plugin.test.ts | 65 +++++++++++++++++++ .../plugins/openrouter-agent-plugin.ts | 28 ++++++++ .../plugins/openrouter-plugin.test.ts | 65 +++++++++++++++++++ .../plugins/openrouter-plugin.ts | 28 ++++++++ js/src/vendor-sdk-types/openrouter-agent.ts | 6 ++ js/src/vendor-sdk-types/openrouter.ts | 6 ++ 9 files changed, 286 insertions(+), 1 deletion(-) diff --git a/e2e/scenarios/openrouter-instrumentation/assertions.ts b/e2e/scenarios/openrouter-instrumentation/assertions.ts index c956e6563..f1784e62d 100644 --- a/e2e/scenarios/openrouter-instrumentation/assertions.ts +++ b/e2e/scenarios/openrouter-instrumentation/assertions.ts @@ -13,6 +13,7 @@ import { CHAT_MODEL, EMBEDDING_MODEL, RERANK_MODEL, + REASONING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -21,7 +22,10 @@ const CHAT_MODEL_NAME = CHAT_MODEL.split("/").at(-1) ?? CHAT_MODEL; const EMBEDDING_MODEL_NAME = EMBEDDING_MODEL.split("/").at(-1) ?? EMBEDDING_MODEL; const RERANK_MODEL_NAME = RERANK_MODEL.split("/").at(-1) ?? RERANK_MODEL; +const REASONING_MODEL_NAME = + REASONING_MODEL.split("/").at(-1) ?? REASONING_MODEL; const OPENROUTER_MODEL_PROVIDER = "openai"; +const OPENROUTER_REASONING_PROVIDER = "deepseek"; const OPENROUTER_RERANK_PROVIDER = "cohere"; type RunOpenRouterScenario = (harness: { @@ -216,6 +220,50 @@ export function defineOpenRouterTraceAssertions(options: { }, ); + test( + "captures reasoning fields for a streamed reasoning chat completion", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "openrouter-chat-reasoning-stream-operation", + ); + const span = findOpenRouterSpan(events, operation?.span.id, [ + "openrouter.chat.send", + ]); + const output = span?.output as + | Array<{ + message?: { + reasoning?: string; + reasoning_content?: string; + reasoning_details?: unknown[]; + }; + }> + | undefined; + const message = output?.[0]?.message; + const reasoning = message?.reasoning ?? message?.reasoning_content; + const hasReasoningText = + typeof reasoning === "string" && reasoning.length > 0; + const hasReasoningDetails = + Array.isArray(message?.reasoning_details) && + message.reasoning_details.length > 0; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: OPENROUTER_REASONING_PROVIDER, + }); + expect(span?.row.metadata?.model).toBe(REASONING_MODEL_NAME); + expect(span?.metrics?.time_to_first_token).toEqual(expect.any(Number)); + expect(span?.metrics?.completion_reasoning_tokens).toEqual( + expect.any(Number), + ); + expect(hasReasoningText || hasReasoningDetails).toBe(true); + }, + ); + test("captures trace for client.embeddings.generate()", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const operation = findLatestSpan( diff --git a/e2e/scenarios/openrouter-instrumentation/constants.mjs b/e2e/scenarios/openrouter-instrumentation/constants.mjs index 2bef0b2e8..623ab57d4 100644 --- a/e2e/scenarios/openrouter-instrumentation/constants.mjs +++ b/e2e/scenarios/openrouter-instrumentation/constants.mjs @@ -1,7 +1,15 @@ const CHAT_MODEL = "openai/gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "openai/text-embedding-3-small"; const RERANK_MODEL = "cohere/rerank-v3.5"; +const REASONING_MODEL = "deepseek/deepseek-r1"; const ROOT_NAME = "openrouter-root"; const SCENARIO_NAME = "openrouter-instrumentation"; -export { CHAT_MODEL, EMBEDDING_MODEL, RERANK_MODEL, ROOT_NAME, SCENARIO_NAME }; +export { + CHAT_MODEL, + EMBEDDING_MODEL, + RERANK_MODEL, + REASONING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +}; diff --git a/e2e/scenarios/openrouter-instrumentation/scenario.impl.mjs b/e2e/scenarios/openrouter-instrumentation/scenario.impl.mjs index a86dfac68..22e61b4b0 100644 --- a/e2e/scenarios/openrouter-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openrouter-instrumentation/scenario.impl.mjs @@ -9,6 +9,7 @@ import { CHAT_MODEL, EMBEDDING_MODEL, RERANK_MODEL, + REASONING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -90,6 +91,36 @@ async function runOpenRouterInstrumentationScenario( }, ); + await runOperation( + "openrouter-chat-reasoning-stream-operation", + "chat-reasoning-stream", + async () => { + const stream = await client.chat.send( + withCompatibleChatRequest({ + model: REASONING_MODEL, + messages: [ + { + role: "user", + content: + "Think briefly, then answer with exactly the number 4.", + }, + ], + maxTokens: 256, + reasoning: { + enabled: true, + exclude: false, + }, + stream: true, + streamOptions: { + includeUsage: true, + }, + temperature: 0, + }), + ); + await collectAsync(stream); + }, + ); + await runOperation( "openrouter-embeddings-operation", "embeddings", diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts b/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts index cb0261670..137232d6a 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts +++ b/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts @@ -335,6 +335,71 @@ describe("OpenRouter Agent Plugin", () => { metrics: {}, }); }); + + it("should aggregate reasoning fields from streaming deltas", () => { + expect( + aggregateOpenRouterChatChunks([ + { + choices: [ + { + delta: { + role: "assistant", + reasoning: "First, ", + reasoning_details: [ + { + type: "reasoning.text", + text: "First, ", + }, + ], + }, + }, + ], + }, + { + choices: [ + { + delta: { + reasoning_content: "check the answer.", + content: "OK", + reasoning_details: [ + { + type: "reasoning.summary", + summary: "Checked the answer.", + }, + ], + }, + finish_reason: "stop", + }, + ], + }, + ]), + ).toEqual({ + output: [ + { + index: 0, + message: { + role: "assistant", + content: "OK", + reasoning: "First, check the answer.", + reasoning_content: "check the answer.", + reasoning_details: [ + { + type: "reasoning.text", + text: "First, ", + }, + { + type: "reasoning.summary", + summary: "Checked the answer.", + }, + ], + }, + logprobs: null, + finish_reason: "stop", + }, + ], + metrics: {}, + }); + }); }); describe("callModel tool patching", () => { diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts b/js/src/instrumentation/plugins/openrouter-agent-plugin.ts index 57ded7931..414d110a5 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts +++ b/js/src/instrumentation/plugins/openrouter-agent-plugin.ts @@ -633,6 +633,11 @@ export function aggregateOpenRouterChatChunks( } { let role: string | undefined; let content = ""; + let reasoning = ""; + let hasReasoning = false; + let reasoningContent = ""; + let hasReasoningContent = false; + let reasoningDetails: unknown[] | undefined; let toolCalls: | Array<{ index?: number; @@ -670,6 +675,24 @@ export function aggregateOpenRouterChatChunks( content += delta.content; } + if (typeof delta.reasoning === "string") { + reasoning += delta.reasoning; + hasReasoning = true; + } + + if (typeof delta.reasoning_content === "string") { + reasoning += delta.reasoning_content; + reasoningContent += delta.reasoning_content; + hasReasoningContent = true; + } + + if (Array.isArray(delta.reasoning_details)) { + reasoningDetails = [ + ...(reasoningDetails || []), + ...delta.reasoning_details, + ]; + } + const choiceFinishReason = choice?.finishReason ?? choice?.finish_reason ?? undefined; const deltaFinishReason = @@ -740,6 +763,11 @@ export function aggregateOpenRouterChatChunks( message: { role, content: content || undefined, + ...(hasReasoning || hasReasoningContent ? { reasoning } : {}), + ...(hasReasoningContent + ? { reasoning_content: reasoningContent } + : {}), + ...(reasoningDetails ? { reasoning_details: reasoningDetails } : {}), ...(toolCalls ? { tool_calls: toolCalls } : {}), }, logprobs: null, diff --git a/js/src/instrumentation/plugins/openrouter-plugin.test.ts b/js/src/instrumentation/plugins/openrouter-plugin.test.ts index 153021161..bc67af92a 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.test.ts +++ b/js/src/instrumentation/plugins/openrouter-plugin.test.ts @@ -282,6 +282,71 @@ describe("OpenRouter Plugin", () => { metrics: {}, }); }); + + it("should aggregate reasoning fields from streaming deltas", () => { + expect( + aggregateOpenRouterChatChunks([ + { + choices: [ + { + delta: { + role: "assistant", + reasoning: "First, ", + reasoning_details: [ + { + type: "reasoning.text", + text: "First, ", + }, + ], + }, + }, + ], + }, + { + choices: [ + { + delta: { + reasoning_content: "check the answer.", + content: "OK", + reasoning_details: [ + { + type: "reasoning.summary", + summary: "Checked the answer.", + }, + ], + }, + finish_reason: "stop", + }, + ], + }, + ]), + ).toEqual({ + output: [ + { + index: 0, + message: { + role: "assistant", + content: "OK", + reasoning: "First, check the answer.", + reasoning_content: "check the answer.", + reasoning_details: [ + { + type: "reasoning.text", + text: "First, ", + }, + { + type: "reasoning.summary", + summary: "Checked the answer.", + }, + ], + }, + logprobs: null, + finish_reason: "stop", + }, + ], + metrics: {}, + }); + }); }); describe("aggregateOpenRouterResponseStreamEvents", () => { diff --git a/js/src/instrumentation/plugins/openrouter-plugin.ts b/js/src/instrumentation/plugins/openrouter-plugin.ts index d1bb68771..2edf46d3c 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.ts +++ b/js/src/instrumentation/plugins/openrouter-plugin.ts @@ -815,6 +815,11 @@ export function aggregateOpenRouterChatChunks( } { let role: string | undefined; let content = ""; + let reasoning = ""; + let hasReasoning = false; + let reasoningContent = ""; + let hasReasoningContent = false; + let reasoningDetails: unknown[] | undefined; let toolCalls: | Array<{ index?: number; @@ -852,6 +857,24 @@ export function aggregateOpenRouterChatChunks( content += delta.content; } + if (typeof delta.reasoning === "string") { + reasoning += delta.reasoning; + hasReasoning = true; + } + + if (typeof delta.reasoning_content === "string") { + reasoning += delta.reasoning_content; + reasoningContent += delta.reasoning_content; + hasReasoningContent = true; + } + + if (Array.isArray(delta.reasoning_details)) { + reasoningDetails = [ + ...(reasoningDetails || []), + ...delta.reasoning_details, + ]; + } + const choiceFinishReason = choice?.finishReason ?? choice?.finish_reason ?? undefined; const deltaFinishReason = @@ -922,6 +945,11 @@ export function aggregateOpenRouterChatChunks( message: { role, content: content || undefined, + ...(hasReasoning || hasReasoningContent ? { reasoning } : {}), + ...(hasReasoningContent + ? { reasoning_content: reasoningContent } + : {}), + ...(reasoningDetails ? { reasoning_details: reasoningDetails } : {}), ...(toolCalls ? { tool_calls: toolCalls } : {}), }, logprobs: null, diff --git a/js/src/vendor-sdk-types/openrouter-agent.ts b/js/src/vendor-sdk-types/openrouter-agent.ts index 289c853a5..b9d8cd1c9 100644 --- a/js/src/vendor-sdk-types/openrouter-agent.ts +++ b/js/src/vendor-sdk-types/openrouter-agent.ts @@ -15,6 +15,9 @@ export type OpenRouterAgentChatChoice = { message?: { role?: string; content?: string | null; + reasoning?: string | null; + reasoning_content?: string | null; + reasoning_details?: unknown[]; tool_calls?: unknown; }; logprobs?: unknown; @@ -26,6 +29,9 @@ export type OpenRouterAgentChatCompletionChunk = { delta?: { role?: string; content?: string; + reasoning?: string; + reasoning_content?: string; + reasoning_details?: unknown[]; tool_calls?: OpenRouterAgentChatToolCallDelta[]; toolCalls?: OpenRouterAgentChatToolCallDelta[]; finish_reason?: string | null; diff --git a/js/src/vendor-sdk-types/openrouter.ts b/js/src/vendor-sdk-types/openrouter.ts index e544721bc..6373e605c 100644 --- a/js/src/vendor-sdk-types/openrouter.ts +++ b/js/src/vendor-sdk-types/openrouter.ts @@ -26,6 +26,9 @@ export type OpenRouterChatChoice = { message?: { role?: string; content?: string | null; + reasoning?: string | null; + reasoning_content?: string | null; + reasoning_details?: unknown[]; tool_calls?: unknown; }; logprobs?: unknown; @@ -47,6 +50,9 @@ export type OpenRouterChatCompletionChunk = { delta?: { role?: string; content?: string; + reasoning?: string; + reasoning_content?: string; + reasoning_details?: unknown[]; tool_calls?: OpenRouterChatToolCallDelta[]; toolCalls?: OpenRouterChatToolCallDelta[]; finish_reason?: string | null; From f6a15d769ba5d295039a703cf489031bbbc048bc Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 4 May 2026 15:35:37 -0700 Subject: [PATCH 2/2] cs --- .changeset/clear-jeans-drive.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/clear-jeans-drive.md diff --git a/.changeset/clear-jeans-drive.md b/.changeset/clear-jeans-drive.md new file mode 100644 index 000000000..94f6e4230 --- /dev/null +++ b/.changeset/clear-jeans-drive.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(openrouter): Capture reasoning