diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/ai-mock-server.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/ai-mock-server.mjs new file mode 100644 index 000000000000..b97e0cb162dc --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/ai-mock-server.mjs @@ -0,0 +1,93 @@ +import { createServer } from 'node:http'; + +// A single mock server standing in for the OpenAI, Anthropic and Google GenAI HTTP APIs, so the real +// SDK clients emit gen_ai spans without any live credentials. Response bodies mirror the mock servers +// in the node-integration tests (suites/tracing/{openai,anthropic,google-genai}). Uses raw `node:http` +// (not express) so the mock doesn't itself get instrumented. + +function readJson(req) { + return new Promise(resolve => { + let body = ''; + req.on('data', chunk => (body += chunk)); + req.on('end', () => { + try { + resolve(JSON.parse(body || '{}')); + } catch { + resolve({}); + } + }); + }); +} + +function sendJson(res, status, obj) { + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(obj)); +} + +let serverPromise; + +/** Lazily starts the shared mock server and resolves to its port. */ +export function getMockAiPort() { + serverPromise ??= new Promise(resolve => { + const server = createServer(async (req, res) => { + const url = req.url || ''; + + // OpenAI: chat completions + if (req.method === 'POST' && url.endsWith('/openai/chat/completions')) { + const { model } = await readJson(req); + sendJson(res, 200, { + id: 'chatcmpl-mock123', + object: 'chat.completion', + created: 1677652288, + model, + choices: [ + { index: 0, message: { role: 'assistant', content: 'Hello from OpenAI mock!' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 10, completion_tokens: 15, total_tokens: 25 }, + }); + return; + } + + // Anthropic: messages + if (req.method === 'POST' && url.endsWith('/anthropic/v1/messages')) { + const { model } = await readJson(req); + sendJson(res, 200, { + id: 'msg_mock123', + type: 'message', + model, + role: 'assistant', + content: [{ type: 'text', text: 'Hello from Anthropic mock!' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 15 }, + }); + return; + } + + // Google GenAI: generateContent (the model name is embedded in the path before `:generateContent`). + // Plain string checks avoid the polynomial-backtracking risk of a `.+` regex on the URL. + if (req.method === 'POST' && url.startsWith('/v1beta/models/') && url.endsWith(':generateContent')) { + await readJson(req); + sendJson(res, 200, { + candidates: [ + { + content: { parts: [{ text: 'Mock response from Google GenAI!' }], role: 'model' }, + finishReason: 'stop', + index: 0, + }, + ], + usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 12, totalTokenCount: 20 }, + }); + return; + } + + res.writeHead(404).end(); + }); + + server.listen(0, () => { + resolve(server.address().port); + }); + }); + + return serverPromise; +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/anthropic/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/anthropic/route.ts new file mode 100644 index 000000000000..f0a811d679c0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/anthropic/route.ts @@ -0,0 +1,21 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { NextResponse } from 'next/server'; +import { getMockAiPort } from '../../../ai-mock-server.mjs'; + +export const dynamic = 'force-dynamic'; + +export async function GET() { + const port = await getMockAiPort(); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${port}/anthropic`, + }); + + await client.messages.create({ + model: 'claude-3-haiku-20240307', + max_tokens: 100, + messages: [{ role: 'user', content: 'What is the capital of France?' }], + }); + + return NextResponse.json({ status: 'ok' }); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/google-genai/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/google-genai/route.ts new file mode 100644 index 000000000000..e6031d59e50a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/google-genai/route.ts @@ -0,0 +1,21 @@ +import { GoogleGenAI } from '@google/genai'; +import { NextResponse } from 'next/server'; +import { getMockAiPort } from '../../../ai-mock-server.mjs'; + +export const dynamic = 'force-dynamic'; + +export async function GET() { + const port = await getMockAiPort(); + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${port}` }, + }); + + await client.models.generateContent({ + model: 'gemini-1.5-flash', + config: { temperature: 0.7, topP: 0.9, maxOutputTokens: 100 }, + contents: [{ role: 'user', parts: [{ text: 'What is the capital of France?' }] }], + }); + + return NextResponse.json({ status: 'ok' }); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/openai/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/openai/route.ts new file mode 100644 index 000000000000..9315f9d079ce --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/openai/route.ts @@ -0,0 +1,20 @@ +import { NextResponse } from 'next/server'; +import OpenAI from 'openai'; +import { getMockAiPort } from '../../../ai-mock-server.mjs'; + +export const dynamic = 'force-dynamic'; + +export async function GET() { + const port = await getMockAiPort(); + const client = new OpenAI({ + baseURL: `http://localhost:${port}/openai`, + apiKey: 'mock-api-key', + }); + + await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'user', content: 'What is the capital of France?' }], + }); + + return NextResponse.json({ status: 'ok' }); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/package.json index 30b78bd745b1..923a6e0d94da 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/package.json @@ -16,6 +16,8 @@ }, "//": "Pin `ioredis` to 5.10.1 and `mysql2` to 3.19.1: both are the last versions before the driver publishes its own native diagnostics channels; orchestrion's configs cover `ioredis <5.11.0` and `mysql2 <3.20.0`.", "dependencies": { + "@anthropic-ai/sdk": "0.63.0", + "@google/genai": "^1.20.0", "@sentry/core": "file:../../packed/sentry-core-packed.tgz", "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", "dataloader": "2.2.2", @@ -28,6 +30,7 @@ "mysql": "^2.18.1", "mysql2": "3.19.1", "next": "16.2.10", + "openai": "5.18.1", "pg": "^8.13.1", "postgres": "^3.4.7", "react": "19.1.0", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/anthropic.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/anthropic.test.ts new file mode 100644 index 000000000000..63adf9ae7aae --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/anthropic.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from '@playwright/test'; +import { waitForStreamedSpans } from '@sentry-internal/test-utils'; + +// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we +// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format. +test('Instruments anthropic-ai automatically via orchestrion', async ({ baseURL }) => { + const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans => + spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.anthropic'), + ); + + await fetch(`${baseURL}/api/anthropic`); + + const spans = await spansPromise; + + const chatSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); + expect(chatSpan).toBeDefined(); + expect(chatSpan?.attributes['sentry.op']?.value).toBe('gen_ai.chat'); + expect(chatSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.anthropic'); + expect(chatSpan?.attributes['gen_ai.system']?.value).toBe('anthropic'); + expect(chatSpan?.attributes['gen_ai.request.model']?.value).toBe('claude-3-haiku-20240307'); + expect(chatSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(10); + expect(chatSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(15); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/google-genai.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/google-genai.test.ts new file mode 100644 index 000000000000..2cdf0acb0e4b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/google-genai.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from '@playwright/test'; +import { waitForStreamedSpans } from '@sentry-internal/test-utils'; + +// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we +// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format. +test('Instruments google-genai automatically via orchestrion', async ({ baseURL }) => { + const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans => + spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.google_genai'), + ); + + await fetch(`${baseURL}/api/google-genai`); + + const spans = await spansPromise; + + const generateSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); + expect(generateSpan).toBeDefined(); + expect(generateSpan?.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.google_genai'); + expect(generateSpan?.attributes['gen_ai.system']?.value).toBe('google_genai'); + expect(generateSpan?.attributes['gen_ai.request.model']?.value).toBe('gemini-1.5-flash'); + expect(generateSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(8); + expect(generateSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(12); + expect(generateSpan?.attributes['gen_ai.usage.total_tokens']?.value).toBe(20); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/openai.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/openai.test.ts new file mode 100644 index 000000000000..88f2b4278f2e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/openai.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from '@playwright/test'; +import { waitForStreamedSpans } from '@sentry-internal/test-utils'; + +// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we +// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format. +test('Instruments openai automatically via orchestrion', async ({ baseURL }) => { + const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans => + spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.openai'), + ); + + await fetch(`${baseURL}/api/openai`); + + const spans = await spansPromise; + + const chatSpan = spans.find(span => span.name === 'chat gpt-3.5-turbo'); + expect(chatSpan).toBeDefined(); + expect(chatSpan?.attributes['sentry.op']?.value).toBe('gen_ai.chat'); + expect(chatSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.openai'); + expect(chatSpan?.attributes['gen_ai.system']?.value).toBe('openai'); + expect(chatSpan?.attributes['gen_ai.request.model']?.value).toBe('gpt-3.5-turbo'); + expect(chatSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(10); + expect(chatSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(15); + expect(chatSpan?.attributes['gen_ai.usage.total_tokens']?.value).toBe(25); +});