test(cloudflare): Proof that AI providers work OOTB on Cloudflare (#22313)

The Cloudflare integration tests were already testing all AI
integrations, but used mocks.

I changed the following in all tests:
- Removed the `mocks.ts` and changed it to the actual package
  - Instead the `.fetch` is being mocked
- Aligned the assertions

Author: JPeer264

dev-packages/cloudflare-integration-tests/package.json

@@ -15,12 +15,17 @@
   },
   "dependencies": {
     "ai": "^6.0.0",
+    "@anthropic-ai/sdk": "0.63.0",
+    "@google/genai": "^1.20.0",
+    "@langchain/core": "^0.3.80",
     "@langchain/langgraph": "^1.0.1",
+    "@langchain/openai": "^0.5.0",
     "@prisma/adapter-d1": "6.15.0",
     "@prisma/client": "6.15.0",
     "@sentry/cloudflare": "10.66.0",
     "@sentry/hono": "10.66.0",
-    "hono": "^4.12.25"
+    "hono": "^4.12.25",
+    "openai": "5.18.1"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20260426.0",

dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts

@@ -1,16 +1,27 @@
+import Anthropic from '@anthropic-ai/sdk';
 import * as Sentry from '@sentry/cloudflare';
-import type { AnthropicAiClient } from '@sentry/core';
-import { MockAnthropic } from './mocks';
 
 interface Env {
   SENTRY_DSN: string;
 }
 
-const mockClient = new MockAnthropic({
-  apiKey: 'mock-api-key',
-});
+// Return a canned response so the `@anthropic-ai/sdk` runs on workerd without hitting the network.
+const mockFetch: typeof fetch = async () =>
+  new Response(
+    JSON.stringify({
+      id: 'msg_mock123',
+      type: 'message',
+      role: 'assistant',
+      model: 'claude-3-haiku-20240307',
+      content: [{ type: 'text', text: 'Hello from Anthropic!' }],
+      stop_reason: 'end_turn',
+      stop_sequence: null,
+      usage: { input_tokens: 10, output_tokens: 15 },
+    }),
+    { status: 200, headers: { 'content-type': 'application/json' } },
+  );
 
-const client: AnthropicAiClient = Sentry.instrumentAnthropicAiClient(mockClient);
+const client = Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch }));
 
 export default Sentry.withSentry(
   (env: Env) => ({
@@ -20,7 +31,7 @@ export default Sentry.withSentry(
   }),
   {
     async fetch(_request, _env, _ctx) {
-      const response = await client.messages?.create({
+      const response = await client.messages.create({
         model: 'claude-3-haiku-20240307',
         messages: [{ role: 'user', content: 'What is the capital of France?' }],
         temperature: 0.7,

dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/mocks.ts

@@ -1,55 +1,57 @@
 import { expect, it } from 'vitest';
 import {
   GEN_AI_OPERATION_NAME_ATTRIBUTE,
+  GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE,
   GEN_AI_REQUEST_MODEL_ATTRIBUTE,
   GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE,
   GEN_AI_RESPONSE_ID_ATTRIBUTE,
   GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
   GEN_AI_SYSTEM_ATTRIBUTE,
   GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
   GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
+  GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
 } from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
 import { createRunner } from '../../../runner';
 
-// These tests are not exhaustive because the instrumentation is
-// already tested in the node integration tests and we merely
-// want to test that the instrumentation does not break in our
-// cloudflare SDK.
+// This test runs the `@anthropic-ai/sdk` on the Workers runtime (with a
+// canned fetch) to verify the instrumentation works end-to-end on Cloudflare,
+// not just against a hand-written mock client.
 
-it('traces a basic message creation request', async ({ signal }) => {
+it('traces a basic message creation request with the anthropic SDK', async ({ signal }) => {
   const runner = createRunner(__dirname)
     .ignore('event')
     .expect(envelope => {
-      // Transaction item (first item in envelope)
       const transactionEvent = envelope[1]?.[0]?.[1] as any;
       expect(transactionEvent.transaction).toBe('GET /');
 
-      // Span container item (second item in same envelope)
       const container = envelope[1]?.[1]?.[1] as any;
       expect(container).toBeDefined();
-
       expect(container.items).toHaveLength(1);
-      const [firstSpan] = container.items;
 
-      // [0] chat claude-3-haiku-20240307
-      expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307');
-      expect(firstSpan!.status).toBe('ok');
-      expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
-      expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
-      expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.anthropic' });
-      expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' });
-      expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
-        type: 'string',
-        value: 'claude-3-haiku-20240307',
-      });
-      expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 });
-      expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({
-        type: 'string',
-        value: 'claude-3-haiku-20240307',
+      expect(container.items[0]).toEqual({
+        trace_id: expect.any(String),
+        span_id: expect.any(String),
+        parent_span_id: expect.any(String),
+        name: 'chat claude-3-haiku-20240307',
+        start_timestamp: expect.any(Number),
+        end_timestamp: expect.any(Number),
+        status: 'ok',
+        is_segment: false,
+        attributes: {
+          'sentry.origin': { value: 'auto.ai.anthropic', type: 'string' },
+          'sentry.op': { value: 'gen_ai.chat', type: 'string' },
+          [GEN_AI_SYSTEM_ATTRIBUTE]: { value: 'anthropic', type: 'string' },
+          [GEN_AI_OPERATION_NAME_ATTRIBUTE]: { value: 'chat', type: 'string' },
+          [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: { value: 'claude-3-haiku-20240307', type: 'string' },
+          [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: { value: 0.7, type: 'double' },
+          [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: { value: 100, type: 'integer' },
+          [GEN_AI_RESPONSE_ID_ATTRIBUTE]: { value: 'msg_mock123', type: 'string' },
+          [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: { value: 'claude-3-haiku-20240307', type: 'string' },
+          [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: { value: 10, type: 'integer' },
+          [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: { value: 15, type: 'integer' },
+          [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: { value: 25, type: 'integer' },
+        },
       });
-      expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'msg_mock123' });
-      expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 });
-      expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 });
     })
     .start(signal);
   await runner.makeRequest('get', '/');

dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts

@@ -1,5 +1,5 @@
 {
-  "name": "worker-name",
+  "name": "anthropic-ai-worker",
   "compatibility_date": "2025-06-17",
   "main": "index.ts",
   "compatibility_flags": ["nodejs_als"],

dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/wrangler.jsonc

@@ -1,16 +1,45 @@
+import { GoogleGenAI } from '@google/genai';
 import * as Sentry from '@sentry/cloudflare';
-import type { GoogleGenAIClient } from '@sentry/core';
-import { MockGoogleGenAI } from './mocks';
 
 interface Env {
   SENTRY_DSN: string;
 }
 
-const mockClient = new MockGoogleGenAI({
-  apiKey: 'mock-api-key',
-});
+// `@google/genai` has no per-client fetch option — it calls the global `fetch`
+// directly. Override it (gated to the Gemini host) so the SDK runs on
+// workerd without hitting the network, while Sentry envelope delivery still
+// uses the original fetch.
+const originalFetch = globalThis.fetch;
+globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+  const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
 
-const client: GoogleGenAIClient = Sentry.instrumentGoogleGenAIClient(mockClient);
+  if (!url.includes('generativelanguage.googleapis.com')) {
+    return originalFetch(input, init);
+  }
+
+  if (url.includes(':embedContent') || url.includes(':batchEmbedContents')) {
+    return new Response(JSON.stringify({ embeddings: [{ values: [0.1, 0.2, 0.3, 0.4, 0.5] }] }), {
+      status: 200,
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  return new Response(
+    JSON.stringify({
+      candidates: [
+        {
+          content: { parts: [{ text: 'Hello from Google GenAI!' }], role: 'model' },
+          finishReason: 'stop',
+          index: 0,
+        },
+      ],
+      usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 12, totalTokenCount: 20 },
+    }),
+    { status: 200, headers: { 'content-type': 'application/json' } },
+  );
+}) as typeof fetch;
+
+const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' }));
 
 export default Sentry.withSentry(
   (env: Env) => ({
@@ -36,12 +65,10 @@ export default Sentry.withSentry(
         ],
       });
 
-      const chatResponse = await chat.sendMessage({
-        message: 'Tell me a joke',
-      });
+      await chat.sendMessage({ message: 'Tell me a joke' });
 
       // Test 2: models.generateContent
-      const modelResponse = await client.models.generateContent({
+      await client.models.generateContent({
         model: 'gemini-1.5-flash',
         config: {
           temperature: 0.7,
@@ -57,12 +84,12 @@ export default Sentry.withSentry(
       });
 
       // Test 3: models.embedContent
-      const embedResponse = await client.models.embedContent({
+      await client.models.embedContent({
         model: 'text-embedding-004',
         contents: 'Hello world',
       });
 
-      return new Response(JSON.stringify({ chatResponse, modelResponse, embedResponse }));
+      return new Response(JSON.stringify({ success: true }));
     },
   },
 );