From 254d5958e0c8285031d076f2ece821ea0b3d9a6e Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 20 Apr 2026 23:17:53 +0200 Subject: [PATCH 1/4] test: Don't use discontinued models --- .../ai-sdk-instrumentation/scenario.impl.mjs | 2 +- .../anthropic-v0273.log-payloads.json | 44 ++++----- .../anthropic-v0273.span-events.json | 12 +-- .../anthropic-v0390.log-payloads.json | 60 ++++++------ .../anthropic-v0390.span-events.json | 16 +-- .../anthropic-v0712.log-payloads.json | 98 ++++++++++--------- .../anthropic-v0712.span-events.json | 22 ++--- .../anthropic-v0730.log-payloads.json | 98 ++++++++++--------- .../anthropic-v0730.span-events.json | 22 ++--- .../anthropic-v0780.log-payloads.json | 98 ++++++++++--------- .../anthropic-v0780.span-events.json | 22 ++--- .../anthropic-v0800.log-payloads.json | 98 ++++++++++--------- .../anthropic-v0800.span-events.json | 22 ++--- .../scenario.impl.mjs | 2 +- .../claude-agent-sdk-v0.1.span-events.json | 16 +-- .../claude-agent-sdk-v0.2.76.span-events.json | 16 +-- .../claude-agent-sdk-v0.2.79.span-events.json | 16 +-- .../claude-agent-sdk-v0.2.81.span-events.json | 16 +-- .../assertions.ts | 6 ++ .../scenario.impl.mjs | 2 +- js/examples/claude-agent-sdk/async_iter.ts | 2 +- js/examples/claude-agent-sdk/index.ts | 2 +- js/src/wrappers/anthropic.test.ts | 17 ++-- .../claude-agent-sdk/claude-agent-sdk.test.ts | 4 +- 24 files changed, 379 insertions(+), 334 deletions(-) diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs index aee9a0030..1f72e0804 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs @@ -143,7 +143,7 @@ async function runAISDKInstrumentationScenario( ) { const instrumentedAI = decorateAI ? decorateAI(options.ai) : options.ai; const openaiModel = options.openai("gpt-4o-mini-2024-07-18"); - const anthropicModel = options.anthropic?.("claude-3-haiku-20240307"); + const anthropicModel = options.anthropic?.("claude-haiku-4-5"); const openaiEmbeddingModel = options.openai.textEmbeddingModel( "text-embedding-3-small", ); diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json index 69590d2f3..4e9a8f8c5 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 26, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1415 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json index 187e089f8..50ae8de47 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json index ba4f1d519..4111987f9 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 26, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1415 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { @@ -315,26 +315,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 6, + "completion_tokens": 5, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "start": 0, "time_to_first_token": 0, - "tokens": 19 + "tokens": 18 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "BETA.", + "text": "BETA", "type": "text" } ], @@ -361,23 +361,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" } ] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json index 0bfc89b4c..10d51bfdc 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -259,7 +259,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -297,7 +297,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json index a782f61c0..d4a50d277 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 27, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1416 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { @@ -365,26 +365,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 6, + "completion_tokens": 5, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "start": 0, "time_to_first_token": 0, - "tokens": 19 + "tokens": 18 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "BETA.", + "text": "BETA", "type": "text" } ], @@ -411,23 +411,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -450,25 +450,30 @@ ], "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 58, + "completion_tokens": 72, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 829, + "prompt_tokens": 1294, "start": 0, "time_to_first_token": 0, - "tokens": 887 + "tokens": 1366 }, "name": "anthropic.beta.messages.toolRunner", "output": { - "content": [], + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], "role": "assistant" }, "type": "task" @@ -505,20 +510,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 375, + "prompt_tokens": 607, "start": 0, "time_to_first_token": 0, - "tokens": 430 + "tokens": 663 }, "name": "anthropic.messages.create", "output": { @@ -570,24 +575,29 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 16, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 457 + "tokens": 703 }, "name": "anthropic.messages.create", "output": { - "content": [], + "content": [ + { + "text": "", + "type": "text" + } + ], "role": "assistant" }, "type": "llm" diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json index 9cd6b6844..0acff38b6 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -297,7 +297,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -335,7 +335,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -374,7 +374,7 @@ "has_output": true, "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic" }, @@ -426,7 +426,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -449,7 +449,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json index a782f61c0..9099fa25b 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 26, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1415 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { @@ -365,26 +365,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 6, + "completion_tokens": 5, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "start": 0, "time_to_first_token": 0, - "tokens": 19 + "tokens": 18 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "BETA.", + "text": "BETA", "type": "text" } ], @@ -411,23 +411,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -450,25 +450,30 @@ ], "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 58, + "completion_tokens": 72, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 829, + "prompt_tokens": 1294, "start": 0, "time_to_first_token": 0, - "tokens": 887 + "tokens": 1366 }, "name": "anthropic.beta.messages.toolRunner", "output": { - "content": [], + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], "role": "assistant" }, "type": "task" @@ -505,20 +510,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 375, + "prompt_tokens": 607, "start": 0, "time_to_first_token": 0, - "tokens": 430 + "tokens": 663 }, "name": "anthropic.messages.create", "output": { @@ -570,24 +575,29 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 16, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 457 + "tokens": 703 }, "name": "anthropic.messages.create", "output": { - "content": [], + "content": [ + { + "text": "", + "type": "text" + } + ], "role": "assistant" }, "type": "llm" diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json index 9cd6b6844..0acff38b6 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -297,7 +297,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -335,7 +335,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -374,7 +374,7 @@ "has_output": true, "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic" }, @@ -426,7 +426,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -449,7 +449,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json index a782f61c0..d4a50d277 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 27, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1416 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { @@ -365,26 +365,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 6, + "completion_tokens": 5, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "start": 0, "time_to_first_token": 0, - "tokens": 19 + "tokens": 18 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "BETA.", + "text": "BETA", "type": "text" } ], @@ -411,23 +411,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -450,25 +450,30 @@ ], "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 58, + "completion_tokens": 72, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 829, + "prompt_tokens": 1294, "start": 0, "time_to_first_token": 0, - "tokens": 887 + "tokens": 1366 }, "name": "anthropic.beta.messages.toolRunner", "output": { - "content": [], + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], "role": "assistant" }, "type": "task" @@ -505,20 +510,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 375, + "prompt_tokens": 607, "start": 0, "time_to_first_token": 0, - "tokens": 430 + "tokens": 663 }, "name": "anthropic.messages.create", "output": { @@ -570,24 +575,29 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 16, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 457 + "tokens": 703 }, "name": "anthropic.messages.create", "output": { - "content": [], + "content": [ + { + "text": "", + "type": "text" + } + ], "role": "assistant" }, "type": "llm" diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json index 9cd6b6844..0acff38b6 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -297,7 +297,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -335,7 +335,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -374,7 +374,7 @@ "has_output": true, "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic" }, @@ -426,7 +426,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -449,7 +449,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json index a782f61c0..9099fa25b 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json @@ -29,26 +29,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 5, + "completion_tokens": 4, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 12, "start": 0, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "OK.", + "text": "OK", "type": "text" } ], @@ -93,20 +93,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 29, + "completion_tokens": 26, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1418 + "tokens": 1415 }, "name": "anthropic.messages.create", "output": { @@ -139,23 +139,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -177,23 +177,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -215,7 +215,7 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null @@ -225,10 +225,10 @@ "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 480 + "tokens": 713 }, "name": "anthropic.messages.create", "output": { @@ -265,20 +265,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 357, + "prompt_tokens": 589, "start": 0, "time_to_first_token": 0, - "tokens": 412 + "tokens": 645 }, "name": "anthropic.messages.create", "output": { @@ -365,26 +365,26 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 6, + "completion_tokens": 5, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "start": 0, "time_to_first_token": 0, - "tokens": 19 + "tokens": 18 }, "name": "anthropic.messages.create", "output": { "content": [ { - "text": "BETA.", + "text": "BETA", "type": "text" } ], @@ -411,23 +411,23 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 18, + "completion_tokens": 15, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 24, "start": 0, "time_to_first_token": 0, - "tokens": 42 + "tokens": 39 }, "name": "anthropic.messages.create", - "output": "1 - one\n2 - two\n3 - three", + "output": "1 one\n2 two\n3 three", "type": "llm" }, { @@ -450,25 +450,30 @@ ], "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 58, + "completion_tokens": 72, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 829, + "prompt_tokens": 1294, "start": 0, "time_to_first_token": 0, - "tokens": 887 + "tokens": 1366 }, "name": "anthropic.beta.messages.toolRunner", "output": { - "content": [], + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], "role": "assistant" }, "type": "task" @@ -505,20 +510,20 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "tool_use", "stop_sequence": null }, "metrics": { - "completion_tokens": 55, + "completion_tokens": 56, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 375, + "prompt_tokens": 607, "start": 0, "time_to_first_token": 0, - "tokens": 430 + "tokens": 663 }, "name": "anthropic.messages.create", "output": { @@ -570,24 +575,29 @@ } ], "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic", "stop_reason": "end_turn", "stop_sequence": null }, "metrics": { - "completion_tokens": 3, + "completion_tokens": 16, "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 454, + "prompt_tokens": 687, "start": 0, "time_to_first_token": 0, - "tokens": 457 + "tokens": 703 }, "name": "anthropic.messages.create", "output": { - "content": [], + "content": [ + { + "text": "", + "type": "text" + } + ], "role": "assistant" }, "type": "llm" diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json index 9cd6b6844..0acff38b6 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json @@ -31,7 +31,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -69,7 +69,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -107,7 +107,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -145,7 +145,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -183,7 +183,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -221,7 +221,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -297,7 +297,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -335,7 +335,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -374,7 +374,7 @@ "has_output": true, "metadata": { "anthropic_tool_runner_iterations": 2, - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "operation": "toolRunner", "provider": "anthropic" }, @@ -426,7 +426,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ @@ -449,7 +449,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "provider": "anthropic" }, "metric_keys": [ diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs index ab6d7f2ba..7bd22954a 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs @@ -6,7 +6,7 @@ import { runTracedScenario, } from "../../helpers/provider-runtime.mjs"; -const ANTHROPIC_MODEL = "claude-3-haiku-20240307"; +const ANTHROPIC_MODEL = "claude-haiku-4-5"; const ROOT_NAME = "anthropic-instrumentation-root"; const SCENARIO_NAME = "anthropic-instrumentation"; const WEATHER_TOOL = { diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.1.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.1.span-events.json index 68fbc65d5..78a9015fa 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.1.span-events.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.1.span-events.json @@ -8,7 +8,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -46,7 +46,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -66,7 +66,7 @@ "Use the calculator tool to multiply 15 by 7. Do not answer from memory." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -100,7 +100,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -136,7 +136,7 @@ "Use the calculator tool to divide 2 by 0. Do not recover from the error." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -170,7 +170,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -235,7 +235,7 @@ "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -286,7 +286,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json index e947c64e5..1a12cdb33 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json @@ -8,7 +8,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -46,7 +46,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -66,7 +66,7 @@ "Use the calculator tool to multiply 15 by 7. Do not answer from memory." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -100,7 +100,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -136,7 +136,7 @@ "Use the calculator tool to divide 2 by 0. Do not recover from the error." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -170,7 +170,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -235,7 +235,7 @@ "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -288,7 +288,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json index e947c64e5..1a12cdb33 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json @@ -8,7 +8,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -46,7 +46,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -66,7 +66,7 @@ "Use the calculator tool to multiply 15 by 7. Do not answer from memory." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -100,7 +100,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -136,7 +136,7 @@ "Use the calculator tool to divide 2 by 0. Do not recover from the error." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -170,7 +170,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -235,7 +235,7 @@ "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -288,7 +288,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json index e947c64e5..1a12cdb33 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json @@ -8,7 +8,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -46,7 +46,7 @@ "Part 2" ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -66,7 +66,7 @@ "Use the calculator tool to multiply 15 by 7. Do not answer from memory." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -100,7 +100,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -136,7 +136,7 @@ "Use the calculator tool to divide 2 by 0. Do not recover from the error." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -170,7 +170,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", @@ -235,7 +235,7 @@ "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." ], "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [ "completion_tokens", @@ -288,7 +288,7 @@ "has_input": true, "has_output": true, "metadata": { - "model": "claude-haiku-4-5-20251001" + "model": "claude-haiku-4-5" }, "metric_keys": [], "name": "Claude Agent", diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts index 11efcb451..ffc8e6b06 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts @@ -101,6 +101,12 @@ function summarizeSpan( } if (summary.metadata && typeof summary.metadata === "object") { const metadata = summary.metadata as Record; + if ( + typeof metadata.model === "string" && + metadata.model.startsWith("claude-haiku-4-5-") + ) { + metadata.model = "claude-haiku-4-5"; + } if (typeof metadata["claude_agent_sdk.description"] === "string") { metadata["claude_agent_sdk.description"] = ""; } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs index f9cd8d392..683918c11 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs @@ -6,7 +6,7 @@ import { } from "../../helpers/provider-runtime.mjs"; import { z } from "zod"; -const CLAUDE_AGENT_MODEL = "claude-haiku-4-5-20251001"; +const CLAUDE_AGENT_MODEL = "claude-haiku-4-5"; export const ROOT_NAME = "claude-agent-sdk-root"; export const SCENARIO_NAME = "claude-agent-sdk-traces"; diff --git a/js/examples/claude-agent-sdk/async_iter.ts b/js/examples/claude-agent-sdk/async_iter.ts index 26c2088ef..207e21b25 100644 --- a/js/examples/claude-agent-sdk/async_iter.ts +++ b/js/examples/claude-agent-sdk/async_iter.ts @@ -105,7 +105,7 @@ async function main() { for await (const message of query({ prompt: conversationPrompt, options: { - model: "claude-haiku-4-5-20251001", + model: "claude-haiku-4-5", permissionMode: "bypassPermissions", maxTurns: 1, }, diff --git a/js/examples/claude-agent-sdk/index.ts b/js/examples/claude-agent-sdk/index.ts index 2a7b1fa94..fcc807e96 100644 --- a/js/examples/claude-agent-sdk/index.ts +++ b/js/examples/claude-agent-sdk/index.ts @@ -117,7 +117,7 @@ Report all results.`; description: "Math specialist for calculations and explanations", prompt: "You are a math expert. Perform calculations step by step.", - model: "haiku", + model: "claude-haiku-4-5", }, }, mcpServers: { diff --git a/js/src/wrappers/anthropic.test.ts b/js/src/wrappers/anthropic.test.ts index 5002f6e3e..e8f1c3f5d 100644 --- a/js/src/wrappers/anthropic.test.ts +++ b/js/src/wrappers/anthropic.test.ts @@ -14,8 +14,8 @@ import { _exportsForTestingOnly, Attachment, initLogger } from "../logger"; import { configureNode } from "../node/config"; import { getCurrentUnixTimestamp } from "../util"; -// use the cheapest model for tests -const TEST_MODEL = "claude-3-haiku-20240307"; +// use a low-cost current Haiku alias for tests +const TEST_MODEL = "claude-haiku-4-5"; interface TextBlock { type: "text"; @@ -131,7 +131,7 @@ describe("anthropic client unit tests", { retry: 3 }, () => { expect(message.content[0].type).toBe("text"); const content = message.content[0] as unknown; if (typeof content === "object" && content !== null && "text" in content) { - expect(content.text).toContain("old pond"); + expect(content.text.toLowerCase()).toContain("old pond"); } else { throw new Error("Content is not a text block"); } @@ -387,12 +387,11 @@ describe("anthropic client unit tests", { retry: 3 }, () => { .toLowerCase() .replace(/\n/g, " ") .replace(/'/g, ""); - // Validate we collected all the text, so check the first, line, the last line - // and a few others too. + // Validate we collected the streamed text without relying on one exact phrasing. expect(output).toContain("shall i compare thee to a summers day"); - expect(output).toContain("too hot the eye of heaven shines"); - expect(output).toContain("so long as men can breathe or eyes can see"); - expect(output).toContain("so long lives this and this gives life to thee"); + expect(output).toContain("shakespeare"); + expect(output).toContain("summer"); + expect(output.length).toBeGreaterThan(200); expect(span["span_attributes"].type).toBe("llm"); expect(span["span_attributes"].name).toBe("anthropic.messages.create"); @@ -706,7 +705,7 @@ describe("anthropic client unit tests", { retry: 3 }, () => { "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; const response = await client.messages.create({ - model: "claude-haiku-4-5-20251001", + model: TEST_MODEL, messages: [ { role: "user", diff --git a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.test.ts b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.test.ts index 4fb558a38..8db8aebf4 100644 --- a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.test.ts +++ b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.test.ts @@ -631,7 +631,7 @@ try { // Initialize Braintrust state once per process } -const TEST_MODEL = "claude-haiku-4-5-20251001"; +const TEST_MODEL = "claude-haiku-4-5"; describe.skipIf(!claudeSDK)("claude-agent-sdk integration tests", () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -919,7 +919,7 @@ describe.skipIf(!claudeSDK)("claude-agent-sdk integration tests", () => { "Math specialist. Use the calculator tool for calculations.", prompt: "You are a math expert. Use the calculator tool to perform the requested calculation. Be concise.", - model: "haiku", + model: TEST_MODEL, }, }, mcpServers: { From 9b6f52b0b8c32dd347aae63b8aa4abd9fb28a352 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 21 Apr 2026 10:36:14 +0200 Subject: [PATCH 2/4] switch google flash model --- js/examples/auto-instrumentation/google-genai-example.js | 2 +- js/src/wrappers/google-genai.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/js/examples/auto-instrumentation/google-genai-example.js b/js/examples/auto-instrumentation/google-genai-example.js index 772b3ca3d..7aa7aaa63 100644 --- a/js/examples/auto-instrumentation/google-genai-example.js +++ b/js/examples/auto-instrumentation/google-genai-example.js @@ -33,7 +33,7 @@ async function main() { // This call will be automatically traced to Braintrust const response = await ai.models.generateContent({ - model: "gemini-2.0-flash-001", + model: "gemini-2.5-flash-lite", contents: "Explain what Gemini models are in one sentence.", }); diff --git a/js/src/wrappers/google-genai.test.ts b/js/src/wrappers/google-genai.test.ts index f6d3a3305..8723debfa 100644 --- a/js/src/wrappers/google-genai.test.ts +++ b/js/src/wrappers/google-genai.test.ts @@ -18,7 +18,7 @@ import { import { wrapGoogleGenAI } from "./google-genai"; import { getCurrentUnixTimestamp } from "../util"; -const TEST_MODEL = "gemini-2.0-flash-001"; +const TEST_MODEL = "gemini-2.5-flash-lite"; const TEST_SUITE_OPTIONS = { timeout: 10000, retry: 3 }; async function drainRawEvents(backgroundLogger: TestBackgroundLogger) { From 90d73abb6a90abfb2f7c9f71bb8d98a7ae7e0edd Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 21 Apr 2026 11:12:45 +0200 Subject: [PATCH 3/4] normalization --- .../huggingface-v281.log-payloads.json | 18 +++++++++--------- .../huggingface-v3150.log-payloads.json | 18 +++++++++--------- .../huggingface-v41315.log-payloads.json | 18 +++++++++--------- .../huggingface-instrumentation/assertions.ts | 9 ++++++++- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json index 0543ea40d..5d8ada30c 100644 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json @@ -119,10 +119,10 @@ "completion_accepted_prediction_tokens": 0, "completion_reasoning_tokens": 0, "completion_rejected_prediction_tokens": 0, - "completion_tokens": 3, + "completion_tokens": "", "prompt_cached_tokens": 0, - "prompt_tokens": 40, - "tokens": 43 + "prompt_tokens": "", + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -253,11 +253,11 @@ "completion_accepted_prediction_tokens": 0, "completion_reasoning_tokens": 0, "completion_rejected_prediction_tokens": 0, - "completion_tokens": 3, + "completion_tokens": "", "prompt_cached_tokens": 0, - "prompt_tokens": 40, + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 43 + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -381,10 +381,10 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 4, - "prompt_tokens": 6, + "completion_tokens": "", + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 10 + "tokens": "" }, "project_id": "", "root_span_id": "", diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json index 1e9d02b65..3ab9d6929 100644 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json @@ -115,9 +115,9 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 1, - "prompt_tokens": 40, - "tokens": 41 + "completion_tokens": "", + "prompt_tokens": "", + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -244,10 +244,10 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 1, - "prompt_tokens": 40, + "completion_tokens": "", + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 41 + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -475,10 +475,10 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 4, - "prompt_tokens": 5, + "completion_tokens": "", + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 9 + "tokens": "" }, "project_id": "", "root_span_id": "", diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json index 1e9d02b65..3ab9d6929 100644 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json @@ -115,9 +115,9 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 1, - "prompt_tokens": 40, - "tokens": 41 + "completion_tokens": "", + "prompt_tokens": "", + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -244,10 +244,10 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 1, - "prompt_tokens": 40, + "completion_tokens": "", + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 41 + "tokens": "" }, "project_id": "", "root_span_id": "", @@ -475,10 +475,10 @@ "id": "", "log_id": "g", "metrics": { - "completion_tokens": 4, - "prompt_tokens": 5, + "completion_tokens": "", + "prompt_tokens": "", "time_to_first_token": "", - "tokens": 9 + "tokens": "" }, "project_id": "", "root_span_id": "", diff --git a/e2e/scenarios/huggingface-instrumentation/assertions.ts b/e2e/scenarios/huggingface-instrumentation/assertions.ts index 209587de7..60b9df63b 100644 --- a/e2e/scenarios/huggingface-instrumentation/assertions.ts +++ b/e2e/scenarios/huggingface-instrumentation/assertions.ts @@ -121,7 +121,14 @@ function normalizeMetrics(value: Json): Json { for (const [key, entry] of Object.entries(value)) { if ( typeof entry === "number" && - ["end", "start", "time_to_first_token"].includes(key) + [ + "completion_tokens", + "end", + "prompt_tokens", + "start", + "time_to_first_token", + "tokens", + ].includes(key) ) { normalized[key] = ""; continue; From edf5726c07a0227141c3d62e88180ac423c308b9 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 21 Apr 2026 11:41:31 +0200 Subject: [PATCH 4/4] more normalization --- .../anthropic-v0273.log-payloads.json | 4 +-- .../anthropic-v0390.log-payloads.json | 4 +-- .../anthropic-v0712.log-payloads.json | 4 +-- .../anthropic-v0730.log-payloads.json | 4 +-- .../anthropic-v0780.log-payloads.json | 4 +-- .../anthropic-v0800.log-payloads.json | 4 +-- .../anthropic-instrumentation/assertions.ts | 33 +++++++++++++++---- 7 files changed, 39 insertions(+), 18 deletions(-) diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json index 4e9a8f8c5..bd580cbc8 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 26, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1415 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json index 4111987f9..748d47113 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 26, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1415 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json index d4a50d277..7b7fd46d3 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 27, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1416 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json index 9099fa25b..7b7fd46d3 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 26, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1415 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json index d4a50d277..7b7fd46d3 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 27, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1416 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json index 9099fa25b..7b7fd46d3 100644 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json @@ -99,14 +99,14 @@ "stop_sequence": null }, "metrics": { - "completion_tokens": 26, + "completion_tokens": "", "end": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 1389, "start": 0, "time_to_first_token": 0, - "tokens": 1415 + "tokens": "" }, "name": "anthropic.messages.create", "output": { diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index 01974fca8..2401efc27 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -73,6 +73,23 @@ function pickMetadata( return Object.keys(picked).length > 0 ? (picked as Json) : null; } +function normalizeMetricValues( + metrics: Json, + keys: string[], + replacement: Json, +): void { + if (!metrics || typeof metrics !== "object" || Array.isArray(metrics)) { + return; + } + + const metricsRecord = metrics as Record; + for (const key of keys) { + if (typeof metricsRecord[key] === "number") { + metricsRecord[key] = replacement; + } + } +} + function summarizeAnthropicPayload(event: CapturedLogEvent): Json { const normalizeToolResultIds = ( messages: @@ -166,12 +183,11 @@ function summarizeAnthropicPayload(event: CapturedLogEvent): Json { // Thinking token counts vary per run (temperature=1, variable thinking depth). // Zero them out so the payload snapshot is stable. if (summary.metrics && typeof summary.metrics === "object") { - const metrics = summary.metrics as Record; - for (const key of ["completion_tokens", "tokens"]) { - if (key in metrics) { - metrics[key] = 0; - } - } + normalizeMetricValues( + summary.metrics, + ["completion_tokens", "tokens"], + 0, + ); } return summary; } @@ -216,6 +232,11 @@ function summarizeAnthropicPayload(event: CapturedLogEvent): Json { if (hasAttachmentInput && textBlock) { textBlock.text = ""; summary.output = output as Json; + normalizeMetricValues( + summary.metrics, + ["completion_tokens", "tokens"], + "", + ); } const hasToolResultInput = input?.some(