diff --git a/.ai-sdk-factory/reproduction-replay.json b/.ai-sdk-factory/reproduction-replay.json new file mode 100644 index 000000000000..6595fb62a087 --- /dev/null +++ b/.ai-sdk-factory/reproduction-replay.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "issueId": 18331, + "command": "pnpm -C examples/ai-functions exec tsx src/reproduction/issue-18331-anthropic-spliced-message-start.ts", + "artifactPaths": [ + "examples/ai-functions/package.json", + "examples/ai-functions/src/reproduction/issue-18331-anthropic-spliced-message-start.ts" + ], + "expectedFailure": { + "exitCode": 1, + "signal": "ISSUE_18331_REPRODUCED: different message_start was silently merged into one completed streamText step" + } +} diff --git a/examples/ai-functions/package.json b/examples/ai-functions/package.json new file mode 100644 index 000000000000..e7ca7d097254 --- /dev/null +++ b/examples/ai-functions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/ai-functions", + "version": "0.0.0", + "private": true +} diff --git a/examples/ai-functions/src/reproduction/issue-18331-anthropic-spliced-message-start.ts b/examples/ai-functions/src/reproduction/issue-18331-anthropic-spliced-message-start.ts new file mode 100644 index 000000000000..e0d8d59f8eac --- /dev/null +++ b/examples/ai-functions/src/reproduction/issue-18331-anthropic-spliced-message-start.ts @@ -0,0 +1,276 @@ +import { createAnthropic } from '../../../../packages/anthropic/dist/index.mjs'; +import { + InvalidResponseDataError, + streamText, +} from '../../../../packages/ai/dist/index.mjs'; + +type StreamPart = { + type: string; + id?: string; + text?: string; + toolCallId?: string; + error?: unknown; + providerMetadata?: { + anthropic?: { + signature?: string; + }; + }; + usage?: { + inputTokens?: number; + }; +}; + +const splicedEvents = [ + { + type: 'message_start', + message: { + id: 'msg_first', + model: 'claude-sonnet-4-20250514', + usage: { input_tokens: 10, output_tokens: 1 }, + }, + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'thinking', thinking: '' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'thinking_delta', thinking: 'first reasoning' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'signature_delta', signature: 'sig_first' }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_first', + name: 'lookup', + input: {}, + }, + }, + { + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: '{"value":"Spark' }, + }, + { + type: 'message_start', + message: { + id: 'msg_second', + model: 'claude-sonnet-4-20250514', + usage: { input_tokens: 99, output_tokens: 1 }, + }, + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'thinking', thinking: '' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'thinking_delta', thinking: 'second reasoning' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'signature_delta', signature: 'sig_second' }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_second', + name: 'lookup', + input: {}, + }, + }, + { + type: 'content_block_delta', + index: 1, + delta: { + type: 'input_json_delta', + partial_json: '{"value":"Spark"}', + }, + }, + { type: 'content_block_stop', index: 1 }, + { + type: 'message_delta', + delta: { stop_reason: 'tool_use', stop_sequence: null }, + usage: { output_tokens: 20 }, + }, + { type: 'message_stop' }, +]; + +const duplicateStartEvents = [ + { + type: 'message_start', + message: { + id: 'msg_duplicate', + model: 'claude-sonnet-4-20250514', + usage: { input_tokens: 10, output_tokens: 1 }, + }, + }, + { + type: 'message_start', + message: { + id: 'msg_duplicate', + model: 'claude-sonnet-4-20250514', + usage: { input_tokens: 99, output_tokens: 1 }, + }, + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'ok' }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 2 }, + }, + { type: 'message_stop' }, +]; + +function createSseFetch(events: unknown[]) { + return async () => + new Response( + events.map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), + { + headers: { 'content-type': 'text/event-stream' }, + status: 200, + }, + ); +} + +async function readProviderParts(events: unknown[]): Promise { + const model = createAnthropic({ + apiKey: 'test-api-key', + fetch: createSseFetch(events), + })('claude-sonnet-4-20250514'); + const { stream } = await model.doStream({ + prompt: [{ role: 'user', content: [{ type: 'text', text: 'Hello' }] }], + }); + const parts: StreamPart[] = []; + for await (const part of stream) { + parts.push(part as StreamPart); + } + return parts; +} + +async function main() { + const providerParts = await readProviderParts(splicedEvents); + const providerErrors = providerParts.filter(part => part.type === 'error'); + const metadataIds = providerParts + .filter(part => part.type === 'response-metadata') + .map(part => part.id); + const completedToolIds = providerParts + .filter(part => part.type === 'tool-call') + .map(part => part.toolCallId); + + const model = createAnthropic({ + apiKey: 'test-api-key', + fetch: createSseFetch(splicedEvents), + })('claude-sonnet-4-20250514'); + const result = streamText({ + model, + prompt: 'Hello', + maxRetries: 0, + }); + const fullStreamParts: StreamPart[] = []; + for await (const part of result.fullStream) { + fullStreamParts.push(part as StreamPart); + } + const steps = await result.steps; + + const duplicateParts = await readProviderParts(duplicateStartEvents); + const duplicateMetadataCount = duplicateParts.filter( + part => part.type === 'response-metadata', + ).length; + const duplicateInputTokens = duplicateParts.find( + part => part.type === 'finish', + )?.usage?.inputTokens; + + const invalidResponseError = providerErrors.some(part => + InvalidResponseDataError.isInstance(part.error), + ); + const mergedReasoning = fullStreamParts + .filter(part => part.type === 'reasoning-delta') + .map(part => part.text) + .join('|'); + const mergedSignatures = fullStreamParts + .filter(part => part.type === 'reasoning-delta') + .map(part => part.providerMetadata?.anthropic?.signature) + .filter(signature => signature != null); + + console.log( + JSON.stringify( + { + metadataIds, + invalidResponseError, + completedToolIds, + mergedReasoning, + mergedSignatures, + recordedStepCount: steps.length, + duplicateMetadataCount, + duplicateInputTokens, + }, + null, + 2, + ), + ); + + if ( + !invalidResponseError && + metadataIds.join(',') === 'msg_first,msg_second' && + completedToolIds.join(',') === 'toolu_second' && + mergedReasoning === 'first reasoning||second reasoning|' && + mergedSignatures.join(',') === 'sig_first,sig_second' && + steps.length === 1 + ) { + console.error( + 'ISSUE_18331_REPRODUCED: different message_start was silently merged into one completed streamText step', + ); + process.exitCode = 1; + return; + } + + if ( + invalidResponseError && + duplicateMetadataCount === 1 && + duplicateInputTokens === 10 + ) { + console.log( + 'Issue behavior is fixed: the splice failed explicitly and the duplicate start was ignored.', + ); + return; + } + + throw new Error( + `Unexpected result: invalidResponseError=${invalidResponseError}, metadataIds=${metadataIds.join( + ',', + )}, completedToolIds=${completedToolIds.join( + ',', + )}, recordedStepCount=${steps.length}, duplicateMetadataCount=${duplicateMetadataCount}, duplicateInputTokens=${duplicateInputTokens}`, + ); +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/anthropic/src/__fixtures__/issue-18331-duplicate-message-start.chunks.txt b/packages/anthropic/src/__fixtures__/issue-18331-duplicate-message-start.chunks.txt new file mode 100644 index 000000000000..defbaad0cd0a --- /dev/null +++ b/packages/anthropic/src/__fixtures__/issue-18331-duplicate-message-start.chunks.txt @@ -0,0 +1,7 @@ +{"type":"message_start","message":{"id":"msg_duplicate","model":"claude-sonnet-4-20250514","usage":{"input_tokens":10,"output_tokens":1}}} +{"type":"message_start","message":{"id":"msg_duplicate","model":"claude-sonnet-4-20250514","usage":{"input_tokens":99,"output_tokens":1}}} +{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} +{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}} +{"type":"content_block_stop","index":0} +{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}} +{"type":"message_stop"} diff --git a/packages/anthropic/src/__fixtures__/issue-18331-sequential-messages.chunks.txt b/packages/anthropic/src/__fixtures__/issue-18331-sequential-messages.chunks.txt new file mode 100644 index 000000000000..c724c5e7ae2d --- /dev/null +++ b/packages/anthropic/src/__fixtures__/issue-18331-sequential-messages.chunks.txt @@ -0,0 +1,12 @@ +{"type":"message_start","message":{"id":"msg_first","model":"claude-sonnet-4-20250514","usage":{"input_tokens":10,"output_tokens":1}}} +{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} +{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"first"}} +{"type":"content_block_stop","index":0} +{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}} +{"type":"message_stop"} +{"type":"message_start","message":{"id":"msg_second","model":"claude-sonnet-4-20250514","usage":{"input_tokens":11,"output_tokens":1}}} +{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} +{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"second"}} +{"type":"content_block_stop","index":0} +{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":3}} +{"type":"message_stop"} diff --git a/packages/anthropic/src/__fixtures__/issue-18331-spliced-message-start.chunks.txt b/packages/anthropic/src/__fixtures__/issue-18331-spliced-message-start.chunks.txt new file mode 100644 index 000000000000..2c31681d05ab --- /dev/null +++ b/packages/anthropic/src/__fixtures__/issue-18331-spliced-message-start.chunks.txt @@ -0,0 +1,17 @@ +{"type":"message_start","message":{"id":"msg_first","model":"claude-sonnet-4-20250514","usage":{"input_tokens":10,"output_tokens":1}}} +{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} +{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"first reasoning"}} +{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_first"}} +{"type":"content_block_stop","index":0} +{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_first","name":"lookup","input":{}}} +{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"value\":\"Spark"}} +{"type":"message_start","message":{"id":"msg_second","model":"claude-sonnet-4-20250514","usage":{"input_tokens":99,"output_tokens":1}}} +{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} +{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"second reasoning"}} +{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_second"}} +{"type":"content_block_stop","index":0} +{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_second","name":"lookup","input":{}}} +{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"value\":\"Spark\"}"}} +{"type":"content_block_stop","index":1} +{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}} +{"type":"message_stop"} diff --git a/packages/anthropic/src/issue-18331.test.ts b/packages/anthropic/src/issue-18331.test.ts new file mode 100644 index 000000000000..dcfbf37741d1 --- /dev/null +++ b/packages/anthropic/src/issue-18331.test.ts @@ -0,0 +1,84 @@ +import { InvalidResponseDataError } from '@ai-sdk/provider'; +import { createTestServer } from '@ai-sdk/test-server/with-vitest'; +import { convertReadableStreamToArray } from '@ai-sdk/provider-utils/test'; +import fs from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { createAnthropic } from './anthropic-provider'; + +const prompt = [ + { + role: 'user' as const, + content: [{ type: 'text' as const, text: 'Hello' }], + }, +]; + +describe('issue 18331', () => { + const server = createTestServer({ + 'https://api.anthropic.com/v1/messages': {}, + }); + const model = createAnthropic({ apiKey: 'test-api-key' })( + 'claude-sonnet-4-20250514', + ); + + function prepareFixture(filename: string) { + const chunks = fs + .readFileSync(`src/__fixtures__/${filename}.chunks.txt`, 'utf8') + .trimEnd() + .split('\n') + .map(line => `data: ${line}\n\n`); + + server.urls['https://api.anthropic.com/v1/messages'].response = { + type: 'stream-chunks', + chunks, + }; + } + + it('fails a different message_start while the previous message is open', async () => { + prepareFixture('issue-18331-spliced-message-start'); + + const { stream } = await model.doStream({ prompt }); + const parts = await convertReadableStreamToArray(stream); + + expect( + parts.some( + part => + part.type === 'error' && + InvalidResponseDataError.isInstance(part.error), + ), + ).toBe(true); + expect(parts.filter(part => part.type === 'response-metadata')).toEqual([ + expect.objectContaining({ id: 'msg_first' }), + ]); + expect(parts.some(part => part.type === 'finish')).toBe(false); + expect(parts.some(part => part.type === 'tool-call')).toBe(false); + }); + + it('ignores a duplicate message_start for the same open message', async () => { + prepareFixture('issue-18331-duplicate-message-start'); + + const { stream } = await model.doStream({ prompt }); + const parts = await convertReadableStreamToArray(stream); + + expect(parts.filter(part => part.type === 'response-metadata')).toEqual([ + expect.objectContaining({ id: 'msg_duplicate' }), + ]); + expect(parts.find(part => part.type === 'finish')).toMatchObject({ + usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 }, + }); + }); + + it('keeps sequential messages separated by message_stop valid', async () => { + prepareFixture('issue-18331-sequential-messages'); + + const { stream } = await model.doStream({ prompt }); + const parts = await convertReadableStreamToArray(stream); + + expect( + parts + .filter(part => part.type === 'response-metadata') + .map(part => part.id), + ).toEqual(['msg_first', 'msg_second']); + expect(parts.filter(part => part.type === 'finish')).toHaveLength(2); + expect(parts.filter(part => part.type === 'error')).toHaveLength(0); + }); +});