From cc2695f74d67e0daae3a4e4495e065440970cfc4 Mon Sep 17 00:00:00 2001 From: lkapadiya-DO Date: Wed, 24 Jun 2026 11:39:55 -0700 Subject: [PATCH 1/7] fix(anthropic-adapter): correct initial content block type for reasoning_content streams --- .../adapters/streaming_iterator.py | 168 ++++--- .../adapters/transformation.py | 7 + ...st_streaming_iterator_reasoning_content.py | 463 ++++++++++++++++++ .../test_streaming_iterator_tool_args.py | 29 +- .../messages/test_parallel_tool_calls.py | 84 ++-- 5 files changed, 634 insertions(+), 117 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 799e8ab9a0a..7d8ebd5086e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -105,13 +105,63 @@ def __next__(self): if self.sent_content_block_start is False: self.sent_content_block_start = True - self.chunk_queue.append( - { - "type": "content_block_start", - "index": self.current_content_block_index, - "content_block": {"type": "text", "text": ""}, - } - ) + # Peek at the first chunk to determine the correct initial + # content block type. Models that use reasoning_content + # (e.g. GLM-5) start with a thinking block, not text. + first_chunk = None + for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + continue + first_chunk = chunk + break + if first_chunk is not None: + ( + block_type, + content_block_start, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( + choices=first_chunk.choices + ) + self.current_content_block_type = block_type + self.current_content_block_start = content_block_start + if block_type == "thinking": + initial_block: dict = {"type": "thinking", "thinking": ""} + elif block_type == "tool_use": + initial_block = dict(content_block_start) + else: + initial_block = {"type": "text", "text": ""} + self.chunk_queue.append( + { + "type": "content_block_start", + "index": self.current_content_block_index, + "content_block": initial_block, + } + ) + processed_first = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( + response=first_chunk, + current_content_block_index=self.current_content_block_index, + ) + # Empty / stop-only first chunk: close the block before the + # terminal message_delta so the sequence stays spec-compliant. + if ( + isinstance(processed_first, dict) + and processed_first.get("type") == "message_delta" + ): + self.chunk_queue.append( + { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + ) + self.sent_content_block_finish = True + self.chunk_queue.append(processed_first) + else: + self.chunk_queue.append( + { + "type": "content_block_start", + "index": self.current_content_block_index, + "content_block": {"type": "text", "text": ""}, + } + ) return self.chunk_queue.popleft() for chunk in self.completion_stream: @@ -128,15 +178,6 @@ def __next__(self): ) if should_start_new_block and not self.sent_content_block_finish: - # Queue the sequence: content_block_stop -> content_block_start - # For text blocks the trigger chunk is not emitted as a separate - # delta because content_block_start carries the information. - # For tool_use blocks we must also emit the trigger chunk's delta - # when it carries input_json_delta data, because some providers - # (e.g. xAI, Gemini) include tool arguments in the same streaming - # chunk as the function name/id. - - # 1. Stop current content block self.chunk_queue.append( { "type": "content_block_stop", @@ -144,7 +185,6 @@ def __next__(self): } ) - # 2. Start new content block self.chunk_queue.append( { "type": "content_block_start", @@ -152,17 +192,7 @@ def __next__(self): "content_block": self.current_content_block_start, } ) - - # 3. If the trigger chunk carries tool argument data, queue it - # so the input_json_delta is not silently dropped. - if ( - processed_chunk.get("type") == "content_block_delta" - and isinstance(processed_chunk.get("delta"), dict) - and processed_chunk["delta"].get("type") == "input_json_delta" - and processed_chunk["delta"].get("partial_json") - ): - self.chunk_queue.append(processed_chunk) - + self.chunk_queue.append(processed_chunk) self.sent_content_block_finish = False return self.chunk_queue.popleft() @@ -245,13 +275,58 @@ async def __anext__(self): # noqa: PLR0915 if self.sent_content_block_start is False: self.sent_content_block_start = True - self.chunk_queue.append( - { - "type": "content_block_start", - "index": self.current_content_block_index, - "content_block": {"type": "text", "text": ""}, - } - ) + first_chunk = None + async for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + continue + first_chunk = chunk + break + if first_chunk is not None: + ( + block_type, + content_block_start, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( + choices=first_chunk.choices + ) + self.current_content_block_type = block_type + self.current_content_block_start = content_block_start + if block_type == "thinking": + initial_block = {"type": "thinking", "thinking": ""} + elif block_type == "tool_use": + initial_block = dict(content_block_start) + else: + initial_block = {"type": "text", "text": ""} + self.chunk_queue.append( + { + "type": "content_block_start", + "index": self.current_content_block_index, + "content_block": initial_block, + } + ) + processed_first = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( + response=first_chunk, + current_content_block_index=self.current_content_block_index, + ) + if ( + isinstance(processed_first, dict) + and processed_first.get("type") == "message_delta" + ): + self.chunk_queue.append( + { + "type": "content_block_stop", + "index": self.current_content_block_index, + } + ) + self.sent_content_block_finish = True + self.chunk_queue.append(processed_first) + else: + self.chunk_queue.append( + { + "type": "content_block_start", + "index": self.current_content_block_index, + "content_block": {"type": "text", "text": ""}, + } + ) return self.chunk_queue.popleft() async for chunk in self.completion_stream: @@ -323,15 +398,6 @@ async def __anext__(self): # noqa: PLR0915 if not self.queued_usage_chunk: if should_start_new_block and not self.sent_content_block_finish: - # Queue the sequence: content_block_stop -> content_block_start - # For text blocks the trigger chunk is not emitted as a separate - # delta because content_block_start carries the information. - # For tool_use blocks we must also emit the trigger chunk's delta - # when it carries input_json_delta data, because some providers - # (e.g. xAI, Gemini) include tool arguments in the same streaming - # chunk as the function name/id. - - # 1. Stop current content block self.chunk_queue.append( { "type": "content_block_stop", @@ -347,19 +413,7 @@ async def __anext__(self): # noqa: PLR0915 "content_block": self.current_content_block_start, } ) - - # 3. If the trigger chunk carries tool argument data, queue it - # so the input_json_delta is not silently dropped. - if ( - processed_chunk.get("type") == "content_block_delta" - and isinstance(processed_chunk.get("delta"), dict) - and processed_chunk["delta"].get("type") - == "input_json_delta" - and processed_chunk["delta"].get("partial_json") - ): - self.chunk_queue.append(processed_chunk) - - # Reset state for new block + self.chunk_queue.append(processed_chunk) self.sent_content_block_finish = False # Return the first queued item diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 072ae7c3bbe..831608a02fa 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1403,6 +1403,13 @@ def _translate_streaming_openai_chunk_to_anthropic_content_block( return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking=thinking, signature=signature ) + elif isinstance(choice, StreamingChoices) and hasattr( + choice.delta, "reasoning_content" + ): + if choice.delta.reasoning_content is not None: + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking="", signature="" + ) return "text", TextBlock(type="text", text="") diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py new file mode 100644 index 00000000000..b9488ba12a0 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py @@ -0,0 +1,463 @@ +""" +Tests for streaming_iterator.py fixes: + +Fix 2 – Peek at first chunk to determine correct initial content_block_start type. + Models that return reasoning_content (e.g. GLM-5 via Vertex AI) start + the stream with a thinking block, not a text block. + +Fix 3 – Queue the processed_chunk (first delta of a new block) when a content + block transition occurs, so the first token is not silently dropped. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, +) +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StreamingChoices, +) + + +# --------------------------------------------------------------------------- +# Mock streams +# --------------------------------------------------------------------------- + + +class MockSyncStream: + """Synchronous mock completion stream yielding a fixed list of chunks.""" + + def __init__(self, chunks: list[ModelResponseStream]): + self._chunks = iter(chunks) + + def __iter__(self): + return self + + def __next__(self): + return next(self._chunks) + + +class MockAsyncStream: + """Asynchronous mock completion stream yielding a fixed list of chunks.""" + + def __init__(self, chunks: list[ModelResponseStream]): + self._chunks = iter(chunks) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._chunks) + except StopIteration: + raise StopAsyncIteration + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_thinking_chunk(text: str) -> ModelResponseStream: + """Create a streaming chunk with reasoning_content (no thinking_blocks).""" + return ModelResponseStream( + choices=[ + StreamingChoices( + delta=Delta( + reasoning_content=text, + content="", + role="assistant", + ), + index=0, + finish_reason=None, + ) + ], + ) + + +def _make_text_chunk(text: str) -> ModelResponseStream: + """Create a streaming chunk with text content.""" + return ModelResponseStream( + choices=[ + StreamingChoices( + delta=Delta( + content=text, + role="assistant", + ), + index=0, + finish_reason=None, + ) + ], + ) + + +def _make_stop_chunk() -> ModelResponseStream: + """Create a streaming chunk signalling end of generation.""" + return ModelResponseStream( + choices=[ + StreamingChoices( + delta=Delta(content=""), + index=0, + finish_reason="stop", + ) + ], + ) + + +def _collect_all_events(wrapper) -> list[dict]: + """Collect all events from a sync AnthropicStreamWrapper.""" + events = [] + for raw in wrapper: + events.append(raw) + return events + + +async def _collect_all_events_async(wrapper) -> list[dict]: + """Collect all events from an async AnthropicStreamWrapper.""" + events = [] + async for raw in wrapper: + events.append(raw) + return events + + +def _assert_monotonic_indices(events: list[dict]) -> None: + """Indices on content_block events must be non-decreasing with matched pairs.""" + open_indices: list[int] = [] + for event in events: + event_type = event.get("type") + if event_type not in ( + "content_block_start", + "content_block_delta", + "content_block_stop", + ): + continue + idx = event.get("index") + assert isinstance(idx, int) + if event_type == "content_block_start": + if open_indices: + assert idx >= open_indices[-1] + open_indices.append(idx) + elif event_type == "content_block_stop": + assert open_indices, f"unexpected content_block_stop at index {idx}" + assert open_indices.pop() == idx + + +# --------------------------------------------------------------------------- +# Fix 2 – Initial content_block_start reflects first chunk type +# --------------------------------------------------------------------------- + + +class TestInitialBlockTypePeek: + """ + When the first chunk from the upstream model contains reasoning_content, + the initial content_block_start must have type "thinking", not "text". + """ + + def test_sync_thinking_first_chunk(self): + chunks = [ + _make_thinking_chunk("Let me think..."), + _make_thinking_chunk(" about this."), + _make_text_chunk("The answer is 42."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="glm-5" + ) + events = _collect_all_events(wrapper) + + assert events[0]["type"] == "message_start" + + content_block_start = events[1] + assert content_block_start["type"] == "content_block_start" + assert content_block_start["content_block"]["type"] == "thinking" + + first_delta = events[2] + assert first_delta["type"] == "content_block_delta" + assert first_delta["delta"]["type"] == "thinking_delta" + + def test_sync_text_first_chunk(self): + """Text-first streams should still emit type 'text' (no regression).""" + chunks = [ + _make_text_chunk("Hello"), + _make_text_chunk(" world"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="gpt-4o" + ) + events = _collect_all_events(wrapper) + + content_block_start = events[1] + assert content_block_start["type"] == "content_block_start" + assert content_block_start["content_block"]["type"] == "text" + + @pytest.mark.asyncio + async def test_async_thinking_first_chunk(self): + chunks = [ + _make_thinking_chunk("Let me think..."), + _make_thinking_chunk(" about this."), + _make_text_chunk("The answer is 42."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="glm-5" + ) + events = await _collect_all_events_async(wrapper) + + assert events[0]["type"] == "message_start" + + content_block_start = events[1] + assert content_block_start["type"] == "content_block_start" + assert content_block_start["content_block"]["type"] == "thinking" + + first_delta = events[2] + assert first_delta["type"] == "content_block_delta" + assert first_delta["delta"]["type"] == "thinking_delta" + + @pytest.mark.asyncio + async def test_async_text_first_chunk(self): + chunks = [ + _make_text_chunk("Hello"), + _make_text_chunk(" world"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="gpt-4o" + ) + events = await _collect_all_events_async(wrapper) + + content_block_start = events[1] + assert content_block_start["type"] == "content_block_start" + assert content_block_start["content_block"]["type"] == "text" + + +# --------------------------------------------------------------------------- +# Fix 3 – Block transition queues the trigger chunk +# --------------------------------------------------------------------------- + + +class TestBlockTransitionIncludesFirstDelta: + """ + When the stream transitions from one block type to another (e.g. thinking + → text), the processed chunk that triggered the transition must be queued + and eventually yielded. Without Fix 3 the first token of the new block + would be silently dropped. + """ + + def test_sync_thinking_to_text_no_token_drop(self): + chunks = [ + _make_thinking_chunk("Reasoning step."), + _make_text_chunk("Answer text."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="glm-5" + ) + events = _collect_all_events(wrapper) + + text_deltas = [ + e + for e in events + if e.get("type") == "content_block_delta" + and e.get("delta", {}).get("type") == "text_delta" + ] + assert len(text_deltas) >= 1, ( + "The first text delta after a thinking→text transition must not be " + "dropped. Got text_delta events: " + repr(text_deltas) + ) + assert text_deltas[0]["delta"]["text"] == "Answer text." + + @pytest.mark.asyncio + async def test_async_thinking_to_text_no_token_drop(self): + chunks = [ + _make_thinking_chunk("Reasoning step."), + _make_text_chunk("Answer text."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="glm-5" + ) + events = await _collect_all_events_async(wrapper) + + text_deltas = [ + e + for e in events + if e.get("type") == "content_block_delta" + and e.get("delta", {}).get("type") == "text_delta" + ] + assert len(text_deltas) >= 1, ( + "The first text delta after a thinking→text transition must not be " + "dropped. Got text_delta events: " + repr(text_deltas) + ) + assert text_deltas[0]["delta"]["text"] == "Answer text." + + def test_sync_event_sequence_is_valid(self): + """ + The full event sequence for a thinking→text stream should follow the + Anthropic SSE spec: + message_start → + content_block_start (thinking) → + content_block_delta (thinking_delta) → + content_block_stop → + content_block_start (text) → + content_block_delta (text_delta) → + content_block_stop → + message_delta → + message_stop + """ + chunks = [ + _make_thinking_chunk("Think."), + _make_text_chunk("Answer."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="glm-5" + ) + events = _collect_all_events(wrapper) + + types = [e["type"] for e in events] + + assert types[0] == "message_start" + assert types[1] == "content_block_start" + assert events[1]["content_block"]["type"] == "thinking" + assert "content_block_delta" in types + idx_first_stop = types.index("content_block_stop") + assert idx_first_stop > 1 + idx_second_start = types.index("content_block_start", idx_first_stop) + assert events[idx_second_start]["content_block"]["type"] == "text" + text_delta_idx = types.index("content_block_delta", idx_second_start) + assert events[text_delta_idx]["delta"]["type"] == "text_delta" + + +# --------------------------------------------------------------------------- +# Extra coverage – text-only full sequence, no duplicates, stop-only guard +# --------------------------------------------------------------------------- + + +class TestTextOnlyFullSequence: + def test_sync_text_only_sequence(self): + chunks = [ + _make_text_chunk("Hello"), + _make_text_chunk(" world"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="gpt-4o" + ) + events = _collect_all_events(wrapper) + types = [e["type"] for e in events] + + assert types == [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + assert events[1]["content_block"]["type"] == "text" + assert events[2]["delta"]["text"] == "Hello" + assert events[3]["delta"]["text"] == " world" + + @pytest.mark.asyncio + async def test_async_text_only_sequence(self): + chunks = [ + _make_text_chunk("Hello"), + _make_text_chunk(" world"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="gpt-4o" + ) + events = await _collect_all_events_async(wrapper) + types = [e["type"] for e in events] + + assert types == [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + + +class TestNoDuplicateFirstDelta: + def test_sync_peeked_first_delta_not_duplicated(self): + chunks = [ + _make_text_chunk("Hello"), + _make_text_chunk(" world"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="gpt-4o" + ) + events = _collect_all_events(wrapper) + + text_deltas = [ + e["delta"]["text"] + for e in events + if e.get("type") == "content_block_delta" + and e.get("delta", {}).get("type") == "text_delta" + ] + assert text_deltas == ["Hello", " world"] + _assert_monotonic_indices(events) + + @pytest.mark.asyncio + async def test_async_peeked_first_delta_not_duplicated(self): + chunks = [ + _make_thinking_chunk("Think."), + _make_text_chunk("Answer."), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="glm-5" + ) + events = await _collect_all_events_async(wrapper) + _assert_monotonic_indices(events) + + +class TestStopOnlyFirstChunk: + """Empty/stop-only first chunk must still emit content_block_stop.""" + + def test_sync_stop_only_first_chunk(self): + chunks = [_make_stop_chunk()] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="gpt-4o" + ) + events = _collect_all_events(wrapper) + types = [e["type"] for e in events] + + assert types == [ + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ] + + @pytest.mark.asyncio + async def test_async_stop_only_first_chunk(self): + chunks = [_make_stop_chunk()] + wrapper = AnthropicStreamWrapper( + completion_stream=MockAsyncStream(chunks), model="gpt-4o" + ) + events = await _collect_all_events_async(wrapper) + types = [e["type"] for e in events] + + assert types == [ + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py index bd39e420607..02e566ab74a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py @@ -151,8 +151,8 @@ async def mock_stream(): async def test_async_stream_no_extra_delta_when_tool_args_empty(): """ When a provider sends tool name/id WITHOUT arguments in the first chunk - (OpenAI-style), the wrapper should NOT emit an extra input_json_delta - after content_block_start. This verifies backward compatibility. + (OpenAI-style), the block transition still queues the trigger chunk's + empty input_json_delta (#25212); the follow-up chunk carries real args. """ # Chunk 1: text text_chunk = _make_chunk(Delta(content="Hi", role="assistant", tool_calls=None)) @@ -221,9 +221,9 @@ async def mock_stream(): assert tool_start_idx is not None - # Count how many input_json_delta events appear after the tool_use block start. - # With empty args in the trigger chunk, only the subsequent tool_args_chunk - # should produce one — not the trigger chunk itself. + # Count input_json_delta events after the tool_use block start. The trigger + # chunk is now queued on block transition (#25212), so an empty partial_json + # delta precedes the follow-up chunk with real arguments. input_json_deltas = [ e for e in events[tool_start_idx + 1 :] @@ -232,11 +232,12 @@ async def mock_stream(): and isinstance(e.get("delta"), dict) and e["delta"].get("type") == "input_json_delta" ] - assert len(input_json_deltas) == 1, ( - f"Expected exactly 1 input_json_delta (from the follow-up chunk), " + assert len(input_json_deltas) == 2, ( + f"Expected trigger empty delta + follow-up args delta, " f"got {len(input_json_deltas)}" ) - assert input_json_deltas[0]["delta"]["partial_json"] == '{"location": "NYC"}' + assert input_json_deltas[0]["delta"]["partial_json"] == "" + assert input_json_deltas[1]["delta"]["partial_json"] == '{"location": "NYC"}' def test_sync_stream_emits_input_json_delta_for_bundled_tool_args(): @@ -308,8 +309,9 @@ def test_sync_stream_emits_input_json_delta_for_bundled_tool_args(): def test_sync_stream_no_extra_delta_when_tool_args_empty(): """ - Sync counterpart: empty args (OpenAI-style) should not emit an extra - input_json_delta from the trigger chunk. + Sync counterpart: empty args on the trigger chunk still emit an + input_json_delta from the block transition (#25212); the follow-up chunk + carries the real arguments. """ text_chunk = _make_chunk(Delta(content="Hi", role="assistant", tool_calls=None)) tool_name_chunk = _make_chunk( @@ -376,8 +378,9 @@ def test_sync_stream_no_extra_delta_when_tool_args_empty(): and isinstance(e.get("delta"), dict) and e["delta"].get("type") == "input_json_delta" ] - assert len(input_json_deltas) == 1, ( - f"Expected exactly 1 input_json_delta (from the follow-up chunk), " + assert len(input_json_deltas) == 2, ( + f"Expected trigger empty delta + follow-up args delta, " f"got {len(input_json_deltas)}" ) - assert input_json_deltas[0]["delta"]["partial_json"] == '{"location": "NYC"}' + assert input_json_deltas[0]["delta"]["partial_json"] == "" + assert input_json_deltas[1]["delta"]["partial_json"] == '{"location": "NYC"}' diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py index 1d25d719384..7dc081f2144 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py @@ -131,22 +131,19 @@ def test_anthropic_stream_wrapper_single_tool_call(): chunks.append(chunk) chunk_types.append(chunk.get("type")) - # Verify the expected sequence of chunk types + # Verify the expected sequence of chunk types. + # The initial content_block_start now peeks at the first upstream chunk + # to determine the correct block type, so we get tool_use directly + # instead of a spurious empty text block. expected_types = [ - "message_start", # Initial message start - # TODO: for future contributors: if the initial content_block_start - # respects the upstream's starting chunk, the initial empty text block - # should be removed (and this test should be updated accordingly) - # --------------------------------------------------------------------- - "content_block_start", # Initial empty text block start - "content_block_stop", # End of empty text block - # --------------------------------------------------------------------- - "content_block_start", # Start of first tool_use content block + "message_start", + "content_block_start", # tool_use (from peek) + "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} - "content_block_stop", # End of first tool_use content block - "message_delta", # Stop reason with merged usage - "message_stop", # Final message stop + "content_block_stop", + "message_delta", + "message_stop", ] assert expected_types == chunk_types @@ -193,26 +190,21 @@ def test_anthropic_stream_wrapper_back_to_back_tool_calls(): chunks.append(chunk) chunk_types.append(chunk.get("type")) - # Verify the expected sequence of chunk types + # Verify the expected sequence of chunk types. expected_types = [ - "message_start", # Initial message start - # TODO: for future contributors: if the initial content_block_start - # respects the upstream's starting chunk, the initial empty text block - # should be removed (and this test should be updated accordingly) - # --------------------------------------------------------------------- - "content_block_start", # Initial empty text block start - "content_block_stop", # End of empty text block - # --------------------------------------------------------------------- - "content_block_start", # Start of first tool_use content block + "message_start", + "content_block_start", # tool_use (from peek) + "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} - "content_block_stop", # End of first tool_use content block - "content_block_start", # Start of second tool_use content block + "content_block_stop", + "content_block_start", # second tool_use + "content_block_delta", # first chunk of second tool "content_block_delta", # {"city": "content_block_delta", # " SF"} - "content_block_stop", # End of second tool_use content block - "message_delta", # Stop reason with merged usage - "message_stop", # Final message stop + "content_block_stop", + "message_delta", + "message_stop", ] assert expected_types == chunk_types @@ -264,34 +256,32 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text(): chunks.append(chunk) chunk_types.append(chunk.get("type")) - # Verify the expected sequence of chunk types + # Verify the expected sequence of chunk types. expected_types = [ - "message_start", # Initial message start - # TODO: for future contributors: if the initial content_block_start - # respects the upstream's starting chunk, the initial empty text block - # should be removed (and this test should be updated accordingly) - # --------------------------------------------------------------------- - "content_block_start", # Initial empty text block start - "content_block_stop", # End of empty text block - # --------------------------------------------------------------------- - "content_block_start", # Start of first tool_use content block + "message_start", + "content_block_start", # tool_use (from peek) + "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} - "content_block_stop", # End of first tool_use content block - "content_block_start", # "The weather is nice today" "content_block_stop", - "content_block_start", # Start of second tool_use content block + "content_block_start", # text + "content_block_delta", # "The weather is nice today." + "content_block_stop", + "content_block_start", # second tool_use + "content_block_delta", # first chunk of second tool "content_block_delta", # {"city": "content_block_delta", # " SF"} - "content_block_stop", # End of second tool_use content block - "content_block_start", # Start of third tool_use content block + "content_block_stop", + "content_block_start", # third tool_use + "content_block_delta", # first chunk of third tool "content_block_delta", # {"city": "content_block_delta", # " CHI"} - "content_block_stop", # End of third tool_use content block - "content_block_start", # "The weather is not so nice today" "content_block_stop", - "message_delta", # Stop reason with merged usage - "message_stop", # Final message stop + "content_block_start", # text + "content_block_delta", # "The weather is not so nice today." + "content_block_stop", + "message_delta", + "message_stop", ] assert expected_types == chunk_types From f42e2448c93f4fbb4efcd2dc8a95916afbfd8550 Mon Sep 17 00:00:00 2001 From: Luke Sorvik Date: Mon, 29 Jun 2026 14:47:20 -0700 Subject: [PATCH 2/7] Empty tool calls being included --- .../experimental_pass_through/adapters/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 831608a02fa..334f36e05ce 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1431,7 +1431,7 @@ def _translate_streaming_openai_chunk_to_anthropic( for choice in choices: if choice.delta.content is not None and len(choice.delta.content) > 0: text += choice.delta.content - if choice.delta.tool_calls is not None: + if choice.delta.tool_calls is not None and len(choice.delta.tool_calls) > 0: partial_json = "" for tool in choice.delta.tool_calls: if ( From 69d8d741f10c346d67266d9c3a295fe85a0954e8 Mon Sep 17 00:00:00 2001 From: Luke Sorvik Date: Mon, 29 Jun 2026 15:19:48 -0700 Subject: [PATCH 3/7] trying to fix empty tool call in stream --- .../experimental_pass_through/adapters/transformation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 334f36e05ce..de976ff4b84 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1432,7 +1432,6 @@ def _translate_streaming_openai_chunk_to_anthropic( if choice.delta.content is not None and len(choice.delta.content) > 0: text += choice.delta.content if choice.delta.tool_calls is not None and len(choice.delta.tool_calls) > 0: - partial_json = "" for tool in choice.delta.tool_calls: if ( tool.function is not None From 6f33962359092288c7af1e03d4f4b520a9447cc6 Mon Sep 17 00:00:00 2001 From: Luke Sorvik Date: Mon, 29 Jun 2026 18:02:23 -0700 Subject: [PATCH 4/7] tests --- ...al_pass_through_adapters_transformation.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index e6e96868f33..00fe54005bf 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -316,6 +316,113 @@ def test_translate_anthropic_messages_to_openai_tool_message_placement(): ), "Tool message should be placed before user message" +def test_translate_streaming_openai_text_delta_no_tool_calls(): + """When tool_calls is None, a text delta with content should be text_delta.""" + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="Hello", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ] + + ( + type_of_content, + content_block_delta, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( + choices=choices # type: ignore[arg-type] + ) + + assert type_of_content == "text_delta" + assert content_block_delta["type"] == "text_delta" + assert content_block_delta["text"] == "Hello" # type: ignore[typeddict-unknown-key] + + +def test_translate_streaming_openai_text_delta_empty_tool_calls(): + """When tool_calls is an empty list (len==0), content should still be text_delta. + + This reproduces the bug where DeepSeek/vLLM sends tool_calls: [] + instead of tool_calls: None. Before the fix, the `is not None` guard + let this into the tool_calls branch, and a stray partial_json="" caused + an empty input_json_delta to be emitted instead of the text content. + """ + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="Hello from DeepSeek", + role="assistant", + function_call=None, + tool_calls=[], # empty list — the exact pattern from vLLM/DeepSeek + audio=None, + ), + logprobs=None, + ) + ] + + ( + type_of_content, + content_block_delta, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( + choices=choices # type: ignore[arg-type] + ) + + assert type_of_content == "text_delta", ( + f"Expected text_delta but got {type_of_content} — " + "empty tool_calls list should not trigger input_json_delta" + ) + assert content_block_delta["type"] == "text_delta" + assert content_block_delta["text"] == "Hello from DeepSeek" # type: ignore[typeddict-unknown-key] + + +def test_translate_streaming_openai_text_delta_tool_calls_empty_list(): + """When tool_calls is an empty list, a text delta should fall through to content_block_stop or text_delta. + + This is the same scenario as empty_tool_calls above but specifically tests + the interaction with _translate_streaming_openai_chunk_to_anthropic_content_block, + which correctly returns "text" when there's text content and tool_calls is []. + """ + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="Let me think about that.", + role="assistant", + function_call=None, + tool_calls=[], # empty list — the exact pattern from vLLM/DeepSeek + audio=None, + ), + logprobs=None, + ) + ] + + ( + type_of_content, + content_block_delta, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( + choices=choices # type: ignore[arg-type] + ) + + assert type_of_content == "text_delta", ( + f"Expected text_delta but got {type_of_content} — " + "empty tool_calls list should not trigger input_json_delta" + ) + assert content_block_delta["type"] == "text_delta" + assert content_block_delta["text"] == "Let me think about that." # type: ignore[typeddict-unknown-key] + + def test_translate_openai_content_to_anthropic_empty_function_arguments(): """Test that empty function arguments are handled safely and don't cause JSON parsing errors.""" From 49e390b3caf3d404ffcaa011aab5a3dc464024d4 Mon Sep 17 00:00:00 2001 From: lkapadiya-DO Date: Tue, 30 Jun 2026 14:11:08 -0700 Subject: [PATCH 5/7] test: merge duplicate empty tool_calls streaming tests into parametrized case --- ...al_pass_through_adapters_transformation.py | 51 ++++--------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 00fe54005bf..e64c28b638d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -346,7 +346,14 @@ def test_translate_streaming_openai_text_delta_no_tool_calls(): assert content_block_delta["text"] == "Hello" # type: ignore[typeddict-unknown-key] -def test_translate_streaming_openai_text_delta_empty_tool_calls(): +@pytest.mark.parametrize( + "content", + [ + "Hello from DeepSeek", + "Let me think about that.", + ], +) +def test_translate_streaming_openai_text_delta_empty_tool_calls(content: str): """When tool_calls is an empty list (len==0), content should still be text_delta. This reproduces the bug where DeepSeek/vLLM sends tool_calls: [] @@ -360,45 +367,7 @@ def test_translate_streaming_openai_text_delta_empty_tool_calls(): index=0, delta=Delta( provider_specific_fields=None, - content="Hello from DeepSeek", - role="assistant", - function_call=None, - tool_calls=[], # empty list — the exact pattern from vLLM/DeepSeek - audio=None, - ), - logprobs=None, - ) - ] - - ( - type_of_content, - content_block_delta, - ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( - choices=choices # type: ignore[arg-type] - ) - - assert type_of_content == "text_delta", ( - f"Expected text_delta but got {type_of_content} — " - "empty tool_calls list should not trigger input_json_delta" - ) - assert content_block_delta["type"] == "text_delta" - assert content_block_delta["text"] == "Hello from DeepSeek" # type: ignore[typeddict-unknown-key] - - -def test_translate_streaming_openai_text_delta_tool_calls_empty_list(): - """When tool_calls is an empty list, a text delta should fall through to content_block_stop or text_delta. - - This is the same scenario as empty_tool_calls above but specifically tests - the interaction with _translate_streaming_openai_chunk_to_anthropic_content_block, - which correctly returns "text" when there's text content and tool_calls is []. - """ - choices = [ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - provider_specific_fields=None, - content="Let me think about that.", + content=content, role="assistant", function_call=None, tool_calls=[], # empty list — the exact pattern from vLLM/DeepSeek @@ -420,7 +389,7 @@ def test_translate_streaming_openai_text_delta_tool_calls_empty_list(): "empty tool_calls list should not trigger input_json_delta" ) assert content_block_delta["type"] == "text_delta" - assert content_block_delta["text"] == "Let me think about that." # type: ignore[typeddict-unknown-key] + assert content_block_delta["text"] == content # type: ignore[typeddict-unknown-key] def test_translate_openai_content_to_anthropic_empty_function_arguments(): From 3c6097b7eb2423d4a2e4b60b951628320208a404 Mon Sep 17 00:00:00 2001 From: lkapadiya-DO Date: Tue, 30 Jun 2026 15:23:44 -0700 Subject: [PATCH 6/7] fix(anthropic-adapter): skip thinking block when reasoning_content is empty --- .../adapters/transformation.py | 4 +- ...al_pass_through_adapters_transformation.py | 59 +++++++++++++++++++ ...st_streaming_iterator_reasoning_content.py | 20 +++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index de976ff4b84..08291dc4448 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1406,7 +1406,7 @@ def _translate_streaming_openai_chunk_to_anthropic_content_block( elif isinstance(choice, StreamingChoices) and hasattr( choice.delta, "reasoning_content" ): - if choice.delta.reasoning_content is not None: + if choice.delta.reasoning_content: return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking="", signature="" ) @@ -1458,7 +1458,7 @@ def _translate_streaming_openai_chunk_to_anthropic( elif isinstance(choice, StreamingChoices) and hasattr( choice.delta, "reasoning_content" ): - if choice.delta.reasoning_content is not None: + if choice.delta.reasoning_content: reasoning_content += choice.delta.reasoning_content if reasoning_content and reasoning_signature: diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index e64c28b638d..81b2b17258b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1609,6 +1609,65 @@ def test_translate_streaming_openai_chunk_to_anthropic_reasoning_content_without assert content_block_delta["thinking"] == "I need to analyze this carefully..." +def test_translate_streaming_openai_chunk_to_anthropic_content_block_empty_reasoning_content(): + """Empty reasoning_content must not open a thinking block without thinking_delta.""" + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + reasoning_content="", + content="", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ] + + ( + block_type, + content_block_start, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic_content_block( + choices=choices + ) + + assert block_type == "text" + assert content_block_start == {"type": "text", "text": ""} + + +def test_translate_streaming_openai_chunk_to_anthropic_empty_reasoning_content_no_thinking_delta(): + """Empty reasoning_content must not emit thinking_delta on a text block.""" + choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + reasoning_content="", + content="", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ] + + ( + type_of_content, + content_block_delta, + ) = LiteLLMAnthropicMessagesAdapter()._translate_streaming_openai_chunk_to_anthropic( + choices=choices + ) + + assert type_of_content == "text_delta" + assert content_block_delta["type"] == "text_delta" + assert content_block_delta["text"] == "" + + def test_translate_openai_response_to_anthropic_with_reasoning_content_only(): """ Test the full response translation when only reasoning_content is present diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py index b9488ba12a0..9ef38df0877 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_reasoning_content.py @@ -198,6 +198,26 @@ def test_sync_text_first_chunk(self): assert content_block_start["type"] == "content_block_start" assert content_block_start["content_block"]["type"] == "text" + def test_sync_empty_reasoning_content_first_chunk_opens_text_block(self): + """Empty reasoning_content placeholder must not open a thinking block.""" + chunks = [ + _make_thinking_chunk(""), + _make_text_chunk("Hello"), + _make_stop_chunk(), + ] + wrapper = AnthropicStreamWrapper( + completion_stream=MockSyncStream(chunks), model="glm-5" + ) + events = _collect_all_events(wrapper) + + content_block_start = events[1] + assert content_block_start["type"] == "content_block_start" + assert content_block_start["content_block"]["type"] == "text" + + first_delta = events[2] + assert first_delta["type"] == "content_block_delta" + assert first_delta["delta"]["type"] == "text_delta" + @pytest.mark.asyncio async def test_async_thinking_first_chunk(self): chunks = [ From 81bb7e14f452bd8bfcf52ca4eeb474ee7b53df55 Mon Sep 17 00:00:00 2001 From: lkapadiya-DO Date: Wed, 1 Jul 2026 09:49:01 -0700 Subject: [PATCH 7/7] fix(anthropic-adapter): suppress empty trigger deltas and restore regression guards --- .../adapters/streaming_iterator.py | 125 +++--- .../adapters/transformation.py | 370 +++++------------- .../test_streaming_iterator_tool_args.py | 28 +- .../messages/test_parallel_tool_calls.py | 6 - 4 files changed, 167 insertions(+), 362 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 7d8ebd5086e..2fae6fc263f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -136,16 +136,15 @@ def __next__(self): "content_block": initial_block, } ) - processed_first = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( - response=first_chunk, - current_content_block_index=self.current_content_block_index, + processed_first = ( + LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( + response=first_chunk, + current_content_block_index=self.current_content_block_index, + ) ) # Empty / stop-only first chunk: close the block before the # terminal message_delta so the sequence stays spec-compliant. - if ( - isinstance(processed_first, dict) - and processed_first.get("type") == "message_delta" - ): + if isinstance(processed_first, dict) and processed_first.get("type") == "message_delta": self.chunk_queue.append( { "type": "content_block_stop", @@ -153,7 +152,9 @@ def __next__(self): } ) self.sent_content_block_finish = True - self.chunk_queue.append(processed_first) + self.chunk_queue.append(processed_first) + elif self._trigger_delta_has_content(processed_first): + self.chunk_queue.append(processed_first) else: self.chunk_queue.append( { @@ -192,14 +193,12 @@ def __next__(self): "content_block": self.current_content_block_start, } ) - self.chunk_queue.append(processed_chunk) + if self._trigger_delta_has_content(processed_chunk): + self.chunk_queue.append(processed_chunk) self.sent_content_block_finish = False return self.chunk_queue.popleft() - if ( - processed_chunk["type"] == "message_delta" - and self.sent_content_block_finish is False - ): + if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False: # Queue both the content_block_stop and the message_delta self.chunk_queue.append( { @@ -240,9 +239,7 @@ def __next__(self): return {"type": "message_stop"} raise StopIteration except Exception as e: - verbose_logger.error( - "Anthropic Adapter - {}\n{}".format(e, traceback.format_exc()) - ) + verbose_logger.error("Anthropic Adapter - {}\n{}".format(e, traceback.format_exc())) raise StopAsyncIteration async def __anext__(self): # noqa: PLR0915 @@ -303,14 +300,13 @@ async def __anext__(self): # noqa: PLR0915 "content_block": initial_block, } ) - processed_first = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( - response=first_chunk, - current_content_block_index=self.current_content_block_index, + processed_first = ( + LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( + response=first_chunk, + current_content_block_index=self.current_content_block_index, + ) ) - if ( - isinstance(processed_first, dict) - and processed_first.get("type") == "message_delta" - ): + if isinstance(processed_first, dict) and processed_first.get("type") == "message_delta": self.chunk_queue.append( { "type": "content_block_stop", @@ -318,7 +314,9 @@ async def __anext__(self): # noqa: PLR0915 } ) self.sent_content_block_finish = True - self.chunk_queue.append(processed_first) + self.chunk_queue.append(processed_first) + elif self._trigger_delta_has_content(processed_first): + self.chunk_queue.append(processed_first) else: self.chunk_queue.append( { @@ -344,10 +342,7 @@ async def __anext__(self): # noqa: PLR0915 ) # Check if this is a usage chunk and we have a held stop_reason chunk - if ( - self.holding_stop_reason_chunk is not None - and getattr(chunk, "usage", None) is not None - ): + if self.holding_stop_reason_chunk is not None and getattr(chunk, "usage", None) is not None: # Merge usage into the held stop_reason chunk merged_chunk = self.holding_stop_reason_chunk.copy() if "delta" not in merged_chunk: @@ -355,16 +350,8 @@ async def __anext__(self): # noqa: PLR0915 # Add usage to the held chunk uncached_input_tokens = chunk.usage.prompt_tokens or 0 - if ( - hasattr(chunk.usage, "prompt_tokens_details") - and chunk.usage.prompt_tokens_details - ): - cached_tokens = ( - getattr( - chunk.usage.prompt_tokens_details, "cached_tokens", 0 - ) - or 0 - ) + if hasattr(chunk.usage, "prompt_tokens_details") and chunk.usage.prompt_tokens_details: + cached_tokens = getattr(chunk.usage.prompt_tokens_details, "cached_tokens", 0) or 0 uncached_input_tokens -= cached_tokens usage_dict: UsageDelta = { @@ -376,16 +363,9 @@ async def __anext__(self): # noqa: PLR0915 hasattr(chunk.usage, "_cache_creation_input_tokens") and chunk.usage._cache_creation_input_tokens > 0 ): - usage_dict["cache_creation_input_tokens"] = ( - chunk.usage._cache_creation_input_tokens - ) - if ( - hasattr(chunk.usage, "_cache_read_input_tokens") - and chunk.usage._cache_read_input_tokens > 0 - ): - usage_dict["cache_read_input_tokens"] = ( - chunk.usage._cache_read_input_tokens - ) + usage_dict["cache_creation_input_tokens"] = chunk.usage._cache_creation_input_tokens + if hasattr(chunk.usage, "_cache_read_input_tokens") and chunk.usage._cache_read_input_tokens > 0: + usage_dict["cache_read_input_tokens"] = chunk.usage._cache_read_input_tokens merged_chunk["usage"] = usage_dict # Queue the merged chunk and reset @@ -413,16 +393,14 @@ async def __anext__(self): # noqa: PLR0915 "content_block": self.current_content_block_start, } ) - self.chunk_queue.append(processed_chunk) + if self._trigger_delta_has_content(processed_chunk): + self.chunk_queue.append(processed_chunk) self.sent_content_block_finish = False # Return the first queued item return self.chunk_queue.popleft() - if ( - processed_chunk["type"] == "message_delta" - and self.sent_content_block_finish is False - ): + if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False: # Queue both the content_block_stop and the holding chunk self.chunk_queue.append( { @@ -431,10 +409,7 @@ async def __anext__(self): # noqa: PLR0915 } ) self.sent_content_block_finish = True - if ( - processed_chunk.get("delta", {}).get("stop_reason") - is not None - ): + if processed_chunk.get("delta", {}).get("stop_reason") is not None: self.holding_stop_reason_chunk = processed_chunk else: self.chunk_queue.append(processed_chunk) @@ -515,6 +490,38 @@ async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: def _increment_content_block_index(self): self.current_content_block_index += 1 + @staticmethod + def _trigger_delta_has_content(processed_chunk: Dict[str, Any]) -> bool: + """Return True if a translated trigger chunk carries a non-empty + ``content_block_delta`` payload that must be re-emitted after a + block transition. + + When an upstream chunk both *triggers* a new content block (its type + differs from the active block) and *carries* delta content, that + content belongs to the new block. The synthesized + ``content_block_start`` only ever carries an empty body — see + ``_translate_streaming_openai_chunk_to_anthropic_content_block``, + which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block — + so the trigger chunk's delta must be re-queued or the first token of + the new block (the first non-empty text/thinking delta, or bundled + tool arguments) is silently dropped. + """ + if processed_chunk.get("type") != "content_block_delta": + return False + delta = processed_chunk.get("delta") + if not isinstance(delta, dict): + return False + delta_type = delta.get("type") + if delta_type == "text_delta": + return bool(delta.get("text")) + if delta_type == "input_json_delta": + return bool(delta.get("partial_json")) + if delta_type == "thinking_delta": + return bool(delta.get("thinking")) + if delta_type == "signature_delta": + return bool(delta.get("signature")) + return False + def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool: """ Determine if we should start a new content block based on the processed chunk. @@ -550,9 +557,7 @@ def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool: if tool_block.get("name"): truncated_name = tool_block["name"] - original_name = self.tool_name_mapping.get( - truncated_name, truncated_name - ) + original_name = self.tool_name_mapping.get(truncated_name, truncated_name) tool_block["name"] = original_name if block_type != self.current_content_block_type: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 08291dc4448..b86c252069b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -129,9 +129,7 @@ class AnthropicAdapter: def __init__(self) -> None: pass - def translate_completion_input_params( - self, kwargs - ) -> Optional[ChatCompletionRequest]: + def translate_completion_input_params(self, kwargs) -> Optional[ChatCompletionRequest]: """ Translate Anthropic request params to OpenAI format. @@ -164,27 +162,19 @@ def translate_completion_input_params_with_tool_mapping( model = kwargs.pop("model") messages = kwargs.pop("messages") if not model: - raise ValueError( - "Bad Request: model is required for Anthropic Messages Request" - ) + raise ValueError("Bad Request: model is required for Anthropic Messages Request") if not messages: - raise ValueError( - "Bad Request: messages is required for Anthropic Messages Request" - ) + raise ValueError("Bad Request: messages is required for Anthropic Messages Request") ######################################################### # Created Typed Request Body ######################################################### - request_body = AnthropicMessagesRequest( - model=model, messages=messages, **kwargs - ) + request_body = AnthropicMessagesRequest(model=model, messages=messages, **kwargs) ( translated_body, tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( - anthropic_message_request=request_body - ) + ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(anthropic_message_request=request_body) return translated_body, tool_name_mapping @@ -243,26 +233,16 @@ def _extract_signature_from_tool_call(self, tool_call: Any) -> Optional[str]: """ signature = None - if ( - hasattr(tool_call, "provider_specific_fields") - and tool_call.provider_specific_fields - ): + if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields: if "thought_signature" in tool_call.provider_specific_fields: signature = tool_call.provider_specific_fields["thought_signature"] - elif ( - hasattr(tool_call.function, "provider_specific_fields") - and tool_call.function.provider_specific_fields - ): + elif hasattr(tool_call.function, "provider_specific_fields") and tool_call.function.provider_specific_fields: if "thought_signature" in tool_call.function.provider_specific_fields: - signature = tool_call.function.provider_specific_fields[ - "thought_signature" - ] + signature = tool_call.function.provider_specific_fields["thought_signature"] return signature - def _extract_signature_from_tool_use_content( - self, content: Dict[str, Any] - ) -> Optional[str]: + def _extract_signature_from_tool_use_content(self, content: Dict[str, Any]) -> Optional[str]: """ Extract signature from a tool_use content block's provider_specific_fields. """ @@ -292,9 +272,7 @@ def _add_cache_control_if_applicable( """ # TypedDict objects are dicts at runtime, so .get() works cache_control = ( - source.get("cache_control") - if isinstance(source, dict) - else getattr(source, "cache_control", None) + source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None) ) if cache_control and model and self.is_anthropic_claude_model(model): # TypedDict objects support dict operations at runtime @@ -335,9 +313,7 @@ def _is_web_search_tool(self, tool: Dict[str, Any]) -> bool: """ tool_type = tool.get("type", "") tool_name = tool.get("name", "") - return ( - isinstance(tool_type, str) and tool_type.startswith("web_search") - ) or tool_name == "web_search" + return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search" def translate_anthropic_messages_to_openai( # noqa: PLR0915 self, @@ -353,66 +329,38 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 for m in messages: user_message: Optional[ChatCompletionUserMessage] = None tool_message_list: List[ChatCompletionToolMessage] = [] - new_user_content_list: List[ - Union[ChatCompletionTextObject, ChatCompletionImageObject] - ] = [] + new_user_content_list: List[Union[ChatCompletionTextObject, ChatCompletionImageObject]] = [] ## USER MESSAGE ## if m["role"] == "user": ## translate user message message_content = m.get("content") if message_content and isinstance(message_content, str): - user_message = ChatCompletionUserMessage( - role="user", content=message_content - ) + user_message = ChatCompletionUserMessage(role="user", content=message_content) elif message_content and isinstance(message_content, list): for content in message_content: if content.get("type") == "text": - text_obj = ChatCompletionTextObject( - type="text", text=content.get("text", "") - ) - self._add_cache_control_if_applicable( - content, text_obj, model - ) + text_obj = ChatCompletionTextObject(type="text", text=content.get("text", "")) + self._add_cache_control_if_applicable(content, text_obj, model) new_user_content_list.append(text_obj) # type: ignore elif content.get("type") == "image": # Convert Anthropic image format to OpenAI format source = content.get("source", {}) - openai_image_url = ( - self._translate_anthropic_image_to_openai( - cast(dict, source) - ) - ) + openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source)) if openai_image_url: - image_url_obj = ChatCompletionImageUrlObject( - url=openai_image_url - ) - image_obj = ChatCompletionImageObject( - type="image_url", image_url=image_url_obj - ) - self._add_cache_control_if_applicable( - content, image_obj, model - ) + image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) + image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) + self._add_cache_control_if_applicable(content, image_obj, model) new_user_content_list.append(image_obj) # type: ignore elif content.get("type") == "document": # Convert Anthropic document format (PDF, etc.) to OpenAI format source = content.get("source", {}) - openai_image_url = ( - self._translate_anthropic_image_to_openai( - cast(dict, source) - ) - ) + openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source)) if openai_image_url: - image_url_obj = ChatCompletionImageUrlObject( - url=openai_image_url - ) - doc_obj = ChatCompletionImageObject( - type="image_url", image_url=image_url_obj - ) - self._add_cache_control_if_applicable( - content, doc_obj, model - ) + image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url) + doc_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj) + self._add_cache_control_if_applicable(content, doc_obj, model) new_user_content_list.append(doc_obj) # type: ignore elif content.get("type") == "tool_result": if "content" not in content: @@ -421,9 +369,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 tool_call_id=content.get("tool_use_id", ""), content="", ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(content.get("content"), str): tool_result = ChatCompletionToolMessage( @@ -431,9 +377,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 tool_call_id=content.get("tool_use_id", ""), content=str(content.get("content", "")), ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(content.get("content"), list): # Combine all content items into a single tool message @@ -450,41 +394,28 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 tool_call_id=content.get("tool_use_id", ""), content=c, ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] elif isinstance(c, dict): if c.get("type") == "text": tool_result = ChatCompletionToolMessage( role="tool", - tool_call_id=content.get( - "tool_use_id", "" - ), + tool_call_id=content.get("tool_use_id", ""), content=c.get("text", ""), ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] elif c.get("type") == "image": source = c.get("source", {}) openai_image_url = ( - self._translate_anthropic_image_to_openai( - cast(dict, source) - ) - or "" + self._translate_anthropic_image_to_openai(cast(dict, source)) or "" ) tool_result = ChatCompletionToolMessage( role="tool", - tool_call_id=content.get( - "tool_use_id", "" - ), + tool_call_id=content.get("tool_use_id", ""), content=openai_image_url, ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] else: # For multiple content items, combine into a single tool message @@ -497,11 +428,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 ] = [] for c in content_items: if isinstance(c, str): - combined_content_parts.append( - ChatCompletionTextObject( - type="text", text=c - ) - ) + combined_content_parts.append(ChatCompletionTextObject(type="text", text=c)) elif isinstance(c, dict): if c.get("type") == "text": combined_content_parts.append( @@ -513,10 +440,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 elif c.get("type") == "image": source = c.get("source", {}) openai_image_url = ( - self._translate_anthropic_image_to_openai( - cast(dict, source) - ) - or "" + self._translate_anthropic_image_to_openai(cast(dict, source)) or "" ) if openai_image_url: combined_content_parts.append( @@ -534,9 +458,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 tool_call_id=content.get("tool_use_id", ""), content=combined_content_parts, # type: ignore ) - self._add_cache_control_if_applicable( - content, tool_result, model - ) + self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) # type: ignore[arg-type] if len(tool_message_list) > 0: @@ -550,14 +472,10 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 ## ASSISTANT MESSAGE ## assistant_message_str: Optional[str] = None - assistant_content_list: List[Dict[str, Any]] = ( - [] - ) # For content blocks with cache_control + assistant_content_list: List[Dict[str, Any]] = [] # For content blocks with cache_control has_cache_control_in_text = False tool_calls: List[ChatCompletionAssistantToolCall] = [] - thinking_blocks: List[ - Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] - ] = [] + thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = [] if m["role"] == "assistant": if isinstance(m.get("content"), str): assistant_message_str = str(m.get("content", "")) @@ -571,9 +489,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 "type": "text", "text": content.get("text", ""), } - self._add_cache_control_if_applicable( - content, text_block, model - ) + self._add_cache_control_if_applicable(content, text_block, model) if "cache_control" in text_block: has_cache_control_in_text = True assistant_content_list.append(text_block) @@ -584,32 +500,21 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 "name": tool_name, "arguments": json.dumps(content.get("input", {})), } - signature = ( - self._extract_signature_from_tool_use_content( - cast(Dict[str, Any], content) - ) - ) + signature = self._extract_signature_from_tool_use_content(cast(Dict[str, Any], content)) if signature: provider_specific_fields: Dict[str, Any] = ( - function_chunk.get("provider_specific_fields") - or {} - ) - provider_specific_fields["thought_signature"] = ( - signature - ) - function_chunk["provider_specific_fields"] = ( - provider_specific_fields + function_chunk.get("provider_specific_fields") or {} ) + provider_specific_fields["thought_signature"] = signature + function_chunk["provider_specific_fields"] = provider_specific_fields tool_call = ChatCompletionAssistantToolCall( id=content.get("id", ""), type="function", function=function_chunk, ) - self._add_cache_control_if_applicable( - content, tool_call, model - ) + self._add_cache_control_if_applicable(content, tool_call, model) tool_calls.append(tool_call) elif content.get("type") == "thinking": thinking_block = ChatCompletionThinkingBlock( @@ -620,12 +525,10 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 ) thinking_blocks.append(thinking_block) elif content.get("type") == "redacted_thinking": - redacted_thinking_block = ( - ChatCompletionRedactedThinkingBlock( - type="redacted_thinking", - data=content.get("data") or "", - cache_control=content.get("cache_control", {}), - ) + redacted_thinking_block = ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=content.get("data") or "", + cache_control=content.get("cache_control", {}), ) thinking_blocks.append(redacted_thinking_block) @@ -640,18 +543,14 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 assistant_content: Any = assistant_content_list elif len(assistant_content_list) > 0 and not has_cache_control_in_text: # Concatenate text blocks into string when no cache_control - assistant_content = "".join( - block.get("text", "") for block in assistant_content_list - ) + assistant_content = "".join(block.get("text", "") for block in assistant_content_list) else: assistant_content = assistant_message_str assistant_message = ChatCompletionAssistantMessage( role="assistant", content=assistant_content, - thinking_blocks=( - thinking_blocks if len(thinking_blocks) > 0 else None - ), + thinking_blocks=(thinking_blocks if len(thinking_blocks) > 0 else None), ) if len(tool_calls) > 0: assistant_message["tool_calls"] = tool_calls # type: ignore @@ -662,9 +561,7 @@ def translate_anthropic_messages_to_openai( # noqa: PLR0915 return new_messages @staticmethod - def translate_anthropic_thinking_to_reasoning_effort( - thinking: Dict[str, Any] - ) -> Optional[str]: + def translate_anthropic_thinking_to_reasoning_effort(thinking: Dict[str, Any]) -> Optional[str]: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -738,9 +635,7 @@ def translate_thinking_for_model( thinking ) if reasoning_effort: - summary = ( - thinking.get("summary") if isinstance(thinking, dict) else None - ) + summary = thinking.get("summary") if isinstance(thinking, dict) else None auto_summary = is_reasoning_auto_summary_enabled() if summary: return { @@ -770,16 +665,10 @@ def translate_anthropic_tool_choice_to_openai( # Truncate tool name if it exceeds OpenAI's 64-char limit original_name = tool_choice.get("name", "") truncated_name = truncate_tool_name(original_name) - tc_function_param = ChatCompletionToolChoiceFunctionParam( - name=truncated_name - ) - return ChatCompletionToolChoiceObjectParam( - type="function", function=tc_function_param - ) + tc_function_param = ChatCompletionToolChoiceFunctionParam(name=truncated_name) + return ChatCompletionToolChoiceObjectParam(type="function", function=tc_function_param) else: - raise ValueError( - "Incompatible tool choice param submitted - {}".format(tool_choice) - ) + raise ValueError("Incompatible tool choice param submitted - {}".format(tool_choice)) def translate_anthropic_tools_to_openai( self, tools: List[AllAnthropicToolsValues], model: Optional[str] = None @@ -805,9 +694,7 @@ def translate_anthropic_tools_to_openai( continue raw_name = tool.get("name") - if raw_name is None or ( - isinstance(raw_name, str) and not str(raw_name).strip() - ): + if raw_name is None or (isinstance(raw_name, str) and not str(raw_name).strip()): original_name = f"litellm_unnamed_tool_{idx}" else: original_name = str(raw_name) @@ -828,17 +715,13 @@ def translate_anthropic_tools_to_openai( for k, v in tool.items(): if k not in mapped_tool_params: # pass additional computer kwargs function_chunk.setdefault("parameters", {}).update({k: v}) - tool_param = ChatCompletionToolParam( - type="function", function=function_chunk - ) + tool_param = ChatCompletionToolParam(type="function", function=function_chunk) self._add_cache_control_if_applicable(tool, tool_param, model) new_tools.append(tool_param) # type: ignore[arg-type] return new_tools, tool_name_mapping # type: ignore[return-value] - def translate_anthropic_output_format_to_openai( - self, output_format: Any - ) -> Optional[Dict[str, Any]]: + def translate_anthropic_output_format_to_openai(self, output_format: Any) -> Optional[Dict[str, Any]]: """ Translate Anthropic's output_format to OpenAI's response_format. @@ -897,25 +780,19 @@ def _add_additional_properties_false(schema: dict) -> None: # Handle array items if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( - schema["items"] - ) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) # Handle anyOf/oneOf/allOf for key in ("anyOf", "oneOf", "allOf"): if key in schema: for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( - sub_schema - ) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) # Handle $defs / definitions for key in ("$defs", "definitions"): if key in schema: for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( - def_schema - ) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) def _add_system_message_to_messages( self, @@ -1035,9 +912,7 @@ def _translate_thinking_to_openai( new_kwargs["thinking"] = thinking # type: ignore return - reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort( - cast(Dict[str, Any], thinking) - ) + reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(cast(Dict[str, Any], thinking)) if not reasoning_effort: return @@ -1073,9 +948,7 @@ def _translate_output_format_to_openai( output_format = anthropic_message_request["output_format"] if not output_format: return - response_format = self.translate_anthropic_output_format_to_openai( - output_format=output_format - ) + response_format = self.translate_anthropic_output_format_to_openai(output_format=output_format) if response_format: new_kwargs["response_format"] = response_format @@ -1106,11 +979,7 @@ def translate_anthropic_to_openai( tool_name_mapping: Dict[str, str] = {} ## CONVERT ANTHROPIC MESSAGES TO OPENAI - messages_list: List[ - Union[ - AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam - ] - ] = cast( + messages_list: List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]] = cast( List[ Union[ AnthropicMessagesUserMessageParam, @@ -1197,10 +1066,7 @@ def _translate_openai_content_to_anthropic( new_content: List[Dict[str, Any]] = [] for choice in choices: # Handle thinking blocks first - if ( - hasattr(choice.message, "thinking_blocks") - and choice.message.thinking_blocks - ): + if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: if thinking_block.get("type") == "thinking": thinking_value = thinking_block.get("thinking", "") @@ -1208,16 +1074,8 @@ def _translate_openai_content_to_anthropic( new_content.append( AnthropicResponseContentBlockThinking( type="thinking", - thinking=( - str(thinking_value) - if thinking_value is not None - else "" - ), - signature=( - str(signature_value) - if signature_value is not None - else None - ), + thinking=(str(thinking_value) if thinking_value is not None else ""), + signature=(str(signature_value) if signature_value is not None else None), ).model_dump() ) elif thinking_block.get("type") == "redacted_thinking": @@ -1229,10 +1087,7 @@ def _translate_openai_content_to_anthropic( ).model_dump() ) # Handle reasoning_content when thinking_blocks is not present - elif ( - hasattr(choice.message, "reasoning_content") - and choice.message.reasoning_content - ): + elif hasattr(choice.message, "reasoning_content") and choice.message.reasoning_content: new_content.append( AnthropicResponseContentBlockThinking( type="thinking", @@ -1244,15 +1099,10 @@ def _translate_openai_content_to_anthropic( # Handle text content if choice.message.content is not None: new_content.append( - AnthropicResponseContentBlockText( - type="text", text=choice.message.content - ).model_dump() + AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) # Handle tool calls (in parallel to text content) - if ( - choice.message.tool_calls is not None - and len(choice.message.tool_calls) > 0 - ): + if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: for tool_call in choice.message.tool_calls: # Extract signature from provider_specific_fields only signature = self._extract_signature_from_tool_call(tool_call) @@ -1264,9 +1114,7 @@ def _translate_openai_content_to_anthropic( # Restore original tool name if it was truncated truncated_name = tool_call.function.name or "" original_name = ( - tool_name_mapping.get(truncated_name, truncated_name) - if tool_name_mapping - else truncated_name + tool_name_mapping.get(truncated_name, truncated_name) if tool_name_mapping else truncated_name ) tool_use_block = AnthropicResponseContentBlockToolUse( @@ -1281,16 +1129,12 @@ def _translate_openai_content_to_anthropic( ) # Add provider_specific_fields if signature is present if provider_specific_fields: - tool_use_block.provider_specific_fields = ( - provider_specific_fields - ) + tool_use_block.provider_specific_fields = provider_specific_fields new_content.append(tool_use_block.model_dump()) return new_content - def _translate_openai_finish_reason_to_anthropic( - self, openai_finish_reason: str - ) -> AnthropicFinishReason: + def _translate_openai_finish_reason_to_anthropic(self, openai_finish_reason: str) -> AnthropicFinishReason: if openai_finish_reason == "stop": return "end_turn" elif openai_finish_reason == "length": @@ -1327,22 +1171,15 @@ def translate_openai_response_to_anthropic( uncached_input_tokens = usage.prompt_tokens or 0 cached_tokens = 0 if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: - cached_tokens = ( - getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 - ) + cached_tokens = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0 uncached_input_tokens -= cached_tokens anthropic_usage = AnthropicUsage( input_tokens=uncached_input_tokens, output_tokens=usage.completion_tokens or 0, ) - if ( - hasattr(usage, "_cache_creation_input_tokens") - and usage._cache_creation_input_tokens > 0 - ): - anthropic_usage["cache_creation_input_tokens"] = ( - usage._cache_creation_input_tokens - ) + if hasattr(usage, "_cache_creation_input_tokens") and usage._cache_creation_input_tokens > 0: + anthropic_usage["cache_creation_input_tokens"] = usage._cache_creation_input_tokens if cached_tokens > 0: anthropic_usage["cache_read_input_tokens"] = cached_tokens @@ -1382,9 +1219,7 @@ def _translate_streaming_openai_chunk_to_anthropic_content_block( ) elif choice.delta.content is not None and len(choice.delta.content) > 0: return "text", TextBlock(type="text", text="") - elif isinstance(choice, StreamingChoices) and hasattr( - choice.delta, "thinking_blocks" - ): + elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): thinking_blocks = choice.delta.thinking_blocks or [] if len(thinking_blocks) > 0: thinking_block = thinking_blocks[0] @@ -1403,13 +1238,9 @@ def _translate_streaming_openai_chunk_to_anthropic_content_block( return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking=thinking, signature=signature ) - elif isinstance(choice, StreamingChoices) and hasattr( - choice.delta, "reasoning_content" - ): + elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"): if choice.delta.reasoning_content: - return "thinking", ChatCompletionThinkingBlock( - type="thinking", thinking="", signature="" - ) + return "thinking", ChatCompletionThinkingBlock(type="thinking", thinking="", signature="") return "text", TextBlock(type="text", text="") @@ -1433,14 +1264,9 @@ def _translate_streaming_openai_chunk_to_anthropic( text += choice.delta.content if choice.delta.tool_calls is not None and len(choice.delta.tool_calls) > 0: for tool in choice.delta.tool_calls: - if ( - tool.function is not None - and tool.function.arguments is not None - ): + if tool.function is not None and tool.function.arguments is not None: partial_json = (partial_json or "") + tool.function.arguments - elif isinstance(choice, StreamingChoices) and hasattr( - choice.delta, "thinking_blocks" - ): + elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): thinking_blocks = choice.delta.thinking_blocks or [] if len(thinking_blocks) > 0: for thinking_block in thinking_blocks: @@ -1455,25 +1281,17 @@ def _translate_streaming_openai_chunk_to_anthropic( reasoning_signature += signature # Handle reasoning_content when thinking_blocks is not present # This handles providers like OpenRouter that return reasoning_content - elif isinstance(choice, StreamingChoices) and hasattr( - choice.delta, "reasoning_content" - ): + elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"): if choice.delta.reasoning_content: reasoning_content += choice.delta.reasoning_content if reasoning_content and reasoning_signature: - raise ValueError( - "Both `reasoning` and `signature` in a single streaming chunk isn't supported." - ) + raise ValueError("Both `reasoning` and `signature` in a single streaming chunk isn't supported.") if partial_json is not None: - return "input_json_delta", ContentJsonBlockDelta( - type="input_json_delta", partial_json=partial_json - ) + return "input_json_delta", ContentJsonBlockDelta(type="input_json_delta", partial_json=partial_json) elif reasoning_content: - return "thinking_delta", ContentThinkingBlockDelta( - type="thinking_delta", thinking=reasoning_content - ) + return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content) elif reasoning_signature: return "signature_delta", ContentThinkingSignatureBlockDelta( type="signature_delta", signature=reasoning_signature @@ -1487,26 +1305,18 @@ def translate_streaming_openai_response_to_anthropic( ## base case - final chunk w/ finish reason if response.choices[0].finish_reason is not None: delta = MessageDelta( - stop_reason=self._translate_openai_finish_reason_to_anthropic( - response.choices[0].finish_reason - ), + stop_reason=self._translate_openai_finish_reason_to_anthropic(response.choices[0].finish_reason), ) if getattr(response, "usage", None) is not None: litellm_usage_chunk: Optional[Usage] = response.usage # type: ignore - elif ( - hasattr(response, "_hidden_params") - and "usage" in response._hidden_params - ): + elif hasattr(response, "_hidden_params") and "usage" in response._hidden_params: litellm_usage_chunk = response._hidden_params["usage"] else: litellm_usage_chunk = None if litellm_usage_chunk is not None: uncached_input_tokens = litellm_usage_chunk.prompt_tokens or 0 cached_tokens = 0 - if ( - hasattr(litellm_usage_chunk, "prompt_tokens_details") - and litellm_usage_chunk.prompt_tokens_details - ): + if hasattr(litellm_usage_chunk, "prompt_tokens_details") and litellm_usage_chunk.prompt_tokens_details: cached_tokens = ( getattr( litellm_usage_chunk.prompt_tokens_details, @@ -1525,15 +1335,15 @@ def translate_streaming_openai_response_to_anthropic( hasattr(litellm_usage_chunk, "_cache_creation_input_tokens") and litellm_usage_chunk._cache_creation_input_tokens > 0 ): - usage_delta["cache_creation_input_tokens"] = ( - litellm_usage_chunk._cache_creation_input_tokens - ) + usage_delta["cache_creation_input_tokens"] = litellm_usage_chunk._cache_creation_input_tokens if cached_tokens > 0: usage_delta["cache_read_input_tokens"] = cached_tokens else: usage_delta = UsageDelta(input_tokens=0, output_tokens=0) return MessageBlockDelta( - type="message_delta", delta=delta, usage=usage_delta # type: ignore + type="message_delta", + delta=delta, + usage=usage_delta, # type: ignore ) ( type_of_content, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py index 02e566ab74a..9c473924ebf 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py @@ -151,8 +151,8 @@ async def mock_stream(): async def test_async_stream_no_extra_delta_when_tool_args_empty(): """ When a provider sends tool name/id WITHOUT arguments in the first chunk - (OpenAI-style), the block transition still queues the trigger chunk's - empty input_json_delta (#25212); the follow-up chunk carries real args. + (OpenAI-style), the wrapper should NOT emit an extra input_json_delta + from the trigger chunk — only the follow-up chunk with real args. """ # Chunk 1: text text_chunk = _make_chunk(Delta(content="Hi", role="assistant", tool_calls=None)) @@ -221,9 +221,8 @@ async def mock_stream(): assert tool_start_idx is not None - # Count input_json_delta events after the tool_use block start. The trigger - # chunk is now queued on block transition (#25212), so an empty partial_json - # delta precedes the follow-up chunk with real arguments. + # Count input_json_delta events after the tool_use block start. Only the + # follow-up chunk with real arguments should produce a delta. input_json_deltas = [ e for e in events[tool_start_idx + 1 :] @@ -232,12 +231,11 @@ async def mock_stream(): and isinstance(e.get("delta"), dict) and e["delta"].get("type") == "input_json_delta" ] - assert len(input_json_deltas) == 2, ( - f"Expected trigger empty delta + follow-up args delta, " + assert len(input_json_deltas) == 1, ( + f"Expected exactly 1 input_json_delta (from the follow-up chunk), " f"got {len(input_json_deltas)}" ) - assert input_json_deltas[0]["delta"]["partial_json"] == "" - assert input_json_deltas[1]["delta"]["partial_json"] == '{"location": "NYC"}' + assert input_json_deltas[0]["delta"]["partial_json"] == '{"location": "NYC"}' def test_sync_stream_emits_input_json_delta_for_bundled_tool_args(): @@ -309,9 +307,8 @@ def test_sync_stream_emits_input_json_delta_for_bundled_tool_args(): def test_sync_stream_no_extra_delta_when_tool_args_empty(): """ - Sync counterpart: empty args on the trigger chunk still emit an - input_json_delta from the block transition (#25212); the follow-up chunk - carries the real arguments. + Sync counterpart: empty args (OpenAI-style) should not emit an extra + input_json_delta from the trigger chunk. """ text_chunk = _make_chunk(Delta(content="Hi", role="assistant", tool_calls=None)) tool_name_chunk = _make_chunk( @@ -378,9 +375,8 @@ def test_sync_stream_no_extra_delta_when_tool_args_empty(): and isinstance(e.get("delta"), dict) and e["delta"].get("type") == "input_json_delta" ] - assert len(input_json_deltas) == 2, ( - f"Expected trigger empty delta + follow-up args delta, " + assert len(input_json_deltas) == 1, ( + f"Expected exactly 1 input_json_delta (from the follow-up chunk), " f"got {len(input_json_deltas)}" ) - assert input_json_deltas[0]["delta"]["partial_json"] == "" - assert input_json_deltas[1]["delta"]["partial_json"] == '{"location": "NYC"}' + assert input_json_deltas[0]["delta"]["partial_json"] == '{"location": "NYC"}' diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py index 7dc081f2144..674e931821b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py @@ -138,7 +138,6 @@ def test_anthropic_stream_wrapper_single_tool_call(): expected_types = [ "message_start", "content_block_start", # tool_use (from peek) - "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} "content_block_stop", @@ -194,12 +193,10 @@ def test_anthropic_stream_wrapper_back_to_back_tool_calls(): expected_types = [ "message_start", "content_block_start", # tool_use (from peek) - "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} "content_block_stop", "content_block_start", # second tool_use - "content_block_delta", # first chunk of second tool "content_block_delta", # {"city": "content_block_delta", # " SF"} "content_block_stop", @@ -260,7 +257,6 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text(): expected_types = [ "message_start", "content_block_start", # tool_use (from peek) - "content_block_delta", # first tool chunk (empty args) "content_block_delta", # {"city": "content_block_delta", # "NY"} "content_block_stop", @@ -268,12 +264,10 @@ def test_anthropic_stream_wrapper_interleaved_tool_calls_and_text(): "content_block_delta", # "The weather is nice today." "content_block_stop", "content_block_start", # second tool_use - "content_block_delta", # first chunk of second tool "content_block_delta", # {"city": "content_block_delta", # " SF"} "content_block_stop", "content_block_start", # third tool_use - "content_block_delta", # first chunk of third tool "content_block_delta", # {"city": "content_block_delta", # " CHI"} "content_block_stop",