From ab99a97afa8c8e9155c632457ad9eac761495a8b Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 29 Apr 2026 12:21:04 -0400 Subject: [PATCH] test(openai): cover chat metadata with store enabled Add cassette-backed regression coverage for wrapped chat completions that pass both OpenAI metadata and store=True. The test verifies the request cassette preserves store=True and that Braintrust span metadata records the OpenAI parameters. --- ..._openai_chat_metadata_with_store_true.yaml | 28 ++++++++++++ ..._openai_chat_metadata_with_store_true.yaml | 28 ++++++++++++ ..._openai_chat_metadata_with_store_true.yaml | 28 ++++++++++++ ..._openai_chat_metadata_with_store_true.yaml | 28 ++++++++++++ .../integrations/openai/test_openai.py | 45 +++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 py/src/braintrust/integrations/openai/cassettes/1.71.0/test_openai_chat_metadata_with_store_true.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/1.77.0/test_openai_chat_metadata_with_store_true.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/1.92.0/test_openai_chat_metadata_with_store_true.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_metadata_with_store_true.yaml diff --git a/py/src/braintrust/integrations/openai/cassettes/1.71.0/test_openai_chat_metadata_with_store_true.yaml b/py/src/braintrust/integrations/openai/cassettes/1.71.0/test_openai_chat_metadata_with_store_true.yaml new file mode 100644 index 00000000..fac1f9a2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/1.71.0/test_openai_chat_metadata_with_store_true.yaml @@ -0,0 +1,28 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","metadata":{"braintrust_test":"store_true"},"store":true}' + headers: + Accept: + - application/json + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.32.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-braintrust-store-true\",\n \"object\": \"chat.completion\",\n \"created\": 1776355234,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"24\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 1,\n \"total_tokens\": 15,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_braintrust\"\n}\n" + headers: + Content-Type: + - application/json + openai-version: + - '2020-10-01' + x-request-id: + - req_braintrust_store_true + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/1.77.0/test_openai_chat_metadata_with_store_true.yaml b/py/src/braintrust/integrations/openai/cassettes/1.77.0/test_openai_chat_metadata_with_store_true.yaml new file mode 100644 index 00000000..fac1f9a2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/1.77.0/test_openai_chat_metadata_with_store_true.yaml @@ -0,0 +1,28 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","metadata":{"braintrust_test":"store_true"},"store":true}' + headers: + Accept: + - application/json + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.32.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-braintrust-store-true\",\n \"object\": \"chat.completion\",\n \"created\": 1776355234,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"24\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 1,\n \"total_tokens\": 15,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_braintrust\"\n}\n" + headers: + Content-Type: + - application/json + openai-version: + - '2020-10-01' + x-request-id: + - req_braintrust_store_true + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/1.92.0/test_openai_chat_metadata_with_store_true.yaml b/py/src/braintrust/integrations/openai/cassettes/1.92.0/test_openai_chat_metadata_with_store_true.yaml new file mode 100644 index 00000000..fac1f9a2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/1.92.0/test_openai_chat_metadata_with_store_true.yaml @@ -0,0 +1,28 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","metadata":{"braintrust_test":"store_true"},"store":true}' + headers: + Accept: + - application/json + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.32.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-braintrust-store-true\",\n \"object\": \"chat.completion\",\n \"created\": 1776355234,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"24\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 1,\n \"total_tokens\": 15,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_braintrust\"\n}\n" + headers: + Content-Type: + - application/json + openai-version: + - '2020-10-01' + x-request-id: + - req_braintrust_store_true + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_metadata_with_store_true.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_metadata_with_store_true.yaml new file mode 100644 index 00000000..fac1f9a2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_metadata_with_store_true.yaml @@ -0,0 +1,28 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","metadata":{"braintrust_test":"store_true"},"store":true}' + headers: + Accept: + - application/json + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.32.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-braintrust-store-true\",\n \"object\": \"chat.completion\",\n \"created\": 1776355234,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"24\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 1,\n \"total_tokens\": 15,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_braintrust\"\n}\n" + headers: + Content-Type: + - application/json + openai-version: + - '2020-10-01' + x-request-id: + - req_braintrust_store_true + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 4848deae..b7b33385 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -1467,6 +1467,51 @@ async def test_openai_async_parallel_requests(memory_logger): assert_metrics_are_valid(span["metrics"]) +@pytest.mark.vcr +def test_openai_chat_metadata_with_store_true(memory_logger, vcr_cassette_dir): + """Test wrapped chat completions preserve OpenAI metadata when store=True is enabled.""" + assert not memory_logger.pop() + + client = wrap_openai(openai.OpenAI()) + + response = client.chat.completions.create( + model=TEST_MODEL, + messages=[{"role": "user", "content": TEST_PROMPT}], + metadata={"braintrust_test": "store_true"}, + store=True, + ) + + assert response + assert response.choices[0].message.content + assert "24" in response.choices[0].message.content or "twenty-four" in response.choices[0].message.content.lower() + + # VCR matches on request body, so playback already fails if Braintrust strips + # store=True before the request reaches OpenAI. Keep this explicit assertion + # to make the regression this test guards obvious. + cassette_path = os.path.join(vcr_cassette_dir, "test_openai_chat_metadata_with_store_true.yaml") + with open(cassette_path, encoding="utf-8") as f: + cassette_text = f.read() + assert '"metadata":{"braintrust_test":"store_true"}' in cassette_text + assert '"store":true' in cassette_text + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + + assert_dict_matches( + span, + { + "input": [{"role": "user", "content": TEST_PROMPT}], + "metadata": { + "model": TEST_MODEL, + "provider": "openai", + "metadata": {"braintrust_test": "store_true"}, + "store": True, + }, + }, + ) + + @pytest.mark.vcr def test_openai_not_given_filtering(memory_logger): """Test that NOT_GIVEN values are filtered out of logged inputs but API call still works."""