From 7f30c3b88d78eaefa8daede53eb323827a06526b Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 1 May 2026 15:16:10 -0400 Subject: [PATCH] fix(mistral): add tool spans for completions Create child TOOL spans for Mistral chat and agent tool calls in both complete and streaming paths. Add VCR regression coverage across the supported mistralai version matrix. --- ...ap_mistral_agents_complete_tool_spans.yaml | 197 +++++++++++++++++ ...wrap_mistral_agents_stream_tool_spans.yaml | 194 ++++++++++++++++ ...wrap_mistral_chat_complete_tool_spans.yaml | 81 +++++++ ...t_wrap_mistral_chat_stream_tool_spans.yaml | 78 +++++++ ...ap_mistral_agents_complete_tool_spans.yaml | 209 ++++++++++++++++++ ...wrap_mistral_agents_stream_tool_spans.yaml | 194 ++++++++++++++++ ...wrap_mistral_chat_complete_tool_spans.yaml | 81 +++++++ ...t_wrap_mistral_chat_stream_tool_spans.yaml | 78 +++++++ .../integrations/mistral/test_mistral.py | 100 ++++++++- .../integrations/mistral/tracing.py | 67 ++++++ 10 files changed, 1278 insertions(+), 1 deletion(-) create mode 100644 py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_complete_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_stream_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_complete_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_stream_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_stream_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_tool_spans.yaml diff --git a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_complete_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_complete_tool_spans.yaml new file mode 100644 index 00000000..b10bfba7 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_complete_tool_spans.yaml @@ -0,0 +1,197 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","name":"braintrust-test-agent-1777662640208","instructions":"You + are concise. Keep responses under five words."}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '144' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/agents + response: + body: + string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"mistral-small-latest","name":"braintrust-test-agent-1777662640208","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_019de4f3414e7434ae6a78a856e4bafd","version":0,"versions":[],"created_at":"2026-05-01T19:10:40.491735Z","updated_at":"2026-05-01T19:10:40.491738Z","deployment_chat":false,"source":"api","version_message":null}' + headers: + CF-RAY: + - 9f51166de93746b5-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:10:40 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '692' + mistral-correlation-id: + - 019de4f3-40f5-7b12-9317-dc66b898f3b1 + set-cookie: + - __cf_bm=IHhDeC7rRK6YDtNcGbJe0_M_e2cgUmbG0oloO5VpkTs-1777662640.3053465-1.0.1.1-jT.JFGuSHJXanjc2Djujkxfx4rE0vcXBs0Hb7OW8W1aNfTIroDsCn_2VrY9kF.m44eU7H8YSwkH.Yh4HOfasl7nS_glA7HOQdvyn2KBdzbcrC0uZkNMzZqcczWqIiCkm; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:40 + GMT + - _cfuvid=4.caIoynBFVOJU5CUbNRmeWN0aEZwc.d_BBtGYy0Ck8-1777662640.3053465-1.0.1.1-40ePzWEHHOdDI.y74_NIOYu42q283fv1RuXUVr3317E; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '95' + x-kong-proxy-latency: + - '41' + x-kong-request-id: + - 019de4f3-40f5-7b12-9317-dc66b898f3b1 + x-kong-upstream-latency: + - '96' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Use get_weather for Paris. Do not answer directly.","role":"user"}],"agent_id":"ag_019de4f3414e7434ae6a78a856e4bafd","max_tokens":100,"stream":false,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '393' + Cookie: + - __cf_bm=IHhDeC7rRK6YDtNcGbJe0_M_e2cgUmbG0oloO5VpkTs-1777662640.3053465-1.0.1.1-jT.JFGuSHJXanjc2Djujkxfx4rE0vcXBs0Hb7OW8W1aNfTIroDsCn_2VrY9kF.m44eU7H8YSwkH.Yh4HOfasl7nS_glA7HOQdvyn2KBdzbcrC0uZkNMzZqcczWqIiCkm; + _cfuvid=4.caIoynBFVOJU5CUbNRmeWN0aEZwc.d_BBtGYy0Ck8-1777662640.3053465-1.0.1.1-40ePzWEHHOdDI.y74_NIOYu42q283fv1RuXUVr3317E + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/agents/completions + response: + body: + string: '{"id":"78cf63e6a47f4e9d8997f88f9d5d09a0","created":1777662640,"model":"mistral-small-latest","usage":{"prompt_tokens":100,"total_tokens":112,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","tool_calls":[{"id":"HpwsuA761","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}],"content":""}}]}' + headers: + CF-RAY: + - 9f5116709c2cac7c-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:10:41 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '447' + mistral-correlation-id: + - 019de4f3-42a6-7bcb-914d-147bd2dfa3ba + x-envoy-upstream-service-time: + - '285' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 019de4f3-42a6-7bcb-914d-147bd2dfa3ba + x-kong-upstream-latency: + - '285' + x-ratelimit-limit-req-minute: + - '400' + x-ratelimit-limit-tokens-minute: + - '1500000' + x-ratelimit-remaining-req-minute: + - '394' + x-ratelimit-remaining-tokens-minute: + - '1499440' + x-ratelimit-tokens-query-cost: + - '112' + status: + code: 200 + message: OK +- request: + body: '' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Cookie: + - __cf_bm=IHhDeC7rRK6YDtNcGbJe0_M_e2cgUmbG0oloO5VpkTs-1777662640.3053465-1.0.1.1-jT.JFGuSHJXanjc2Djujkxfx4rE0vcXBs0Hb7OW8W1aNfTIroDsCn_2VrY9kF.m44eU7H8YSwkH.Yh4HOfasl7nS_glA7HOQdvyn2KBdzbcrC0uZkNMzZqcczWqIiCkm; + _cfuvid=4.caIoynBFVOJU5CUbNRmeWN0aEZwc.d_BBtGYy0Ck8-1777662640.3053465-1.0.1.1-40ePzWEHHOdDI.y74_NIOYu42q283fv1RuXUVr3317E + Host: + - api.mistral.ai + user-agent: + - mistral-client-python/1.12.4 + method: DELETE + uri: https://api.mistral.ai/v1/agents/ag_019de4f3414e7434ae6a78a856e4bafd + response: + body: + string: '' + headers: + CF-RAY: + - 9f5116752c34f80e-YYZ + Connection: + - keep-alive + Date: + - Fri, 01 May 2026 19:10:41 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-4579-71f1-8652-a0ef8f391026 + x-envoy-upstream-service-time: + - '70' + x-kong-proxy-latency: + - '14' + x-kong-request-id: + - 019de4f3-4579-71f1-8652-a0ef8f391026 + x-kong-upstream-latency: + - '71' + status: + code: 204 + message: No Content +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_stream_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_stream_tool_spans.yaml new file mode 100644 index 00000000..abbd3180 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_agents_stream_tool_spans.yaml @@ -0,0 +1,194 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","name":"braintrust-test-agent-1777662671943","instructions":"You + are concise. Keep responses under five words."}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '144' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/agents + response: + body: + string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"mistral-small-latest","name":"braintrust-test-agent-1777662671943","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_019de4f3bd267493989dd4f850ceb55b","version":0,"versions":[],"created_at":"2026-05-01T19:11:12.191782Z","updated_at":"2026-05-01T19:11:12.191786Z","deployment_chat":false,"source":"api","version_message":null}' + headers: + CF-RAY: + - 9f51173449b5cceb-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:11:12 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '692' + mistral-correlation-id: + - 019de4f3-bce2-7629-b11e-cc23c1380f13 + set-cookie: + - __cf_bm=mrhYeiViro6wHtnqEpjGd7hLPJ9mRg8F4nnNLW3ccTY-1777662672.0451913-1.0.1.1-0IRI6JbggwIc5aakVXrqZQm5PPHM.6xVcr90B638zB0obwP4lOxct1IbX4krwFGQFLq46M48D4NWvuD9p8r1RfcGdtWw4nZ.bXBhK9KkndJ_V5.E7_Kh8Kra7DFrwRjI; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:41:12 + GMT + - _cfuvid=n4cckRYwtN0jFPxsF.RUuDU.CMxMRnq6LwMIFs0BFRU-1777662672.0451913-1.0.1.1-cGhPr72ahCa8TzVXUwb_f.ZXxxvaf3e3FaY7n6CAEOk; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '90' + x-kong-proxy-latency: + - '20' + x-kong-request-id: + - 019de4f3-bce2-7629-b11e-cc23c1380f13 + x-kong-upstream-latency: + - '92' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Use get_weather for Paris. Do not answer directly.","role":"user"}],"agent_id":"ag_019de4f3bd267493989dd4f850ceb55b","max_tokens":100,"stream":true,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '392' + Cookie: + - __cf_bm=mrhYeiViro6wHtnqEpjGd7hLPJ9mRg8F4nnNLW3ccTY-1777662672.0451913-1.0.1.1-0IRI6JbggwIc5aakVXrqZQm5PPHM.6xVcr90B638zB0obwP4lOxct1IbX4krwFGQFLq46M48D4NWvuD9p8r1RfcGdtWw4nZ.bXBhK9KkndJ_V5.E7_Kh8Kra7DFrwRjI; + _cfuvid=n4cckRYwtN0jFPxsF.RUuDU.CMxMRnq6LwMIFs0BFRU-1777662672.0451913-1.0.1.1-cGhPr72ahCa8TzVXUwb_f.ZXxxvaf3e3FaY7n6CAEOk + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/agents/completions + response: + body: + string: 'data: {"id":"f0045c73780a49f9baf96638d0c1635b","object":"chat.completion.chunk","created":1777662672,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + + + data: {"id":"f0045c73780a49f9baf96638d0c1635b","object":"chat.completion.chunk","created":1777662672,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"ykk9sb1fw","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"total_tokens":112,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":80}},"p":"ab"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9f5117364caf78a5-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Fri, 01 May 2026 19:11:13 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-be2b-7b2e-8400-692abd2de685 + x-envoy-upstream-service-time: + - '299' + x-kong-proxy-latency: + - '271' + x-kong-request-id: + - 019de4f3-be2b-7b2e-8400-692abd2de685 + x-kong-upstream-latency: + - '300' + status: + code: 200 + message: OK +- request: + body: '' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Cookie: + - __cf_bm=mrhYeiViro6wHtnqEpjGd7hLPJ9mRg8F4nnNLW3ccTY-1777662672.0451913-1.0.1.1-0IRI6JbggwIc5aakVXrqZQm5PPHM.6xVcr90B638zB0obwP4lOxct1IbX4krwFGQFLq46M48D4NWvuD9p8r1RfcGdtWw4nZ.bXBhK9KkndJ_V5.E7_Kh8Kra7DFrwRjI; + _cfuvid=n4cckRYwtN0jFPxsF.RUuDU.CMxMRnq6LwMIFs0BFRU-1777662672.0451913-1.0.1.1-cGhPr72ahCa8TzVXUwb_f.ZXxxvaf3e3FaY7n6CAEOk + Host: + - api.mistral.ai + user-agent: + - mistral-client-python/1.12.4 + method: DELETE + uri: https://api.mistral.ai/v1/agents/ag_019de4f3bd267493989dd4f850ceb55b + response: + body: + string: '' + headers: + CF-RAY: + - 9f51173f188cc109-YYZ + Connection: + - keep-alive + Date: + - Fri, 01 May 2026 19:11:13 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-c3a8-7592-bba9-1a048f558172 + x-envoy-upstream-service-time: + - '68' + x-kong-proxy-latency: + - '22' + x-kong-request-id: + - 019de4f3-c3a8-7592-bba9-1a048f558172 + x-kong-upstream-latency: + - '69' + status: + code: 204 + message: No Content +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_complete_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_complete_tool_spans.yaml new file mode 100644 index 00000000..a2438234 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_complete_tool_spans.yaml @@ -0,0 +1,81 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","messages":[{"content":"Use get_weather + for Paris. Do not answer directly.","role":"user"}],"max_tokens":100,"stream":false,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '375' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/chat/completions + response: + body: + string: '{"id":"ed7d005e9120493387cfefce5dd61075","created":1777662639,"model":"mistral-small-latest","usage":{"prompt_tokens":88,"total_tokens":100,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","tool_calls":[{"id":"fhRlf8eEX","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}],"content":""}}]}' + headers: + CF-RAY: + - 9f511666a8f9ac25-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:10:39 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '446' + mistral-correlation-id: + - 019de4f3-3c6f-7851-aac3-e1ca8cf0681b + set-cookie: + - __cf_bm=53BOO4OeMTHqij1mGNRJ4WHE2c6ddP9JG3I4YgrRvVM-1777662639.1476867-1.0.1.1-Vkwzp4PnwPH.P3Yj2Z7NS8Qi1I0pQQ01ClGvAd8quaDQ7VMY2sJn7q53yoL2NoQ.Y3UNhFYteHCaUda5vUORn0ZpwC1GFL5f14_3k6plERtLRiCk7Lm6_7usCvNy.wtj; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:39 + GMT + - _cfuvid=oJp5KGEHG7pcVYA8PH67ABH4mbNvMkX2C5TVhetwviQ-1777662639.1476867-1.0.1.1-YI0wlXSVybJ7KX7wDD2KY.cCk21d2YIj5nAwmvv40kA; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '301' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 019de4f3-3c6f-7851-aac3-e1ca8cf0681b + x-kong-upstream-latency: + - '302' + x-ratelimit-limit-req-minute: + - '400' + x-ratelimit-limit-tokens-minute: + - '1500000' + x-ratelimit-remaining-req-minute: + - '396' + x-ratelimit-remaining-tokens-minute: + - '1499588' + x-ratelimit-tokens-query-cost: + - '100' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_stream_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_stream_tool_spans.yaml new file mode 100644 index 00000000..1485d4bf --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_wrap_mistral_chat_stream_tool_spans.yaml @@ -0,0 +1,78 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","messages":[{"content":"Use get_weather + for Paris. Do not answer directly.","role":"user"}],"max_tokens":100,"stream":true,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '374' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/1.12.4 + method: POST + uri: https://api.mistral.ai/v1/chat/completions + response: + body: + string: 'data: {"id":"8adace786caf496d8cf6a5227b5af380","object":"chat.completion.chunk","created":1777662639,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + + + data: {"id":"8adace786caf496d8cf6a5227b5af380","object":"chat.completion.chunk","created":1777662639,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"pRtAVK9lN","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":88,"total_tokens":100,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":64}},"p":"abcdefghijklmnopqrstuvwxyz012"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9f51166acea03a53-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Fri, 01 May 2026 19:10:40 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-3f02-743d-a685-a1063b2c72df + set-cookie: + - __cf_bm=ObX8zUJ0Yr9f.fFqiNgEiuhvJX.6ke7CCyWkcfNIuSs-1777662639.805578-1.0.1.1-V.KyPnebnpOXGHvvADWEvVbYKJI.ZClBaMVfMfCEL.mOZxSlWZn4sND1KFeTmVS6QBA_ZRxEnZXJrliAXJ9pw6WLS78ZDVtwlgKaqq2sSvEoTfzua46NyTaDtUwY5aRR; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:40 + GMT + - _cfuvid=e5HeqHfROBY5wHnmhc50Pg32Oscas6BZbfo8k4eZwYI-1777662639.805578-1.0.1.1-kOWZvRUNF.UDeVLYt4eiwavWz2HAh9a1ZPmjqPQT7Mw; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '261' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 019de4f3-3f02-743d-a685-a1063b2c72df + x-kong-upstream-latency: + - '262' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_tool_spans.yaml new file mode 100644 index 00000000..4af6cc26 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_tool_spans.yaml @@ -0,0 +1,209 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","name":"braintrust-test-agent-1777662623421","instructions":"You + are concise. Keep responses under five words."}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '144' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/agents + response: + body: + string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"mistral-small-latest","name":"braintrust-test-agent-1777662623421","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_019de4f30009709abee525d209718f05","version":0,"versions":[],"created_at":"2026-05-01T19:10:23.779196Z","updated_at":"2026-05-01T19:10:23.779200Z","deployment_chat":false,"source":"api","version_message":null}' + headers: + CF-RAY: + - 9f5116059e051117-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:10:23 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '692' + mistral-correlation-id: + - 019de4f2-ffc5-76cf-962e-1048105ba417 + set-cookie: + - __cf_bm=lIlEf60G1s4B9wDiNzvw2IARjvn3WggrWy.oOjMJDyg-1777662623.6203983-1.0.1.1-_JIGF3WKBF3RfpRdqKYeksF1ApxRHb.R0YMMa5BzF0dTAIUr1JerKW_W3B_5x5KD5fvdumzkX7eYCsKGgk3oTISwwkVGJEkxJgYyy6Tqp7lONT2QnMeJgI3L0RkqGfpq; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:23 + GMT + - _cfuvid=0ob6NZHFPHGUjgQK4V.XaKBLjQjTuAyuIoMg6nfilrk-1777662623.6203983-1.0.1.1-KCo54SoR2UC8bdXrXJYhObLo7RCbzCAQJoNd.0XI60g; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '89' + x-kong-proxy-latency: + - '20' + x-kong-request-id: + - 019de4f2-ffc5-76cf-962e-1048105ba417 + x-kong-upstream-latency: + - '91' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Use get_weather for Paris. Do not answer directly.","role":"user"}],"agent_id":"ag_019de4f30009709abee525d209718f05","max_tokens":100,"stream":false,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '393' + Cookie: + - __cf_bm=lIlEf60G1s4B9wDiNzvw2IARjvn3WggrWy.oOjMJDyg-1777662623.6203983-1.0.1.1-_JIGF3WKBF3RfpRdqKYeksF1ApxRHb.R0YMMa5BzF0dTAIUr1JerKW_W3B_5x5KD5fvdumzkX7eYCsKGgk3oTISwwkVGJEkxJgYyy6Tqp7lONT2QnMeJgI3L0RkqGfpq; + _cfuvid=0ob6NZHFPHGUjgQK4V.XaKBLjQjTuAyuIoMg6nfilrk-1777662623.6203983-1.0.1.1-KCo54SoR2UC8bdXrXJYhObLo7RCbzCAQJoNd.0XI60g + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/agents/completions + response: + body: + string: '{"id":"5de45163dfef4c7c99a42adccc3a7dbe","created":1777662624,"model":"mistral-small-latest","usage":{"prompt_tokens":100,"total_tokens":112,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","tool_calls":[{"id":"K7P01JvBF","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}],"content":""}}]}' + headers: + CF-RAY: + - 9f511607fb3c3b8e-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:10:24 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '447' + mistral-correlation-id: + - 019de4f3-0137-73b2-8cc8-76ebdf556a31 + set-cookie: + - __cf_bm=u8V74eetYN.ZO3MA77INKkBR5vwTgHOPz7lqliwhGSs-1777662623.99174-1.0.1.1-DYDBLvxf2H5MXpHvZ.Z0l8bxCXKjaU8bhj_4bJ4M9dW3PJ3C4tO9qBYeieYmY2Hqb11FfBMVp4k_E0G8k6jikOJ80knF3pl5Bb.4Jj5M0EdnnobxftCUMzILq0Iag7Ue; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:24 + GMT + - _cfuvid=Lmhhe7DR0BHu_TGWVhlJnYSsVIlB.aM_M8fr4x75iK0-1777662623.99174-1.0.1.1-.Dy1RnMV7a58kIwtfgZpqEPdWlQMFPidGhSdQnBtQ.k; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '380' + x-kong-proxy-latency: + - '12' + x-kong-request-id: + - 019de4f3-0137-73b2-8cc8-76ebdf556a31 + x-kong-upstream-latency: + - '382' + x-ratelimit-limit-req-minute: + - '400' + x-ratelimit-limit-tokens-minute: + - '1500000' + x-ratelimit-remaining-req-minute: + - '397' + x-ratelimit-remaining-tokens-minute: + - '1499688' + x-ratelimit-tokens-query-cost: + - '112' + status: + code: 200 + message: OK +- request: + body: '' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Cookie: + - __cf_bm=u8V74eetYN.ZO3MA77INKkBR5vwTgHOPz7lqliwhGSs-1777662623.99174-1.0.1.1-DYDBLvxf2H5MXpHvZ.Z0l8bxCXKjaU8bhj_4bJ4M9dW3PJ3C4tO9qBYeieYmY2Hqb11FfBMVp4k_E0G8k6jikOJ80knF3pl5Bb.4Jj5M0EdnnobxftCUMzILq0Iag7Ue; + _cfuvid=Lmhhe7DR0BHu_TGWVhlJnYSsVIlB.aM_M8fr4x75iK0-1777662623.99174-1.0.1.1-.Dy1RnMV7a58kIwtfgZpqEPdWlQMFPidGhSdQnBtQ.k + Host: + - api.mistral.ai + user-agent: + - mistral-client-python/2.4.4 + method: DELETE + uri: https://api.mistral.ai/v1/agents/ag_019de4f30009709abee525d209718f05 + response: + body: + string: '' + headers: + CF-RAY: + - 9f51160cad28aa96-YYZ + Connection: + - keep-alive + Date: + - Fri, 01 May 2026 19:10:24 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-042e-76aa-a8c1-54a8c5d0c8e2 + set-cookie: + - __cf_bm=HfJz7ggN9Yu9fzl0DPmgZ5tXJqfmRSTHQ6K_vqGsFaU-1777662624.7501178-1.0.1.1-mkgsWgJA923pNmkHFQ_U77_WwjSrTDwTkrk4LvJnpyRUDMo0vznYz9Tow5Md2BYhG8IPml0SrCpueQ..0LSkqbbkZgaATvhfIWETsFkYcXZVe_wvGbdV985dsVRsg9Rn; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:40:24 + GMT + - _cfuvid=SrUw9w5OF46oqeazkl1WDjGt.NOmiADYpj.OB43J_vg-1777662624.7501178-1.0.1.1-ucku8NSia0iqm75wVnUkM3MiqZP9MqSWBIXzUzx4VTY; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '61' + x-kong-proxy-latency: + - '18' + x-kong-request-id: + - 019de4f3-042e-76aa-a8c1-54a8c5d0c8e2 + x-kong-upstream-latency: + - '63' + status: + code: 204 + message: No Content +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_stream_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_stream_tool_spans.yaml new file mode 100644 index 00000000..49ddfd4a --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_stream_tool_spans.yaml @@ -0,0 +1,194 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","name":"braintrust-test-agent-1777662665590","instructions":"You + are concise. Keep responses under five words."}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '144' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/agents + response: + body: + string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"mistral-small-latest","name":"braintrust-test-agent-1777662665590","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_019de4f3a44d76848d65aa17474d2023","version":0,"versions":[],"created_at":"2026-05-01T19:11:05.836676Z","updated_at":"2026-05-01T19:11:05.836680Z","deployment_chat":false,"source":"api","version_message":null}' + headers: + CF-RAY: + - 9f51170c99e511fc-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:11:05 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '692' + mistral-correlation-id: + - 019de4f3-a41e-714b-acfa-824d4000ca1d + set-cookie: + - __cf_bm=cGW.3HaCZhtD_IZOzE8R2ZM64w06vyhZSav_1O02Yvw-1777662665.6980782-1.0.1.1-ET4Yx9bfjonMs9td.HrkyTxtV0y3ez1HV_.mRhhAf1kbNtetj_PYhhKHp4OhveXtB3XIo4FLvqlxmvWXEKwfZzMKsg2lqxe180fAtVEPQd6U8BQ8BkUpW.kpg1HS.Ag9; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:41:05 + GMT + - _cfuvid=9Jrd6RtErkXLuzlAMyznjSREN5bXjWaJ5doZa.LinSU-1777662665.6980782-1.0.1.1-04VvCQJUlFDKheXfoXpgh1VbeDONnfE7Nt4N5cGNIx8; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '83' + x-kong-proxy-latency: + - '23' + x-kong-request-id: + - 019de4f3-a41e-714b-acfa-824d4000ca1d + x-kong-upstream-latency: + - '85' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Use get_weather for Paris. Do not answer directly.","role":"user"}],"agent_id":"ag_019de4f3a44d76848d65aa17474d2023","max_tokens":100,"stream":true,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '392' + Cookie: + - __cf_bm=cGW.3HaCZhtD_IZOzE8R2ZM64w06vyhZSav_1O02Yvw-1777662665.6980782-1.0.1.1-ET4Yx9bfjonMs9td.HrkyTxtV0y3ez1HV_.mRhhAf1kbNtetj_PYhhKHp4OhveXtB3XIo4FLvqlxmvWXEKwfZzMKsg2lqxe180fAtVEPQd6U8BQ8BkUpW.kpg1HS.Ag9; + _cfuvid=9Jrd6RtErkXLuzlAMyznjSREN5bXjWaJ5doZa.LinSU-1777662665.6980782-1.0.1.1-04VvCQJUlFDKheXfoXpgh1VbeDONnfE7Nt4N5cGNIx8 + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/agents/completions + response: + body: + string: 'data: {"id":"080abb87139b4de784ae5c199269ffce","object":"chat.completion.chunk","created":1777662666,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + + + data: {"id":"080abb87139b4de784ae5c199269ffce","object":"chat.completion.chunk","created":1777662666,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"OD88wuikZ","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"total_tokens":112,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"p":"abcdefghijklmnopqrstuvwxyz01234"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9f51170e8e54543d-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Fri, 01 May 2026 19:11:06 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-a63a-77e1-b77d-67e4a587cf06 + x-envoy-upstream-service-time: + - '390' + x-kong-proxy-latency: + - '14' + x-kong-request-id: + - 019de4f3-a63a-77e1-b77d-67e4a587cf06 + x-kong-upstream-latency: + - '391' + status: + code: 200 + message: OK +- request: + body: '' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Cookie: + - __cf_bm=cGW.3HaCZhtD_IZOzE8R2ZM64w06vyhZSav_1O02Yvw-1777662665.6980782-1.0.1.1-ET4Yx9bfjonMs9td.HrkyTxtV0y3ez1HV_.mRhhAf1kbNtetj_PYhhKHp4OhveXtB3XIo4FLvqlxmvWXEKwfZzMKsg2lqxe180fAtVEPQd6U8BQ8BkUpW.kpg1HS.Ag9; + _cfuvid=9Jrd6RtErkXLuzlAMyznjSREN5bXjWaJ5doZa.LinSU-1777662665.6980782-1.0.1.1-04VvCQJUlFDKheXfoXpgh1VbeDONnfE7Nt4N5cGNIx8 + Host: + - api.mistral.ai + user-agent: + - mistral-client-python/2.4.4 + method: DELETE + uri: https://api.mistral.ai/v1/agents/ag_019de4f3a44d76848d65aa17474d2023 + response: + body: + string: '' + headers: + CF-RAY: + - 9f5117147f4c1486-YYZ + Connection: + - keep-alive + Date: + - Fri, 01 May 2026 19:11:07 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f3-a907-7bcc-ab7b-29525c142df3 + x-envoy-upstream-service-time: + - '71' + x-kong-proxy-latency: + - '27' + x-kong-request-id: + - 019de4f3-a907-7bcc-ab7b-29525c142df3 + x-kong-upstream-latency: + - '72' + status: + code: 204 + message: No Content +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_tool_spans.yaml new file mode 100644 index 00000000..c6317f82 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_tool_spans.yaml @@ -0,0 +1,81 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","messages":[{"content":"Use get_weather + for Paris. Do not answer directly.","role":"user"}],"max_tokens":100,"stream":false,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '375' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/chat/completions + response: + body: + string: '{"id":"aab6fae0e8db47b99507b7e630f76156","created":1777662585,"model":"mistral-small-latest","usage":{"prompt_tokens":88,"total_tokens":100,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","tool_calls":[{"id":"JJ6iKa60d","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}],"content":""}}]}' + headers: + CF-RAY: + - 9f511515febfa204-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 May 2026 19:09:45 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '446' + mistral-correlation-id: + - 019de4f2-69ff-75db-a6c9-804071321149 + set-cookie: + - __cf_bm=pDVmOBtlEIvxElp5jKzQMCfMdnSh9HzYKLwj3_6envA-1777662585.2773309-1.0.1.1-AmEPVV00gabNnTuWp_ZUDNJa2KQYIzYfNNYGyWCdnyxwM5e0xNzqIBbz5GDL1Skxk0dSj4QYAPqNEarMDDNiRIKGXz1m8PV8qsX7s9aX_QVQAQ8kqM4KV1dMmASfvgTH; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:39:45 + GMT + - _cfuvid=Fz8ContgsXYQ4OcuM8y3FTcjrh0eUuH5GuMOYdDNU_Y-1777662585.2773309-1.0.1.1-372hLjX.B3gHLdc6O5zIBb2W0qykUdNuAU9yl5Fx58M; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '213' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 019de4f2-69ff-75db-a6c9-804071321149 + x-kong-upstream-latency: + - '214' + x-ratelimit-limit-req-minute: + - '400' + x-ratelimit-limit-tokens-minute: + - '1500000' + x-ratelimit-remaining-req-minute: + - '399' + x-ratelimit-remaining-tokens-minute: + - '1499900' + x-ratelimit-tokens-query-cost: + - '100' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_tool_spans.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_tool_spans.yaml new file mode 100644 index 00000000..d8c945b8 --- /dev/null +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_tool_spans.yaml @@ -0,0 +1,78 @@ +interactions: +- request: + body: '{"model":"mistral-small-latest","messages":[{"content":"Use get_weather + for Paris. Do not answer directly.","role":"user"}],"max_tokens":100,"stream":true,"tools":[{"function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]},"description":"Get + the weather for a city."},"type":"function"}],"tool_choice":"any"}' + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '374' + Host: + - api.mistral.ai + content-type: + - application/json + user-agent: + - mistral-client-python/2.4.4 + method: POST + uri: https://api.mistral.ai/v1/chat/completions + response: + body: + string: 'data: {"id":"f261266e6e9a48b5aa2ad2b0a45c06e5","object":"chat.completion.chunk","created":1777662585,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + + + data: {"id":"f261266e6e9a48b5aa2ad2b0a45c06e5","object":"chat.completion.chunk","created":1777662585,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"FDBZmkbqC","function":{"name":"get_weather","arguments":"{\"city\": + \"Paris\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":88,"total_tokens":100,"completion_tokens":12,"prompt_tokens_details":{"cached_tokens":0}},"p":"abcdefghijklmnopqrstuvwxyz"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9f5115198b50ac78-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Fri, 01 May 2026 19:09:46 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 019de4f2-6c33-7b2d-a213-1abeb6dfc29a + set-cookie: + - __cf_bm=oSPh5osVD2X6VF2vwusnTGm1neeZSIluEXM2bhKQEmA-1777662585.843838-1.0.1.1-m3ydINeOzjPhcKZINd9qYr4Jco_ZKkGbtYdf0Eeaz0VHEIsJxEffHSXAENMdXHZAhDlZTYmLrDGfAptGhdOxTWQ3berHApphrSlOv0OptsStvdi8PiGNihAScTvt.YER; + HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 01 May 2026 19:39:46 + GMT + - _cfuvid=Jk04Ic4Ws7RJLJT7wYGrmWJJZq7qT0LQ_X7eSvVWmPM-1777662585.843838-1.0.1.1-MKVms03WNVz96djZZUMO5N2nSkmxEfYMX7ORsc82vBw; + HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai + x-envoy-upstream-service-time: + - '283' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 019de4f2-6c33-7b2d-a213-1abeb6dfc29a + x-kong-upstream-latency: + - '284' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 5c030bb3..37b7161f 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -19,7 +19,7 @@ ) from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.span_types import SpanTypeAttribute -from braintrust.test_helpers import init_test_logger +from braintrust.test_helpers import find_spans_by_type, init_test_logger pytest.importorskip("mistralai") @@ -179,6 +179,21 @@ def _get_client(): return Mistral(api_key=os.environ.get("MISTRAL_API_KEY")) +def _weather_tool(): + return { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + + def _assert_ocr_metrics_are_valid(metrics, start, end): assert metrics["duration"] >= 0 assert metrics["pages_processed"] >= 1 @@ -230,6 +245,47 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): assert_metrics_are_valid(span["metrics"], start, end) +@pytest.mark.vcr +def test_wrap_mistral_chat_complete_tool_spans(memory_logger): + assert not memory_logger.pop() + + client = wrap_mistral(_get_client()) + response = client.chat.complete( + model=CHAT_MODEL, + messages=[{"role": "user", "content": "Use get_weather for Paris. Do not answer directly."}], + tools=[_weather_tool()], + tool_choice="any", + max_tokens=100, + ) + + assert response.choices[0].message.tool_calls + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + spans = memory_logger.pop() + assert len(find_spans_by_type(spans, SpanTypeAttribute.LLM)) == 1 + assert len(find_spans_by_type(spans, SpanTypeAttribute.TOOL)) == 1 + + +@pytest.mark.vcr +def test_wrap_mistral_chat_stream_tool_spans(memory_logger): + assert not memory_logger.pop() + + client = wrap_mistral(_get_client()) + with client.chat.stream( + model=CHAT_MODEL, + messages=[{"role": "user", "content": "Use get_weather for Paris. Do not answer directly."}], + tools=[_weather_tool()], + tool_choice="any", + max_tokens=100, + ) as stream: + chunks = list(stream) + + assert chunks + spans = memory_logger.pop() + assert len(find_spans_by_type(spans, SpanTypeAttribute.LLM)) == 1 + assert len(find_spans_by_type(spans, SpanTypeAttribute.TOOL)) == 1 + + @pytest.mark.vcr def test_wrap_mistral_chat_stream_sync(memory_logger): assert not memory_logger.pop() @@ -411,6 +467,48 @@ def test_wrap_mistral_beta_conversations_restart_sync(memory_logger): ) +@pytest.mark.vcr +def test_wrap_mistral_agents_complete_tool_spans(memory_logger): + assert not memory_logger.pop() + + client = wrap_mistral(_get_client()) + with _temporary_agent(client) as agent_id: + response = client.agents.complete( + agent_id=agent_id, + messages=[{"role": "user", "content": "Use get_weather for Paris. Do not answer directly."}], + tools=[_weather_tool()], + tool_choice="any", + max_tokens=100, + ) + + assert response.choices[0].message.tool_calls + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + spans = memory_logger.pop() + assert len(find_spans_by_type(spans, SpanTypeAttribute.LLM)) == 1 + assert len(find_spans_by_type(spans, SpanTypeAttribute.TOOL)) == 1 + + +@pytest.mark.vcr +def test_wrap_mistral_agents_stream_tool_spans(memory_logger): + assert not memory_logger.pop() + + client = wrap_mistral(_get_client()) + with _temporary_agent(client) as agent_id: + with client.agents.stream( + agent_id=agent_id, + messages=[{"role": "user", "content": "Use get_weather for Paris. Do not answer directly."}], + tools=[_weather_tool()], + tool_choice="any", + max_tokens=100, + ) as stream: + chunks = list(stream) + + assert chunks + spans = memory_logger.pop() + assert len(find_spans_by_type(spans, SpanTypeAttribute.LLM)) == 1 + assert len(find_spans_by_type(spans, SpanTypeAttribute.TOOL)) == 1 + + @pytest.mark.vcr def test_wrap_mistral_agents_complete_sync(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index 11046872..6928170a 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -1,5 +1,6 @@ """Mistral-specific tracing helpers.""" +import json import logging import re import time @@ -17,6 +18,7 @@ ) from braintrust.logger import start_span from braintrust.span_types import SpanTypeAttribute +from braintrust.util import clean_nones logger = logging.getLogger(__name__) @@ -952,11 +954,75 @@ def _aggregate_conversation_events(items: list[Any]) -> dict[str, Any]: return result +def _maybe_parse_tool_arguments(arguments: Any) -> Any: + if not isinstance(arguments, str): + return arguments + try: + return json.loads(arguments) + except json.JSONDecodeError: + return arguments + + +def _completion_tool_calls(response_data: dict[str, Any] | None) -> list[tuple[int, int, dict[str, Any]]]: + if not response_data: + return [] + + tool_calls = [] + choices = response_data.get("choices") + if not isinstance(choices, list): + return [] + + for choice_index, choice in enumerate(choices): + if not isinstance(choice, dict): + continue + message = choice.get("message") + if not isinstance(message, dict): + continue + choice_tool_calls = message.get("tool_calls") + if not isinstance(choice_tool_calls, list): + continue + for tool_index, tool_call in enumerate(choice_tool_calls): + if isinstance(tool_call, dict): + tool_calls.append((choice_index, tool_index, tool_call)) + return tool_calls + + +def _log_completion_tool_spans(response_data: dict[str, Any] | None, *, parent_span: Any) -> None: + tool_calls = _completion_tool_calls(response_data) + if not tool_calls: + return + + parent_export = parent_span.export() if parent_span is not None else None + for choice_index, tool_index, tool_call in tool_calls: + function = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {} + tool_type = tool_call.get("type") or ("function" if function else None) + name = function.get("name") or tool_type or "tool" + span_args = { + "name": f"tool: {name}", + "type": SpanTypeAttribute.TOOL, + "input": _maybe_parse_tool_arguments(function.get("arguments")), + "metadata": clean_nones( + { + "tool_call_id": tool_call.get("id"), + "tool_type": tool_type, + "tool_index": tool_call.get("index", tool_index), + "choice_index": choice_index, + } + ) + or None, + } + if parent_export is not None: + span_args["parent"] = parent_export + with start_span(**span_args): + pass + + def _finalize_completion_response(span: Any, request_metadata: dict[str, Any], response: Any, start_time: float): response_data = _normalized_mistral_dict(response) response_metadata = _response_data_to_metadata(response_data) usage = response_data.get("usage") if response_data else None + _log_completion_tool_spans(response_data, parent_span=span) _log_and_end_span( span, output=response_data.get("choices") if response_data else None, @@ -1028,6 +1094,7 @@ def _finalize_completion_stream( ): response = _aggregate_completion_events(items) response_metadata = _response_data_to_metadata(response) + _log_completion_tool_spans(response, parent_span=span) _log_and_end_span( span, output=response.get("choices"),