Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion py/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ latest = "autoevals==0.2.0"
"0.0.129" = "autoevals==0.0.129"

[tool.braintrust.matrix.google-genai]
latest = "google-genai==1.73.1"
latest = "google-genai==1.74.0"
"1.30.0" = "google-genai==1.30.0"

[tool.braintrust.matrix.dspy]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Test auto_instrument for Google GenAI (no uninstrument available)."""

import os

from braintrust.auto import auto_instrument
from braintrust.integrations.test_utils import autoinstrument_test_context

Expand All @@ -19,7 +21,11 @@

client = Client()
response = client.models.generate_content(
model="gemini-2.0-flash-001",
model=(
"gemini-3.1-flash-lite-preview"
if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest"
else "gemini-2.0-flash-001"
),
contents="Say hi",
config=types.GenerateContentConfig(max_output_tokens=100),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ interactions:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate, zstd
- gzip, deflate
Connection:
- keep-alive
Content-Length:
Expand All @@ -16,34 +16,33 @@ interactions:
Host:
- generativelanguage.googleapis.com
user-agent:
- google-genai-sdk/1.60.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
x-goog-api-client:
- google-genai-sdk/1.60.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
method: POST
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:generateContent
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-lite-preview:generateContent
response:
body:
string: !!binary |
H4sIAAAAAAAC/61Ry07DMBC85yuMz03lpA8eF4QKUgtUVBDxRpUh28TCtSPboURV/x0naVoHrvhg
rXZmd3Zn1x5C+IOKmMXUgMYn6MVmEFpXf4lJYUAYCzQpm8yoMntu/dZObCkGvssiPGbIpKDgAI3l
ClklNEEp8AwVMkdGxrQ4fRXYqd3s4rfOXlFJDmW7pYyBN/RNQ8ALJphOb4FqKUraXXQzwzuUfiXX
MsmUfC+H9kmXBIMjQsKgPwyHg8Ow1z/2GulKFOeaJjAFQ60rdLc7ti2WmYnkJ4iRzCtXwlrF8bAF
B8EWN9JQ3oZ6nT9d9bnVZNz11rHdLk85M0W5YXTxGGHHINMaqjHIc3z8PeI/aQVBW8zb3qU+1T0o
zeqbJLC0V/LDLvEXnOrUJySoumIFOpNCwyQuebNBVtDnq4fLp3k6z6czrUaKnCXY23g/hhi4Mq4C
AAA=
string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\":
[\n {\n \"text\": \"Hi! How can I help you today?\",\n
\ \"thoughtSignature\": \"EjQKMgEMOdbHN1lJWA89gycYPMD3z+d6uLyVv8JHe6Ez0OHI/luhPiV5HI8Oo7aM8zfWN9oH\"\n
\ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\":
\"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\":
3,\n \"candidatesTokenCount\": 9,\n \"totalTokenCount\": 12,\n \"promptTokensDetails\":
[\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 3\n }\n
\ ]\n },\n \"modelVersion\": \"gemini-3.1-flash-lite-preview\",\n \"responseId\":
\"X3PzaaCuFKy6_uMP-fHewQs\"\n}\n"
headers:
Alt-Svc:
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
Content-Encoding:
- gzip
Content-Type:
- application/json; charset=UTF-8
Date:
- Thu, 22 Jan 2026 21:44:31 GMT
- Thu, 30 Apr 2026 15:21:03 GMT
Server:
- scaffolding on HTTPServer2
Server-Timing:
- gfet4t7; dur=430
- gfet4t7; dur=1029
Transfer-Encoding:
- chunked
Vary:
Expand All @@ -54,8 +53,12 @@ interactions:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Gemini-Service-Tier:
- standard
X-XSS-Protection:
- '0'
content-length:
- '659'
status:
code: 200
message: OK
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,40 +3,41 @@ interactions:
body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role":
"user"}], "generationConfig": {"maxOutputTokens": 100}}'
headers:
accept:
Accept:
- '*/*'
accept-encoding:
Accept-Encoding:
- gzip, deflate
connection:
Connection:
- keep-alive
content-length:
Content-Length:
- '133'
content-type:
Content-Type:
- application/json
host:
Host:
- generativelanguage.googleapis.com
user-agent:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
x-goog-api-client:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
method: POST
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:streamGenerateContent?alt=sse
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-lite-preview:streamGenerateContent?alt=sse
response:
body:
string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The\"}],\"role\":
\"model\"}}],\"usageMetadata\": {\"promptTokenCount\": 8,\"totalTokenCount\":
8,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 8}]},\"modelVersion\":
\"gemini-2.0-flash-001\",\"responseId\": \"_qTiaLjwK6vpgbUP2uvA-QU\"}\r\n\r\ndata:
{\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" capital of France\"}],\"role\":
\"model\"}}],\"usageMetadata\": {\"promptTokenCount\": 8,\"totalTokenCount\":
8,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 8}]},\"modelVersion\":
\"gemini-2.0-flash-001\",\"responseId\": \"_qTiaLjwK6vpgbUP2uvA-QU\"}\r\n\r\ndata:
{\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" is Paris.\\n\"}],\"role\":
\"model\"},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\":
7,\"candidatesTokenCount\": 8,\"totalTokenCount\": 15,\"promptTokensDetails\":
[{\"modality\": \"TEXT\",\"tokenCount\": 7}],\"candidatesTokensDetails\":
[{\"modality\": \"TEXT\",\"tokenCount\": 8}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\":
\"_qTiaLjwK6vpgbUP2uvA-QU\"}\r\n\r\n"
string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The
capital\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\":
8,\"candidatesTokenCount\": 2,\"totalTokenCount\": 10,\"promptTokensDetails\":
[{\"modality\": \"TEXT\",\"tokenCount\": 8}]},\"modelVersion\": \"gemini-3.1-flash-lite-preview\",\"responseId\":
\"InPzaZ3qM_OW_uMPgIPKgAg\"}\r\n\r\ndata: {\"candidates\": [{\"content\":
{\"parts\": [{\"text\": \" of France is Paris.\"}],\"role\": \"model\"},\"index\":
0}],\"usageMetadata\": {\"promptTokenCount\": 8,\"candidatesTokenCount\":
7,\"totalTokenCount\": 15,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\":
8}]},\"modelVersion\": \"gemini-3.1-flash-lite-preview\",\"responseId\": \"InPzaZ3qM_OW_uMPgIPKgAg\"}\r\n\r\ndata:
{\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\",\"thoughtSignature\":
\"EjQKMgEMOdbHSvZ6i6U0AqmiTPMv1nuMqGtV/EjgT0eIv2dKVM/WLFZe+TeYLcWCrlHdMp9v\"}],\"role\":
\"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\":
8,\"candidatesTokenCount\": 7,\"totalTokenCount\": 15,\"promptTokensDetails\":
[{\"modality\": \"TEXT\",\"tokenCount\": 8}]},\"modelVersion\": \"gemini-3.1-flash-lite-preview\",\"responseId\":
\"InPzaZ3qM_OW_uMPgIPKgAg\"}\r\n\r\n"
headers:
Alt-Svc:
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
Expand All @@ -45,11 +46,11 @@ interactions:
Content-Type:
- text/event-stream
Date:
- Sun, 05 Oct 2025 17:03:58 GMT
- Thu, 30 Apr 2026 15:20:02 GMT
Server:
- scaffolding on HTTPServer2
Server-Timing:
- gfet4t7; dur=391
- gfet4t7; dur=635
Transfer-Encoding:
- chunked
Vary:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,46 @@ interactions:
body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role":
"user"}], "generationConfig": {"maxOutputTokens": 100}}'
headers:
accept:
Accept:
- '*/*'
accept-encoding:
Accept-Encoding:
- gzip, deflate
connection:
Connection:
- keep-alive
content-length:
Content-Length:
- '133'
content-type:
Content-Type:
- application/json
host:
Host:
- generativelanguage.googleapis.com
user-agent:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
x-goog-api-client:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
method: POST
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:generateContent
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-lite-preview:generateContent
response:
body:
string: !!binary |
H4sIAAAAAAAC/61RTU/DMAy991dEOa9T1jFWcYMNJhCICSqEBAiZ1uuitUlJsvEx7b+TtmuXwpUc
Isvv2c9+3nqE0BhEwhMwqOkJebIZQrbVX2JSGBTGAk3KJgtQ5sCt39aJLcXgZ1lEoyWSGApuICNy
QS4UiBgJ12QOiuv+s6BO3a6NX3oHNSUzLFvlMsGsoe8aAl1wwfXyDkFLUdLuo9s5bVHYpNcyLZR8
Kwf2WZ8Nw+NwfDRkwyAYjFkwCr1GuhKlaw0p3qAB6wi0e1PbIi9MJFcoJnJdOTKuVRz/OnC4h420
y3eQwaj3p6meWkmeubY6jtvdIePmq7L0/DGijj+mM1Pjj+fY+HvCf9IKu1re/ir1oR5QaV5fJMXc
3sgP+sxfZKCXPmODqilVqAspNF4mJe/1PeJwNZudQf6dm9WcbSbTj1NGvZ33A7z5p7+oAgAA
string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\":
[\n {\n \"text\": \"The capital of France is Paris.\",\n
\ \"thoughtSignature\": \"EjQKMgEMOdbHr79RjAy6QLknd/Xh1snrpBl9Cp5L5kUqjtOkUYJDeTCClNymmjJOiu0eWCen\"\n
\ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\":
\"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\":
8,\n \"candidatesTokenCount\": 7,\n \"totalTokenCount\": 15,\n \"promptTokensDetails\":
[\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 8\n }\n
\ ]\n },\n \"modelVersion\": \"gemini-3.1-flash-lite-preview\",\n \"responseId\":
\"InPzaf9869b-4w_ZjtWACA\"\n}\n"
headers:
Alt-Svc:
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
Content-Encoding:
- gzip
Content-Type:
- application/json; charset=UTF-8
Date:
- Sun, 05 Oct 2025 17:03:58 GMT
- Thu, 30 Apr 2026 15:20:02 GMT
Server:
- scaffolding on HTTPServer2
Server-Timing:
- gfet4t7; dur=390
- gfet4t7; dur=2097
Transfer-Encoding:
- chunked
Vary:
Expand All @@ -53,8 +53,12 @@ interactions:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Gemini-Service-Tier:
- standard
X-XSS-Protection:
- '0'
content-length:
- '660'
status:
code: 200
message: OK
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,46 @@ interactions:
body: '{"contents": [{"parts": [{"text": "What is the capital of France?"}], "role":
"user"}], "generationConfig": {"maxOutputTokens": 100}}'
headers:
accept:
Accept:
- '*/*'
accept-encoding:
Accept-Encoding:
- gzip, deflate
connection:
Connection:
- keep-alive
content-length:
Content-Length:
- '133'
content-type:
Content-Type:
- application/json
host:
Host:
- generativelanguage.googleapis.com
user-agent:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
x-goog-api-client:
- google-genai-sdk/1.41.0 gl-python/3.13.3
- google-genai-sdk/1.74.0 gl-python/3.12.12
method: POST
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:generateContent
uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-lite-preview:generateContent
response:
body:
string: !!binary |
H4sIAAAAAAAC/61RUUvDMBB+768IeV5H2rG1+iZTQdjY0CLCFDnXaxfWJjXJxDH2303bdUv11TyE
477v7rv77uARQtcgUp6CQU2vycpmCDk0f41JYVAYC3Qpm6xAmQu3fQcnthSD33URTTZI1lBxAwWR
GblXINZIuCZLUFwPXwV16o7n+G1wUVOywLpVKVMsOvqxI9CMC643jwhaipr2lCyW9IzCVz6TeaXk
Rz2wz4ZsHMfhKIhHQRRfsWAShV4n3YjSnYYc52jAOgLnvaltUVYmkVsUU7lrHIlaFce/HhyfYCPt
8j0kGA/+NNW3VpIXrq2O43Z3KLjZN5bevSTU8cf0Zur88Rwbf0/4T1pxX8s7XaU91DMqzduL5Fja
G/nhkPlZAXrjMxY0TalCXUmh8SGtee+ThMOinE8/J/vSbJcRwmx/w6h39H4AjvRKC6gCAAA=
string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\":
[\n {\n \"text\": \"The capital of France is Paris.\",\n
\ \"thoughtSignature\": \"EjQKMgEMOdbHMOZK43pYvf3A+807eMoGls15SwPG82ofiMkBO4hBMLEb5q8R+1pbHqySdVGP\"\n
\ }\n ],\n \"role\": \"model\"\n },\n \"finishReason\":
\"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\":
8,\n \"candidatesTokenCount\": 7,\n \"totalTokenCount\": 15,\n \"promptTokensDetails\":
[\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 8\n }\n
\ ]\n },\n \"modelVersion\": \"gemini-3.1-flash-lite-preview\",\n \"responseId\":
\"I3PzacShL_ac_uMP_rqQiQg\"\n}\n"
headers:
Alt-Svc:
- h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
Content-Encoding:
- gzip
Content-Type:
- application/json; charset=UTF-8
Date:
- Sun, 05 Oct 2025 17:03:59 GMT
- Thu, 30 Apr 2026 15:20:03 GMT
Server:
- scaffolding on HTTPServer2
Server-Timing:
- gfet4t7; dur=321
- gfet4t7; dur=808
Transfer-Encoding:
- chunked
Vary:
Expand All @@ -53,8 +53,12 @@ interactions:
- nosniff
X-Frame-Options:
- SAMEORIGIN
X-Gemini-Service-Tier:
- standard
X-XSS-Protection:
- '0'
content-length:
- '661'
status:
code: 200
message: OK
Expand Down
Loading