Skip to content

Commit ea712e2

Browse files
realarkrealark
committed
wip -- initial spec test runner for oai + anthropic
1 parent bbbd6ec commit ea712e2

21 files changed

Lines changed: 2899 additions & 0 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,6 @@ Dockerfile.local
2222
CLAUDE.local.md
2323
.zed
2424
tsconfig.tsbuildinfo
25+
26+
# BTX spec cache
27+
py/src/braintrust/btx/.spec-cache/

‎py/noxfile.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ def _pinned_python_version():
123123
CONTRIB_DIR = "braintrust/contrib"
124124
DEVSERVER_DIR = "braintrust/devserver"
125125
TYPE_TESTS_DIR = "braintrust/type_tests"
126+
BTX_DIR = "braintrust/btx"
126127

127128

128129
SILENT_INSTALLS = True
@@ -183,6 +184,16 @@ def test_openai_http2_streaming(session, version):
183184
_run_tests(session, f"{INTEGRATION_DIR}/openai/test_openai_http2.py", version=version)
184185

185186

187+
@nox.session()
188+
def test_btx(session):
189+
"""Run the BTX cross-language LLM-span spec tests (OpenAI + Anthropic providers)."""
190+
_install_test_deps(session)
191+
_install(session, "openai")
192+
_install(session, "anthropic")
193+
session.install("pyyaml")
194+
_run_tests(session, "braintrust/btx")
195+
196+
186197
OPENAI_AGENTS_VERSIONS = _get_matrix_versions("openai-agents")
187198

188199

@@ -533,6 +544,7 @@ def _run_core_tests(session):
533544
CONTRIB_DIR,
534545
DEVSERVER_DIR,
535546
TYPE_TESTS_DIR,
547+
BTX_DIR,
536548
],
537549
)
538550

‎py/src/braintrust/btx/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# BTX: Cross-language SDK spec tests for the Braintrust Python SDK.
2+
# Specs are fetched from braintrustdata/braintrust-spec at a pinned ref.

‎py/src/braintrust/btx/cassettes/anthropic/.gitkeep‎

Whitespace-only changes.

‎py/src/braintrust/btx/cassettes/anthropic/attachments.yaml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
interactions:
2+
- request:
3+
body: '{"max_tokens":128,"messages":[{"role":"user","content":[{"type":"text","text":"What
4+
color is this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"claude-haiku-4-5-20251001","temperature":0.0}'
5+
headers:
6+
Accept:
7+
- application/json
8+
Accept-Encoding:
9+
- gzip, deflate
10+
Connection:
11+
- keep-alive
12+
Content-Length:
13+
- '339'
14+
Content-Type:
15+
- application/json
16+
Host:
17+
- api.anthropic.com
18+
User-Agent:
19+
- Anthropic/Python 0.95.0
20+
X-Stainless-Arch:
21+
- arm64
22+
X-Stainless-Async:
23+
- 'false'
24+
X-Stainless-Lang:
25+
- python
26+
X-Stainless-OS:
27+
- MacOS
28+
X-Stainless-Package-Version:
29+
- 0.95.0
30+
X-Stainless-Runtime:
31+
- CPython
32+
X-Stainless-Runtime-Version:
33+
- 3.14.3
34+
anthropic-version:
35+
- '2023-06-01'
36+
x-stainless-read-timeout:
37+
- '600'
38+
x-stainless-retry-count:
39+
- '0'
40+
x-stainless-timeout:
41+
- '600'
42+
method: POST
43+
uri: https://api.anthropic.com/v1/messages
44+
response:
45+
body:
46+
string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01FL9Ud1XCdeD4speKTR3aLJ","type":"message","role":"assistant","content":[{"type":"text","text":"This
47+
image appears to be **red** (or a reddish color). It looks like a small red
48+
dot or mark against a white background."}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":33,"service_tier":"standard","inference_geo":"not_available"}}'
49+
headers:
50+
CF-RAY:
51+
- 9ece8971ffea763c-SEA
52+
Connection:
53+
- keep-alive
54+
Content-Security-Policy:
55+
- default-src 'none'; frame-ancestors 'none'
56+
Content-Type:
57+
- application/json
58+
Date:
59+
- Wed, 15 Apr 2026 22:55:17 GMT
60+
Server:
61+
- cloudflare
62+
Transfer-Encoding:
63+
- chunked
64+
X-Robots-Tag:
65+
- none
66+
anthropic-organization-id:
67+
- 27796668-7351-40ac-acc4-024aee8995a5
68+
anthropic-ratelimit-input-tokens-limit:
69+
- '4000000'
70+
anthropic-ratelimit-input-tokens-remaining:
71+
- '4000000'
72+
anthropic-ratelimit-input-tokens-reset:
73+
- '2026-04-15T22:55:17Z'
74+
anthropic-ratelimit-output-tokens-limit:
75+
- '800000'
76+
anthropic-ratelimit-output-tokens-remaining:
77+
- '800000'
78+
anthropic-ratelimit-output-tokens-reset:
79+
- '2026-04-15T22:55:17Z'
80+
anthropic-ratelimit-requests-limit:
81+
- '20000'
82+
anthropic-ratelimit-requests-remaining:
83+
- '19999'
84+
anthropic-ratelimit-requests-reset:
85+
- '2026-04-15T22:55:16Z'
86+
anthropic-ratelimit-tokens-limit:
87+
- '4800000'
88+
anthropic-ratelimit-tokens-remaining:
89+
- '4800000'
90+
anthropic-ratelimit-tokens-reset:
91+
- '2026-04-15T22:55:17Z'
92+
cf-cache-status:
93+
- DYNAMIC
94+
content-length:
95+
- '577'
96+
request-id:
97+
- req_011Ca6P43j9FEqwivw6NbeSH
98+
server-timing:
99+
- x-originResponse;dur=885
100+
set-cookie:
101+
- _cfuvid=VGZH1MlyDgLD7xOBW_ZCz4E85rqWdya2rabKjiBH41E-1776293716.799078-1.0.1.1-sa5vbAN.0Z3I6YHhQujQU01DrCf381zZyP_iGm_qo60;
102+
HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com
103+
strict-transport-security:
104+
- max-age=31536000; includeSubDomains; preload
105+
vary:
106+
- Accept-Encoding
107+
x-envoy-upstream-service-time:
108+
- '882'
109+
status:
110+
code: 200
111+
message: OK
112+
- request:
113+
body: null
114+
headers:
115+
Accept:
116+
- '*/*'
117+
Accept-Encoding:
118+
- gzip, deflate, zstd
119+
Connection:
120+
- keep-alive
121+
Content-Length:
122+
- '0'
123+
User-Agent:
124+
- python-requests/2.33.1
125+
method: POST
126+
uri: https://www.braintrust.dev/api/apikey/login
127+
response:
128+
body:
129+
string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}'
130+
headers:
131+
Access-Control-Allow-Credentials:
132+
- 'true'
133+
Access-Control-Allow-Headers:
134+
- X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5,
135+
Content-Type, Date, X-Api-Version
136+
Access-Control-Allow-Methods:
137+
- GET,OPTIONS,PATCH,DELETE,POST,PUT
138+
Access-Control-Allow-Origin:
139+
- '*'
140+
Cache-Control:
141+
- public, max-age=0, must-revalidate
142+
Content-Length:
143+
- '395'
144+
Content-Security-Policy:
145+
- 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic''
146+
''nonce-MDNhNGViOWItNWFlMC00MzEzLTlkMWUtMjc0NDM4ZmEwMmY5'' *.js.stripe.com
147+
js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev
148+
btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com
149+
d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com
150+
cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net
151+
fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'';
152+
frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16;
153+
report-to csp-endpoint-0'
154+
Content-Type:
155+
- application/json; charset=utf-8
156+
Date:
157+
- Wed, 15 Apr 2026 22:55:18 GMT
158+
Etag:
159+
- '"12n7ok4b5phaz"'
160+
Reporting-Endpoints:
161+
- csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16"
162+
Server:
163+
- Vercel
164+
Strict-Transport-Security:
165+
- max-age=63072000
166+
X-Clerk-Auth-Message:
167+
- Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid,
168+
token-carrier=header)
169+
X-Clerk-Auth-Reason:
170+
- token-invalid
171+
X-Clerk-Auth-Status:
172+
- signed-out
173+
X-Content-Type-Options:
174+
- nosniff
175+
X-Frame-Options:
176+
- SAMEORIGIN
177+
X-Matched-Path:
178+
- /api/apikey/login
179+
X-Nonce:
180+
- MDNhNGViOWItNWFlMC00MzEzLTlkMWUtMjc0NDM4ZmEwMmY5
181+
X-Vercel-Cache:
182+
- MISS
183+
X-Vercel-Id:
184+
- pdx1::iad1::pjcvq-1776293718158-43f44aced35e
185+
status:
186+
code: 200
187+
message: OK
188+
version: 1

0 commit comments

Comments
 (0)