feat(framework): add classifier support to Eval (#282)

Add a new `classifiers` parameter to `Eval`/`EvalAsync`/`Evaluator` that
runs classification functions alongside scorers. Classifier results are
recorded under a dedicated `classifications` field on `EvalResult` and
logged to classifier-typed spans.

Based on JS PR:
braintrustdata/braintrust-sdk-javascript#1553
and the spec:
https://github.com/braintrustdata/braintrust-spec/blob/main/docs/telemetry/classifier.md

Author: AbhiPrasad

py/src/braintrust/cassettes/test_classifier_spans_are_logged.yaml

@@ -0,0 +1,76 @@
+interactions:
+- request:
+    body: '{"id": "test-classifier-span"}'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '30'
+      Content-Type:
+      - application/json
+      User-Agent:
+      - python-requests/2.32.5
+    method: POST
+    uri: https://www.braintrust.dev/api/base_experiment/get_id
+  response:
+    body:
+      string: "[\n  {\n    \"validation\": \"uuid\",\n    \"code\": \"invalid_string\",\n
+        \   \"message\": \"Invalid uuid\",\n    \"path\": [\n      \"id\"\n    ]\n
+        \ }\n] [user_email=___braintrust_anon_user___@braintrustdata.com] [timestamp=1776185951.284]
+        [request_id=yul1::rlkb8-1776185950926-a394af342bfa]"
+    headers:
+      Cache-Control:
+      - public, max-age=0, must-revalidate
+      Content-Length:
+      - '267'
+      Content-Security-Policy:
+      - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic''
+        ''nonce-NGNlMGVjNTUtNmE2MC00ZDA3LWE3OGMtMDQ3NWExNWVkZGUz''  *.js.stripe.com
+        js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev
+        btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com
+        d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com
+        cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net
+        fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'';
+        frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16;
+        report-to csp-endpoint-0'
+      Content-Type:
+      - text/plain; charset=utf-8
+      Date:
+      - Tue, 14 Apr 2026 16:59:11 GMT
+      Etag:
+      - '"ox95rpt6sr7f"'
+      Reporting-Endpoints:
+      - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16"
+      Server:
+      - Vercel
+      Set-Cookie:
+      - __Host-authjs.csrf-token=29800db0ea46edba4ca7714d00cba09854ccf08e98f28666bd7a8f816dc260ea%7C4f5ce689e487bc9972f79ed54f75a99deeb00baec720e82e1efa5e58f551d316;
+        Path=/; HttpOnly; Secure; SameSite=Lax
+      - __Secure-authjs.callback-url=https%3A%2F%2Fwww.braintrustdata.com; Path=/;
+        HttpOnly; Secure; SameSite=Lax
+      Strict-Transport-Security:
+      - max-age=63072000
+      X-Clerk-Auth-Reason:
+      - session-token-and-uat-missing
+      X-Clerk-Auth-Status:
+      - signed-out
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Matched-Path:
+      - /api/base_experiment/get_id
+      X-Nonce:
+      - NGNlMGVjNTUtNmE2MC00ZDA3LWE3OGMtMDQ3NWExNWVkZGUz
+      X-Vercel-Cache:
+      - MISS
+      X-Vercel-Id:
+      - yul1::iad1::rlkb8-1776185950926-a394af342bfa
+    status:
+      code: 400
+      message: Bad Request
+version: 1

py/src/braintrust/cli/push.py

@@ -17,7 +17,7 @@
 
 import requests
 import slugify
-from braintrust.framework import _evals, _scorer_name, _set_lazy_load
+from braintrust.framework import _classifier_name, _evals, _scorer_name, _set_lazy_load
 
 from .. import api_conn, login, org_id, proxy_conn
 from ..framework2 import ProjectIdCache, global_
@@ -303,8 +303,11 @@ def _collect_evaluator_defs(
         evaluator = eval_instance.evaluator
         project_id = project_ids.get_by_name(evaluator.project_name)
 
-        scores = [{"name": _scorer_name(scorer, i)} for i, scorer in enumerate(evaluator.scores)]
-        evaluator_definition: dict[str, Any] = {"scores": scores}
+        scores = [{"name": _scorer_name(scorer, i)} for i, scorer in enumerate(evaluator.scores or [])]
+        classifiers = [
+            {"name": _classifier_name(classifier, i)} for i, classifier in enumerate(evaluator.classifiers or [])
+        ]
+        evaluator_definition: dict[str, Any] = {"scores": scores, "classifiers": classifiers}
         if evaluator.parameters is not None:
             evaluator_definition["parameters"] = serialize_remote_eval_parameters_container(evaluator.parameters)
 

py/src/braintrust/cli/test_push_evaluator.py

@@ -15,10 +15,11 @@ def _make_scorer(name):
     return scorer
 
 
-def _make_evaluator(project_name, scorer_names, parameters=None):
+def _make_evaluator(project_name, scorer_names, parameters=None, classifier_names=None):
     evaluator = MagicMock()
     evaluator.project_name = project_name
     evaluator.scores = [_make_scorer(n) for n in scorer_names]
+    evaluator.classifiers = [_make_scorer(n) for n in (classifier_names or [])]
     evaluator.parameters = parameters
 
     instance = MagicMock()
@@ -50,7 +51,10 @@ def test_basic_evaluator_def_structure(self, mock_project_ids):
                             "sandbox_spec": {"provider": "lambda"},
                             "entrypoints": ["evals/my_eval.py"],
                             "eval_name": "my_eval",
-                            "evaluator_definition": {"scores": [{"name": "accuracy"}]},
+                            "evaluator_definition": {
+                                "scores": [{"name": "accuracy"}],
+                                "classifiers": [],
+                            },
                         },
                         "bundle_id": "bundle-abc",
                     },
@@ -98,6 +102,16 @@ def test_evaluator_with_parameters(self, mock_project_ids):
         assert parameters["source"] is None
         assert parameters["schema"]["prompt"]["type"] == "prompt"
 
+    def test_evaluator_with_classifiers(self, mock_project_ids):
+        evaluators = {"eval1": _make_evaluator("test-project", ["accuracy"], classifier_names=["category"])}
+
+        functions = []
+        _collect_evaluator_defs(mock_project_ids, functions, "bundle-1", "replace", "eval.py", evaluators)
+
+        eval_def = functions[0]["function_data"]["data"]["location"]["evaluator_definition"]
+        assert eval_def["scores"] == [{"name": "accuracy"}]
+        assert eval_def["classifiers"] == [{"name": "category"}]
+
     def test_slug_from_source_file(self, mock_project_ids):
         evaluators = {"Test Eval": _make_evaluator("test-project", ["accuracy"])}
 

py/src/braintrust/devserver/server.py

@@ -32,6 +32,8 @@
     Evaluator,
     ExperimentSummary,
     SSEProgressEvent,
+    _classifier_name,
+    _scorer_name,
 )
 from ..generated_types import FunctionId
 from ..logger import BraintrustState, bt_iscoroutinefunction
@@ -123,7 +125,10 @@ async def list_evaluators(request: Request) -> JSONResponse:
             "parameters": (
                 serialize_remote_eval_parameters_container(evaluator.parameters) if evaluator.parameters else None
             ),
-            "scores": [{"name": getattr(score, "name", f"score_{i}")} for i, score in enumerate(evaluator.scores)],
+            "scores": [{"name": _scorer_name(score, i)} for i, score in enumerate(evaluator.scores or [])],
+            "classifiers": [
+                {"name": _classifier_name(classifier, i)} for i, classifier in enumerate(evaluator.classifiers or [])
+            ],
         }
 
     return JSONResponse(evaluator_list)
@@ -227,7 +232,7 @@ def stream_fn(event: SSEProgressEvent):
                 **{
                     **eval_kwargs,
                     "state": state,
-                    "scores": evaluator.scores
+                    "scores": (evaluator.scores or [])
                     + [
                         make_scorer(state, score["name"], score["function_id"], ctx.project_id)
                         for score in eval_data.get("scores", [])

py/src/braintrust/devserver/test_server_integration.py

@@ -49,6 +49,9 @@ def scorer(input: str, output: str, expected: str) -> float:
         """Simple exact match scorer."""
         return 1.0 if output == expected else 0.0
 
+    def classifier(input: str, output: str, expected: str) -> dict[str, str]:
+        return {"id": "correct" if output == expected else "incorrect", "name": "answer_type"}
+
     evaluator = Evaluator(
         project_name="test-math-eval",
         eval_name="simple-math-eval",
@@ -59,6 +62,7 @@ def scorer(input: str, output: str, expected: str) -> float:
         ],
         task=task,
         scores=[scorer],
+        classifiers=[classifier],
         experiment_name=None,
         metadata=None,
     )
@@ -114,6 +118,8 @@ def test_devserver_list_evaluators(client, api_key, org_name):
     assert response.status_code == 200
     evaluators = response.json()
     assert "simple-math-eval" in evaluators
+    assert evaluators["simple-math-eval"]["scores"] == [{"name": "scorer"}]
+    assert evaluators["simple-math-eval"]["classifiers"] == [{"name": "classifier"}]
 
 
 def parse_sse_events(response_text: str) -> list[dict[str, Any]]: