Merge branch 'main' into link-to-local-ui

Author: Dylan Huang

eval_protocol/__init__.py

@@ -37,19 +37,19 @@
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
 from .typed_interface import reward_function
-from .quickstart import aha_judge, split_multi_turn_rows
+from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
 from .pytest import evaluation_test, SingleTurnRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
 
 from .adapters import OpenAIResponsesAdapter
 
 try:
-    from .adapters import LangfuseAdapter
+    from .adapters import LangfuseAdapter, create_langfuse_adapter
 except ImportError:
     LangfuseAdapter = None
 
 try:
-    from .adapters import BraintrustAdapter
+    from .adapters import BraintrustAdapter, create_braintrust_adapter
 except ImportError:
     BraintrustAdapter = None
 
@@ -64,12 +64,15 @@
 __all__ = [
     "DefaultParameterIdGenerator",
     "aha_judge",
-    "split_multi_turn_rows",
+    "multi_turn_assistant_to_ground_truth",
+    "assistant_to_ground_truth",
     "evaluation_test",
     "SingleTurnRolloutProcessor",
     "OpenAIResponsesAdapter",
     "LangfuseAdapter",
+    "create_langfuse_adapter",
     "BraintrustAdapter",
+    "create_braintrust_adapter",
     "LangSmithAdapter",
     # Core interfaces
     "Message",

eval_protocol/adapters/base.py

@@ -19,3 +19,7 @@ def get_evaluation_rows(self, *args, **kwargs) -> List[EvaluationRow]:
     def upload_scores(self, rows: List[EvaluationRow], model_name: str, mean_score: float) -> None:
         """Upload evaluation scores back to the data source for tracking and analysis."""
         pass
+
+    def upload_score(self, row: EvaluationRow, model_name: str) -> None:
+        """Upload evaluation score for a single row back to the data source."""
+        pass

eval_protocol/adapters/braintrust.py

@@ -264,6 +264,40 @@ def upload_scores(self, rows: List[EvaluationRow], model_name: str, mean_score:
         except Exception as e:
             logger.warning("Failed to push scores to Braintrust: %s", e)
 
+    def upload_score(self, row: EvaluationRow, model_name: str) -> None:
+        """Upload evaluation score for a single row back to Braintrust.
+
+        Args:
+            row: Single EvaluationRow with evaluation_result and session_data containing trace ID
+            model_name: Name of the model (used as the score name in Braintrust)
+        """
+        try:
+            if (
+                row.evaluation_result
+                and row.evaluation_result.is_score_valid
+                and row.input_metadata
+                and row.input_metadata.session_data
+                and "braintrust_trace_id" in row.input_metadata.session_data
+            ):
+                headers = {
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                }
+
+                trace_id = row.input_metadata.session_data["braintrust_trace_id"]
+                if trace_id:
+                    feedback_items = [{"id": trace_id, "scores": {model_name: row.evaluation_result.score}}]
+
+                    response = requests.post(
+                        f"{self.api_url}/v1/feedback",
+                        headers=headers,
+                        json={"feedback": feedback_items},
+                        timeout=30,
+                    )
+                    response.raise_for_status()
+        except Exception as e:
+            logger.warning("Failed to upload single score to Braintrust: %s", e)
+
 
 def create_braintrust_adapter(
     api_key: Optional[str] = None,