supabase example

Author: Dylan Huang

.github/workflows/ci.yml

@@ -86,6 +86,11 @@ jobs:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
           FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
+          SUPABASE_PASSWORD: ${{ secrets.SUPABASE_PASSWORD }}
+          SUPABASE_HOST: ${{ secrets.SUPABASE_HOST }}
+          SUPABASE_PORT: ${{ secrets.SUPABASE_PORT }}
+          SUPABASE_DATABASE: ${{ secrets.SUPABASE_DATABASE }}
+          SUPABASE_USER: ${{ secrets.SUPABASE_USER }}
           PYTHONWARNINGS: "ignore::DeprecationWarning,ignore::RuntimeWarning"
         run: |
           # Run most tests in parallel, but explicitly ignore tests that manage their own servers or are slow

eval_protocol/models.py

@@ -572,6 +572,13 @@ def get_assistant_messages(self) -> List[Message]:
         """Returns only the assistant messages from the conversation."""
         return [msg for msg in self.messages if msg.role == "assistant"]
 
+    def last_assistant_message(self) -> Optional[Message]:
+        """Returns the last assistant message from the conversation. Returns None if none found."""
+        assistant_messages = self.get_assistant_messages()
+        if not assistant_messages:
+            return None
+        return assistant_messages[-1]
+
     def get_user_messages(self) -> List[Message]:
         """Returns only the user messages from the conversation."""
         return [msg for msg in self.messages if msg.role == "user"]

eval_protocol/pytest/default_pydantic_ai_rollout_processor.py

@@ -3,6 +3,7 @@
 import types
 from typing import List
 
+from attr import dataclass
 from openai.types.chat.chat_completion_assistant_message_param import ChatCompletionAssistantMessageParam
 
 from eval_protocol.models import EvaluationRow, Message
@@ -23,10 +24,17 @@
     UserPromptPart,
 )
 from pydantic_ai.providers.openai import OpenAIProvider
+from typing_extensions import TypedDict
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class ModelConfig(TypedDict):
+    model: str
+    provider: str
+
+
 class PydanticAgentRolloutProcessor(RolloutProcessor):
     """Rollout processor for Pydantic AI agents. Mainly converts
     EvaluationRow.messages to and from Pydantic AI ModelMessage format."""
@@ -58,10 +66,10 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) ->
                     "completion_params['model'] must be a dict mapping agent argument names to model config dicts (with 'model' and 'provider' keys)"
                 )
             kwargs = {}
-            for model_name, model_config in config.completion_params["model"].items():
-                kwargs[model_name] = OpenAIModel(
-                    model_config["model"],
-                    provider=model_config["provider"],
+            for k, v in config.completion_params["model"].items():
+                kwargs[k] = OpenAIModel(
+                    v["model"],
+                    provider=v["provider"],
                 )
             agent = setup_agent(**kwargs)
             model = None

pyproject.toml

@@ -125,6 +125,12 @@ svgbench = [
 pydantic = [
     "pydantic-ai",
 ]
+supabase = [
+    "supabase>=2.18.1",
+]
+chinook = [
+    "psycopg2-binary>=2.9.10",
+]
 
 [tool.pytest.ini_options]
 addopts = "-q"