From e2287a5cc4b8bfb7bced2eae9dbc0f41934ba22c Mon Sep 17 00:00:00 2001 From: Stefano Maffeis Date: Thu, 16 Jul 2026 06:06:36 +0000 Subject: [PATCH 1/2] feat(exceptions): standardize provider error __str__ format Add error_type class attribute and __str__ override to ProviderError and subclasses, producing consistent [provider] type: message output across all LLM providers. Co-Authored-By: Paperclip --- openagent_eval/exceptions/provider.py | 14 +++++++++++ tests/unit/test_exceptions.py | 34 +++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/openagent_eval/exceptions/provider.py b/openagent_eval/exceptions/provider.py index 0d47e3a..8782025 100644 --- a/openagent_eval/exceptions/provider.py +++ b/openagent_eval/exceptions/provider.py @@ -12,8 +12,11 @@ class ProviderError(OpenAgentEvalError): message: Human-readable error message. provider_name: Name of the provider that caused the error. details: Additional context about the error. + error_type: Human-readable error type label for standardized formatting. """ + error_type: str = "Provider Error" + def __init__( self, message: str, @@ -34,10 +37,17 @@ def __init__( super().__init__(message=message, details=error_details) self.provider_name = provider_name + def __str__(self) -> str: + """Return standardized string: [provider] error_type: message.""" + provider_prefix = f"[{self.provider_name}] " if self.provider_name else "" + return f"{provider_prefix}{self.error_type}: {self.message}" + class ProviderNotFoundError(ProviderError): """Raised when a requested provider cannot be found.""" + error_type: str = "Provider Not Found" + def __init__( self, provider_name: str, @@ -70,6 +80,8 @@ class ProviderConnectionError(ProviderError): original_error: The original exception that caused the connection failure. """ + error_type: str = "Connection Error" + def __init__( self, message: str, @@ -100,6 +112,8 @@ class ProviderExecutionError(ProviderError): original_error: The original exception that caused the failure. """ + error_type: str = "Execution Error" + def __init__( self, message: str, diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 448a096..4d4596d 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -134,27 +134,53 @@ class TestProviderError: """Tests for provider errors.""" def test_basic_error(self) -> None: - """Test basic provider error.""" + """Test basic provider error with standardized format.""" error = ProviderError("Invalid provider") - assert str(error) == "Invalid provider" + assert str(error) == "Provider Error: Invalid provider" + assert error.error_type == "Provider Error" + + def test_error_with_provider_name(self) -> None: + """Test provider error with provider name in standardized format.""" + error = ProviderError( + "API key not found", + provider_name="openai", + ) + assert str(error) == "[openai] Provider Error: API key not found" def test_not_found_error(self) -> None: """Test provider not found error.""" error = ProviderNotFoundError("my_provider", available_providers=["openai", "gemini"]) assert error.provider_name == "my_provider" assert error.available_providers == ["openai", "gemini"] + assert error.error_type == "Provider Not Found" + assert ( + str(error) + == "[my_provider] Provider Not Found: Provider not found: my_provider. Available providers: openai, gemini" + ) def test_connection_error(self) -> None: """Test provider connection error.""" original = ConnectionError("Connection failed") - error = ProviderConnectionError("Failed to connect", original_error=original) + error = ProviderConnectionError( + "Failed to connect", + provider_name="gemini", + original_error=original, + ) assert error.original_error is original + assert error.error_type == "Connection Error" + assert str(error) == "[gemini] Connection Error: Failed to connect" def test_execution_error(self) -> None: """Test provider execution error.""" original = RuntimeError("Execution failed") - error = ProviderExecutionError("Failed", original_error=original) + error = ProviderExecutionError( + "API call failed", + provider_name="anthropic", + original_error=original, + ) assert error.original_error is original + assert error.error_type == "Execution Error" + assert str(error) == "[anthropic] Execution Error: API call failed" class TestPluginError: From 7e4510a9c7ce94ae29b2e18fe09d7701103752e6 Mon Sep 17 00:00:00 2001 From: Stefano Maffeis Date: Thu, 16 Jul 2026 21:46:47 +0000 Subject: [PATCH 2/2] fix(exceptions): preserve details in ProviderError.__str__ The __str__ override dropped self.details from the output, which broke existing tests that match on details keywords (e.g., 'available_embedders', 'available_retrievers'). Include non-provider_name details in the standardized format, matching the base OpenAgentEvalError behavior. Co-Authored-By: Paperclip --- openagent_eval/exceptions/provider.py | 10 ++++++++-- tests/unit/test_exceptions.py | 12 +++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/openagent_eval/exceptions/provider.py b/openagent_eval/exceptions/provider.py index 8782025..a6ed894 100644 --- a/openagent_eval/exceptions/provider.py +++ b/openagent_eval/exceptions/provider.py @@ -38,9 +38,15 @@ def __init__( self.provider_name = provider_name def __str__(self) -> str: - """Return standardized string: [provider] error_type: message.""" + """Return standardized string: [provider] error_type: message (details).""" provider_prefix = f"[{self.provider_name}] " if self.provider_name else "" - return f"{provider_prefix}{self.error_type}: {self.message}" + base = f"{provider_prefix}{self.error_type}: {self.message}" + if self.details: + details_items = {k: v for k, v in self.details.items() if k != "provider_name"} + if details_items: + details_str = ", ".join(f"{k}={v}" for k, v in details_items.items()) + return f"{base} ({details_str})" + return base class ProviderNotFoundError(ProviderError): diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 4d4596d..387a625 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -153,10 +153,12 @@ def test_not_found_error(self) -> None: assert error.provider_name == "my_provider" assert error.available_providers == ["openai", "gemini"] assert error.error_type == "Provider Not Found" - assert ( - str(error) - == "[my_provider] Provider Not Found: Provider not found: my_provider. Available providers: openai, gemini" + expected = ( + "[my_provider] Provider Not Found: " + "Provider not found: my_provider. Available providers: openai, gemini " + "(available_providers=['openai', 'gemini'])" ) + assert str(error) == expected def test_connection_error(self) -> None: """Test provider connection error.""" @@ -168,7 +170,7 @@ def test_connection_error(self) -> None: ) assert error.original_error is original assert error.error_type == "Connection Error" - assert str(error) == "[gemini] Connection Error: Failed to connect" + assert str(error) == "[gemini] Connection Error: Failed to connect (original_error=Connection failed)" def test_execution_error(self) -> None: """Test provider execution error.""" @@ -180,7 +182,7 @@ def test_execution_error(self) -> None: ) assert error.original_error is original assert error.error_type == "Execution Error" - assert str(error) == "[anthropic] Execution Error: API call failed" + assert str(error) == "[anthropic] Execution Error: API call failed (original_error=Execution failed)" class TestPluginError: