diff --git a/tests/errors_test.py b/tests/errors_test.py
index 6507c29..fda2a5c 100644
--- a/tests/errors_test.py
+++ b/tests/errors_test.py
@@ -1,9 +1,10 @@
 from datetime import datetime
+from unittest.mock import Mock, AsyncMock
 
 import pytest
 
 from traveltimepy import Client, AsyncClient
-from traveltimepy.errors import TravelTimeApiError
+from traveltimepy.errors import TravelTimeApiError, TravelTimeError
 from traveltimepy.requests.common import Coordinates
 from traveltimepy.requests.time_map import TimeMapDepartureSearch
 from traveltimepy.requests.transportation import Driving
@@ -65,3 +66,24 @@ def test_invalid_request_sync(client: Client):
             intersections=[],
         )
     assert e.value.status_code == 422
+
+
+def test_unexpected_error_response_sync():
+    mock_response = Mock()
+    mock_response.status_code = 400
+    mock_response.json.return_value = {"unexpected": "format"}
+
+    with Client("test", "test") as client:
+        with pytest.raises(TravelTimeError, match="unexpected response"):
+            client._handle_response(mock_response, Mock)
+
+
+@pytest.mark.asyncio
+async def test_unexpected_error_response_async():
+    mock_response = AsyncMock()
+    mock_response.status = 400
+    mock_response.text.return_value = '{"unexpected": "format"}'
+
+    async with AsyncClient("test", "test") as client:
+        with pytest.raises(TravelTimeError, match="unexpected response"):
+            await client._handle_response(mock_response, Mock)
diff --git a/traveltimepy/async_base_client.py b/traveltimepy/async_base_client.py
index 816c669..7b6f649 100644
--- a/traveltimepy/async_base_client.py
+++ b/traveltimepy/async_base_client.py
@@ -5,7 +5,7 @@
 import aiohttp
 from aiohttp import ClientSession, ClientResponse, BasicAuth, TCPConnector
 from aiolimiter import AsyncLimiter
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
 from tenacity import (
     retry,
     stop_after_attempt,
@@ -25,6 +25,7 @@
 from traveltimepy.accept_type import AcceptType
 from traveltimepy.base_client import BaseClient, __version__
 from traveltimepy.errors import (
+    TravelTimeError,
     TravelTimeJsonError,
     TravelTimeServerError,
 )
@@ -265,7 +266,13 @@ async def _handle_response(
         text = await response.text()
         json_data = json.loads(text)
         if response.status != 200:
-            error = ResponseError.model_validate_json(json.dumps(json_data))
+            try:
+                error = ResponseError.model_validate_json(json.dumps(json_data))
+            except ValidationError:
+                raise TravelTimeError(
+                    f"Server returned status code {response.status} "
+                    f"with unexpected response: {json_data}"
+                )
             if response.status >= 500:
                 raise TravelTimeServerError(error.description)
             else:
diff --git a/traveltimepy/sync_base_client.py b/traveltimepy/sync_base_client.py
index 50f621b..f0453c7 100644
--- a/traveltimepy/sync_base_client.py
+++ b/traveltimepy/sync_base_client.py
@@ -3,7 +3,7 @@
 from typing import Optional, Dict, TypeVar, Type, List, cast
 
 import requests
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
 from requests.adapters import HTTPAdapter
 from requests.auth import HTTPBasicAuth
 from requests_ratelimiter import LimiterSession
@@ -26,6 +26,7 @@
 from traveltimepy.accept_type import AcceptType
 from traveltimepy.base_client import BaseClient, __version__
 from traveltimepy.errors import (
+    TravelTimeError,
     TravelTimeJsonError,
     TravelTimeServerError,
 )
@@ -316,7 +317,13 @@ def _handle_response(
             json_data = {"error": "Invalid JSON response"}
 
         if response.status_code != 200:
-            error = ResponseError.model_validate_json(json.dumps(json_data))
+            try:
+                error = ResponseError.model_validate_json(json.dumps(json_data))
+            except ValidationError:
+                raise TravelTimeError(
+                    f"Server returned status code {response.status_code} "
+                    f"with unexpected response: {json_data}"
+                )
             if response.status_code >= 500:
                 raise TravelTimeServerError(error.description)
             else: