Merge pull request #20 from datamasque/discovery-run-type

feat: Add DiscoveryConfigType and DiscoveryConfigNotFoundError

Author: cph-datamasque

datamasque/client/__init__.py

@@ -17,6 +17,7 @@
     DataMasqueNotReadyError,
     DataMasqueTransportError,
     DataMasqueUserError,
+    DiscoveryConfigNotFoundError,
     FailedToStartError,
     IfmAuthError,
     InvalidDiscoveryConfigError,
@@ -77,7 +78,7 @@
     SchemaDiscoveryResult,
     TableConstraints,
 )
-from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId
+from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
 from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
 from datamasque.client.models.files import (
     DataMasqueFile,
@@ -146,6 +147,8 @@
     "DatabricksConnectionConfig",
     "DiscoveryConfig",
     "DiscoveryConfigId",
+    "DiscoveryConfigNotFoundError",
+    "DiscoveryConfigType",
     "DiscoveryMatch",
     "DynamoConnectionConfig",
     "FailedToStartError",

datamasque/client/discovery.py

@@ -10,6 +10,7 @@
 from datamasque.client.exceptions import (
     AsyncRulesetGenerationInProgressError,
     DataMasqueException,
+    DiscoveryConfigNotFoundError,
     FailedToStartError,
     InvalidDiscoveryConfigError,
 )
@@ -362,15 +363,29 @@ def start_file_data_discovery_run_from_config(self, request: FileDataDiscoveryFr
     # (a `{"message", "line_number", "column_number"}` dict per error).
     DISCOVERY_CONFIG_ERROR_FIELDS = ("discovery_config", "config_yaml")
 
+    # The server emits this phrase when the referenced config id cannot be found
+    # (it was never created, or it has since been deleted).
+    # There is no structured error code in the body,
+    # so this stable message is the only signal
+    # that separates a bad reference from a present-but-unusable config
+    # (e.g. a non-`valid` validation state, which is also a string under `discovery_config`).
+    MISSING_DISCOVERY_CONFIG_SIGNATURE = "object does not exist"
+
     @classmethod
     def _maybe_raise_discovery_config_error(cls, run_data: object, response: Response, run_kind: str) -> None:
-        """Raise `InvalidDiscoveryConfigError` if the server's 400 body cites the discovery config."""
+        """Raise a discovery-config error if the server's 400 body cites the discovery config."""
         if not isinstance(run_data, dict):
             return
 
         for field in cls.DISCOVERY_CONFIG_ERROR_FIELDS:
             if field in run_data:
                 detail = cls._format_discovery_config_error(run_data[field])
+                if cls.MISSING_DISCOVERY_CONFIG_SIGNATURE in detail:
+                    raise DiscoveryConfigNotFoundError(
+                        f"{run_kind} run failed to start: the referenced discovery config does not exist: {detail}",
+                        response=response,
+                    )
+
                 raise InvalidDiscoveryConfigError(
                     f"{run_kind} run failed to start due to discovery config error: {detail}",
                     response=response,

datamasque/client/exceptions.py

@@ -43,9 +43,24 @@ class InvalidLibraryError(FailedToStartError):
 
 class InvalidDiscoveryConfigError(FailedToStartError):
     """
-    Raised when a discovery run fails to start due to an unusable discovery config.
+    Raised when a discovery run fails to start because the referenced config exists but is unusable.
 
-    The referenced config is missing, archived, or not in a `valid` validation state.
+    The config is present but not in a `valid` validation state,
+    is the wrong type for the connection,
+    or carries YAML the server rejects at trigger time.
+    A config that cannot be found at all raises `DiscoveryConfigNotFoundError` instead.
+    """
+
+
+class DiscoveryConfigNotFoundError(FailedToStartError):
+    """
+    Raised when a discovery run references a discovery config that the server cannot find.
+
+    The config does not exist — it was never created, or it has since been deleted.
+    Unlike `InvalidDiscoveryConfigError`,
+    this signals a bad reference (a deleted or mistyped id)
+    rather than a present but unusable config,
+    so callers can treat it as a caller/setup error.
     """
 
 

datamasque/client/models/discovery_config.py

@@ -1,3 +1,4 @@
+import enum
 from datetime import datetime
 from typing import Any, NewType, Optional
 
@@ -8,6 +9,13 @@
 DiscoveryConfigId = NewType("DiscoveryConfigId", str)
 
 
+class DiscoveryConfigType(enum.Enum):
+    """Which discovery config variant a config targets: database (qualified columns) or file (locators)."""
+
+    database = "database"
+    file = "file"
+
+
 def unwrap_discovery_config_id(value: Any) -> Any:
     """
     Coerce a `DiscoveryConfig` to its `id`; pass other values through unchanged.
@@ -33,6 +41,7 @@ class DiscoveryConfig(BaseModel):
 
     name: str
     yaml: Optional[str] = Field(default=None, alias="config_yaml")
+    config_type: DiscoveryConfigType
     id: Optional[DiscoveryConfigId] = None
     # Server-managed validation surface, populated by the DataMasque server.
     # `is_valid` may be `in_progress` immediately after creating a large config,

tests/test_discovery.py

@@ -30,6 +30,7 @@
     AsyncRulesetGenerationInProgressError,
     DataMasqueApiError,
     DataMasqueException,
+    DiscoveryConfigNotFoundError,
     FailedToStartError,
     InvalidDiscoveryConfigError,
 )
@@ -864,14 +865,14 @@ def test_schema_discovery_from_config_request_accepts_discovery_config_id():
 
 def test_schema_discovery_from_config_request_unwraps_discovery_config_model():
     """Passing a full `DiscoveryConfig` object substitutes its `id` for the wire payload."""
-    config = DiscoveryConfig(name="my_cfg", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID))
+    config = DiscoveryConfig(name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID))
     req = SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config)
     assert req.model_dump(exclude_none=True, mode="json")["discovery_config"] == DISCOVERY_CONFIG_ID
 
 
 def test_schema_discovery_from_config_request_rejects_unsaved_discovery_config():
     """A `DiscoveryConfig` without an `id` cannot be used yet — raises immediately."""
-    config = DiscoveryConfig(name="my_cfg")
+    config = DiscoveryConfig(name="my_cfg", config_type="database")
     with pytest.raises(ValueError, match="id is None"):
         SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config)
 
@@ -974,7 +975,7 @@ def test_start_schema_discovery_run_from_config_sends_schemas(client):
 
 def test_start_file_data_discovery_run_from_config_sends_discovery_config(client):
     """`start_file_data_discovery_run_from_config` posts only the connection and discovery_config id."""
-    config = DiscoveryConfig(name="my_cfg", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID))
+    config = DiscoveryConfig(name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID))
     req = FileDataDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config)
     with requests_mock.Mocker() as m:
         m.post("http://test-server/api/run-file-data-discovery/v2/", json={"id": 99}, status_code=201)
@@ -1042,21 +1043,45 @@ def test_start_schema_discovery_run_from_config_raises_invalid_discovery_config_
             )
 
 
-def test_start_schema_discovery_run_from_config_raises_invalid_discovery_config_when_missing(client):
-    """A 400 from DRF's PrimaryKeyRelatedField (config not found / archived) is also classified."""
+def test_start_schema_discovery_run_from_config_raises_not_found_when_missing(client):
+    """
+    A 400 for a config that cannot be found raises `DiscoveryConfigNotFoundError`.
+
+    This is a bad reference rather than an unusable-but-present config,
+    so it must not be conflated with `InvalidDiscoveryConfigError`.
+    The not-found subclass still inherits `FailedToStartError` for callers that catch the base.
+    """
     with requests_mock.Mocker() as m:
         m.post(
             "http://test-server/api/schema-discovery/v2/",
             json={"discovery_config": [f'Invalid pk "{DISCOVERY_CONFIG_ID}" - object does not exist.']},
             status_code=400,
         )
-        with pytest.raises(InvalidDiscoveryConfigError, match="object does not exist"):
+        with pytest.raises(DiscoveryConfigNotFoundError, match="object does not exist") as exc_info:
             client.start_schema_discovery_run_from_config(
                 SchemaDiscoveryFromConfigRequest(
                     connection="conn-1", discovery_config=DiscoveryConfigId(DISCOVERY_CONFIG_ID)
                 ),
             )
 
+    assert isinstance(exc_info.value, FailedToStartError)
+
+
+def test_start_file_data_discovery_run_from_config_raises_not_found_when_missing(client):
+    """A not-found saved config on the file-data trigger also raises `DiscoveryConfigNotFoundError`."""
+    with requests_mock.Mocker() as m:
+        m.post(
+            "http://test-server/api/run-file-data-discovery/v2/",
+            json={"discovery_config": [f'Invalid pk "{DISCOVERY_CONFIG_ID}" - object does not exist.']},
+            status_code=400,
+        )
+        with pytest.raises(DiscoveryConfigNotFoundError, match="object does not exist"):
+            client.start_file_data_discovery_run_from_config(
+                FileDataDiscoveryFromConfigRequest(
+                    connection="conn-1", discovery_config=DiscoveryConfigId(DISCOVERY_CONFIG_ID)
+                ),
+            )
+
 
 def test_start_schema_discovery_run_from_config_raises_invalid_discovery_config_on_broken_yaml(client):
     """

tests/test_discovery_configs.py

@@ -29,13 +29,15 @@ def sample_config_list_response() -> dict[str, Any]:
             {
                 "id": CONFIG_ID_1,
                 "name": "my_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-02T12:00:00Z",
             },
             {
                 "id": CONFIG_ID_2,
                 "name": "another_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-02-01T12:00:00Z",
                 "modified": "2025-02-02T12:00:00Z",
@@ -50,6 +52,7 @@ def sample_config_detail_response() -> dict[str, Any]:
         "id": CONFIG_ID_1,
         "name": "my_config",
         "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        "config_type": "database",
         "archived": False,
         "created": "2025-01-01T12:00:00Z",
         "modified": "2025-01-02T12:00:00Z",
@@ -61,6 +64,7 @@ def discovery_config() -> DiscoveryConfig:
     return DiscoveryConfig(
         name="test_config",
         yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        config_type="database",
     )
 
 
@@ -89,13 +93,15 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None:
             {
                 "id": CONFIG_ID_1,
                 "name": "c1",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-01T12:00:00Z",
             },
             {
                 "id": CONFIG_ID_2,
                 "name": "c2",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-01T12:00:00Z",
@@ -110,6 +116,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None:
             {
                 "id": "cccccccc-1111-2222-3333-444444444444",
                 "name": "c3",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-01T12:00:00Z",
@@ -162,6 +169,7 @@ def test_get_discovery_config_by_name_found(
             {
                 "id": CONFIG_ID_1,
                 "name": "my_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-02T12:00:00Z",
@@ -209,6 +217,7 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa
         "results": [
             {
                 "name": "my_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-02T12:00:00Z",
@@ -231,6 +240,7 @@ def test_create_discovery_config(client: DataMasqueClient, discovery_config: Dis
         "id": CONFIG_ID_1,
         "name": "test_config",
         "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        "config_type": "database",
         "archived": False,
         "created": "2025-06-01T10:00:00Z",
         "modified": "2025-06-01T10:00:00Z",
@@ -252,6 +262,7 @@ def test_create_discovery_config(client: DataMasqueClient, discovery_config: Dis
     request_body = m.last_request.json()
     assert request_body["name"] == "test_config"
     assert request_body["config_yaml"] == "labels: []\nmetadata_rules: []\nidd_rules: []\n"
+    assert request_body["config_type"] == "database"
 
 
 def test_update_discovery_config(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None:
@@ -261,6 +272,7 @@ def test_update_discovery_config(client: DataMasqueClient, discovery_config: Dis
         "id": CONFIG_ID_1,
         "name": "test_config",
         "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        "config_type": "database",
         "archived": False,
         "created": "2025-06-01T10:00:00Z",
         "modified": "2025-06-02T10:00:00Z",
@@ -293,6 +305,7 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc
         "id": CONFIG_ID_1,
         "name": "test_config",
         "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        "config_type": "database",
         "archived": False,
         "created": "2025-06-01T10:00:00Z",
         "modified": "2025-06-01T10:00:00Z",
@@ -317,6 +330,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc
             {
                 "id": CONFIG_ID_1,
                 "name": "test_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-06-01T10:00:00Z",
                 "modified": "2025-06-01T10:00:00Z",
@@ -327,6 +341,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc
         "id": CONFIG_ID_1,
         "name": "test_config",
         "config_yaml": "labels: []",
+        "config_type": "database",
         "archived": False,
         "created": "2025-06-01T10:00:00Z",
         "modified": "2025-06-01T10:00:00Z",
@@ -335,6 +350,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc
         "id": CONFIG_ID_1,
         "name": "test_config",
         "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n",
+        "config_type": "database",
         "archived": False,
         "created": "2025-06-01T10:00:00Z",
         "modified": "2025-06-02T10:00:00Z",
@@ -403,6 +419,7 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat
         "results": [
             {
                 "name": "my_config",
+                "config_type": "database",
                 "archived": False,
                 "created": "2025-01-01T12:00:00Z",
                 "modified": "2025-01-02T12:00:00Z",
@@ -441,6 +458,7 @@ def test_discovery_config_parses_validation_fields() -> None:
         {
             "id": CONFIG_ID_1,
             "name": "my_config",
+            "config_type": "database",
             "config_yaml": "labels: []",
             "is_valid": "invalid",
             "validation_error": "bad shape on line 3",
@@ -459,6 +477,7 @@ def test_discovery_config_validation_fields_optional() -> None:
         {
             "id": CONFIG_ID_1,
             "name": "my_config",
+            "config_type": "database",
             "created": "2025-01-01T12:00:00Z",
             "modified": "2025-01-02T12:00:00Z",
         }
@@ -474,11 +493,11 @@ def test_unwrap_discovery_config_id_passes_through_strings() -> None:
 
 
 def test_unwrap_discovery_config_id_extracts_id_from_model() -> None:
-    config = DiscoveryConfig(name="x", id=DiscoveryConfigId(CONFIG_ID_1))
+    config = DiscoveryConfig(name="x", config_type="database", id=DiscoveryConfigId(CONFIG_ID_1))
     assert unwrap_discovery_config_id(config) == CONFIG_ID_1
 
 
 def test_unwrap_discovery_config_id_raises_without_id() -> None:
-    config = DiscoveryConfig(name="x")
+    config = DiscoveryConfig(name="x", config_type="database")
     with pytest.raises(ValueError, match="id is None"):
         unwrap_discovery_config_id(config)