diff --git a/docs/CUSTOM_FIELDS.md b/docs/CUSTOM_FIELDS.md index baad498..c3ebe1e 100644 --- a/docs/CUSTOM_FIELDS.md +++ b/docs/CUSTOM_FIELDS.md @@ -23,9 +23,9 @@ Multiple datasets and fields can be defined in the files. However, the files nee The first contains the definitions of the datasets you want to create. It should have following structure: -| workspace_id | dataset_id | dataset_name | dataset_datasource_id | dataset_source_table | dataset_source_sql | parent_dataset_reference | parent_dataset_reference_attribute_id | dataset_reference_source_colum | wdf_id | -| -------------------- | ----------------- | -------------------- | --------------------- | -------------------- | ------------------ | ------------------------ | ------------------------------------- | ------------------------------ | ------ | -| child_workspace_id_1 | custom_dataset_id | Custom Dataset Title | datasource_id | dataset_source_table | | parent_dataset_id | parent_dataset.reference_field | custom_dataset.reference_field | wdf_id | +| workspace_id | dataset_id | dataset_name | dataset_datasource_id | dataset_source_table | dataset_source_sql | parent_dataset_reference | parent_dataset_reference_attribute_id | dataset_reference_source_column | wdf_id | wdf_column_name | +| -------------------- | ----------------- | -------------------- | --------------------- | -------------------- | ------------------ | ------------------------ | ------------------------------------- | ------------------------------- | ------ | --------------- | +| child_workspace_id_1 | custom_dataset_id | Custom Dataset Title | datasource_id | dataset_source_table | | parent_dataset_id | parent_dataset.reference_field | custom_dataset.reference_field | wdf_id | wdf_column_name | #### Validity constraints @@ -47,8 +47,9 @@ For readability, here is the data structure in JSON format with comments. Howeve "dataset_source_sql": null, // SQL query defining the dataset "parent_dataset_reference": "products", // ID of the parent dataset to which the custom one will be connected "parent_dataset_reference_attribute_id": "products.product_id", // parent dataset column name used fot the "join" - "dataset_reference_source_colum": "product_id", // custom dataset column name used for the "join" - "wdf_id": "x__client_id" // workspace data filter id + "dataset_reference_source_column": "product_id", // custom dataset column name used for the "join" + "wdf_id": "x__client_id", // workspace data filter id + "wdf_column_name": "client_id" // name of the column used for filtering } ``` diff --git a/scripts/custom_fields/input_processor.py b/scripts/custom_fields/input_processor.py index f0b87fa..e75dcf9 100644 --- a/scripts/custom_fields/input_processor.py +++ b/scripts/custom_fields/input_processor.py @@ -221,7 +221,7 @@ def datasets_to_ldm( multivalue=True, sources=[ CatalogDeclarativeReferenceSource( - column=dataset.definition.dataset_reference_source_colum, + column=dataset.definition.dataset_reference_source_column, target=CatalogGrainIdentifier( id=dataset.definition.parent_dataset_reference_attribute_id, type=CustomFieldType.ATTRIBUTE.value, @@ -247,7 +247,7 @@ def datasets_to_ldm( filter_id=CatalogDatasetWorkspaceDataFilterIdentifier( id=dataset.definition.wdf_id ), - filter_column=dataset.definition.wdf_id, + filter_column=dataset.definition.wdf_column_name, filter_column_data_type=ColumnDataType.STRING.value, ) ], diff --git a/scripts/custom_fields/models/custom_data_object.py b/scripts/custom_fields/models/custom_data_object.py index 538cd80..07ec606 100644 --- a/scripts/custom_fields/models/custom_data_object.py +++ b/scripts/custom_fields/models/custom_data_object.py @@ -62,8 +62,9 @@ class CustomDatasetDefinition(BaseModel): dataset_source_sql: str | None parent_dataset_reference: str parent_dataset_reference_attribute_id: str - dataset_reference_source_colum: str + dataset_reference_source_column: str wdf_id: str + wdf_column_name: str @model_validator(mode="after") def check_source(self) -> "CustomDatasetDefinition": diff --git a/tests/test_custom_fields/test_input_processor.py b/tests/test_custom_fields/test_input_processor.py index 76e4d2d..679412b 100644 --- a/tests/test_custom_fields/test_input_processor.py +++ b/tests/test_custom_fields/test_input_processor.py @@ -68,8 +68,9 @@ def mock_dataset_definition(): dataset_source_sql=None, parent_dataset_reference="parent_ds", parent_dataset_reference_attribute_id="parent_attr", - dataset_reference_source_colum="ref_col", + dataset_reference_source_column="ref_col", wdf_id="wdf1", + wdf_column_name="col1", ) diff --git a/tests/test_custom_fields/test_input_validator.py b/tests/test_custom_fields/test_input_validator.py index f300bd1..b360f49 100644 --- a/tests/test_custom_fields/test_input_validator.py +++ b/tests/test_custom_fields/test_input_validator.py @@ -28,9 +28,9 @@ def valid_dataset_definitions(): "dataset_source_sql": None, "parent_dataset_reference": "parent1", "parent_dataset_reference_attribute_id": "parent1.id", - "dataset_reference_source_colum": "id", + "dataset_reference_source_column": "id", "wdf_id": "wdf1", - "wdf_value": "val1", + "wdf_column_name": "id", }, { "workspace_id": "ws2", @@ -41,9 +41,9 @@ def valid_dataset_definitions(): "dataset_source_sql": None, "parent_dataset_reference": "parent2", "parent_dataset_reference_attribute_id": "parent2.id", - "dataset_reference_source_colum": "id", + "dataset_reference_source_column": "id", "wdf_id": "wdf2", - "wdf_value": "val2", + "wdf_column_name": "id", }, ] @@ -107,9 +107,9 @@ def test_duplicate_dataset_raises(valid_dataset_definitions): "dataset_source_sql": None, "parent_dataset_reference": "parent1", "parent_dataset_reference_attribute_id": "parent1.id", - "dataset_reference_source_colum": "id", + "dataset_reference_source_column": "id", "wdf_id": "wdf1", - "wdf_value": "val1", + "wdf_column_name": "id", } ] validator = CustomFieldsDataValidator() diff --git a/tests/test_custom_fields/test_models/test_custom_data_object.py b/tests/test_custom_fields/test_models/test_custom_data_object.py index 1663ce3..cd10a4f 100644 --- a/tests/test_custom_fields/test_models/test_custom_data_object.py +++ b/tests/test_custom_fields/test_models/test_custom_data_object.py @@ -42,9 +42,9 @@ def make_valid_dataset_def(**kwargs): "dataset_source_sql": None, "parent_dataset_reference": "parent_ds", "parent_dataset_reference_attribute_id": "parent_attr", - "dataset_reference_source_colum": "src_col", + "dataset_reference_source_column": "src_col", "wdf_id": "wdf1", - "wdf_value": "val1", + "wdf_column_name": "col1", } data.update(kwargs) return data