From 34b2a34514e89a7d91065dfcddba7740037ac480 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 16:46:20 -0700 Subject: [PATCH 01/10] chore: point fideslang to branch with data_purposes support Update fideslang dependency to use the feat/add-data-purposes-to-dataset-models branch which adds data_purposes at dataset, collection, field, and sub-field levels. Dependency: ethyca/fideslang#39 Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5790328d99e..d6eaec0e93f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "fastapi-cli~=0.0.16", "fastapi-pagination[sqlalchemy]==0.15.0", "fastapi[all]==0.123.3", - "fideslang==3.1.4a1", + "fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models", "fideslog==1.2.15", "firebase-admin==5.3.0", "flower==2.0.1", From 9c0240ebdb3d094b3e5b6cc9e9512535e67eb7f2 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 17:49:38 -0700 Subject: [PATCH 02/10] fix: allow direct references in hatch metadata for fideslang git dep Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d6eaec0e93f..3a752695e54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,9 @@ dev = [ fides = "fides.cli:cli" webserver = "fides.api.main:start_webserver" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] source = "vcs" fallback-version = "0.0.0+dev" From 5662d04f236591a63a589eb38e740d3f5bf45cee Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 18:01:16 -0700 Subject: [PATCH 03/10] fix: move fideslang git ref to requirements.txt to avoid hatchling direct-reference error Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 5 +---- requirements.txt | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 3a752695e54..5790328d99e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "fastapi-cli~=0.0.16", "fastapi-pagination[sqlalchemy]==0.15.0", "fastapi[all]==0.123.3", - "fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models", + "fideslang==3.1.4a1", "fideslog==1.2.15", "firebase-admin==5.3.0", "flower==2.0.1", @@ -180,9 +180,6 @@ dev = [ fides = "fides.cli:cli" webserver = "fides.api.main:start_webserver" -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.version] source = "vcs" fallback-version = "0.0.0+dev" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000..72766131c14 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models From d05cded6df48d2a4cc58b9a52f08761314f200d7 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 10:47:28 -0700 Subject: [PATCH 04/10] Bumping Fideslang version --- pyproject.toml | 2 +- requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index acc50850228..fe6643e2b4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ "fastapi-cli~=0.0.16", "fastapi-pagination[sqlalchemy]==0.15.0", "fastapi[all]==0.123.3", - "fideslang==3.1.4a1", + "fideslang==3.1.3", "fideslog==1.2.15", "firebase-admin==5.3.0", "flower==2.0.1", diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 72766131c14..00000000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models From a2867de920b953b44b2fc79f5a451c17114e7c5a Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 10:52:23 -0700 Subject: [PATCH 05/10] Add changelog entry for dataset data_purposes support Co-Authored-By: Claude Opus 4.6 (1M context) --- changelog/7674-dataset-data-purposes.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/7674-dataset-data-purposes.yaml diff --git a/changelog/7674-dataset-data-purposes.yaml b/changelog/7674-dataset-data-purposes.yaml new file mode 100644 index 00000000000..430a22e04ff --- /dev/null +++ b/changelog/7674-dataset-data-purposes.yaml @@ -0,0 +1,4 @@ +type: Changed +description: Updated fideslang to 3.1.3 to add `data_purposes` support at dataset, collection, field, and sub-field levels +pr: 7674 +labels: [] From 0dc3be1314601e8139e536a69040308bf4956cf1 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 10:56:51 -0700 Subject: [PATCH 06/10] Updating uv.lock --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 1327024e87b..229e8a77763 100644 --- a/uv.lock +++ b/uv.lock @@ -1137,7 +1137,7 @@ requires-dist = [ { name = "fastapi", extras = ["all"], specifier = "==0.123.3" }, { name = "fastapi-cli", specifier = "~=0.0.16" }, { name = "fastapi-pagination", extras = ["sqlalchemy"], specifier = "==0.15.0" }, - { name = "fideslang", specifier = "==3.1.4a1" }, + { name = "fideslang", specifier = "==3.1.3" }, { name = "fideslog", specifier = "==1.2.15" }, { name = "filelock", specifier = "~=3.20.0" }, { name = "firebase-admin", specifier = "==5.3.0" }, @@ -1436,16 +1436,16 @@ wheels = [ [[package]] name = "fideslang" -version = "3.1.4a1" +version = "3.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "pydantic" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/25/f40b2357daf0ebefa234b301a0c33154ea3f76030b82dffa5b39f5e44067/fideslang-3.1.4a1.tar.gz", hash = "sha256:d03f07f4a83fa3ef733c17d2e4891611beec9598288a17e2371403240b5412c4", size = 201457, upload-time = "2025-12-18T02:01:27.318Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/b3/7f7867a074975af22b4f48f19fac52ef1eda362cef74ea00285bc73324d2/fideslang-3.1.3.tar.gz", hash = "sha256:e1a1b377da21b7521df3aeacaadd0c19c3f6367d65a5f97291f9aefcf498fed2", size = 201399, upload-time = "2026-03-25T17:36:28.501Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/12/9563420acabd163519bac3a735ea9776ed2c0e87463742c5a3626045509f/fideslang-3.1.4a1-py3-none-any.whl", hash = "sha256:0f82282593a6ebcef0965e66b9296fbf89a42b2177e77a8252ee755f18013813", size = 47723, upload-time = "2025-12-18T02:01:25.959Z" }, + { url = "https://files.pythonhosted.org/packages/ee/3a/8882bada1bf5c65909e5db43bdebcf57db9bded8677a4129b4a1fe382a7a/fideslang-3.1.3-py3-none-any.whl", hash = "sha256:6741f88b05c924beb103a1a2cd742f64749145f5fe706dcd2b150fafa9cf6365", size = 47592, upload-time = "2026-03-25T17:36:27.121Z" }, ] [[package]] From a18df6eeabfe04b2df0ba11842ec59fe3f962327 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 12:06:39 -0700 Subject: [PATCH 07/10] Update uv.lock --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index edaa8c4f3fd..f192dae6af3 100644 --- a/uv.lock +++ b/uv.lock @@ -1102,7 +1102,7 @@ requires-dist = [ { name = "fastapi", extras = ["all"], specifier = "==0.123.3" }, { name = "fastapi-cli", specifier = "~=0.0.16" }, { name = "fastapi-pagination", extras = ["sqlalchemy"], specifier = "==0.15.0" }, - { name = "fideslang", specifier = "==3.1.3" } + { name = "fideslang", specifier = "==3.1.3" }, { name = "filelock", specifier = "~=3.20.0" }, { name = "firebase-admin", specifier = "==5.3.0" }, { name = "flower", specifier = "==2.0.1" }, From 76cba4fc2f690593a5de8aadb25eacae445db6fe Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 13:05:04 -0700 Subject: [PATCH 08/10] Add data_purposes column to Dataset model and migration The fideslang 3.1.3 bump added a data_purposes field to the Dataset Pydantic model, but the SQLAlchemy model was missing the corresponding column, causing TypeError on dataset create/update. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...3a9b1d4f2_add_data_purposes_to_datasets.py | 27 +++++++++++++++++++ src/fides/api/models/sql_models.py | 1 + 2 files changed, 28 insertions(+) create mode 100644 src/fides/api/alembic/migrations/versions/xx_2026_03_25_1500_c7e3a9b1d4f2_add_data_purposes_to_datasets.py diff --git a/src/fides/api/alembic/migrations/versions/xx_2026_03_25_1500_c7e3a9b1d4f2_add_data_purposes_to_datasets.py b/src/fides/api/alembic/migrations/versions/xx_2026_03_25_1500_c7e3a9b1d4f2_add_data_purposes_to_datasets.py new file mode 100644 index 00000000000..77b8bd3d6ed --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/xx_2026_03_25_1500_c7e3a9b1d4f2_add_data_purposes_to_datasets.py @@ -0,0 +1,27 @@ +"""add data_purposes column to ctl_datasets + +Revision ID: c7e3a9b1d4f2 +Revises: 25ffa8bf6a95 +Create Date: 2026-03-25 15:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c7e3a9b1d4f2" +down_revision = "25ffa8bf6a95" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "ctl_datasets", + sa.Column("data_purposes", sa.ARRAY(sa.String()), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("ctl_datasets", "data_purposes") diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 316a845dddd..bc7e48eca0e 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -406,6 +406,7 @@ class Dataset(Base, FidesBase): meta = Column(JSON) data_categories = Column(ARRAY(String)) + data_purposes = Column(ARRAY(String)) collections = Column(JSON) fides_meta = Column(JSON) From fcdf4743432071f0822b5f61d0a57ad3dc41095d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 13:14:33 -0700 Subject: [PATCH 09/10] Add data_purposes annotation to db_dataset.yml for fides_db_scan Co-Authored-By: Claude Opus 4.6 (1M context) --- .fides/db_dataset.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 573599401e3..5659af9d491 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -697,6 +697,9 @@ dataset: - name: data_categories data_categories: - system.operations + - name: data_purposes + data_categories: + - system.operations - name: description data_categories: - system.operations From 4056b0e8f2a4c67309e5e6c4df9cf9532bdd2a63 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 25 Mar 2026 13:25:30 -0700 Subject: [PATCH 10/10] Normalize customer_dataset in merge_datasets to prevent spurious diffs The customer_dataset was not being normalized through FideslangDataset like stored and upcoming datasets were, causing DeepDiff to detect false changes when fideslang adds new fields (e.g. data_purposes). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/fides/service/connection/merge_configs_util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fides/service/connection/merge_configs_util.py b/src/fides/service/connection/merge_configs_util.py index 0649c2f4ae0..7815c5304b5 100644 --- a/src/fides/service/connection/merge_configs_util.py +++ b/src/fides/service/connection/merge_configs_util.py @@ -328,14 +328,17 @@ def merge_datasets( """ stored_dataset_copy = copy.deepcopy(stored_dataset) upcoming_dataset_copy = copy.deepcopy(upcoming_dataset) + customer_dataset_copy = copy.deepcopy(customer_dataset) normalized_stored_dataset = normalize_dataset(stored_dataset_copy, "stored") normalized_upcoming_dataset = normalize_dataset(upcoming_dataset_copy, "upcoming") + normalized_customer_dataset = normalize_dataset(customer_dataset_copy, "customer") if normalized_stored_dataset is None or normalized_upcoming_dataset is None: return upcoming_dataset stored_dataset_copy = normalized_stored_dataset upcoming_dataset_copy = normalized_upcoming_dataset + customer_dataset = normalized_customer_dataset or customer_dataset # convert dataset to yaml string and then replace the instance key placeholder wrapped_dataset = {"dataset": [stored_dataset_copy]}