Make task_always_eager configurable for Fides workers

.github/pull_request_template.md

@@ -12,5 +12,4 @@ Closes ###
 * [ ] Documentation Updated
 * [ ] Increment Applicable Chart Versions
 * [ ] Relevant Follow-Up Issues Created
-* [ ] Update the Fides chart [CHANGELOG.md](https://github.com/ethyca/fides-helm/blob/main/fides/CHANGELOG.md)
-* [ ] Update the Fides-minimal chart [CHANGELOG.md](https://github.com/ethyca/fides-helm/blob/main/fides-minimal/CHANGELOG.md)
+* [ ] Update the [CHANGELOG.md](https://github.com/ethyca/fides-helm/blob/main/CHANGELOG.md)

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/)
 
-To view the Changelog for the Fides code, see the [CHANGELOG.md of the ethyca/fides repository](https://github.com/ethyca/fides/blob/main/CHANGELOG.md).
+To see what's new in Fides, see the [CHANGELOG.md of the ethyca/fides repository](https://github.com/ethyca/fides/blob/main/CHANGELOG.md) and the [Fides release notes](https://www.ethyca.com/docs/releases).
 
 The types of changes are:
 
@@ -16,12 +16,23 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [Unreleased](https://github.com/ethyca/fides-helm/compare/fides-0.18.0...main)
+## [Unreleased](https://github.com/ethyca/fides-helm/compare/fides-0.19.1...main)
 
 ### Added
 
 ### Fixed
 
+## [0.19.1](https://github.com/ethyca/fides-helm/compare/fides-0.19.0...fides-0.19.1)
+
+### Added
+
+- Worker queue and exclude queue annotations: `fid.es/worker-queues` and `fid.es/worker-exclude-queues`
+- Make `task_always_eager` configurable on Fides workers, defaults to `false`.
+
+### Changed
+
+- Upgrade default Fides version to [`2.84.2`](https://github.com/ethyca/fides/releases/tag/2.84.2) (from `2.82.1`)
+
 ## [0.19.0](https://github.com/ethyca/fides-helm/compare/fides-0.18.0...fides-0.19.0)
 
 ### Added

fides-minimal/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: fides-minimal
-version: 0.17.0
+version: 0.19.1
 appVersion: "2.48.1"
 description: Fides is an open-source privacy engineering platform for managing the fulfillment of data privacy requests in your runtime environment, and the enforcement of privacy regulations in your code. This version of the Helm chart removes some excess features such as the lookup function which may not be available in all cases.
 type: application

fides-minimal/templates/fides/worker-config.yaml

@@ -16,7 +16,7 @@ data:
     [celery]
     event_queue_prefix = "fides_worker"
     task_default_queue = "fides"
-    task_always_eager = false
+    task_always_eager = {{ .Values.fides.workerConfiguration.taskAlwaysEager }}
     redis_socket_keepalive = true
 
     [security]

fides-minimal/templates/fides/worker-deployment.yaml

@@ -63,9 +63,17 @@ spec:
     {{- include "fides.deploymentStrategy" $ | nindent 4 }}
   template:
     metadata:
-      {{- with $.Values.podAnnotations }}
+      {{- if or $.Values.podAnnotations (hasKey . "queues") (hasKey . "excludeQueues") }}
       annotations:
+        {{- with $.Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- if hasKey . "queues" }}
+        fid.es/worker-queues: {{ .queues | toJson | quote }}
+        {{- end }}
+        {{- if hasKey . "excludeQueues" }}
+        fid.es/worker-exclude-queues: {{ .excludeQueues | toJson | quote }}
+        {{- end }}
       {{- end }}
       labels:
         {{- include "fides.worker.selectorLabels" $ | nindent 8 }}

fides-minimal/values.yaml

@@ -70,6 +70,10 @@ fides:
   # To override defaults, explicitly define workers below. To disable a worker, set count: 0.
   # For more information, see: https://www.ethyca.com/docs/dev-docs/get-started/advanced#running-workers
   workerConfiguration:
+    # When true, the worker's fides.toml pins celery.task_always_eager = true,
+    # so subtasks spawned within a worker run inline instead of being requeued.
+    # Default false enables proper fan-out across the worker pool.
+    taskAlwaysEager: false
     workers: []
     # Example worker override:
     # - name: other

fides/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: fides
-version: 0.19.0
-appVersion: "2.82.1"
+version: 0.19.1
+appVersion: "2.84.2"
 description: Fides is an open-source privacy engineering platform for managing the fulfillment of data privacy requests in your runtime environment, and the enforcement of privacy regulations in your code.
 type: application
 keywords:

fides/templates/fides/worker-config.yaml

@@ -15,7 +15,7 @@ data:
     [celery]
     event_queue_prefix = "fides_worker"
     task_default_queue = "fides"
-    task_always_eager = false
+    task_always_eager = {{ .Values.fides.workerConfiguration.taskAlwaysEager }}
     redis_socket_keepalive = true
 
     [security]

fides/templates/fides/worker-deployment.yaml

@@ -61,9 +61,17 @@ spec:
     {{- include "fides.deploymentStrategy" $ | nindent 4 }}
   template:
     metadata:
-      {{- with $.Values.podAnnotations }}
+      {{- if or $.Values.podAnnotations (hasKey . "queues") (hasKey . "excludeQueues") }}
       annotations:
+        {{- with $.Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- if hasKey . "queues" }}
+        fid.es/worker-queues: {{ .queues | toJson | quote }}
+        {{- end }}
+        {{- if hasKey . "excludeQueues" }}
+        fid.es/worker-exclude-queues: {{ .excludeQueues | toJson | quote }}
+        {{- end }}
       {{- end }}
       labels:
         {{- include "fides.worker.selectorLabels" $ | nindent 8 }}

fides/values.yaml

@@ -75,6 +75,10 @@ fides:
   # To override defaults, explicitly define workers below. To disable a worker, set count: 0.
   # For more information, see: https://www.ethyca.com/docs/dev-docs/get-started/advanced#running-workers
   workerConfiguration:
+    # When true, the worker's fides.toml pins celery.task_always_eager = true,
+    # so subtasks spawned within a worker run inline instead of being requeued.
+    # Default false enables proper fan-out across the worker pool.
+    taskAlwaysEager: false
     # useHttpProbe switches the worker liveness probe from a Celery exec command (default)
     # to an HTTP GET against the in-process health check server on port 9000.
     # The exec probe spawns a full celery process on each check, which may cause