diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index 16286b84..1c3a58ea 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -737,6 +737,13 @@ spec: from watcher-api format: int32 type: integer + applicationCredentialSecret: + description: |- + ApplicationCredentialSecret - the AC secret watcher is currently + consuming and protecting with the openstack.org/watcher-ac-consumer + finalizer. Tracked so the controller can remove its finalizer from the + old secret when the openstack-operator rotates the reference. + type: string applierServiceReadyCount: description: ApplierServiceReadyCount defines the number or replicas ready from watcher-applier diff --git a/api/go.mod b/api/go.mod index 248ac0c9..21711ef7 100644 --- a/api/go.mod +++ b/api/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.6 require ( github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 k8s.io/api v0.31.14 k8s.io/apimachinery v0.31.14 k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d diff --git a/api/go.sum b/api/go.sum index ba7a7303..629b2ed7 100644 --- a/api/go.sum +++ b/api/go.sum @@ -76,12 +76,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps= github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= -github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= -github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= +github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 h1:p03uEXoSreyu7LpFmb9YyYM8tEx2D2+7qqhLXNWHTq0= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:JC04T5G4E/he5ukonV1oCqa0QzFkLv761VbLruVghJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/api/v1beta1/watcher_types.go b/api/v1beta1/watcher_types.go index aace7d54..7863857b 100644 --- a/api/v1beta1/watcher_types.go +++ b/api/v1beta1/watcher_types.go @@ -61,6 +61,12 @@ type WatcherStatus struct { // DecisionEngineServiceReadyCount defines the number or replicas ready from watcher-decision-engine DecisionEngineServiceReadyCount int32 `json:"decisionengineServiceReadyCount,omitempty"` + + // ApplicationCredentialSecret - the AC secret watcher is currently + // consuming and protecting with the openstack.org/watcher-ac-consumer + // finalizer. Tracked so the controller can remove its finalizer from the + // old secret when the openstack-operator rotates the reference. + ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` } // WatcherDBPurge defines the parameters for the Watcher database purging cron job diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index 16286b84..1c3a58ea 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -737,6 +737,13 @@ spec: from watcher-api format: int32 type: integer + applicationCredentialSecret: + description: |- + ApplicationCredentialSecret - the AC secret watcher is currently + consuming and protecting with the openstack.org/watcher-ac-consumer + finalizer. Tracked so the controller can remove its finalizer from the + old secret when the openstack-operator rotates the reference. + type: string applierServiceReadyCount: description: ApplierServiceReadyCount defines the number or replicas ready from watcher-applier diff --git a/go.mod b/go.mod index dfd07f0d..35afbf1a 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,12 @@ require ( github.com/go-logr/logr v1.4.3 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.28.2 - github.com/onsi/gomega v1.40.0 + github.com/onsi/gomega v1.41.0 github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 - github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f - github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 - github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 + github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf + github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 + github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587 github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65 github.com/openstack-k8s-operators/watcher-operator/api v0.0.0-00010101000000-000000000000 go.uber.org/zap v1.28.0 @@ -62,8 +62,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 // indirect - github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162 // indirect + github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0 // indirect + github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect diff --git a/go.sum b/go.sum index 18c61fa5..5897d17c 100644 --- a/go.sum +++ b/go.sum @@ -114,22 +114,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps= github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= -github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= -github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= +github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f h1:28WYAUIef3uion0Pps6doCSSbgZtIcodGzwG6BHhCOw= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260507114237-f0b612d6c21f/go.mod h1:4ryvbSYuoN522BIPijnm0wMemPgJVKf7jCv8BNDq46I= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162 h1:kUfZlcl+EbUBEWe6EGLXjzlUeYj7xZ21QsPA5jMJlwE= -github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260430090237-a4265c18a162/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162 h1:b/E0t3klHOxMUzpp/TaMB6OwLliv9mwsHxE94i8Rifc= -github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260430090237-a4265c18a162/go.mod h1:xSRQQt2ygoUdjq0pJelQ7X2aeKPIgAj71O9NpV10CQY= -github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 h1:mG3QhS/QWv9Y/AkZZ5OzO6hu6+l5oDXnI/Q5ZUbj6Zs= -github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:ZYG9CQe7cOePOKQbenEZFA28kPdkUOe9QKbDRwGhEV0= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf h1:FoKK0zNo48i4ZMFxScupCK/YAmy6Ps4IILz3CK4BCTI= +github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260520090027-4d7b7a01c0bf/go.mod h1:VNX1Mda2u5+yGxycIyVrgABucitMDR9ct3Lj6ROS92I= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587 h1:p03uEXoSreyu7LpFmb9YyYM8tEx2D2+7qqhLXNWHTq0= +github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:JC04T5G4E/he5ukonV1oCqa0QzFkLv761VbLruVghJM= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0 h1:kMie+G0aHlGwDHjimjj8AUxTl2R7LGfai/8pev2T+TY= +github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0 h1:wuzSibIT9F/5RbMmxvBVFj6fy2vtKo58nibzmk5L4PM= +github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:tft3oDiN+v6wX3ILPXGUM/gCLJz6QtrPN63hxpJ3E24= +github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587 h1:jpouKcgs2Kc5z2JHIpvsXMxEonfXLgzX3KswuBoeKQ0= +github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260518125357-72bdd580c587/go.mod h1:nLS2oK4pBo756JNN1cPgr44S0X9V11QScgVla89Ojok= github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65 h1:MNjmU326MKwYU6xT6AL2aKbr/0ids87wP5B+s5CL2O0= github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260503164939-40728ae44d65/go.mod h1:I2LO+wGIzbirVczQ9qo/49y2F5Zo/MEg/pabkXOrY2M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/controller/watcher_controller.go b/internal/controller/watcher_controller.go index 76b19c17..aa0df5ed 100644 --- a/internal/controller/watcher_controller.go +++ b/internal/controller/watcher_controller.go @@ -478,6 +478,24 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return ctrl.Result{}, err } + // Add consumer finalizer to the new AC secret early, before deployment. + // The old secret's finalizer is removed later (after all services deploy) + // so that rapid rotations don't revoke a credential still in use by pods. + if instance.Spec.Auth.ApplicationCredentialSecret != "" { + if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + instance.Spec.Auth.ApplicationCredentialSecret, + "", + watcher.ACConsumerFinalizer); err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) // End of config generation for dbsync @@ -523,6 +541,27 @@ func (r *WatcherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re } // End of Watcher Applier deploy + // Manage the old AC secret's finalizer and status tracking. + // On rotation (old != new), only remove the old secret's finalizer after + // all sub-services are ready with the new credentials. This prevents + // premature revocation during rapid rotations. + isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret + + if isRotation { + allServicesReady := instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherAPIReadyCondition) && + instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherApplierReadyCondition) && + instance.Status.Conditions.IsTrue(watcherv1beta1.WatcherDecisionEngineReadyCondition) + if allServicesReady { + if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, watcher.ACConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + } + } else { + instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + } + // // remove finalizers from unused MariaDBAccount records // this assumes all database-depedendent deployments are up and @@ -1347,6 +1386,17 @@ func (r *WatcherReconciler) reconcileDelete(ctx context.Context, instance *watch } // + // Remove consumer finalizer from AC secrets watcher was consuming. + for _, secretName := range []string{ + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + } { + if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, watcher.ACConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil diff --git a/internal/watcher/constants.go b/internal/watcher/constants.go index 5babba7c..c2e1ff0c 100644 --- a/internal/watcher/constants.go +++ b/internal/watcher/constants.go @@ -50,4 +50,7 @@ const ( // scriptVolume is the name of the volume used to ship scripts into pods scriptVolume = "scripts-volume" + + // ACConsumerFinalizer is added to AC secrets that watcher is actively consuming + ACConsumerFinalizer = "openstack.org/watcher-ac-consumer" ) diff --git a/test/functional/watcher_controller_test.go b/test/functional/watcher_controller_test.go index 95365828..18ae3b12 100644 --- a/test/functional/watcher_controller_test.go +++ b/test/functional/watcher_controller_test.go @@ -17,6 +17,7 @@ import ( . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" + "github.com/openstack-k8s-operators/watcher-operator/internal/watcher" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -1858,6 +1859,206 @@ var _ = Describe("Watcher controller", func() { }) }) + When("ApplicationCredential consumer finalizer is managed", func() { + var acSecretName string + + BeforeEach(func() { + acSecretName = "ac-watcher-consumer-fnz-secret" //nolint:gosec + + acSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: acSecretName, + Namespace: watcherTest.Instance.Namespace, + }, + Data: map[string][]byte{ + keystonev1beta1.ACIDSecretKey: []byte("consumer-test-ac-id"), + keystonev1beta1.ACSecretSecretKey: []byte("consumer-test-ac-secret"), //nolint:gosec + }, + } + Expect(k8sClient.Create(ctx, acSecret)).To(Succeed()) + DeferCleanup(k8sClient.Delete, ctx, acSecret) + + DeferCleanup(k8sClient.Delete, ctx, CreateWatcherMessageBusSecret(watcherTest.Instance.Namespace, "rabbitmq-secret")) + + memcachedSpec := memcachedv1.MemcachedSpec{ + MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ + Replicas: ptr.To(int32(1)), + }, + } + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.Watcher.Namespace, MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: "metric-storage-prometheus-endpoint"}, + map[string][]byte{ + "host": []byte("prometheus.example.com"), + "port": []byte("9090"), + }, + )) + + DeferCleanup( + k8sClient.Delete, ctx, th.CreateSecret( + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: SecretName}, + map[string][]byte{ + "WatcherPassword": []byte("password"), + }, + )) + + // Create Watcher CR after all secrets and dependencies are in place + // so sub-CR controllers don't enter long exponential backoff. + spec := GetDefaultWatcherSpec() + spec["auth"] = map[string]any{"applicationCredentialSecret": acSecretName} + DeferCleanup(th.DeleteInstance, CreateWatcher(watcherTest.Instance, spec)) + + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.Instance.Namespace, + *GetWatcher(watcherTest.Instance).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + + mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount) + mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName) + infra.SimulateTransportURLReady(watcherTest.WatcherTransportURL) + + keystone.SimulateKeystoneServiceReady(watcherTest.KeystoneServiceName) + th.SimulateJobSuccess(watcherTest.WatcherDBSync) + }) + + It("should add the consumer finalizer to the AC secret", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(watcher.ACConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + }) + + It("should track the consumed AC secret in status", func() { + Eventually(func(g Gomega) { + w := GetWatcher(watcherTest.Instance) + g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName)) + }, timeout, interval).Should(Succeed()) + }) + + It("should move the finalizer from the old to the new secret on rotation", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(watcher.ACConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Simulate all watcher services deploying successfully + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet) + keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName) + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet) + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet) + + Eventually(func(g Gomega) { + w := GetWatcher(watcherTest.Instance) + g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(acSecretName)) + }, timeout, interval).Should(Succeed()) + + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + newACSecretName := "ac-watcher-consumer-rotated-secret" //nolint:gosec + newSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: watcherTest.Instance.Namespace, + Name: newACSecretName, + }, + Data: map[string][]byte{ + keystonev1beta1.ACIDSecretKey: []byte("rotated-ac-id"), + keystonev1beta1.ACSecretSecretKey: []byte("rotated-ac-secret-value"), //nolint:gosec + }, + } + DeferCleanup(k8sClient.Delete, ctx, newSecret) + Expect(k8sClient.Create(ctx, newSecret)).To(Succeed()) + + Eventually(func(g Gomega) { + w := GetWatcher(watcherTest.Instance) + w.Spec.Auth.ApplicationCredentialSecret = newACSecretName + g.Expect(k8sClient.Update(ctx, w)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + + // New secret gets the consumer finalizer immediately (early in reconcile) + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: newACSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(watcher.ACConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Old secret keeps the finalizer until all services deploy (split pattern) + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + Expect(secret.Finalizers).To( + ContainElement(watcher.ACConsumerFinalizer)) + + // Simulate all watcher services deploying successfully + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherAPIStatefulSet) + keystone.SimulateKeystoneEndpointReady(watcherTest.WatcherKeystoneEndpointName) + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherApplierStatefulSet) + th.SimulateStatefulSetReplicaReady(watcherTest.WatcherDecisionEngineStatefulSet) + + // Now the old secret's finalizer is removed and status updated + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(watcher.ACConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + w := GetWatcher(watcherTest.Instance) + g.Expect(w.Status.ApplicationCredentialSecret).To(Equal(newACSecretName)) + }, timeout, interval).Should(Succeed()) + }) + + It("should remove the consumer finalizer from AC secret on CR deletion", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(watcher.ACConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + th.DeleteInstance(GetWatcher(watcherTest.Instance)) + + secret := th.GetSecret(types.NamespacedName{ + Namespace: watcherTest.Instance.Namespace, + Name: acSecretName, + }) + Expect(secret.Finalizers).NotTo( + ContainElement(watcher.ACConsumerFinalizer)) + }) + }) + When("ApplicationCredential is adopted on existing deployment", func() { var appCredSecretName string var appCredID string diff --git a/test/kuttl/test-suites/default/appcred-tests/00-cleanup.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/00-cleanup.yaml similarity index 100% rename from test/kuttl/test-suites/default/appcred-tests/00-cleanup.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/00-cleanup.yaml diff --git a/test/kuttl/test-suites/default/appcred-tests/01-assert.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/01-assert.yaml similarity index 100% rename from test/kuttl/test-suites/default/appcred-tests/01-assert.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/01-assert.yaml diff --git a/test/kuttl/test-suites/default/appcred-tests/01-deploy.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/01-deploy.yaml similarity index 100% rename from test/kuttl/test-suites/default/appcred-tests/01-deploy.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/01-deploy.yaml diff --git a/test/kuttl/test-suites/default/appcred-tests/02-assert.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/02-assert.yaml similarity index 66% rename from test/kuttl/test-suites/default/appcred-tests/02-assert.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/02-assert.yaml index e22f53e0..b4bd1d7a 100644 --- a/test/kuttl/test-suites/default/appcred-tests/02-assert.yaml +++ b/test/kuttl/test-suites/default/zz-appcred-tests/02-assert.yaml @@ -11,13 +11,20 @@ commands: ac_id=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.acID}') if [ -z "${ac_id}" ]; then - echo "ERROR: appcred/ac-watcher.status.acID is empty" + echo "ERROR: appcred/ac-watcher-test.status.acID is empty" exit 1 fi - echo "ac-watcher.status.acID=${ac_id}" + echo "ac-watcher-test.status.acID=${ac_id}" - oc get -n "${NS}" secret/ac-watcher-test-secret >/dev/null - secret_ac_id=$(oc get -n "${NS}" secret/ac-watcher-test-secret -o jsonpath='{.data.AC_ID}' | base64 -d) + ac_secret_name=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.secretName}') + if [ -z "${ac_secret_name}" ]; then + echo "ERROR: appcred/ac-watcher-test.status.secretName is empty" + exit 1 + fi + echo "AC secret name: ${ac_secret_name}" + + oc get -n "${NS}" "secret/${ac_secret_name}" >/dev/null + secret_ac_id=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.data.AC_ID}' | base64 -d) if [ "${secret_ac_id}" != "${ac_id}" ]; then echo "ERROR: Secret AC_ID (${secret_ac_id}) != appcred.status.acID (${ac_id})" exit 1 @@ -46,6 +53,23 @@ commands: fi echo "✓ watcher pods restarted after appcred secret became available" + echo "Checking consumer finalizer on AC secret..." + finalizers=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.metadata.finalizers}') + if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then + echo "ERROR: AC secret missing watcher consumer finalizer" + echo " finalizers: ${finalizers}" + exit 1 + fi + echo "✓ AC secret has openstack.org/watcher-ac-consumer finalizer" + + echo "Checking watcher status tracks the consumed AC secret..." + status_ac=$(oc get -n "${NS}" watcher/watcher-kuttl -o jsonpath='{.status.applicationCredentialSecret}') + if [ "${status_ac}" != "${ac_secret_name}" ]; then + echo "ERROR: watcher.status.applicationCredentialSecret expected ${ac_secret_name}, got ${status_ac}" + exit 1 + fi + echo "✓ watcher.status.applicationCredentialSecret = ${status_ac}" + echo "Checking watcher config contains application_credential_id..." oc exec -n "${NS}" pod/watcher-kuttl-api-0 -c watcher-api -- \ bash -c "grep -q \"^application_credential_id = ${ac_id}$\" /etc/watcher/watcher.conf.d/00-default.conf" diff --git a/test/kuttl/test-suites/default/appcred-tests/02-deploy-appcred.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/02-deploy-appcred.yaml similarity index 67% rename from test/kuttl/test-suites/default/appcred-tests/02-deploy-appcred.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/02-deploy-appcred.yaml index 3d8ea35a..7038a616 100644 --- a/test/kuttl/test-suites/default/appcred-tests/02-deploy-appcred.yaml +++ b/test/kuttl/test-suites/default/zz-appcred-tests/02-deploy-appcred.yaml @@ -29,4 +29,14 @@ commands: unrestricted: false EOF - oc patch watcher watcher-kuttl -n "${NS}" --type=merge -p '{"spec":{"auth":{"applicationCredentialSecret":"ac-watcher-test-secret"}}}' + echo "Waiting for KeystoneApplicationCredential ac-watcher-test to be Ready..." + oc wait -n "${NS}" appcred/ac-watcher-test --for=condition=Ready --timeout=300s + + ac_secret_name=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.secretName}') + if [ -z "${ac_secret_name}" ]; then + echo "ERROR: appcred/ac-watcher-test.status.secretName is empty" + exit 1 + fi + echo "AC secret name: ${ac_secret_name}" + + oc patch watcher watcher-kuttl -n "${NS}" --type=merge -p "{\"spec\":{\"auth\":{\"applicationCredentialSecret\":\"${ac_secret_name}\"}}}" diff --git a/test/kuttl/test-suites/default/appcred-tests/03-assert.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/03-assert.yaml similarity index 73% rename from test/kuttl/test-suites/default/appcred-tests/03-assert.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/03-assert.yaml index a61fce0c..239626c4 100644 --- a/test/kuttl/test-suites/default/appcred-tests/03-assert.yaml +++ b/test/kuttl/test-suites/default/zz-appcred-tests/03-assert.yaml @@ -12,7 +12,7 @@ commands: exit 1 fi - echo "Waiting for ac-watcher.status.acID to change..." + echo "Waiting for ac-watcher-test.status.acID to change..." for _ in $(seq 1 60); do ac_id=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.acID}') if [ -n "${ac_id}" ] && [ "${ac_id}" != "${prev_acid}" ]; then @@ -29,7 +29,18 @@ commands: fi echo "✓ ACID rotated: ${prev_acid} -> ${ac_id}" - secret_ac_id=$(oc get -n "${NS}" secret/ac-watcher-test-secret -o jsonpath='{.data.AC_ID}' | base64 -d) + ac_secret_name=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.secretName}') + if [ -z "${ac_secret_name}" ]; then + echo "ERROR: appcred/ac-watcher-test.status.secretName is empty after rotation" + exit 1 + fi + echo "AC secret after rotation: ${ac_secret_name}" + + # Patch the service CR with the new secret name + oc patch watcher watcher-kuttl -n "${NS}" --type=merge \ + -p "{\"spec\":{\"auth\":{\"applicationCredentialSecret\":\"${ac_secret_name}\"}}}" + + secret_ac_id=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.data.AC_ID}' | base64 -d) if [ "${secret_ac_id}" != "${ac_id}" ]; then echo "ERROR: Secret AC_ID (${secret_ac_id}) != rotated appcred.status.acID (${ac_id})" exit 1 @@ -58,6 +69,15 @@ commands: fi echo "✓ watcher pods restarted after rotation" + echo "Checking consumer finalizer present on current AC secret after rotation..." + finalizers=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.metadata.finalizers}') + if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then + echo "ERROR: AC secret lost watcher consumer finalizer after rotation" + echo " finalizers: ${finalizers}" + exit 1 + fi + echo "✓ AC secret has openstack.org/watcher-ac-consumer finalizer after rotation" + echo "Checking watcher config contains rotated application_credential_id..." oc exec -n "${NS}" pod/watcher-kuttl-api-0 -c watcher-api -- \ bash -c "grep -q \"^application_credential_id = ${ac_id}$\" /etc/watcher/watcher.conf.d/00-default.conf" diff --git a/test/kuttl/test-suites/default/appcred-tests/03-rotate-appcred.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/03-rotate-appcred.yaml similarity index 53% rename from test/kuttl/test-suites/default/appcred-tests/03-rotate-appcred.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/03-rotate-appcred.yaml index fcf9ef4f..606139fc 100644 --- a/test/kuttl/test-suites/default/appcred-tests/03-rotate-appcred.yaml +++ b/test/kuttl/test-suites/default/zz-appcred-tests/03-rotate-appcred.yaml @@ -19,3 +19,23 @@ commands: oc patch -n "${NS}" keystoneapplicationcredential ac-watcher-test --type=merge --subresource=status \ -p '{"status":{"expiresAt":"2001-05-19T00:00:00Z"}}' + + echo "Waiting for AC to rotate (new acID)..." + for _ in $(seq 1 60); do + ac_id=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.acID}') + if [ -n "${ac_id}" ] && [ "${ac_id}" != "${old_acid}" ]; then + break + fi + sleep 5 + done + ac_id=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.acID}') + if [ -z "${ac_id}" ] || [ "${ac_id}" = "${old_acid}" ]; then + echo "ERROR: ACID did not change after forcing rotation" + exit 1 + fi + + new_secret_name=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.secretName}') + echo "New AC secret after rotation: ${new_secret_name}" + + oc patch watcher watcher-kuttl -n "${NS}" --type=merge \ + -p "{\"spec\":{\"auth\":{\"applicationCredentialSecret\":\"${new_secret_name}\"}}}" diff --git a/test/kuttl/test-suites/default/zz-appcred-tests/04-delete-watcher.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/04-delete-watcher.yaml new file mode 100644 index 00000000..0a0a8153 --- /dev/null +++ b/test/kuttl/test-suites/default/zz-appcred-tests/04-delete-watcher.yaml @@ -0,0 +1,29 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -euo pipefail + NS="${NAMESPACE}" + + ac_secret_name=$(oc get -n "${NS}" appcred/ac-watcher-test -o jsonpath='{.status.secretName}') + echo "Current AC secret: ${ac_secret_name}" + + echo "Deleting Watcher CR..." + oc delete -n "${NS}" watcher watcher-kuttl --timeout=300s + + echo "Verifying consumer finalizer was removed from AC secret..." + for _ in $(seq 1 30); do + finalizers=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.metadata.finalizers}' 2>/dev/null || true) + if [[ "${finalizers}" != *"openstack.org/watcher-ac-consumer"* ]]; then + break + fi + sleep 2 + done + + finalizers=$(oc get -n "${NS}" "secret/${ac_secret_name}" -o jsonpath='{.metadata.finalizers}' 2>/dev/null || true) + if [[ "${finalizers}" == *"openstack.org/watcher-ac-consumer"* ]]; then + echo "ERROR: consumer finalizer still present on AC secret after Watcher deletion" + echo " finalizers: ${finalizers}" + exit 1 + fi + echo "✓ consumer finalizer removed from AC secret after Watcher deletion" diff --git a/test/kuttl/test-suites/default/appcred-tests/04-cleanup.yaml b/test/kuttl/test-suites/default/zz-appcred-tests/05-cleanup.yaml similarity index 83% rename from test/kuttl/test-suites/default/appcred-tests/04-cleanup.yaml rename to test/kuttl/test-suites/default/zz-appcred-tests/05-cleanup.yaml index a4aa41ba..8b85cd1c 100644 --- a/test/kuttl/test-suites/default/appcred-tests/04-cleanup.yaml +++ b/test/kuttl/test-suites/default/zz-appcred-tests/05-cleanup.yaml @@ -5,10 +5,6 @@ delete: kind: KeystoneApplicationCredential name: ac-watcher-test namespace: watcher-kuttl-default - - apiVersion: v1 - kind: Secret - name: ac-watcher-test-secret - namespace: watcher-kuttl-default - apiVersion: watcher.openstack.org/v1beta1 kind: Watcher name: watcher-kuttl