From 92d68c0262ff993f917022e53f21d3f8f0046ae4 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 11 Oct 2024 13:27:14 -0400 Subject: [PATCH 1/2] Reconcile SecretProvider for the CSO on ARO HCP Reconcile the SecretProviderClass for the cluster storage operator (CSO) for ARO HCP deployments. The SecretProviderClass is used by the Secrets Store CSI driver to mount a certificate to a volume in the azure-disk-csi-controller and azure-file-csi-controller pod deployments. Signed-off-by: Bryan Cox --- .../hostedcontrolplane_controller.go | 19 +++++++++++++++++++ .../hostedcontrolplane/storage/operator.go | 19 +++++++++++++++++++ .../hostedcontrolplane/storage/params.go | 8 ++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 0d88997afa1..8456304e21e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -4969,6 +4969,25 @@ func (r *HostedControlPlaneReconciler) reconcileCSISnapshotControllerOperator(ct func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider, userReleaseImageProvider *imageprovider.SimpleReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error { params := storage.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext) + if hyperazureutil.IsAroHCP() { + // Reconcile SecretProviderClasses + azureDiskSecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureDiskCSISecretStoreProviderClassName, hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskSecretProviderClass, func() error { + secretproviderclass.ReconcileManagedAzureSecretProviderClass(azureDiskSecretProviderClass, hcp, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Disk.CertificateName) + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile Azure Disk Secret Provider Class: %w", err) + } + + azureFileSecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureFileCSISecretStoreProviderClassName, hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureFileSecretProviderClass, func() error { + secretproviderclass.ReconcileManagedAzureSecretProviderClass(azureFileSecretProviderClass, hcp, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.File.CertificateName) + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile Azure File Secret Provider Class: %w", err) + } + } + if hcp.Spec.Platform.Type == hyperv1.AzurePlatform { credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace) if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go index 03f559846de..45a3275226b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go @@ -6,6 +6,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage/assets" assets2 "github.com/openshift/hypershift/support/assets" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -31,6 +32,24 @@ func ReconcileOperatorDeployment( case "cluster-storage-operator": deployment.Spec.Template.Spec.Containers[i].Image = params.StorageOperatorImage params.ImageReplacer.replaceEnvVars(deployment.Spec.Template.Spec.Containers[i].Env) + + // For managed Azure, we need to supply a couple of environment variables for CSO to pass on to the CSI controllers for disk and file. + // CSO passes those on to the CSI deployment here - https://github.com/openshift/cluster-storage-operator/pull/517/files. + // CSI then mounts the Secrets Provider Class here - https://github.com/openshift/csi-operator/pull/309/files. + if azureutil.IsAroHCP() { + if deployment.Spec.Template.Spec.Containers[i].Env == nil { + deployment.Spec.Template.Spec.Containers[i].Env = make([]corev1.EnvVar, 0) + } + deployment.Spec.Template.Spec.Containers[i].Env = append(deployment.Spec.Template.Spec.Containers[i].Env, + corev1.EnvVar{ + Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_DISK", + Value: params.AzureDiskSecretProviderClassName, + }, + corev1.EnvVar{ + Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_FILE", + Value: params.AzureFileSecretProviderClassName, + }) + } } } diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go index 94b53d154e6..2cbfa37f3bd 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go @@ -33,10 +33,10 @@ func NewParams( ir.setOperatorImageReferences(releaseImageProvider, userReleaseImageProvider) params := Params{ - OwnerRef: config.OwnerRefFrom(hcp), - StorageOperatorImage: releaseImageProvider.GetImage(storageOperatorImageName), - AvailabilityProberImage: releaseImageProvider.GetImage(util.AvailabilityProberImageName), - ImageReplacer: ir, + OwnerRef: config.OwnerRefFrom(hcp), + StorageOperatorImage: releaseImageProvider.GetImage(storageOperatorImageName), + AvailabilityProberImage: releaseImageProvider.GetImage(util.AvailabilityProberImageName), + ImageReplacer: ir, } params.DeploymentConfig = config.DeploymentConfig{ AdditionalLabels: map[string]string{ From 74287710a908357c17902185ca2cf5f3f58886c9 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 11 Oct 2024 13:58:56 -0400 Subject: [PATCH 2/2] Reconcile Secret Data for Azure Disk and File CSI Reconcile the secret data needed for the azure-disk and azure-file CSI controllers. The format is the same as the Cloud Provider. More info on the configuration can be found here: https://cloud-provider-azure.sigs.k8s.io/install/configs/ Signed-off-by: Bryan Cox --- .../cloud/azure/providerconfig.go | 1 + .../hostedcontrolplane_controller.go | 25 ++++++++ .../hostedcontrolplane/storage/azure.go | 61 +++++++++++++++++++ .../hostedcontrolplane/storage/operator.go | 5 +- .../hostedcontrolplane/storage/params.go | 8 +-- 5 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/storage/azure.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go index 0cf682ebfe4..c4eb918fee6 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go @@ -123,4 +123,5 @@ type AzureConfig struct { LoadBalancerSku string `json:"loadBalancerSku"` DisableOutboundSNAT bool `json:"disableOutboundSNAT"` LoadBalancerName string `json:"loadBalancerName"` + AADClientCertPath string `json:"aadClientCertPath"` } diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 8456304e21e..75662d3e299 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -4986,6 +4986,31 @@ func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx conte }); err != nil { return fmt.Errorf("failed to reconcile Azure File Secret Provider Class: %w", err) } + + // Get the credentials secret so we can retrieve the tenant ID for the configuration + credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace) + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) + + // Reconcile the secret needed for azure-disk-csi-controller + // This is related to https://github.com/openshift/csi-operator/pull/290. + azureDiskCSISecret := manifests.AzureDiskConfigWithCredentials(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureDiskCSISecret(azureDiskCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure Disk CSI config: %w", err) + } + + // Reconcile the secret needed for azure-disk-csi-controller + // This is related to https://github.com/openshift/csi-operator/pull/290. + azureFileCSISecret := manifests.AzureFileConfigWithCredentials(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureFileCSISecret(azureFileCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure File CSI config: %w", err) + } } if hcp.Spec.Platform.Type == hyperv1.AzurePlatform { diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go new file mode 100644 index 00000000000..e77cc9afc2d --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go @@ -0,0 +1,61 @@ +package storage + +import ( + "encoding/json" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" + + corev1 "k8s.io/api/core/v1" +) + +// initializeAzureCSIControllerConfig initializes an AzureConfig object which will be used to populate the secrets +// needed by azure-disk-csi-controller and azure-file-csi-controller. +func initializeAzureCSIControllerConfig(hcp *hyperv1.HostedControlPlane, tenantID string) azure.AzureConfig { + azureConfig := azure.AzureConfig{ + Cloud: hcp.Spec.Platform.Azure.Cloud, + TenantID: tenantID, + SubscriptionID: hcp.Spec.Platform.Azure.SubscriptionID, + ResourceGroup: hcp.Spec.Platform.Azure.ResourceGroupName, + Location: hcp.Spec.Platform.Azure.Location, + } + + return azureConfig +} + +// ReconcileAzureDiskCSISecret reconciles the configuration for the secret as expected by azure-disk-csi-controller +func ReconcileAzureDiskCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.AADClientID = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Disk.ClientID + config.AADClientCertPath = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Disk.CertificateName + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} + +// ReconcileAzureFileCSISecret reconciles the configuration for the secret as expected by azure-file-csi-controller +func ReconcileAzureFileCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.AADClientID = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.File.ClientID + config.AADClientCertPath = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.File.CertificateName + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go index 45a3275226b..8b7d04a0175 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go @@ -7,6 +7,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage/assets" assets2 "github.com/openshift/hypershift/support/assets" "github.com/openshift/hypershift/support/azureutil" + "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -43,11 +44,11 @@ func ReconcileOperatorDeployment( deployment.Spec.Template.Spec.Containers[i].Env = append(deployment.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_DISK", - Value: params.AzureDiskSecretProviderClassName, + Value: config.ManagedAzureDiskCSISecretStoreProviderClassName, }, corev1.EnvVar{ Name: "ARO_HCP_SECRET_PROVIDER_CLASS_FOR_FILE", - Value: params.AzureFileSecretProviderClassName, + Value: config.ManagedAzureFileCSISecretStoreProviderClassName, }) } } diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go index 2cbfa37f3bd..94b53d154e6 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go @@ -33,10 +33,10 @@ func NewParams( ir.setOperatorImageReferences(releaseImageProvider, userReleaseImageProvider) params := Params{ - OwnerRef: config.OwnerRefFrom(hcp), - StorageOperatorImage: releaseImageProvider.GetImage(storageOperatorImageName), - AvailabilityProberImage: releaseImageProvider.GetImage(util.AvailabilityProberImageName), - ImageReplacer: ir, + OwnerRef: config.OwnerRefFrom(hcp), + StorageOperatorImage: releaseImageProvider.GetImage(storageOperatorImageName), + AvailabilityProberImage: releaseImageProvider.GetImage(util.AvailabilityProberImageName), + ImageReplacer: ir, } params.DeploymentConfig = config.DeploymentConfig{ AdditionalLabels: map[string]string{