From 0e47832e9f6d4b41017e07a4bc5095267473e4c3 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Wed, 6 Nov 2024 15:35:33 -0500 Subject: [PATCH] Authenticate Azure KMS with cert authentication This commit authenticates Azure KMS with certificate authentication in order to communicate with Azure Cloud API. The certificate is stored in an Azure key vault and mounted into the KAS pod through a Secrets Store CSI driver SecretProviderClass. Signed-off-by: Bryan Cox --- .../hostedcontrolplane_controller.go | 76 ++++++++++++++++++- .../hostedcontrolplane/kas/kms/azure.go | 7 ++ .../hostedcontrolplane/kas/kms/kms.go | 21 +++++ support/config/constants.go | 13 ++++ support/filewatcher/filewatcher.go | 69 +++++++++++++++++ 5 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 support/filewatcher/filewatcher.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 327862bd1df..658201f3d82 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -80,12 +80,14 @@ import ( pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" "github.com/openshift/hypershift/support/config" component "github.com/openshift/hypershift/support/controlplane-component" "github.com/openshift/hypershift/support/events" + "github.com/openshift/hypershift/support/filewatcher" "github.com/openshift/hypershift/support/globalconfig" "github.com/openshift/hypershift/support/metrics" "github.com/openshift/hypershift/support/proxy" @@ -3046,6 +3048,28 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex }); err != nil { return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err) } + + if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error { + return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS) + }); err != nil { + return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err) + } + + if hyperazureutil.IsAroHCP() { + // Reconcile the SecretProviderClass + kmsSecretProviderClass := manifests.ManagedAzureKeyVaultSecretProviderClass( + config.ManagedAzureKMSSecretProviderClassName, + hcp.Namespace, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID, + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName) + + if _, err := createOrUpdate(ctx, r, kmsSecretProviderClass, func() error { + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile KMS SecretProviderClass: %w", err) + } + } } } @@ -5370,6 +5394,7 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex } azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure + // Pull the credentials secret so we can retrieve the tenant ID used in authenticating with Azure Cloud API credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { condition := metav1.Condition{ @@ -5382,12 +5407,55 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex meta.SetStatusCondition(&hcp.Status.Conditions, condition) return } - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) + // Retrieve the KMS certificate + certPath := config.ManagedAzureCertificateMountPath + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName + certsContent, err := os.ReadFile(certPath) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to retrieve KMS authentication certificate", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + // Watch the KMS certificate for changes; if the certificate changes, the pod will be restarted + err = filewatcher.WatchFileForChanges(certPath) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to watch KMS authentication certificate for changes", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + // Authenticate to Azure with the certificate + parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to parse KMS authentication certificate", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + options := &azidentity.ClientCertificateCredentialOptions{ + SendCertificateChain: true, + } + cred, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.SecretEncryption.KMS.Azure.KMS.ClientID, parsedCertificate, key, options) if err != nil { conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason, fmt.Sprintf("failed to obtain azure client credential: %v", err)) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go index 81ee7b4dacb..538378e3aaa 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go @@ -118,6 +118,7 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error { podSpec.Volumes = append(podSpec.Volumes, util.BuildVolume(kasVolumeAzureKMSCredentials(), buildVolumeAzureKMSCredentials), util.BuildVolume(kasVolumeKMSSocket(), buildVolumeKMSSocket), + util.BuildVolume(kasVolumeKMSSecretStore(), buildVolumeKMSSecretStore), ) podSpec.Containers = append(podSpec.Containers, @@ -146,6 +147,12 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error { container.VolumeMounts = append(container.VolumeMounts, azureKMSVolumeMounts.ContainerMounts(KasMainContainerName)...) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: config.ManagedAzureKMSSecretStoreVolumeName, + MountPath: config.ManagedAzureCertificateMountPath, + ReadOnly: true, + }) + return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go index 9c31a35ccd0..24a18ede484 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go @@ -1,8 +1,11 @@ package kms import ( + "github.com/openshift/hypershift/support/config" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/utils/ptr" ) type IKMSProvider interface { @@ -27,3 +30,21 @@ func kasVolumeKMSSocket() *corev1.Volume { func buildVolumeKMSSocket(v *corev1.Volume) { v.EmptyDir = &corev1.EmptyDirVolumeSource{} } + +func kasVolumeKMSSecretStore() *corev1.Volume { + return &corev1.Volume{ + Name: config.ManagedAzureKMSSecretStoreVolumeName, + } +} + +func buildVolumeKMSSecretStore(v *corev1.Volume) { + v.VolumeSource = corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: config.ManagedAzureSecretsStoreCSIDriver, + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{ + config.ManagedAzureSecretProviderClass: config.ManagedAzureKMSSecretStoreVolumeName, + }, + }, + } +} diff --git a/support/config/constants.go b/support/config/constants.go index 07bfdb1b77b..24af7a8cc8f 100644 --- a/support/config/constants.go +++ b/support/config/constants.go @@ -49,9 +49,22 @@ const ( CPOOverridesEnvVar = "ENABLE_CPO_OVERRIDES" AuditWebhookService = "audit-webhook" +) +// Azure related constants +const ( // AROHCPKeyVaultManagedIdentityClientID captures the client ID of the managed identity created on an ARO HCP // management cluster. This managed identity is used to pull secrets and certificates out of Azure Key Vaults in the // management cluster's resource group in Azure. AROHCPKeyVaultManagedIdentityClientID = "ARO_HCP_KEY_VAULT_USER_CLIENT_ID" + + ManagedAzureClientIdEnvVarKey = "ARO_HCP_MI_CLIENT_ID" + ManagedAzureTenantIdEnvVarKey = "ARO_HCP_TENANT_ID" + ManagedAzureCertificatePathEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_PATH" + ManagedAzureCertificateMountPath = "/mnt/certs" + ManagedAzureSecretsStoreCSIDriver = "secrets-store.csi.k8s.io" + ManagedAzureSecretProviderClass = "secretProviderClass" + + ManagedAzureKMSSecretStoreVolumeName = "azure-kms-cert" + ManagedAzureKMSSecretProviderClassName = "managed-azure-kms" ) diff --git a/support/filewatcher/filewatcher.go b/support/filewatcher/filewatcher.go new file mode 100644 index 00000000000..6ca75272937 --- /dev/null +++ b/support/filewatcher/filewatcher.go @@ -0,0 +1,69 @@ +package filewatcher + +import ( + "os" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" + ctrl "sigs.k8s.io/controller-runtime" +) + +var ( + watchCertificateFileOnce sync.Once + log = ctrl.Log.WithName("file-change-watcher") +) + +// WatchFileForChanges watches the file, fileToWatch. If the file contents have changed, the pod this function is +// running on will be restarted. +func WatchFileForChanges(fileToWatch string) error { + var err error + + // This starts only one occurrence of the file watcher, which watches the file, fileToWatch, for changes every interval. + // In addition, it also captures an initial hash of the file contents to use to monitor the file for changes. + watchCertificateFileOnce.Do(func() { + log.Info("Starting the file change watcher...") + + // Update the file path to watch in case this is a symlink + fileToWatch, err = filepath.EvalSymlinks(fileToWatch) + if err != nil { + return + } + log.Info("Watching file...", "file", fileToWatch) + + // Start the file watcher to monitor file changes + go checkForFileChanges(fileToWatch) + }) + return err +} + +// checkForFileChanges starts a new file watcher. If the file is changed, the pod running this function will exit. +func checkForFileChanges(path string) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + + go func() { + for { + select { + case event, ok := <-watcher.Events: + if ok && (event.Has(fsnotify.Write) || event.Has(fsnotify.Chmod) || event.Has(fsnotify.Remove)) { + log.Info("file was modified, exiting", "event name", event.Name, "event operation", event.Op) + os.Exit(0) + } + case err, ok := <-watcher.Errors: + if ok { + log.Error(err, "file watcher error") + } + } + } + }() + + err = watcher.Add(path) + if err != nil { + return err + } + + return nil +}