diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 327862bd1df..658201f3d82 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -80,12 +80,14 @@ import ( pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" "github.com/openshift/hypershift/support/config" component "github.com/openshift/hypershift/support/controlplane-component" "github.com/openshift/hypershift/support/events" + "github.com/openshift/hypershift/support/filewatcher" "github.com/openshift/hypershift/support/globalconfig" "github.com/openshift/hypershift/support/metrics" "github.com/openshift/hypershift/support/proxy" @@ -3046,6 +3048,28 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex }); err != nil { return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err) } + + if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error { + return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS) + }); err != nil { + return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err) + } + + if hyperazureutil.IsAroHCP() { + // Reconcile the SecretProviderClass + kmsSecretProviderClass := manifests.ManagedAzureKeyVaultSecretProviderClass( + config.ManagedAzureKMSSecretProviderClassName, + hcp.Namespace, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID, + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName) + + if _, err := createOrUpdate(ctx, r, kmsSecretProviderClass, func() error { + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile KMS SecretProviderClass: %w", err) + } + } } } @@ -5370,6 +5394,7 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex } azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure + // Pull the credentials secret so we can retrieve the tenant ID used in authenticating with Azure Cloud API credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { condition := metav1.Condition{ @@ -5382,12 +5407,55 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex meta.SetStatusCondition(&hcp.Status.Conditions, condition) return } - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) + // Retrieve the KMS certificate + certPath := config.ManagedAzureCertificateMountPath + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName + certsContent, err := os.ReadFile(certPath) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to retrieve KMS authentication certificate", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + // Watch the KMS certificate for changes; if the certificate changes, the pod will be restarted + err = filewatcher.WatchFileForChanges(certPath) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to watch KMS authentication certificate for changes", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + // Authenticate to Azure with the certificate + parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil) + if err != nil { + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionFalse, + Message: "Failed to parse KMS authentication certificate", + Reason: err.Error(), + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return + } + + options := &azidentity.ClientCertificateCredentialOptions{ + SendCertificateChain: true, + } + cred, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.SecretEncryption.KMS.Azure.KMS.ClientID, parsedCertificate, key, options) if err != nil { conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason, fmt.Sprintf("failed to obtain azure client credential: %v", err)) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go index 81ee7b4dacb..538378e3aaa 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go @@ -118,6 +118,7 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error { podSpec.Volumes = append(podSpec.Volumes, util.BuildVolume(kasVolumeAzureKMSCredentials(), buildVolumeAzureKMSCredentials), util.BuildVolume(kasVolumeKMSSocket(), buildVolumeKMSSocket), + util.BuildVolume(kasVolumeKMSSecretStore(), buildVolumeKMSSecretStore), ) podSpec.Containers = append(podSpec.Containers, @@ -146,6 +147,12 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error { container.VolumeMounts = append(container.VolumeMounts, azureKMSVolumeMounts.ContainerMounts(KasMainContainerName)...) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: config.ManagedAzureKMSSecretStoreVolumeName, + MountPath: config.ManagedAzureCertificateMountPath, + ReadOnly: true, + }) + return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go index 9c31a35ccd0..24a18ede484 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go @@ -1,8 +1,11 @@ package kms import ( + "github.com/openshift/hypershift/support/config" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/utils/ptr" ) type IKMSProvider interface { @@ -27,3 +30,21 @@ func kasVolumeKMSSocket() *corev1.Volume { func buildVolumeKMSSocket(v *corev1.Volume) { v.EmptyDir = &corev1.EmptyDirVolumeSource{} } + +func kasVolumeKMSSecretStore() *corev1.Volume { + return &corev1.Volume{ + Name: config.ManagedAzureKMSSecretStoreVolumeName, + } +} + +func buildVolumeKMSSecretStore(v *corev1.Volume) { + v.VolumeSource = corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: config.ManagedAzureSecretsStoreCSIDriver, + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{ + config.ManagedAzureSecretProviderClass: config.ManagedAzureKMSSecretStoreVolumeName, + }, + }, + } +} diff --git a/support/config/constants.go b/support/config/constants.go index 07bfdb1b77b..24af7a8cc8f 100644 --- a/support/config/constants.go +++ b/support/config/constants.go @@ -49,9 +49,22 @@ const ( CPOOverridesEnvVar = "ENABLE_CPO_OVERRIDES" AuditWebhookService = "audit-webhook" +) +// Azure related constants +const ( // AROHCPKeyVaultManagedIdentityClientID captures the client ID of the managed identity created on an ARO HCP // management cluster. This managed identity is used to pull secrets and certificates out of Azure Key Vaults in the // management cluster's resource group in Azure. AROHCPKeyVaultManagedIdentityClientID = "ARO_HCP_KEY_VAULT_USER_CLIENT_ID" + + ManagedAzureClientIdEnvVarKey = "ARO_HCP_MI_CLIENT_ID" + ManagedAzureTenantIdEnvVarKey = "ARO_HCP_TENANT_ID" + ManagedAzureCertificatePathEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_PATH" + ManagedAzureCertificateMountPath = "/mnt/certs" + ManagedAzureSecretsStoreCSIDriver = "secrets-store.csi.k8s.io" + ManagedAzureSecretProviderClass = "secretProviderClass" + + ManagedAzureKMSSecretStoreVolumeName = "azure-kms-cert" + ManagedAzureKMSSecretProviderClassName = "managed-azure-kms" ) diff --git a/support/filewatcher/filewatcher.go b/support/filewatcher/filewatcher.go new file mode 100644 index 00000000000..6ca75272937 --- /dev/null +++ b/support/filewatcher/filewatcher.go @@ -0,0 +1,69 @@ +package filewatcher + +import ( + "os" + "path/filepath" + "sync" + + "github.com/fsnotify/fsnotify" + ctrl "sigs.k8s.io/controller-runtime" +) + +var ( + watchCertificateFileOnce sync.Once + log = ctrl.Log.WithName("file-change-watcher") +) + +// WatchFileForChanges watches the file, fileToWatch. If the file contents have changed, the pod this function is +// running on will be restarted. +func WatchFileForChanges(fileToWatch string) error { + var err error + + // This starts only one occurrence of the file watcher, which watches the file, fileToWatch, for changes every interval. + // In addition, it also captures an initial hash of the file contents to use to monitor the file for changes. + watchCertificateFileOnce.Do(func() { + log.Info("Starting the file change watcher...") + + // Update the file path to watch in case this is a symlink + fileToWatch, err = filepath.EvalSymlinks(fileToWatch) + if err != nil { + return + } + log.Info("Watching file...", "file", fileToWatch) + + // Start the file watcher to monitor file changes + go checkForFileChanges(fileToWatch) + }) + return err +} + +// checkForFileChanges starts a new file watcher. If the file is changed, the pod running this function will exit. +func checkForFileChanges(path string) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + + go func() { + for { + select { + case event, ok := <-watcher.Events: + if ok && (event.Has(fsnotify.Write) || event.Has(fsnotify.Chmod) || event.Has(fsnotify.Remove)) { + log.Info("file was modified, exiting", "event name", event.Name, "event operation", event.Op) + os.Exit(0) + } + case err, ok := <-watcher.Errors: + if ok { + log.Error(err, "file watcher error") + } + } + } + }() + + err = watcher.Add(path) + if err != nil { + return err + } + + return nil +}