OCPBUGS-42434: Authenticate Azure KMS with cert authentication

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -80,12 +80,14 @@ import (
 	pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests"
 	sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress"
 	supportawsutil "github.com/openshift/hypershift/support/awsutil"
+	hyperazureutil "github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/capabilities"
 	"github.com/openshift/hypershift/support/certs"
 	"github.com/openshift/hypershift/support/conditions"
 	"github.com/openshift/hypershift/support/config"
 	component "github.com/openshift/hypershift/support/controlplane-component"
 	"github.com/openshift/hypershift/support/events"
+	"github.com/openshift/hypershift/support/filewatcher"
 	"github.com/openshift/hypershift/support/globalconfig"
 	"github.com/openshift/hypershift/support/metrics"
 	"github.com/openshift/hypershift/support/proxy"
@@ -3046,6 +3048,28 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 			}); err != nil {
 				return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err)
 			}
+
+			if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error {
+				return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS)
+			}); err != nil {
+				return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err)
+			}
+
+			if hyperazureutil.IsAroHCP() {
+				// Reconcile the SecretProviderClass
+				kmsSecretProviderClass := manifests.ManagedAzureKeyVaultSecretProviderClass(
+					config.ManagedAzureKMSSecretProviderClassName,
+					hcp.Namespace,
+					hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name,
+					hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID,
+					hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName)
+
+				if _, err := createOrUpdate(ctx, r, kmsSecretProviderClass, func() error {
+					return nil
+				}); err != nil {
+					return fmt.Errorf("failed to reconcile KMS SecretProviderClass: %w", err)
+				}
+			}
 		}
 	}
 
@@ -5370,6 +5394,7 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex
 	}
 	azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure
 
+	// Pull the credentials secret so we can retrieve the tenant ID used in authenticating with Azure Cloud API
 	credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
 	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
 		condition := metav1.Condition{
@@ -5382,12 +5407,55 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex
 		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
 		return
 	}
-
 	tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"])
-	clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"])
-	clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"])
 
-	cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil)
+	// Retrieve the KMS certificate
+	certPath := config.ManagedAzureCertificateMountPath + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName
+	certsContent, err := os.ReadFile(certPath)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to retrieve KMS authentication certificate",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	// Watch the KMS certificate for changes; if the certificate changes, the pod will be restarted
+	err = filewatcher.WatchFileForChanges(certPath)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to watch KMS authentication certificate for changes",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	// Authenticate to Azure with the certificate
+	parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to parse KMS authentication certificate",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	options := &azidentity.ClientCertificateCredentialOptions{
+		SendCertificateChain: true,
+	}
+	cred, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.SecretEncryption.KMS.Azure.KMS.ClientID, parsedCertificate, key, options)
 	if err != nil {
 		conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason,
 			fmt.Sprintf("failed to obtain azure client credential: %v", err))

control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go

@@ -118,6 +118,7 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error {
 	podSpec.Volumes = append(podSpec.Volumes,
 		util.BuildVolume(kasVolumeAzureKMSCredentials(), buildVolumeAzureKMSCredentials),
 		util.BuildVolume(kasVolumeKMSSocket(), buildVolumeKMSSocket),
+		util.BuildVolume(kasVolumeKMSSecretStore(), buildVolumeKMSSecretStore),
 	)
 
 	podSpec.Containers = append(podSpec.Containers,
@@ -146,6 +147,12 @@ func (p *azureKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error {
 	container.VolumeMounts = append(container.VolumeMounts,
 		azureKMSVolumeMounts.ContainerMounts(KasMainContainerName)...)
 
+	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+		Name:      config.ManagedAzureKMSSecretStoreVolumeName,
+		MountPath: config.ManagedAzureCertificateMountPath,
+		ReadOnly:  true,
+	})
+
 	return nil
 }
 

control-plane-operator/controllers/hostedcontrolplane/kas/kms/kms.go

@@ -1,8 +1,11 @@
 package kms
 
 import (
+	"github.com/openshift/hypershift/support/config"
+
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
+	"k8s.io/utils/ptr"
 )
 
 type IKMSProvider interface {
@@ -27,3 +30,21 @@ func kasVolumeKMSSocket() *corev1.Volume {
 func buildVolumeKMSSocket(v *corev1.Volume) {
 	v.EmptyDir = &corev1.EmptyDirVolumeSource{}
 }
+
+func kasVolumeKMSSecretStore() *corev1.Volume {
+	return &corev1.Volume{
+		Name: config.ManagedAzureKMSSecretStoreVolumeName,
+	}
+}
+
+func buildVolumeKMSSecretStore(v *corev1.Volume) {
+	v.VolumeSource = corev1.VolumeSource{
+		CSI: &corev1.CSIVolumeSource{
+			Driver:   config.ManagedAzureSecretsStoreCSIDriver,
+			ReadOnly: ptr.To(true),
+			VolumeAttributes: map[string]string{
+				config.ManagedAzureSecretProviderClass: config.ManagedAzureKMSSecretStoreVolumeName,
+			},
+		},
+	}
+}

support/config/constants.go

@@ -49,9 +49,22 @@ const (
 	CPOOverridesEnvVar = "ENABLE_CPO_OVERRIDES"
 
 	AuditWebhookService = "audit-webhook"
+)
 
+// Azure related constants
+const (
 	// AROHCPKeyVaultManagedIdentityClientID captures the client ID of the managed identity created on an ARO HCP
 	// management cluster. This managed identity is used to pull secrets and certificates out of Azure Key Vaults in the
 	// management cluster's resource group in Azure.
 	AROHCPKeyVaultManagedIdentityClientID = "ARO_HCP_KEY_VAULT_USER_CLIENT_ID"
+
+	ManagedAzureClientIdEnvVarKey        = "ARO_HCP_MI_CLIENT_ID"
+	ManagedAzureTenantIdEnvVarKey        = "ARO_HCP_TENANT_ID"
+	ManagedAzureCertificatePathEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_PATH"
+	ManagedAzureCertificateMountPath     = "/mnt/certs"
+	ManagedAzureSecretsStoreCSIDriver    = "secrets-store.csi.k8s.io"
+	ManagedAzureSecretProviderClass      = "secretProviderClass"
+
+	ManagedAzureKMSSecretStoreVolumeName   = "azure-kms-cert"
+	ManagedAzureKMSSecretProviderClassName = "managed-azure-kms"
 )

support/filewatcher/filewatcher.go

@@ -0,0 +1,69 @@
+package filewatcher
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+var (
+	watchCertificateFileOnce sync.Once
+	log                      = ctrl.Log.WithName("file-change-watcher")
+)
+
+// WatchFileForChanges watches the file, fileToWatch. If the file contents have changed, the pod this function is
+// running on will be restarted.
+func WatchFileForChanges(fileToWatch string) error {
+	var err error
+
+	// This starts only one occurrence of the file watcher, which watches the file, fileToWatch, for changes every interval.
+	// In addition, it also captures an initial hash of the file contents to use to monitor the file for changes.
+	watchCertificateFileOnce.Do(func() {
+		log.Info("Starting the file change watcher...")
+
+		// Update the file path to watch in case this is a symlink
+		fileToWatch, err = filepath.EvalSymlinks(fileToWatch)
+		if err != nil {
+			return
+		}
+		log.Info("Watching file...", "file", fileToWatch)
+
+		// Start the file watcher to monitor file changes
+		go checkForFileChanges(fileToWatch)
+	})
+	return err
+}
+
+// checkForFileChanges starts a new file watcher. If the file is changed, the pod running this function will exit.
+func checkForFileChanges(path string) error {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if ok && (event.Has(fsnotify.Write) || event.Has(fsnotify.Chmod) || event.Has(fsnotify.Remove)) {
+					log.Info("file was modified, exiting", "event name", event.Name, "event operation", event.Op)
+					os.Exit(0)
+				}
+			case err, ok := <-watcher.Errors:
+				if ok {
+					log.Error(err, "file watcher error")
+				}
+			}
+		}
+	}()
+
+	err = watcher.Add(path)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}