[release-4.18] OCPBUGS-45524: Implement Managed Identities in HyperShift

api/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect
 	github.com/stretchr/testify v1.9.0 // indirect
 	golang.org/x/net v0.31.0 // indirect

api/go.sum

@@ -30,8 +30,8 @@ github.com/openshift/api v0.0.0-20240524162738-d899f8877d22/go.mod h1:7Hm1kLJGxW
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA=
 github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

api/vendor/modules.txt

@@ -25,8 +25,8 @@ github.com/modern-go/reflect2
 github.com/openshift/api/config/v1
 # github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 ## explicit
-# github.com/rogpeppe/go-internal v1.12.0
-## explicit; go 1.20
+# github.com/rogpeppe/go-internal v1.13.1
+## explicit; go 1.22
 # github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace
 ## explicit; go 1.12
 # github.com/stretchr/testify v1.9.0

cmd/infra/azure/create.go

@@ -428,8 +428,13 @@ func buildCreateServicePrincipalCommand(subscriptionID, managedResourceGroupName
 	switch component {
 	case cloudProvider:
 		scopes = fmt.Sprintf("%s %s", scopes, nsgRG)
+	case cpo:
+		scopes = fmt.Sprintf("%s %s", scopes, nsgRG)
+		scopes = fmt.Sprintf("%s %s", scopes, vnetRG)
 	case ingress:
 		scopes = fmt.Sprintf("%s %s %s", scopes, vnetRG, dnsZoneRG)
+	case nodePoolMgmt:
+		scopes = fmt.Sprintf("%s %s", scopes, vnetRG)
 	}
 
 	// The command creates a Service Principal with a role(s) over resource group(s), create a new certificate for it, and store it in an existing keyvault

cmd/install/assets/cluster-api-provider-azure/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml

@@ -122,6 +122,11 @@ spec:
                     type: object
                     x-kubernetes-map-type: atomic
                 type: object
+              certPath:
+                description: CertPath is the path where certificates exist. When set,
+                  it takes precedence over ClientSecret for types that use certs like
+                  ServicePrincipalCertificate.
+                type: string
               clientID:
                 description: |-
                   ClientID is the service principal client ID.

cmd/install/assets/cluster-api-provider-azure/infrastructure.cluster.x-k8s.io_azuremachines.yaml

@@ -281,8 +281,8 @@ spec:
               image:
                 description: |-
                   Image is used to provide details of an image to use during VM creation.
-                  If image details are omitted the image will default the Azure Marketplace "capi" offer,
-                  which is based on Ubuntu.
+                  If image details are omitted, the default is to use an Azure Compute Gallery Image
+                  from CAPZ's community gallery.
                 properties:
                   computeGallery:
                     description: ComputeGallery specifies an image to use from the

cmd/install/assets/cluster-api-provider-azure/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml

@@ -292,8 +292,8 @@ spec:
                       image:
                         description: |-
                           Image is used to provide details of an image to use during VM creation.
-                          If image details are omitted the image will default the Azure Marketplace "capi" offer,
-                          which is based on Ubuntu.
+                          If image details are omitted, the default is to use an Azure Compute Gallery Image
+                          from CAPZ's community gallery.
                         properties:
                           computeGallery:
                             description: ComputeGallery specifies an image to use

cmd/install/assets/hypershift_operator.go

@@ -1268,6 +1268,7 @@ func (o HyperShiftOperatorClusterRole) Build() *rbacv1.ClusterRole {
 				Verbs: []string{
 					"list",
 					"create",
+					"update",
 				},
 			})
 	}

control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go

@@ -11,10 +11,12 @@ import (
 )
 
 type AzureParams struct {
-	ClusterID        string                  `json:"clusterID"`
-	ClusterNetwork   string                  `json:"clusterNetwork"`
-	OwnerRef         *metav1.OwnerReference  `json:"ownerRef"`
-	DeploymentConfig config.DeploymentConfig `json:"deploymentConfig"`
+	ClusterID               string                  `json:"clusterID"`
+	ClusterNetwork          string                  `json:"clusterNetwork"`
+	OwnerRef                *metav1.OwnerReference  `json:"ownerRef"`
+	DeploymentConfig        config.DeploymentConfig `json:"deploymentConfig"`
+	TenantID                string                  `json:"tenantID"`
+	SecretProviderClassName string                  `json:"secretProviderClassName"`
 }
 
 func NewAzureParams(hcp *hyperv1.HostedControlPlane) *AzureParams {
@@ -26,6 +28,7 @@ func NewAzureParams(hcp *hyperv1.HostedControlPlane) *AzureParams {
 		ClusterNetwork: hcp.Spec.Networking.ClusterNetwork[0].CIDR.String(),
 	}
 	p.OwnerRef = config.ControllerOwnerRef(hcp)
+	p.SecretProviderClassName = config.ManagedAzureCloudProviderSecretProviderClassName
 
 	p.DeploymentConfig.Resources = config.ResourcesSpec{
 		ccmContainer().Name: {

control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go

@@ -6,6 +6,7 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/support/azureutil"
+	"github.com/openshift/hypershift/support/config"
 
 	corev1 "k8s.io/api/core/v1"
 )
@@ -42,10 +43,11 @@ func ReconcileCloudConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.Hos
 		return err
 	}
 
-	cfg.AADClientID = string(credentialsSecret.Data["AZURE_CLIENT_ID"])
-	cfg.AADClientSecret = string(credentialsSecret.Data["AZURE_CLIENT_SECRET"])
+	cfg.AADClientID = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.CloudProvider.ClientID
+	cfg.AADClientCertPath = config.ManagedAzureCertificatePath + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.CloudProvider.CertificateName
 	cfg.UseManagedIdentityExtension = false
 	cfg.UseInstanceMetadata = false
+
 	serializedConfig, err := json.MarshalIndent(cfg, "", "  ")
 	if err != nil {
 		return fmt.Errorf("failed to serialize cloudconfig: %w", err)
@@ -103,12 +105,14 @@ func azureConfigWithoutCredentials(hcp *hyperv1.HostedControlPlane, credentialsS
 // Now the source is https://github.com/kubernetes-sigs/cloud-provider-azure/blob/e5d670328a51e31787fc949ddf41a3efcd90d651/examples/out-of-tree/cloud-controller-manager.yaml#L232
 // https://github.com/kubernetes-sigs/cloud-provider-azure/tree/e5d670328a51e31787fc949ddf41a3efcd90d651/pkg/provider/config
 type AzureConfig struct {
-	Cloud                        string `json:"cloud"`
-	TenantID                     string `json:"tenantId"`
-	UseManagedIdentityExtension  bool   `json:"useManagedIdentityExtension"`
-	SubscriptionID               string `json:"subscriptionId"`
-	AADClientID                  string `json:"aadClientId"`
+	Cloud                       string `json:"cloud"`
+	TenantID                    string `json:"tenantId"`
+	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension"`
+	SubscriptionID              string `json:"subscriptionId"`
+	AADClientID                 string `json:"aadClientId"`
+	// TODO HOSTEDCP-1542 - Bryan - drop client secret once we have WorkloadIdentity working
 	AADClientSecret              string `json:"aadClientSecret"`
+	AADClientCertPath            string `json:"aadClientCertPath"`
 	ResourceGroup                string `json:"resourceGroup"`
 	Location                     string `json:"location"`
 	VnetName                     string `json:"vnetName"`

control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
+	"github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/config"
 	"github.com/openshift/hypershift/support/proxy"
 	"github.com/openshift/hypershift/support/util"
@@ -46,6 +47,14 @@ func ReconcileDeployment(deployment *appsv1.Deployment, hcp *hyperv1.HostedContr
 
 	addVolumes(deployment)
 
+	deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts,
+		azureutil.CreateVolumeMountForAzureSecretStoreProviderClass(config.ManagedAzureCloudProviderSecretStoreVolumeName),
+	)
+
+	deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes,
+		azureutil.CreateVolumeForAzureSecretStoreProviderClass(config.ManagedAzureCloudProviderSecretStoreVolumeName, config.ManagedAzureCloudProviderSecretProviderClassName),
+	)
+
 	config.OwnerRefFrom(hcp).ApplyTo(deployment)
 	p.DeploymentConfig.ApplyTo(deployment)
 	return nil

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -93,6 +93,7 @@ import (
 	"github.com/openshift/hypershift/support/config"
 	component "github.com/openshift/hypershift/support/controlplane-component"
 	"github.com/openshift/hypershift/support/events"
+	"github.com/openshift/hypershift/support/filewatcher"
 	"github.com/openshift/hypershift/support/globalconfig"
 	"github.com/openshift/hypershift/support/metrics"
 	"github.com/openshift/hypershift/support/proxy"
@@ -412,7 +413,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			Type:               string(hyperv1.ValidHostedControlPlaneConfiguration),
 			ObservedGeneration: hostedControlPlane.Generation,
 		}
-		if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil {
+		if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil {
 			condition.Status = metav1.ConditionFalse
 			condition.Message = err.Error()
 			condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason
@@ -865,12 +866,26 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error {
 	return nil
 }
 
-func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error {
-	for _, svc := range hc.Spec.Services {
+func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
+	for _, svc := range hcp.Spec.Services {
 		if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) {
 			return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service)
 		}
 	}
+
+	if hyperazureutil.IsAroHCP() {
+		credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
+		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
+			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
+		}
+
+		tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"])
+
+		if err := verifyResourceGroupLocationsMatch(ctx, hcp, tenantID); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -3068,6 +3083,23 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
 			}); err != nil {
 				return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err)
 			}
+
+			if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error {
+				return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS)
+			}); err != nil {
+				return fmt.Errorf("failed to reconcile kms encryption config secret: %w", err)
+			}
+
+			if hyperazureutil.IsAroHCP() {
+				// Reconcile the SecretProviderClass
+				kmsSecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureKMSSecretProviderClassName, hcp.Namespace)
+				if _, err := createOrUpdate(ctx, r, kmsSecretProviderClass, func() error {
+					secretproviderclass.ReconcileManagedAzureSecretProviderClass(kmsSecretProviderClass, hcp, hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName)
+					return nil
+				}); err != nil {
+					return fmt.Errorf("failed to reconcile KMS SecretProviderClass: %w", err)
+				}
+			}
 		}
 	}
 
@@ -4799,6 +4831,10 @@ func (r *HostedControlPlaneReconciler) reconcileCloudControllerManager(ctx conte
 			return fmt.Errorf("failed to reconcile %s cloud controller manager deployment: %w", hcp.Spec.Platform.Type, err)
 		}
 	case hyperv1.AzurePlatform:
+		// Set up the params
+		p := azure.NewAzureParams(hcp)
+
+		// Reconcile CCM ServiceAccount
 		ownerRef := config.OwnerRefFrom(hcp)
 		sa := azure.CCMServiceAccount(hcp.Namespace)
 		if _, err := createOrUpdate(ctx, r, sa, func() error {
@@ -4807,7 +4843,23 @@ func (r *HostedControlPlaneReconciler) reconcileCloudControllerManager(ctx conte
 			return fmt.Errorf("failed to reconcile %s cloud provider service account: %w", hcp.Spec.Platform.Type, err)
 		}
 
-		p := azure.NewAzureParams(hcp)
+		// Reconcile SecretProviderClass
+		azureCloudProviderSecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureCloudProviderSecretProviderClassName, hcp.Namespace)
+		if _, err := createOrUpdate(ctx, r, azureCloudProviderSecretProviderClass, func() error {
+			secretproviderclass.ReconcileManagedAzureSecretProviderClass(azureCloudProviderSecretProviderClass, hcp, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.CloudProvider.CertificateName)
+			return nil
+		}); err != nil {
+			return fmt.Errorf("failed to reconcile azure cloud provider secret provider class: %w", err)
+		}
+
+		// Retrieve the credentials secret to get the tenant ID
+		credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace)
+		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
+			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
+		}
+		p.TenantID = string(credentialsSecret.Data["AZURE_TENANT_ID"])
+
+		// Reconcile the CCM Deployment
 		deployment := azure.CCMDeployment(hcp.Namespace)
 		if _, err := createOrUpdate(ctx, r, deployment, func() error {
 			return azure.ReconcileDeployment(deployment, hcp, p, sa.Name, releaseImageProvider)
@@ -5512,12 +5564,55 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex
 		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
 		return
 	}
-
 	tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"])
-	clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"])
-	clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"])
 
-	cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil)
+	// Retrieve the KMS certificate
+	certPath := config.ManagedAzureCertificateMountPath + hcp.Spec.SecretEncryption.KMS.Azure.KMS.CertificateName
+	certsContent, err := os.ReadFile(certPath)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to retrieve KMS authentication certificate",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	// Watch the KMS certificate for changes; if the certificate changes, the pod will be restarted
+	err = filewatcher.WatchFileForChanges(certPath)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to watch KMS authentication certificate for changes",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	// Authenticate to Azure with the certificate
+	parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil)
+	if err != nil {
+		condition := metav1.Condition{
+			Type:               string(hyperv1.ValidAzureKMSConfig),
+			ObservedGeneration: hcp.Generation,
+			Status:             metav1.ConditionFalse,
+			Message:            "Failed to parse KMS authentication certificate",
+			Reason:             err.Error(),
+		}
+		meta.SetStatusCondition(&hcp.Status.Conditions, condition)
+		return
+	}
+
+	options := &azidentity.ClientCertificateCredentialOptions{
+		SendCertificateChain: true,
+	}
+	cred, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.SecretEncryption.KMS.Azure.KMS.ClientID, parsedCertificate, key, options)
 	if err != nil {
 		conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason,
 			fmt.Sprintf("failed to obtain azure client credential: %v", err))
@@ -5619,3 +5714,48 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli
 	}
 	return false, nil
 }
+
+// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups
+func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane, tenantID string) error {
+	// Retrieve the CPO certificate
+	certPath := config.ManagedAzureCertificatePath + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName
+	certsContent, err := os.ReadFile(certPath)
+	if err != nil {
+		return fmt.Errorf("failed to read certificate: %v", err)
+	}
+
+	// Authenticate to Azure with the certificate
+	parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil)
+	if err != nil {
+		return err
+	}
+
+	options := &azidentity.ClientCertificateCredentialOptions{
+		SendCertificateChain: true,
+	}
+	creds, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.ClientID, parsedCertificate, key, options)
+	if err != nil {
+		return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err)
+	}
+
+	// Retrieve full vnet information from the VNET ID
+	vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get vnet info to verify its location: %v", err)
+	}
+	// Retrieve full network security group information from the network security group ID
+	nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get network security group info to verify its location: %v", err)
+	}
+	// Retrieve full resource group information from the resource group name
+	rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get resource group info to verify its location: %v", err)
+	}
+	// Verify the vnet resource group location, network security group resource group location, and the managed resource group location match
+	if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") {
+		return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, ""))
+	}
+	return nil
+}