OCPBUGS-42434: Reconcile SecretProvider for CPO on ARO HCP

cmd/infra/azure/create.go

@@ -377,6 +377,9 @@ func buildCreateServicePrincipalCommand(subscriptionID, managedResourceGroupName
 	switch component {
 	case cloudProvider:
 		scopes = fmt.Sprintf("%s %s", scopes, nsgRG)
+	case cpo:
+		scopes = fmt.Sprintf("%s %s", scopes, nsgRG)
+		scopes = fmt.Sprintf("%s %s", scopes, vnetRG)
 	case ingress:
 		scopes = fmt.Sprintf("%s %s", scopes, vnetRG)
 	}

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -73,6 +73,7 @@ import (
 	pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests"
 	sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress"
 	supportawsutil "github.com/openshift/hypershift/support/awsutil"
+	hyperazureutil "github.com/openshift/hypershift/support/azureutil"
 	"github.com/openshift/hypershift/support/capabilities"
 	"github.com/openshift/hypershift/support/certs"
 	"github.com/openshift/hypershift/support/conditions"
@@ -385,7 +386,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			Type:               string(hyperv1.ValidHostedControlPlaneConfiguration),
 			ObservedGeneration: hostedControlPlane.Generation,
 		}
-		if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil {
+		if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil {
 			condition.Status = metav1.ConditionFalse
 			condition.Message = err.Error()
 			condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason
@@ -838,12 +839,26 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error {
 	return nil
 }
 
-func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error {
-	for _, svc := range hc.Spec.Services {
+func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
+	for _, svc := range hcp.Spec.Services {
 		if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) {
 			return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service)
 		}
 	}
+
+	if hyperazureutil.IsAroHCP() {
+		credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}}
+		if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil {
+			return fmt.Errorf("failed to get Azure credentials secret: %w", err)
+		}
+
+		tenantID := strings.TrimSpace(string(credentialsSecret.Data["AZURE_TENANT_ID"]))
+
+		if err := verifyResourceGroupLocationsMatch(ctx, hcp, tenantID); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -5465,3 +5480,48 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli
 	}
 	return false, nil
 }
+
+// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups
+func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane, tenantID string) error {
+	// Retrieve the CPO certificate
+	certPath := "/mnt/certs/" + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName
+	certsContent, err := os.ReadFile(certPath)
+	if err != nil {
+		return fmt.Errorf("failed to read certificate: %v", err)
+	}
+
+	// Authenticate to Azure with the certificate
+	parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil)
+	if err != nil {
+		return err
+	}
+
+	options := &azidentity.ClientCertificateCredentialOptions{
+		SendCertificateChain: true,
+	}
+	creds, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.ClientID, parsedCertificate, key, options)
+	if err != nil {
+		return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err)
+	}
+
+	// Retrieve full vnet information from the VNET ID
+	vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get vnet info to verify its location: %v", err)
+	}
+	// Retrieve full network security group information from the network security group ID
+	nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get network security group info to verify its location: %v", err)
+	}
+	// Retrieve full resource group information from the resource group name
+	rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds)
+	if err != nil {
+		return fmt.Errorf("failed to get resource group info to verify its location: %v", err)
+	}
+	// Verify the vnet resource group location, network security group resource group location, and the managed resource group location match
+	if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") {
+		return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, ""))
+	}
+	return nil
+}

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -141,6 +141,8 @@ const (
 	etcdCheckRequeueInterval = 10 * time.Second
 
 	awsEndpointDeletionGracePeriod = 10 * time.Minute
+
+	CPOSecretProviderClassName = "managed-azure-cpo"
 )
 
 var (
@@ -1785,9 +1787,10 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, fmt.Errorf("failed to reconcile network policies: %w", err)
 	}
 
-	// Reconcile the AWS OIDC discovery
+	// Reconcile platform specific items
 	switch hcluster.Spec.Platform.Type {
 	case hyperv1.AWSPlatform:
+		// Reconcile the AWS OIDC discovery
 		if err := r.reconcileAWSOIDCDocuments(ctx, log, hcluster, hcp); err != nil {
 			meta.SetStatusCondition(&hcluster.Status.Conditions, metav1.Condition{
 				Type:               string(hyperv1.ValidOIDCConfiguration),
@@ -1811,6 +1814,18 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 		if err := r.Client.Status().Update(ctx, hcluster); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err)
 		}
+	case hyperv1.AzurePlatform:
+		cpoSecretProviderClass := cpomanifests.ManagedAzureKeyVaultSecretProviderClass(
+			CPOSecretProviderClassName,
+			hcp.Namespace,
+			hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name,
+			hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID,
+			hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName)
+		if _, err = createOrUpdate(ctx, r, cpoSecretProviderClass, func() error {
+			return nil
+		}); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator secret provider class: %w", err)
+		}
 	}
 
 	log.Info("successfully reconciled")
@@ -2721,15 +2736,6 @@ func reconcileControlPlaneOperatorDeployment(
 		)
 	}
 
-	aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID)
-	if ok {
-		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env,
-			corev1.EnvVar{
-				Name:  config.AROHCPKeyVaultManagedIdentityClientID,
-				Value: aroHCPKVMIClientID,
-			})
-	}
-
 	mainContainer = hyperutil.FindContainer("control-plane-operator", deployment.Spec.Template.Spec.Containers)
 	proxy.SetEnvVars(&mainContainer.Env)
 
@@ -2801,6 +2807,40 @@ func reconcileControlPlaneOperatorDeployment(
 				},
 			},
 		})
+	case hyperv1.AzurePlatform:
+		// Add the client ID of the managed Azure key vault as an environment variable on the CPO. This is used in
+		// configuring the SecretProviderClass CRs for OpenShift components on the HCP needing to authenticate with
+		// Azure cloud API.
+		aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID)
+		if ok {
+			deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env,
+				corev1.EnvVar{
+					Name:  config.AROHCPKeyVaultManagedIdentityClientID,
+					Value: aroHCPKVMIClientID,
+				})
+		}
+
+		// Mount the control plane operator's certificate from the managed Azure key vault. The CPO authenticates with
+		// the Azure cloud API for validating resource group locations.
+		deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts,
+			corev1.VolumeMount{
+				Name:      "cpo-cert",
+				MountPath: "/mnt/certs",
+				ReadOnly:  true,
+			})
+		deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes,
+			corev1.Volume{
+				Name: "cpo-cert",
+				VolumeSource: corev1.VolumeSource{
+					CSI: &corev1.CSIVolumeSource{
+						Driver:   "secrets-store.csi.k8s.io",
+						ReadOnly: ptr.To(true),
+						VolumeAttributes: map[string]string{
+							"secretProviderClass": CPOSecretProviderClassName,
+						},
+					},
+				},
+			})
 	}
 
 	if hcp.Spec.AdditionalTrustBundle != nil {
@@ -4367,12 +4407,6 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h
 		}
 	}
 
-	// Verify the resource group locations match
-	err := azureutil.VerifyResourceGroupLocationsMatch(ctx, hc, credentialsSecret)
-	if err != nil {
-		errs = append(errs, err)
-	}
-
 	return utilerrors.NewAggregate(errs)
 }
 

support/azureutil/azureutil.go

@@ -9,12 +9,8 @@ import (
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/support/config"
 
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/utils/ptr"
-
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 )
@@ -147,8 +143,8 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro
 	return vnet, nil
 }
 
-// getNetworkSecurityGroupInfo gets the full information on a network security group based on its ID
-func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) {
+// GetNetworkSecurityGroupInfo gets the full information on a network security group based on its ID
+func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) {
 	partialNSGInfo, err := arm.ParseResourceID(nsgID)
 	if err != nil {
 		return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err)
@@ -167,8 +163,8 @@ func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscription
 	return nsg, nil
 }
 
-// getResourceGroupInfo gets the full information on a resource group based on its name
-func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) {
+// GetResourceGroupInfo gets the full information on a resource group based on its name
+func GetResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) {
 	resourceGroupClient, err := armresources.NewResourceGroupsClient(subscriptionID, azureCreds, nil)
 	if err != nil {
 		return armresources.ResourceGroupsClientGetResponse{}, fmt.Errorf("failed to create new resource groups client: %w", err)
@@ -182,44 +178,6 @@ func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID str
 	return rg, nil
 }
 
-// VerifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups
-func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCluster, credentialsSecret *corev1.Secret) error {
-	// Setup azureCreds so we can retrieve the locations of the resource groups
-	tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"])
-	clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"])
-	clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"])
-
-	creds, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil)
-	if err != nil {
-		return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err)
-	}
-
-	// Retrieve full vnet information from the VNET ID
-	vnet, err := GetVnetInfoFromVnetID(ctx, hc.Spec.Platform.Azure.VnetID, hc.Spec.Platform.Azure.SubscriptionID, creds)
-	if err != nil {
-		return fmt.Errorf("failed to get vnet info to verify its location: %v", err)
-	}
-
-	// Retrieve full network security group information from the network security group ID
-	nsg, err := getNetworkSecurityGroupInfo(ctx, hc.Spec.Platform.Azure.SecurityGroupID, hc.Spec.Platform.Azure.SubscriptionID, creds)
-	if err != nil {
-		return fmt.Errorf("failed to get network security group info to verify its location: %v", err)
-	}
-
-	// Retrieve full resource group information from the resource group name
-	rg, err := getResourceGroupInfo(ctx, hc.Spec.Platform.Azure.ResourceGroupName, hc.Spec.Platform.Azure.SubscriptionID, creds)
-	if err != nil {
-		return fmt.Errorf("failed to get resource group info to verify its location: %v", err)
-	}
-
-	// Verify the vnet resource group location, network security group resource group location, and the managed resource group location match
-	if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") {
-		return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, ""))
-	}
-
-	return nil
-}
-
 // IsAroHCP returns true if the managed service environment variable is set to ARO-HCP
 func IsAroHCP() bool {
 	return os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP