From 5ab0c51c1e2f8c6ee01ac570c7dcf2d98d5a1628 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 29 Oct 2024 15:06:09 -0400 Subject: [PATCH] Reconcile SecretProvider for CPO on ARO HCP Reconcile the SecretProviderClass for the control plane operator for ARO HCP deployments. The SecretProviderClass is used by the Secrets Store CSI driver to mount a certificate to a volume in the control plane operator pod deployment. Signed-off-by: Bryan Cox --- cmd/infra/azure/create.go | 3 + .../hostedcontrolplane_controller.go | 66 ++++++++++++++++++- .../hostedcluster/hostedcluster_controller.go | 66 ++++++++++++++----- support/azureutil/azureutil.go | 50 ++------------ 4 files changed, 120 insertions(+), 65 deletions(-) diff --git a/cmd/infra/azure/create.go b/cmd/infra/azure/create.go index a073f2c429b..d5f4b3d1576 100644 --- a/cmd/infra/azure/create.go +++ b/cmd/infra/azure/create.go @@ -377,6 +377,9 @@ func buildCreateServicePrincipalCommand(subscriptionID, managedResourceGroupName switch component { case cloudProvider: scopes = fmt.Sprintf("%s %s", scopes, nsgRG) + case cpo: + scopes = fmt.Sprintf("%s %s", scopes, nsgRG) + scopes = fmt.Sprintf("%s %s", scopes, vnetRG) case ingress: scopes = fmt.Sprintf("%s %s", scopes, vnetRG) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 9ea6a542df2..dec913225d3 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -73,6 +73,7 @@ import ( pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" @@ -385,7 +386,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R Type: string(hyperv1.ValidHostedControlPlaneConfiguration), ObservedGeneration: hostedControlPlane.Generation, } - if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil { + if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil { condition.Status = metav1.ConditionFalse condition.Message = err.Error() condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason @@ -838,12 +839,26 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error { return nil } -func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error { - for _, svc := range hc.Spec.Services { +func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + for _, svc := range hcp.Spec.Services { if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) { return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service) } } + + if hyperazureutil.IsAroHCP() { + credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + + tenantID := strings.TrimSpace(string(credentialsSecret.Data["AZURE_TENANT_ID"])) + + if err := verifyResourceGroupLocationsMatch(ctx, hcp, tenantID); err != nil { + return err + } + } + return nil } @@ -5465,3 +5480,48 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli } return false, nil } + +// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups +func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane, tenantID string) error { + // Retrieve the CPO certificate + certPath := "/mnt/certs/" + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName + certsContent, err := os.ReadFile(certPath) + if err != nil { + return fmt.Errorf("failed to read certificate: %v", err) + } + + // Authenticate to Azure with the certificate + parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil) + if err != nil { + return err + } + + options := &azidentity.ClientCertificateCredentialOptions{ + SendCertificateChain: true, + } + creds, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.ClientID, parsedCertificate, key, options) + if err != nil { + return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + } + + // Retrieve full vnet information from the VNET ID + vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + } + // Retrieve full network security group information from the network security group ID + nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get network security group info to verify its location: %v", err) + } + // Retrieve full resource group information from the resource group name + rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get resource group info to verify its location: %v", err) + } + // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match + if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { + return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) + } + return nil +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 5498807ea83..17fef89993d 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -141,6 +141,8 @@ const ( etcdCheckRequeueInterval = 10 * time.Second awsEndpointDeletionGracePeriod = 10 * time.Minute + + CPOSecretProviderClassName = "managed-azure-cpo" ) var ( @@ -1785,9 +1787,10 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, fmt.Errorf("failed to reconcile network policies: %w", err) } - // Reconcile the AWS OIDC discovery + // Reconcile platform specific items switch hcluster.Spec.Platform.Type { case hyperv1.AWSPlatform: + // Reconcile the AWS OIDC discovery if err := r.reconcileAWSOIDCDocuments(ctx, log, hcluster, hcp); err != nil { meta.SetStatusCondition(&hcluster.Status.Conditions, metav1.Condition{ Type: string(hyperv1.ValidOIDCConfiguration), @@ -1811,6 +1814,18 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques if err := r.Client.Status().Update(ctx, hcluster); err != nil { return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err) } + case hyperv1.AzurePlatform: + cpoSecretProviderClass := cpomanifests.ManagedAzureKeyVaultSecretProviderClass( + CPOSecretProviderClassName, + hcp.Namespace, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName) + if _, err = createOrUpdate(ctx, r, cpoSecretProviderClass, func() error { + return nil + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator secret provider class: %w", err) + } } log.Info("successfully reconciled") @@ -2721,15 +2736,6 @@ func reconcileControlPlaneOperatorDeployment( ) } - aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID) - if ok { - deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, - corev1.EnvVar{ - Name: config.AROHCPKeyVaultManagedIdentityClientID, - Value: aroHCPKVMIClientID, - }) - } - mainContainer = hyperutil.FindContainer("control-plane-operator", deployment.Spec.Template.Spec.Containers) proxy.SetEnvVars(&mainContainer.Env) @@ -2801,6 +2807,40 @@ func reconcileControlPlaneOperatorDeployment( }, }, }) + case hyperv1.AzurePlatform: + // Add the client ID of the managed Azure key vault as an environment variable on the CPO. This is used in + // configuring the SecretProviderClass CRs for OpenShift components on the HCP needing to authenticate with + // Azure cloud API. + aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID) + if ok { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: config.AROHCPKeyVaultManagedIdentityClientID, + Value: aroHCPKVMIClientID, + }) + } + + // Mount the control plane operator's certificate from the managed Azure key vault. The CPO authenticates with + // the Azure cloud API for validating resource group locations. + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "cpo-cert", + MountPath: "/mnt/certs", + ReadOnly: true, + }) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "cpo-cert", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{ + "secretProviderClass": CPOSecretProviderClassName, + }, + }, + }, + }) } if hcp.Spec.AdditionalTrustBundle != nil { @@ -4367,12 +4407,6 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } } - // Verify the resource group locations match - err := azureutil.VerifyResourceGroupLocationsMatch(ctx, hc, credentialsSecret) - if err != nil { - errs = append(errs, err) - } - return utilerrors.NewAggregate(errs) } diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index 54f62568f67..eb840ea3572 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -9,12 +9,8 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/config" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) @@ -147,8 +143,8 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro return vnet, nil } -// getNetworkSecurityGroupInfo gets the full information on a network security group based on its ID -func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { +// GetNetworkSecurityGroupInfo gets the full information on a network security group based on its ID +func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { partialNSGInfo, err := arm.ParseResourceID(nsgID) if err != nil { return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err) @@ -167,8 +163,8 @@ func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscription return nsg, nil } -// getResourceGroupInfo gets the full information on a resource group based on its name -func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { +// GetResourceGroupInfo gets the full information on a resource group based on its name +func GetResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { resourceGroupClient, err := armresources.NewResourceGroupsClient(subscriptionID, azureCreds, nil) if err != nil { return armresources.ResourceGroupsClientGetResponse{}, fmt.Errorf("failed to create new resource groups client: %w", err) @@ -182,44 +178,6 @@ func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID str return rg, nil } -// VerifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups -func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCluster, credentialsSecret *corev1.Secret) error { - // Setup azureCreds so we can retrieve the locations of the resource groups - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - - creds, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) - if err != nil { - return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) - } - - // Retrieve full vnet information from the VNET ID - vnet, err := GetVnetInfoFromVnetID(ctx, hc.Spec.Platform.Azure.VnetID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get vnet info to verify its location: %v", err) - } - - // Retrieve full network security group information from the network security group ID - nsg, err := getNetworkSecurityGroupInfo(ctx, hc.Spec.Platform.Azure.SecurityGroupID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get network security group info to verify its location: %v", err) - } - - // Retrieve full resource group information from the resource group name - rg, err := getResourceGroupInfo(ctx, hc.Spec.Platform.Azure.ResourceGroupName, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get resource group info to verify its location: %v", err) - } - - // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match - if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { - return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) - } - - return nil -} - // IsAroHCP returns true if the managed service environment variable is set to ARO-HCP func IsAroHCP() bool { return os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP