diff --git a/cmd/infra/azure/create.go b/cmd/infra/azure/create.go index a073f2c429b..d5f4b3d1576 100644 --- a/cmd/infra/azure/create.go +++ b/cmd/infra/azure/create.go @@ -377,6 +377,9 @@ func buildCreateServicePrincipalCommand(subscriptionID, managedResourceGroupName switch component { case cloudProvider: scopes = fmt.Sprintf("%s %s", scopes, nsgRG) + case cpo: + scopes = fmt.Sprintf("%s %s", scopes, nsgRG) + scopes = fmt.Sprintf("%s %s", scopes, vnetRG) case ingress: scopes = fmt.Sprintf("%s %s", scopes, vnetRG) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 9ea6a542df2..dec913225d3 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -73,6 +73,7 @@ import ( pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" @@ -385,7 +386,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R Type: string(hyperv1.ValidHostedControlPlaneConfiguration), ObservedGeneration: hostedControlPlane.Generation, } - if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil { + if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil { condition.Status = metav1.ConditionFalse condition.Message = err.Error() condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason @@ -838,12 +839,26 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error { return nil } -func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error { - for _, svc := range hc.Spec.Services { +func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + for _, svc := range hcp.Spec.Services { if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) { return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service) } } + + if hyperazureutil.IsAroHCP() { + credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + + tenantID := strings.TrimSpace(string(credentialsSecret.Data["AZURE_TENANT_ID"])) + + if err := verifyResourceGroupLocationsMatch(ctx, hcp, tenantID); err != nil { + return err + } + } + return nil } @@ -5465,3 +5480,48 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli } return false, nil } + +// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups +func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane, tenantID string) error { + // Retrieve the CPO certificate + certPath := "/mnt/certs/" + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName + certsContent, err := os.ReadFile(certPath) + if err != nil { + return fmt.Errorf("failed to read certificate: %v", err) + } + + // Authenticate to Azure with the certificate + parsedCertificate, key, err := azidentity.ParseCertificates(certsContent, nil) + if err != nil { + return err + } + + options := &azidentity.ClientCertificateCredentialOptions{ + SendCertificateChain: true, + } + creds, err := azidentity.NewClientCertificateCredential(tenantID, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.ClientID, parsedCertificate, key, options) + if err != nil { + return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + } + + // Retrieve full vnet information from the VNET ID + vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + } + // Retrieve full network security group information from the network security group ID + nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get network security group info to verify its location: %v", err) + } + // Retrieve full resource group information from the resource group name + rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get resource group info to verify its location: %v", err) + } + // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match + if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { + return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) + } + return nil +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 5498807ea83..17fef89993d 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -141,6 +141,8 @@ const ( etcdCheckRequeueInterval = 10 * time.Second awsEndpointDeletionGracePeriod = 10 * time.Minute + + CPOSecretProviderClassName = "managed-azure-cpo" ) var ( @@ -1785,9 +1787,10 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, fmt.Errorf("failed to reconcile network policies: %w", err) } - // Reconcile the AWS OIDC discovery + // Reconcile platform specific items switch hcluster.Spec.Platform.Type { case hyperv1.AWSPlatform: + // Reconcile the AWS OIDC discovery if err := r.reconcileAWSOIDCDocuments(ctx, log, hcluster, hcp); err != nil { meta.SetStatusCondition(&hcluster.Status.Conditions, metav1.Condition{ Type: string(hyperv1.ValidOIDCConfiguration), @@ -1811,6 +1814,18 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques if err := r.Client.Status().Update(ctx, hcluster); err != nil { return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err) } + case hyperv1.AzurePlatform: + cpoSecretProviderClass := cpomanifests.ManagedAzureKeyVaultSecretProviderClass( + CPOSecretProviderClassName, + hcp.Namespace, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.Name, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ManagedIdentitiesKeyVault.TenantID, + hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.ControlPlaneOperator.CertificateName) + if _, err = createOrUpdate(ctx, r, cpoSecretProviderClass, func() error { + return nil + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator secret provider class: %w", err) + } } log.Info("successfully reconciled") @@ -2721,15 +2736,6 @@ func reconcileControlPlaneOperatorDeployment( ) } - aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID) - if ok { - deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, - corev1.EnvVar{ - Name: config.AROHCPKeyVaultManagedIdentityClientID, - Value: aroHCPKVMIClientID, - }) - } - mainContainer = hyperutil.FindContainer("control-plane-operator", deployment.Spec.Template.Spec.Containers) proxy.SetEnvVars(&mainContainer.Env) @@ -2801,6 +2807,40 @@ func reconcileControlPlaneOperatorDeployment( }, }, }) + case hyperv1.AzurePlatform: + // Add the client ID of the managed Azure key vault as an environment variable on the CPO. This is used in + // configuring the SecretProviderClass CRs for OpenShift components on the HCP needing to authenticate with + // Azure cloud API. + aroHCPKVMIClientID, ok := os.LookupEnv(config.AROHCPKeyVaultManagedIdentityClientID) + if ok { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: config.AROHCPKeyVaultManagedIdentityClientID, + Value: aroHCPKVMIClientID, + }) + } + + // Mount the control plane operator's certificate from the managed Azure key vault. The CPO authenticates with + // the Azure cloud API for validating resource group locations. + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "cpo-cert", + MountPath: "/mnt/certs", + ReadOnly: true, + }) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "cpo-cert", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{ + "secretProviderClass": CPOSecretProviderClassName, + }, + }, + }, + }) } if hcp.Spec.AdditionalTrustBundle != nil { @@ -4367,12 +4407,6 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } } - // Verify the resource group locations match - err := azureutil.VerifyResourceGroupLocationsMatch(ctx, hc, credentialsSecret) - if err != nil { - errs = append(errs, err) - } - return utilerrors.NewAggregate(errs) } diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index 54f62568f67..eb840ea3572 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -9,12 +9,8 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/config" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) @@ -147,8 +143,8 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro return vnet, nil } -// getNetworkSecurityGroupInfo gets the full information on a network security group based on its ID -func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { +// GetNetworkSecurityGroupInfo gets the full information on a network security group based on its ID +func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { partialNSGInfo, err := arm.ParseResourceID(nsgID) if err != nil { return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err) @@ -167,8 +163,8 @@ func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscription return nsg, nil } -// getResourceGroupInfo gets the full information on a resource group based on its name -func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { +// GetResourceGroupInfo gets the full information on a resource group based on its name +func GetResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { resourceGroupClient, err := armresources.NewResourceGroupsClient(subscriptionID, azureCreds, nil) if err != nil { return armresources.ResourceGroupsClientGetResponse{}, fmt.Errorf("failed to create new resource groups client: %w", err) @@ -182,44 +178,6 @@ func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID str return rg, nil } -// VerifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups -func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCluster, credentialsSecret *corev1.Secret) error { - // Setup azureCreds so we can retrieve the locations of the resource groups - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - - creds, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) - if err != nil { - return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) - } - - // Retrieve full vnet information from the VNET ID - vnet, err := GetVnetInfoFromVnetID(ctx, hc.Spec.Platform.Azure.VnetID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get vnet info to verify its location: %v", err) - } - - // Retrieve full network security group information from the network security group ID - nsg, err := getNetworkSecurityGroupInfo(ctx, hc.Spec.Platform.Azure.SecurityGroupID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get network security group info to verify its location: %v", err) - } - - // Retrieve full resource group information from the resource group name - rg, err := getResourceGroupInfo(ctx, hc.Spec.Platform.Azure.ResourceGroupName, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get resource group info to verify its location: %v", err) - } - - // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match - if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { - return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) - } - - return nil -} - // IsAroHCP returns true if the managed service environment variable is set to ARO-HCP func IsAroHCP() bool { return os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP