From d4053ca898ff0d45ff4bf9e6e5437b5590e7396f Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 11 Oct 2024 12:07:47 -0400 Subject: [PATCH 1/2] Reconcile SecretProvider for CNCC on ARO HCP Reconcile the SecretProviderClass for the cloud network config controller (CNCC) for ARO HCP deployments. The SecretProviderClass is used by the Secrets Store CSI driver to mount a certificate to a volume in the CNCC pod deployment. Signed-off-by: Bryan Cox --- .../cno/clusternetworkoperator.go | 36 +++++++++++++++---- .../hostedcontrolplane_controller.go | 18 ++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go index aab94d80ff5..663c2653263 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go @@ -8,19 +8,18 @@ import ( "strconv" "text/template" - "github.com/openshift/hypershift/support/proxy" - "github.com/openshift/hypershift/support/rhobsmonitoring" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/blang/semver" - routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/proxy" + "github.com/openshift/hypershift/support/rhobsmonitoring" "github.com/openshift/hypershift/support/util" + + routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -29,6 +28,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/blang/semver" ) const ( @@ -113,6 +115,9 @@ type Params struct { DeploymentConfig config.DeploymentConfig IsPrivate bool DefaultIngressDomain string + AzureClientID string + AzureTenantID string + AzureCertificateName string } func init() { @@ -161,6 +166,11 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv CAConfigMapKey: caConfigMapKey, } + if azureutil.IsAroHCP() { + p.AzureClientID = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Network.ClientID + p.AzureCertificateName = hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Network.CertificateName + } + p.DeploymentConfig.AdditionalLabels = map[string]string{ config.NeedManagementKASAccessLabel: "true", } @@ -608,6 +618,20 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat {Name: "ca-bundle", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: manifests.RootCASecret("").Name, DefaultMode: ptr.To[int32](0640)}}}, } + // For ARO HCP deployments, we pass the env variable for the SecretProviderClass for the Secrets Store CSI driver + // to use on the CNCC deployment. + if azureutil.IsAroHCP() { + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, + azureutil.CreateEnvVarsForAzureManagedIdentity(params.AzureClientID, params.AzureTenantID, params.AzureCertificateName)...) + + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "ARO_HCP_SECRET_PROVIDER_CLASS", + Value: config.ManagedAzureNetworkSecretStoreProviderClassName, + }, + ) + } + params.DeploymentConfig.ApplyTo(dep) util.AvailabilityProber(kas.InClusterKASReadyURL(platformType), params.AvailabilityProberImage, &dep.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) { o.KubeconfigVolumeName = "hosted-etc-kube" diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 0768786a490..1e7e15337e6 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -3619,6 +3619,24 @@ func (r *HostedControlPlaneReconciler) reconcileClusterVersionOperator(ctx conte func (r *HostedControlPlaneReconciler) reconcileClusterNetworkOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider, userReleaseImageProvider *imageprovider.SimpleReleaseImageProvider, hasRouteCap bool, createOrUpdate upsert.CreateOrUpdateFN) error { p := cno.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext, r.DefaultIngressDomain) + // Create SecretProviderClass when deploying on ARO HCP + if hyperazureutil.IsAroHCP() { + cnccSecretProviderClass := manifests.ManagedAzureSecretProviderClass(config.ManagedAzureNetworkSecretStoreProviderClassName, hcp.Namespace) + if _, err := createOrUpdate(ctx, r, cnccSecretProviderClass, func() error { + secretproviderclass.ReconcileManagedAzureSecretProviderClass(cnccSecretProviderClass, hcp, hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlane.Network.CertificateName) + return nil + }); err != nil { + return fmt.Errorf("failed to reconcile ingressoperator secret provider class: %w", err) + } + + credentialsSecret := manifests.AzureCredentialInformation(hcp.Namespace) + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + + p.AzureTenantID = string(credentialsSecret.Data["AZURE_TENANT_ID"]) + } + sa := manifests.ClusterNetworkOperatorServiceAccount(hcp.Namespace) if _, err := createOrUpdate(ctx, r.Client, sa, func() error { return cno.ReconcileServiceAccount(sa, p.OwnerRef) From 403475780beb8740006c43fe42c072e66c2ac5e7 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Mon, 2 Dec 2024 15:32:11 -0500 Subject: [PATCH 2/2] Pass only the certificate name for CNO deployment Pass only the certificate name for the CNO deployment for managed Azure. CNO uses its own certificate path here, https://github.com/openshift/cluster-network-operator/blob/7736bfe37f1276f771fcef03077f6d840eb6b862/pkg/network/cloud_network.go#L23. This is combined with the certificate name here, https://github.com/openshift/cluster-network-operator/blob/7736bfe37f1276f771fcef03077f6d840eb6b862/pkg/network/cloud_network.go#L111. --- .../cno/clusternetworkoperator.go | 17 +++++++++++++---- support/config/constants.go | 16 +++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go index 663c2653263..16e1d9315a3 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go @@ -622,11 +622,20 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat // to use on the CNCC deployment. if azureutil.IsAroHCP() { dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, - azureutil.CreateEnvVarsForAzureManagedIdentity(params.AzureClientID, params.AzureTenantID, params.AzureCertificateName)...) - - dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "ARO_HCP_SECRET_PROVIDER_CLASS", + Name: config.ManagedAzureClientIdEnvVarKey, + Value: params.AzureClientID, + }, + corev1.EnvVar{ + Name: config.ManagedAzureTenantIdEnvVarKey, + Value: params.AzureTenantID, + }, + corev1.EnvVar{ + Name: config.ManagedAzureCertificateNameEnvVarKey, + Value: params.AzureCertificateName, + }, + corev1.EnvVar{ + Name: config.ManagedAzureSecretProviderClassEnvVarKey, Value: config.ManagedAzureNetworkSecretStoreProviderClassName, }, ) diff --git a/support/config/constants.go b/support/config/constants.go index 7ab94e87e00..011288e5455 100644 --- a/support/config/constants.go +++ b/support/config/constants.go @@ -61,13 +61,15 @@ const ( // management cluster's resource group in Azure. AROHCPKeyVaultManagedIdentityClientID = "ARO_HCP_KEY_VAULT_USER_CLIENT_ID" - ManagedAzureClientIdEnvVarKey = "ARO_HCP_MI_CLIENT_ID" - ManagedAzureTenantIdEnvVarKey = "ARO_HCP_TENANT_ID" - ManagedAzureCertificatePathEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_PATH" - ManagedAzureCertificateMountPath = "/mnt/certs" - ManagedAzureCertificatePath = "/mnt/certs/" - ManagedAzureSecretsStoreCSIDriver = "secrets-store.csi.k8s.io" - ManagedAzureSecretProviderClass = "secretProviderClass" + ManagedAzureClientIdEnvVarKey = "ARO_HCP_MI_CLIENT_ID" + ManagedAzureTenantIdEnvVarKey = "ARO_HCP_TENANT_ID" + ManagedAzureCertificatePathEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_PATH" + ManagedAzureCertificateNameEnvVarKey = "ARO_HCP_CLIENT_CERTIFICATE_NAME" + ManagedAzureSecretProviderClassEnvVarKey = "ARO_HCP_SECRET_PROVIDER_CLASS" + ManagedAzureCertificateMountPath = "/mnt/certs" + ManagedAzureCertificatePath = "/mnt/certs/" + ManagedAzureSecretsStoreCSIDriver = "secrets-store.csi.k8s.io" + ManagedAzureSecretProviderClass = "secretProviderClass" ManagedAzureCPOSecretProviderClassName = "managed-azure-cpo" ManagedAzureCPOSecretStoreVolumeName = "cpo-cert"