diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment.go index 08f318d8231..f173e2c92ad 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment.go @@ -62,6 +62,10 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep } }) + if hcp.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(hcp.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } + // Set replicas based on whether termination handler is needed // If the disable annotation is present, scale to 0 replicas deployment.Spec.Replicas = ptr.To[int32](1) diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment_test.go index ee9a5052630..6889e57546e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment_test.go @@ -9,9 +9,20 @@ import ( assets "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/assets" controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + func TestAdaptDeployment(t *testing.T) { testCases := []struct { name string @@ -181,3 +192,91 @@ func TestGetTerminationHandlerQueueURL(t *testing.T) { }) } } + +func TestAdaptDeploymentAWSCABundle(t *testing.T) { + testCases := []struct { + name string + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When additional trust bundle is set it should add combined CA bundle with init container", + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + { + name: "When no additional trust bundle is set it should not add CA bundle resources", + additionalTrust: nil, + expectCABundle: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + InfraID: "test-infra-id", + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + AWS: &hyperv1.AWSPlatformSpec{ + Region: "us-east-1", + }, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment, err := assets.LoadDeploymentManifest(ComponentName) + g.Expect(err).ToNot(HaveOccurred()) + + err = adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment.go index b858efe0a7b..4d187aa2676 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment.go @@ -1,8 +1,10 @@ package capiprovider import ( + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" component "github.com/openshift/hypershift/support/controlplane-component" "github.com/openshift/hypershift/support/k8sutil" + "github.com/openshift/hypershift/support/podspec" "github.com/openshift/hypershift/support/proxy" appsv1 "k8s.io/api/apps/v1" @@ -20,6 +22,10 @@ func (capi *CAPIProviderOptions) adaptDeployment(cpContext component.WorkloadCon proxy.SetEnvVars(&deployment.Spec.Template.Spec.Containers[0].Env) + if cpContext.HCP.Spec.Platform.Type == hyperv1.AWSPlatform && cpContext.HCP.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(cpContext.HCP.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } + if deployment.Annotations == nil { deployment.Annotations = make(map[string]string) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment_test.go index 3b1570e78a4..2126471bc1a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment_test.go @@ -231,3 +231,130 @@ func TestAdaptDeployment_WithNilAnnotations(t *testing.T) { g.Expect(deployment.Annotations).ToNot(BeNil()) g.Expect(deployment.Annotations[k8sutil.HostedClusterAnnotation]).To(Equal("test-namespace/test-cluster")) } + +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + +func TestAdaptDeploymentAWSCABundle(t *testing.T) { + testCases := []struct { + name string + platformType hyperv1.PlatformType + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When AWS platform with additional trust bundle it should add combined CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + { + name: "When AWS platform without additional trust bundle it should not add CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: nil, + expectCABundle: false, + }, + { + name: "When non-AWS platform with additional trust bundle it should not add CA bundle", + platformType: hyperv1.KubevirtPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + Annotations: map[string]string{ + "hypershift.openshift.io/cluster": "clusters/test-cluster", + }, + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tc.platformType, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + deploymentSpec := &appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "manager", + Image: "test-image", + }, + }, + }, + }, + } + + capi := &CAPIProviderOptions{ + deploymentSpec: deploymentSpec, + } + + cpContext := component.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ComponentName, + Namespace: hcp.Namespace, + }, + } + + err := capi.adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/component.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/component.go index 0d41bad5a3b..07434a33240 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/component.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/component.go @@ -31,6 +31,7 @@ func (c *awsOptions) NeedsManagementKASAccess() bool { func NewComponent() component.ControlPlaneComponent { return component.NewDeploymentComponent(ComponentName, &awsOptions{}). + WithAdaptFunction(adaptDeployment). WithPredicate(predicate). WithManifestAdapter( "config.yaml", diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment.go new file mode 100644 index 00000000000..286cf7bcb05 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment.go @@ -0,0 +1,15 @@ +package aws + +import ( + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/podspec" + + appsv1 "k8s.io/api/apps/v1" +) + +func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Deployment) error { + if cpContext.HCP.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(cpContext.HCP.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment_test.go new file mode 100644 index 00000000000..7fc6934037f --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment_test.go @@ -0,0 +1,108 @@ +package aws + +import ( + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/assets" + controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + +func TestAdaptDeploymentTrustBundle(t *testing.T) { + testCases := []struct { + name string + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When no additional trust bundle is set it should not add CA bundle resources", + additionalTrust: nil, + expectCABundle: false, + }, + { + name: "When additional trust bundle is set it should add combined CA bundle with init container", + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment, err := assets.LoadDeploymentManifest(ComponentName) + g.Expect(err).ToNot(HaveOccurred()) + + err = adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment.go index a7283b9e79e..de8429e3bd5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment.go @@ -1,6 +1,7 @@ package ingressoperator import ( + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" component "github.com/openshift/hypershift/support/controlplane-component" @@ -45,5 +46,9 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep } }) + if cpContext.HCP.Spec.Platform.Type == hyperv1.AWSPlatform && cpContext.HCP.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(cpContext.HCP.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } + return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment_test.go index c5f206efc68..a8f20598dd5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/ingressoperator/deployment_test.go @@ -77,3 +77,107 @@ func TestAdaptDeployment(t *testing.T) { }) } } + +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + +func TestAdaptDeploymentAWSCABundle(t *testing.T) { + testCases := []struct { + name string + platformType hyperv1.PlatformType + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When AWS platform with additional trust bundle it should add combined CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + { + name: "When AWS platform without additional trust bundle it should not add CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: nil, + expectCABundle: false, + }, + { + name: "When non-AWS platform with additional trust bundle it should not add CA bundle", + platformType: hyperv1.KubevirtPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tc.platformType, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + cpContext := component.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + UserReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment, err := assets.LoadDeploymentManifest(ComponentName) + g.Expect(err).ToNot(HaveOccurred()) + + err = adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment.go index dea07bfac87..9bee29afd29 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment.go @@ -43,6 +43,10 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep } }) + if hcp.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(hcp.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, corev1.Container{ Name: "token-minter", diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment_test.go new file mode 100644 index 00000000000..78bac688201 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment_test.go @@ -0,0 +1,112 @@ +package karpenter + +import ( + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/assets" + controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + +func TestAdaptDeploymentAWSCABundle(t *testing.T) { + testCases := []struct { + name string + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When additional trust bundle is set it should add combined CA bundle with init container", + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + { + name: "When no additional trust bundle is set it should not add CA bundle resources", + additionalTrust: nil, + expectCABundle: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + InfraID: "test-infra-id", + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + AWS: &hyperv1.AWSPlatformSpec{ + Region: "us-east-1", + }, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment, err := assets.LoadDeploymentManifest(ComponentName) + g.Expect(err).ToNot(HaveOccurred()) + + err = adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment.go index a398adf71cb..44c7a945716 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment.go @@ -70,6 +70,10 @@ func (karp *KarpenterOperatorOptions) adaptDeployment(cpContext component.Worklo ) } }) + + if hcp.Spec.AdditionalTrustBundle != nil { + podspec.DeploymentAddAWSCABundleVolume(hcp.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName)) + } } return nil diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment_test.go index 0b78a0df9f4..f47315548c5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment_test.go @@ -231,3 +231,120 @@ func TestAdaptDeployment(t *testing.T) { }) } } + +type fakeReleaseProvider struct{} + +func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" } +func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false } +func (f *fakeReleaseProvider) Version() string { return "4.17.0" } +func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) { + return nil, nil +} +func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil } + +func TestAdaptDeploymentAWSCABundle(t *testing.T) { + testCases := []struct { + name string + platformType hyperv1.PlatformType + additionalTrust *corev1.LocalObjectReference + expectCABundle bool + }{ + { + name: "When AWS platform with additional trust bundle it should add combined CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: true, + }, + { + name: "When AWS platform without additional trust bundle it should not add CA bundle", + platformType: hyperv1.AWSPlatform, + additionalTrust: nil, + expectCABundle: false, + }, + { + name: "When non-AWS platform with additional trust bundle it should not add CA bundle", + platformType: hyperv1.KubevirtPlatform, + additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"}, + expectCABundle: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + awsSpec := &hyperv1.AWSPlatformSpec{ + Region: "us-east-1", + } + if tc.platformType != hyperv1.AWSPlatform { + awsSpec = nil + } + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters-test-cluster", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tc.platformType, + AWS: awsSpec, + }, + AdditionalTrustBundle: tc.additionalTrust, + }, + } + + karp := &KarpenterOperatorOptions{ + HyperShiftOperatorImage: "test-hypershift-operator-image", + ControlPlaneOperatorImage: "test-cpo-image", + IgnitionEndpoint: "https://ignition.example.com", + } + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + ReleaseImageProvider: &fakeReleaseProvider{}, + } + + deployment, err := assets.LoadDeploymentManifest(ComponentName) + g.Expect(err).ToNot(HaveOccurred()) + + err = karp.adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + volumes := deployment.Spec.Template.Spec.Volumes + initContainers := deployment.Spec.Template.Spec.InitContainers + container := deployment.Spec.Template.Spec.Containers[0] + + if tc.expectCABundle { + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "user-ca-bundle"), + HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"), + ))) + g.Expect(volumes).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("VolumeSource.EmptyDir", Not(BeNil())), + ))) + g.Expect(initContainers).To(ContainElement(SatisfyAll( + HaveField("Name", "setup-aws-ca-bundle"), + HaveField("Image", "test-cpo-image"), + ))) + g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll( + HaveField("Name", "aws-ca-bundle"), + HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"), + HaveField("ReadOnly", true), + ))) + g.Expect(container.Env).To(ContainElement(SatisfyAll( + HaveField("Name", "AWS_CA_BUNDLE"), + HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"), + ))) + } else { + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle"))) + g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle"))) + g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle"))) + g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE"))) + } + }) + } +} diff --git a/support/podspec/volumes.go b/support/podspec/volumes.go index c768f46c507..0bc3b7eacac 100644 --- a/support/podspec/volumes.go +++ b/support/podspec/volumes.go @@ -3,6 +3,8 @@ package podspec import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" ) func BuildVolume(volume *corev1.Volume, buildFn func(*corev1.Volume)) corev1.Volume { @@ -32,6 +34,89 @@ func DeploymentAddTrustBundleVolume(trustBundleConfigMap *corev1.LocalObjectRefe }) } +// DeploymentAddAWSCABundleVolume creates a combined CA bundle containing both the system CAs from +// the container image and the user-provided additionalTrustBundle CAs, then sets AWS_CA_BUNDLE on +// the first container. An init container concatenates /etc/pki/tls/certs/ca-bundle.crt (system CAs) +// with the user CAs into a single file. This is necessary because the AWS SDK replaces the default +// system CA bundle when AWS_CA_BUNDLE is set, rather than appending to it. +// +// This helper is used by both external third-party binaries (karpenter, aws-cloud-controller-manager, +// aws-node-termination-handler) and internal HyperShift binaries (karpenter-operator, capi-provider, +// ingress-operator). Even though internal binaries could handle CA loading in Go code, using the +// same environment-variable-based approach keeps the fix uniform across all AWS components. +// +// The initContainerImage should be a RHEL-based image that has /bin/sh and cat available +// (e.g. the control-plane-operator image). +func DeploymentAddAWSCABundleVolume(trustBundleConfigMap *corev1.LocalObjectReference, deployment *appsv1.Deployment, initContainerImage string) { + const ( + userCAVolumeName = "user-ca-bundle" + combinedCAVolumeName = "aws-ca-bundle" + userCAMountPath = "/user-ca" + combinedCAMountPath = "/etc/pki/ca-trust/extracted/hypershift" + userCAFileName = "user-ca-bundle.pem" + combinedCAFileName = "combined-ca-bundle.pem" + systemCABundlePath = "/etc/pki/tls/certs/ca-bundle.crt" + initContainerName = "setup-aws-ca-bundle" + ) + + // Volume for user CAs from additionalTrustBundle ConfigMap. + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: userCAVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: *trustBundleConfigMap, + Items: []corev1.KeyToPath{{Key: "ca-bundle.crt", Path: userCAFileName}}, + }, + }, + }) + + // EmptyDir volume for the combined (system + user) CA bundle. + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: combinedCAVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + // Init container concatenates system CAs with user CAs into the combined bundle. + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: initContainerName, + Image: initContainerImage, + Command: []string{"/bin/sh", "-c", + "cat " + systemCABundlePath + " " + userCAMountPath + "/" + userCAFileName + + " > " + combinedCAMountPath + "/" + combinedCAFileName}, + VolumeMounts: []corev1.VolumeMount{ + {Name: userCAVolumeName, MountPath: userCAMountPath, ReadOnly: true}, + {Name: combinedCAVolumeName, MountPath: combinedCAMountPath}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }) + + // Mount the combined CA bundle in the main container. + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: combinedCAVolumeName, + MountPath: combinedCAMountPath, + ReadOnly: true, + }) + + // Point AWS_CA_BUNDLE to the combined CA file so the AWS SDK trusts both system and user CAs. + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "AWS_CA_BUNDLE", + Value: combinedCAMountPath + "/" + combinedCAFileName, + }) +} + func UpdateVolume(name string, volumes []corev1.Volume, update func(v *corev1.Volume)) { for i, v := range volumes { if v.Name == name { diff --git a/support/podspec/volumes_test.go b/support/podspec/volumes_test.go new file mode 100644 index 00000000000..d5d34df7c09 --- /dev/null +++ b/support/podspec/volumes_test.go @@ -0,0 +1,150 @@ +package podspec + +import ( + "strings" + "testing" + + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +// initDeployment creates a base deployment for testing. When existing is non-empty, +// it populates volumes, init containers, volume mounts, and env vars using the +// string as a naming prefix. When empty, those fields are left uninitialized. +func initDeployment(existing string) *appsv1.Deployment { + dep := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "main"}, + }, + }, + }, + }, + } + if existing != "" { + dep.Spec.Template.Spec.Volumes = []corev1.Volume{ + {Name: existing + "-volume"}, + } + dep.Spec.Template.Spec.InitContainers = []corev1.Container{ + {Name: existing + "-init"}, + } + dep.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + {Name: existing + "-mount", MountPath: "/data"}, + } + dep.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: strings.ToUpper(existing) + "_VAR", Value: "value"}, + } + } + return dep +} + +func TestDeploymentAddAWSCABundleVolume(t *testing.T) { + testCases := []struct { + name string + trustBundleConfigMap *corev1.LocalObjectReference + existing string + initContainerImage string + }{ + { + name: "When a trust bundle ConfigMap is provided it should add volumes, init container, volume mount, and AWS_CA_BUNDLE env var", + trustBundleConfigMap: &corev1.LocalObjectReference{Name: "my-trust-bundle"}, + existing: "", + initContainerImage: "registry.example.com/cpo:latest", + }, + { + name: "When the deployment already has existing resources it should append without removing them", + trustBundleConfigMap: &corev1.LocalObjectReference{Name: "custom-ca"}, + existing: "existing", + initContainerImage: "registry.example.com/cpo:v2", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + deployment := initDeployment(tc.existing) + existingVolumeCount := len(deployment.Spec.Template.Spec.Volumes) + existingInitContainerCount := len(deployment.Spec.Template.Spec.InitContainers) + existingVolumeMountCount := len(deployment.Spec.Template.Spec.Containers[0].VolumeMounts) + existingEnvCount := len(deployment.Spec.Template.Spec.Containers[0].Env) + + DeploymentAddAWSCABundleVolume(tc.trustBundleConfigMap, deployment, tc.initContainerImage) + + spec := deployment.Spec.Template.Spec + + // It should add exactly two new volumes (user-ca-bundle and aws-ca-bundle). + g.Expect(spec.Volumes).To(HaveLen(existingVolumeCount + 2)) + + // Verify user-ca-bundle volume references the ConfigMap. + var userCAVolume *corev1.Volume + for i := range spec.Volumes { + if spec.Volumes[i].Name == "user-ca-bundle" { + userCAVolume = &spec.Volumes[i] + break + } + } + g.Expect(userCAVolume).NotTo(BeNil(), "user-ca-bundle volume should exist") + g.Expect(userCAVolume.VolumeSource.ConfigMap).NotTo(BeNil()) + g.Expect(userCAVolume.VolumeSource.ConfigMap.LocalObjectReference.Name).To(Equal(tc.trustBundleConfigMap.Name)) + g.Expect(userCAVolume.VolumeSource.ConfigMap.Items).To(ConsistOf( + corev1.KeyToPath{Key: "ca-bundle.crt", Path: "user-ca-bundle.pem"}, + )) + + // Verify aws-ca-bundle volume is an EmptyDir. + var combinedCAVolume *corev1.Volume + for i := range spec.Volumes { + if spec.Volumes[i].Name == "aws-ca-bundle" { + combinedCAVolume = &spec.Volumes[i] + break + } + } + g.Expect(combinedCAVolume).NotTo(BeNil(), "aws-ca-bundle volume should exist") + g.Expect(combinedCAVolume.VolumeSource.EmptyDir).NotTo(BeNil()) + + // It should add exactly one init container. + g.Expect(spec.InitContainers).To(HaveLen(existingInitContainerCount + 1)) + + initContainer := spec.InitContainers[len(spec.InitContainers)-1] + g.Expect(initContainer.Name).To(Equal("setup-aws-ca-bundle")) + g.Expect(initContainer.Image).To(Equal(tc.initContainerImage)) + g.Expect(initContainer.Command).To(Equal([]string{ + "/bin/sh", "-c", + "cat /etc/pki/tls/certs/ca-bundle.crt /user-ca/user-ca-bundle.pem > /etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem", + })) + g.Expect(initContainer.VolumeMounts).To(ConsistOf( + corev1.VolumeMount{Name: "user-ca-bundle", MountPath: "/user-ca", ReadOnly: true}, + corev1.VolumeMount{Name: "aws-ca-bundle", MountPath: "/etc/pki/ca-trust/extracted/hypershift"}, + )) + + // It should set resource requests on the init container. + g.Expect(initContainer.Resources.Requests).To(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("10m"))) + g.Expect(initContainer.Resources.Requests).To(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("10Mi"))) + + // It should set a restricted security context on the init container. + g.Expect(initContainer.SecurityContext).NotTo(BeNil()) + g.Expect(initContainer.SecurityContext.AllowPrivilegeEscalation).To(Equal(ptr.To(false))) + g.Expect(initContainer.SecurityContext.Capabilities).NotTo(BeNil()) + g.Expect(initContainer.SecurityContext.Capabilities.Drop).To(ConsistOf(corev1.Capability("ALL"))) + + // It should add exactly one volume mount to the main container. + g.Expect(spec.Containers[0].VolumeMounts).To(HaveLen(existingVolumeMountCount + 1)) + addedMount := spec.Containers[0].VolumeMounts[len(spec.Containers[0].VolumeMounts)-1] + g.Expect(addedMount.Name).To(Equal("aws-ca-bundle")) + g.Expect(addedMount.MountPath).To(Equal("/etc/pki/ca-trust/extracted/hypershift")) + g.Expect(addedMount.ReadOnly).To(BeTrue()) + + // It should set AWS_CA_BUNDLE env var on the main container. + g.Expect(spec.Containers[0].Env).To(HaveLen(existingEnvCount + 1)) + addedEnv := spec.Containers[0].Env[len(spec.Containers[0].Env)-1] + g.Expect(addedEnv.Name).To(Equal("AWS_CA_BUNDLE")) + g.Expect(addedEnv.Value).To(Equal("/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem")) + }) + } +} diff --git a/test/e2e/nodepool_additionalTrustBundlePropagation_test.go b/test/e2e/nodepool_additionalTrustBundlePropagation_test.go index befa16b1c93..59def4771b4 100644 --- a/test/e2e/nodepool_additionalTrustBundlePropagation_test.go +++ b/test/e2e/nodepool_additionalTrustBundlePropagation_test.go @@ -131,6 +131,68 @@ func (k *AdditionalTrustBundlePropagationTest) Run(t *testing.T, nodePool hyperv ) } + // Verify AWS_CA_BUNDLE wiring on the aws-cloud-controller-manager deployment + if k.hostedCluster.Spec.Platform.Type == hyperv1.AWSPlatform { + hcpNamespace := manifests.HostedControlPlaneNamespace(k.hostedCluster.Namespace, k.hostedCluster.Name) + awsCCMDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aws-cloud-controller-manager", + Namespace: hcpNamespace, + }, + } + e2eutil.EventuallyObject(t, k.ctx, "Waiting for aws-cloud-controller-manager to have AWS_CA_BUNDLE wiring", + func(ctx context.Context) (*appsv1.Deployment, error) { + err := k.mgmtClient.Get(ctx, crclient.ObjectKeyFromObject(awsCCMDeployment), awsCCMDeployment) + return awsCCMDeployment, err + }, + []e2eutil.Predicate[*appsv1.Deployment]{ + func(obj *appsv1.Deployment) (bool, string, error) { + // Check AWS_CA_BUNDLE env var on the first container. + hasEnv := false + for _, env := range obj.Spec.Template.Spec.Containers[0].Env { + if env.Name == "AWS_CA_BUNDLE" && env.Value == "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem" { + hasEnv = true + break + } + } + if !hasEnv { + return false, "AWS_CA_BUNDLE env var not found on first container", nil + } + + // Check setup-aws-ca-bundle init container. + hasInitContainer := false + for _, ic := range obj.Spec.Template.Spec.InitContainers { + if ic.Name == "setup-aws-ca-bundle" { + hasInitContainer = true + break + } + } + if !hasInitContainer { + return false, "setup-aws-ca-bundle init container not found", nil + } + + // Check aws-ca-bundle volume. + hasVolume := false + for _, v := range obj.Spec.Template.Spec.Volumes { + if v.Name == "aws-ca-bundle" && v.EmptyDir != nil { + hasVolume = true + break + } + } + if !hasVolume { + return false, "aws-ca-bundle volume not found", nil + } + + if ready := util.IsDeploymentReady(k.ctx, obj); !ready { + return false, "Deployment is not ready", nil + } + return true, "AWS_CA_BUNDLE wiring is present and deployment is ready", nil + }, + }, + e2eutil.WithInterval(10*time.Second), e2eutil.WithTimeout(5*time.Minute), + ) + } + t.Logf("Updating hosted cluster by removing additional trust bundle.") if err = e2eutil.UpdateObject(t, k.ctx, k.mgmtClient, k.hostedCluster, func(obj *hyperv1.HostedCluster) { obj.Spec.AdditionalTrustBundle = nil @@ -166,6 +228,45 @@ func (k *AdditionalTrustBundlePropagationTest) Run(t *testing.T, nodePool hyperv }, ) + // Verify AWS_CA_BUNDLE wiring is removed from the aws-cloud-controller-manager deployment + if k.hostedCluster.Spec.Platform.Type == hyperv1.AWSPlatform { + hcpNamespace := manifests.HostedControlPlaneNamespace(k.hostedCluster.Namespace, k.hostedCluster.Name) + awsCCMDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aws-cloud-controller-manager", + Namespace: hcpNamespace, + }, + } + e2eutil.EventuallyObject(t, k.ctx, "Waiting for aws-cloud-controller-manager to have AWS_CA_BUNDLE wiring removed", + func(ctx context.Context) (*appsv1.Deployment, error) { + err := k.mgmtClient.Get(ctx, crclient.ObjectKeyFromObject(awsCCMDeployment), awsCCMDeployment) + return awsCCMDeployment, err + }, + []e2eutil.Predicate[*appsv1.Deployment]{ + func(obj *appsv1.Deployment) (bool, string, error) { + // Ensure AWS_CA_BUNDLE env var is gone. + for _, env := range obj.Spec.Template.Spec.Containers[0].Env { + if env.Name == "AWS_CA_BUNDLE" { + return false, "AWS_CA_BUNDLE env var is still present", nil + } + } + + // Ensure setup-aws-ca-bundle init container is gone. + for _, ic := range obj.Spec.Template.Spec.InitContainers { + if ic.Name == "setup-aws-ca-bundle" { + return false, "setup-aws-ca-bundle init container is still present", nil + } + } + + if ready := util.IsDeploymentReady(k.ctx, obj); !ready { + return false, "Deployment is not ready", nil + } + return true, "AWS_CA_BUNDLE wiring is removed and deployment is ready", nil + }, + }, + ) + } + e2eutil.EventuallyObject(t, k.ctx, fmt.Sprintf("Waiting for NodePool %s/%s to begin updating", nodePool.Namespace, nodePool.Name), func(ctx context.Context) (*hyperv1.NodePool, error) { err := k.mgmtClient.Get(ctx, crclient.ObjectKeyFromObject(&nodePool), &nodePool)