From c49aacbe040d8ccfb145209cb75aa630ecb85212 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Mon, 9 Sep 2024 09:43:35 -0400 Subject: [PATCH 1/5] Add ability to add MI sidecar containers to pods Adds a function to include the managed identity sidecar containers in a deployment. The adapter-init is added as an init container and the adapter-server is added as a sidecar container in the related pod deployment it is included in. Signed-off-by: Bryan Cox --- support/azureutil/azureutil.go | 70 ++++++++++++++++ support/azureutil/azureutil_test.go | 121 ++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index 89ed85aa60d..f1c2e4d0a25 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -9,6 +9,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" @@ -17,6 +18,14 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) +// We received this images directly from Microsoft; we should expect them to change as Microsoft continues development on both containers. +// We are scheduled to receive new updates to these containers in October 2024 to support Managed Identities. They currently only support Service Principal. +// TODO past October, will we receive new versions? +const ( + AdapterInitImage = "aromiwi.azurecr.io/artifact/b8e9ef87-cd63-4085-ab14-1c637806568c/buddy/adapter-init:20240905.9" + AdapterServerImage = "aromiwi.azurecr.io/artifact/b8e9ef87-cd63-4085-ab14-1c637806568c/buddy/adapter-server:20240905.5" +) + // GetSubnetNameFromSubnetID extracts the subnet name from a subnet ID // Example subnet ID: /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/ func GetSubnetNameFromSubnetID(subnetID string) (string, error) { @@ -217,3 +226,64 @@ func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCl return nil } + +// GetAzureCredentialsFromSecret gets the Service Principal client ID, client secret, and tenant ID from the credentials +// secret. This function will be modified a bit once the Microsoft sidecar containers support Managed Identity are +// delivered (expected Oct 2024). +func GetAzureCredentialsFromSecret(ctx context.Context, c client.Client, namespace, credsName string) (*corev1.Secret, error) { + var azureCredentials corev1.Secret + + // Retrieve the Azure credentials secret to extract the needed fields for the managed identity containers + credentialsSecretName := client.ObjectKey{Namespace: namespace, Name: credsName} + if err := c.Get(ctx, credentialsSecretName, &azureCredentials); err != nil { + return nil, fmt.Errorf("failed to get secret %s: %w", credentialsSecretName, err) + } + + for _, expectedKey := range []string{"AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", "AZURE_TENANT_ID"} { + if _, found := azureCredentials.Data[expectedKey]; !found { + return nil, fmt.Errorf("credentials secret for cluster doesn't have required key %s", expectedKey) + } + } + + return &azureCredentials, nil +} + +// AdapterInitContainer returns the Microsoft adapter-init init container. This container needs the NET_ADMIN permission +// so the adapter-server sidecar container can intercept the Managed Identity Azure API authentication calls. +func AdapterInitContainer() corev1.Container { + return corev1.Container{ + Name: "adapter-init", + Image: AdapterInitImage, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + }, + }, + }} +} + +// AdapterServerContainer returns the Microsoft adapter-server sidecar container. Currently, this container mimics Azure +// Managed Identity approval and returns an authentication token. The container currently needs a Service Principal to +// do this. Future versions of this container will be able to take a Managed Identity instead. +func AdapterServerContainer(clientID, clientSecret, tenantID string) corev1.Container { + return corev1.Container{Name: "adapter-server", + Image: AdapterServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"sp"}, + Env: []corev1.EnvVar{ + { + Name: "AZURE_CLIENT_ID", + Value: clientID, + }, + { + Name: "AZURE_CLIENT_SECRET", + Value: clientSecret, + }, + { + Name: "AZURE_TENANT_ID", + Value: tenantID, + }, + }} +} diff --git a/support/azureutil/azureutil_test.go b/support/azureutil/azureutil_test.go index 6a5da3ad2da..9882749ef2d 100644 --- a/support/azureutil/azureutil_test.go +++ b/support/azureutil/azureutil_test.go @@ -1,9 +1,17 @@ package azureutil import ( + "context" "testing" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + . "github.com/onsi/gomega" + "github.com/openshift/hypershift/support/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestGetSubnetNameFromSubnetID(t *testing.T) { @@ -138,3 +146,116 @@ func TestGetVnetNameAndResourceGroupFromVnetID(t *testing.T) { }) } } + +func TestGetAzureCredentialsFromSecret(t *testing.T) { + tests := []struct { + testCaseName string + hc *hyperv1.HostedCluster + secret *corev1.Secret + expectedErr bool + }{ + { + testCaseName: "nominal test case", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentials"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_CLIENT_SECRET": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: false, + }, + { + testCaseName: "wrong secret name, err", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentialss"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_CLIENT_SECRET": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: true, + }, + { + testCaseName: "missing date from secret, err", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentialss"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.testCaseName, func(t *testing.T) { + g := NewGomegaWithT(t) + + objs := []crclient.Object{tc.hc, tc.secret} + + client := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(objs...).Build() + + creds, err := GetAzureCredentialsFromSecret(context.TODO(), client, tc.hc.Namespace, tc.hc.Spec.Platform.Azure.Credentials.Name) + if !tc.expectedErr { + g.Expect(err).To(BeNil()) + g.Expect(creds.Name).To(Equal(tc.hc.Spec.Platform.Azure.Credentials.Name)) + } else { + g.Expect(err).To(Not(BeNil())) + } + }) + } +} From da39c6232f098798325bc41eecaad1fe0c2f0090 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Mon, 5 Aug 2024 09:30:34 -0400 Subject: [PATCH 2/5] Add Managed Identity Support in Azure HC API Adds fields in the Azure HostedCluster API for the client IDs related to the managed service identities used for the following control plane components: azure cloud provider, KMS, CAPZ, the control plane operator, the image registry operator, cluster ingress operator, cluster network operator/cluster-network-config-controller, azure-disk-controller and azure-file-controller. Signed-off-by: Bryan Cox --- api/hypershift/v1beta1/hostedcluster_types.go | 94 +++++++- .../v1beta1/zz_generated.deepcopy.go | 32 +++ .../v1beta1/azureresourcemanagedidentities.go | 38 ++++ .../v1beta1/controlplanemanagedidentities.go | 114 ++++++++++ ...ypershift.openshift.io_hostedclusters.yaml | 122 ++++++++++ ...hift.openshift.io_hostedcontrolplanes.yaml | 122 ++++++++++ docs/content/reference/api.md | 212 ++++++++++++++++++ .../hypershift/v1beta1/hostedcluster_types.go | 94 +++++++- .../v1beta1/zz_generated.deepcopy.go | 32 +++ 9 files changed, 856 insertions(+), 4 deletions(-) create mode 100644 client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go create mode 100644 client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index 42a98277c51..2aa9d9960fc 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -1805,7 +1805,7 @@ type AzurePlatformSpec struct { // // Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. // - //Example: if your resource group ID is /subscriptions//resourceGroups/, your + // Example: if your resource group ID is /subscriptions//resourceGroups/, your // ResourceGroupName is . // // +kubebuilder:default:=default @@ -1857,8 +1857,98 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="SecurityGroupID is immutable" // +kubebuilder:validation:Required // +immutable - // +required SecurityGroupID string `json:"securityGroupID,omitempty"` + + // managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + // and data plane components that authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ManagedIdentities AzureResourceManagedIdentities `json:"managedIdentities,omitempty"` +} + +// AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +// and data plane components that authenticate with Azure's API. +type AzureResourceManagedIdentities struct { + // ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + // authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentities ControlPlaneManagedIdentities `json:"controlPlaneManagedIdentities"` + + // Future placeholder - DataPlaneMIs * DataPlaneManagedIdentities +} + +// ManagedIdentityClientID is a client ID of a managed identity +// +kubebuilder:validation:XValidation:rule="self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$')",message="the client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12." +type ManagedIdentityClientID string + +// ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +// to authenticate with Azure's API. +// Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +// The format a managed identity should be `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}`. +type ControlPlaneManagedIdentities struct { + // azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + // cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureCloudProviderManagedIdentityClientID ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID"` + + // clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + // azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ClusterAPIAzureManagedIdentityClientID ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID"` + + // controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + // operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentityClientID ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID"` + + // azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + // ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + // in the form 8-4-4-4-12. + // + // +optional + AzureKMSManagedIdentityClientID ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + + // imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + // of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ImageRegistryManagedIdentityClientID ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID"` + + // ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + IngressManagedIdentityClientID ManagedIdentityClientID `json:"ingressManagedIdentityClientID"` + + // networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + NetworkManagedIdentityClientID ManagedIdentityClientID `json:"networkManagedIdentityClientID"` + + // azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureDiskManagedIdentityClientID ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID"` + + // azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureFileManagedIdentityClientID ManagedIdentityClientID `json:"azureFileManagedIdentityClientID"` } // OpenStackPlatformSpec specifies configuration for clusters running on OpenStack. diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 14e4e51e549..ff6b0f7894b 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -564,6 +564,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in out.Credentials = in.Credentials + out.ManagedIdentities = in.ManagedIdentities } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzurePlatformSpec. @@ -576,6 +577,22 @@ func (in *AzurePlatformSpec) DeepCopy() *AzurePlatformSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureResourceManagedIdentities) DeepCopyInto(out *AzureResourceManagedIdentities) { + *out = *in + out.ControlPlaneManagedIdentities = in.ControlPlaneManagedIdentities +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureResourceManagedIdentities. +func (in *AzureResourceManagedIdentities) DeepCopy() *AzureResourceManagedIdentities { + if in == nil { + return nil + } + out := new(AzureResourceManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureVMImage) DeepCopyInto(out *AzureVMImage) { *out = *in @@ -879,6 +896,21 @@ func (in *ClusterVersionStatus) DeepCopy() *ClusterVersionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlaneManagedIdentities) DeepCopyInto(out *ControlPlaneManagedIdentities) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneManagedIdentities. +func (in *ControlPlaneManagedIdentities) DeepCopy() *ControlPlaneManagedIdentities { + if in == nil { + return nil + } + out := new(ControlPlaneManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSSpec) DeepCopyInto(out *DNSSpec) { *out = *in diff --git a/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go b/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go new file mode 100644 index 00000000000..c662e88f3e8 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AzureResourceManagedIdentitiesApplyConfiguration represents an declarative configuration of the AzureResourceManagedIdentities type for use +// with apply. +type AzureResourceManagedIdentitiesApplyConfiguration struct { + ControlPlaneManagedIdentities *ControlPlaneManagedIdentitiesApplyConfiguration `json:"controlPlaneManagedIdentities,omitempty"` +} + +// AzureResourceManagedIdentitiesApplyConfiguration constructs an declarative configuration of the AzureResourceManagedIdentities type for use with +// apply. +func AzureResourceManagedIdentities() *AzureResourceManagedIdentitiesApplyConfiguration { + return &AzureResourceManagedIdentitiesApplyConfiguration{} +} + +// WithControlPlaneManagedIdentities sets the ControlPlaneManagedIdentities field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControlPlaneManagedIdentities field is set to the value of the last call. +func (b *AzureResourceManagedIdentitiesApplyConfiguration) WithControlPlaneManagedIdentities(value *ControlPlaneManagedIdentitiesApplyConfiguration) *AzureResourceManagedIdentitiesApplyConfiguration { + b.ControlPlaneManagedIdentities = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go b/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go new file mode 100644 index 00000000000..1c7d273cb17 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go @@ -0,0 +1,114 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// ControlPlaneManagedIdentitiesApplyConfiguration represents an declarative configuration of the ControlPlaneManagedIdentities type for use +// with apply. +type ControlPlaneManagedIdentitiesApplyConfiguration struct { + AzureCloudProviderManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID,omitempty"` + ClusterAPIAzureManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID,omitempty"` + ControlPlaneManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID,omitempty"` + AzureKMSManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + ImageRegistryManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID,omitempty"` + IngressManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"ingressManagedIdentityClientID,omitempty"` + NetworkManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"networkManagedIdentityClientID,omitempty"` + AzureDiskManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID,omitempty"` + AzureFileManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureFileManagedIdentityClientID,omitempty"` +} + +// ControlPlaneManagedIdentitiesApplyConfiguration constructs an declarative configuration of the ControlPlaneManagedIdentities type for use with +// apply. +func ControlPlaneManagedIdentities() *ControlPlaneManagedIdentitiesApplyConfiguration { + return &ControlPlaneManagedIdentitiesApplyConfiguration{} +} + +// WithAzureCloudProviderManagedIdentityClientID sets the AzureCloudProviderManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureCloudProviderManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureCloudProviderManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureCloudProviderManagedIdentityClientID = &value + return b +} + +// WithClusterAPIAzureManagedIdentityClientID sets the ClusterAPIAzureManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ClusterAPIAzureManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithClusterAPIAzureManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ClusterAPIAzureManagedIdentityClientID = &value + return b +} + +// WithControlPlaneManagedIdentityClientID sets the ControlPlaneManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControlPlaneManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithControlPlaneManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ControlPlaneManagedIdentityClientID = &value + return b +} + +// WithAzureKMSManagedIdentityClientID sets the AzureKMSManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureKMSManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureKMSManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureKMSManagedIdentityClientID = &value + return b +} + +// WithImageRegistryManagedIdentityClientID sets the ImageRegistryManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ImageRegistryManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithImageRegistryManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ImageRegistryManagedIdentityClientID = &value + return b +} + +// WithIngressManagedIdentityClientID sets the IngressManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the IngressManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithIngressManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.IngressManagedIdentityClientID = &value + return b +} + +// WithNetworkManagedIdentityClientID sets the NetworkManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NetworkManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithNetworkManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.NetworkManagedIdentityClientID = &value + return b +} + +// WithAzureDiskManagedIdentityClientID sets the AzureDiskManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureDiskManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureDiskManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureDiskManagedIdentityClientID = &value + return b +} + +// WithAzureFileManagedIdentityClientID sets the AzureFileManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureFileManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureFileManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureFileManagedIdentityClientID = &value + return b +} diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml index 72c396dbcdc..94fd151abbc 100644 --- a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml +++ b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml @@ -8255,6 +8255,128 @@ spec: x-kubernetes-validations: - message: Location is immutable rule: self == oldSelf + managedIdentities: + description: |- + managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + and data plane components that authenticate with Azure's API. + properties: + controlPlaneManagedIdentities: + description: |- + ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + authenticate with Azure's API. + properties: + azureCloudProviderManagedIdentityClientID: + description: |- + azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureDiskManagedIdentityClientID: + description: |- + azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureFileManagedIdentityClientID: + description: |- + azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureKMSManagedIdentityClientID: + description: |- + azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + clusterAPIAzureManagedIdentityClientID: + description: |- + clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + controlPlaneManagedIdentityClientID: + description: |- + controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + imageRegistryManagedIdentityClientID: + description: |- + imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + ingressManagedIdentityClientID: + description: |- + ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + networkManagedIdentityClientID: + description: |- + networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + required: + - azureCloudProviderManagedIdentityClientID + - azureDiskManagedIdentityClientID + - azureFileManagedIdentityClientID + - clusterAPIAzureManagedIdentityClientID + - controlPlaneManagedIdentityClientID + - imageRegistryManagedIdentityClientID + - ingressManagedIdentityClientID + - networkManagedIdentityClientID + type: object + required: + - controlPlaneManagedIdentities + type: object resourceGroup: default: default description: |- diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml index 684cb311f3c..07bbf77c4f4 100644 --- a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml +++ b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml @@ -8219,6 +8219,128 @@ spec: x-kubernetes-validations: - message: Location is immutable rule: self == oldSelf + managedIdentities: + description: |- + managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + and data plane components that authenticate with Azure's API. + properties: + controlPlaneManagedIdentities: + description: |- + ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + authenticate with Azure's API. + properties: + azureCloudProviderManagedIdentityClientID: + description: |- + azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureDiskManagedIdentityClientID: + description: |- + azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureFileManagedIdentityClientID: + description: |- + azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureKMSManagedIdentityClientID: + description: |- + azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + clusterAPIAzureManagedIdentityClientID: + description: |- + clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + controlPlaneManagedIdentityClientID: + description: |- + controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + imageRegistryManagedIdentityClientID: + description: |- + imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + ingressManagedIdentityClientID: + description: |- + ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + networkManagedIdentityClientID: + description: |- + networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + required: + - azureCloudProviderManagedIdentityClientID + - azureDiskManagedIdentityClientID + - azureFileManagedIdentityClientID + - clusterAPIAzureManagedIdentityClientID + - controlPlaneManagedIdentityClientID + - imageRegistryManagedIdentityClientID + - ingressManagedIdentityClientID + - networkManagedIdentityClientID + type: object + required: + - controlPlaneManagedIdentities + type: object resourceGroup: default: default description: |- diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 673584152de..626d16df52b 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -2737,6 +2737,53 @@ configuration for the Azure cloud provider, aka Azure cloud controller manager ( expected to exist under the same subscription as SubscriptionID.

+ + +managedIdentities
+ + +AzureResourceManagedIdentities + + + + +

managedIdentities contains the client IDs related to the managed identities needed for HCP control plane +and data plane components that authenticate with Azure’s API.

+ + + + +###AzureResourceManagedIdentities { #hypershift.openshift.io/v1beta1.AzureResourceManagedIdentities } +

+(Appears on: +AzurePlatformSpec) +

+

+

AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +and data plane components that authenticate with Azure’s API.

+

+ + + + + + + + + + + +
FieldDescription
+controlPlaneManagedIdentities
+ + +ControlPlaneManagedIdentities + + +		 +

ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to +authenticate with Azure’s API.

+
###AzureVMImage { #hypershift.openshift.io/v1beta1.AzureVMImage } @@ -3542,6 +3589,163 @@ and reports missing images if any.

+###ControlPlaneManagedIdentities { #hypershift.openshift.io/v1beta1.ControlPlaneManagedIdentities } +

+(Appears on: +AzureResourceManagedIdentities) +

+

+

ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +to authenticate with Azure’s API. +Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +The format a managed identity should be /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+azureCloudProviderManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure +cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+clusterAPIAzureManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api +azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated +hexadecimal characters in the form 8-4-4-4-12.

+
+controlPlaneManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane +operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated +hexadecimal characters in the form 8-4-4-4-12.

+
+azureKMSManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +(Optional) +

azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client +ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters +in the form 8-4-4-4-12.

+
+imageRegistryManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups +of hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+ingressManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+networkManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+azureDiskManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen +separated hexadecimal characters in the form 8-4-4-4-12.

+
+azureFileManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen +separated hexadecimal characters in the form 8-4-4-4-12.

+
###DNSSpec { #hypershift.openshift.io/v1beta1.DNSSpec }

(Appears on: @@ -6729,6 +6933,14 @@ is empty.

+###ManagedIdentityClientID { #hypershift.openshift.io/v1beta1.ManagedIdentityClientID } +

+(Appears on: +ControlPlaneManagedIdentities) +

+

+

ManagedIdentityClientID is a client ID of a managed identity

+

###MarketplaceImage { #hypershift.openshift.io/v1beta1.MarketplaceImage }

(Appears on: diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index 42a98277c51..2aa9d9960fc 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -1805,7 +1805,7 @@ type AzurePlatformSpec struct { // // Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. // - //Example: if your resource group ID is /subscriptions//resourceGroups/, your + // Example: if your resource group ID is /subscriptions//resourceGroups/, your // ResourceGroupName is . // // +kubebuilder:default:=default @@ -1857,8 +1857,98 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="SecurityGroupID is immutable" // +kubebuilder:validation:Required // +immutable - // +required SecurityGroupID string `json:"securityGroupID,omitempty"` + + // managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + // and data plane components that authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ManagedIdentities AzureResourceManagedIdentities `json:"managedIdentities,omitempty"` +} + +// AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +// and data plane components that authenticate with Azure's API. +type AzureResourceManagedIdentities struct { + // ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + // authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentities ControlPlaneManagedIdentities `json:"controlPlaneManagedIdentities"` + + // Future placeholder - DataPlaneMIs * DataPlaneManagedIdentities +} + +// ManagedIdentityClientID is a client ID of a managed identity +// +kubebuilder:validation:XValidation:rule="self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$')",message="the client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12." +type ManagedIdentityClientID string + +// ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +// to authenticate with Azure's API. +// Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +// The format a managed identity should be `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}`. +type ControlPlaneManagedIdentities struct { + // azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + // cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureCloudProviderManagedIdentityClientID ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID"` + + // clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + // azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ClusterAPIAzureManagedIdentityClientID ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID"` + + // controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + // operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentityClientID ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID"` + + // azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + // ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + // in the form 8-4-4-4-12. + // + // +optional + AzureKMSManagedIdentityClientID ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + + // imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + // of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ImageRegistryManagedIdentityClientID ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID"` + + // ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + IngressManagedIdentityClientID ManagedIdentityClientID `json:"ingressManagedIdentityClientID"` + + // networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + NetworkManagedIdentityClientID ManagedIdentityClientID `json:"networkManagedIdentityClientID"` + + // azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureDiskManagedIdentityClientID ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID"` + + // azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureFileManagedIdentityClientID ManagedIdentityClientID `json:"azureFileManagedIdentityClientID"` } // OpenStackPlatformSpec specifies configuration for clusters running on OpenStack. diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 14e4e51e549..ff6b0f7894b 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -564,6 +564,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in out.Credentials = in.Credentials + out.ManagedIdentities = in.ManagedIdentities } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzurePlatformSpec. @@ -576,6 +577,22 @@ func (in *AzurePlatformSpec) DeepCopy() *AzurePlatformSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureResourceManagedIdentities) DeepCopyInto(out *AzureResourceManagedIdentities) { + *out = *in + out.ControlPlaneManagedIdentities = in.ControlPlaneManagedIdentities +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureResourceManagedIdentities. +func (in *AzureResourceManagedIdentities) DeepCopy() *AzureResourceManagedIdentities { + if in == nil { + return nil + } + out := new(AzureResourceManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureVMImage) DeepCopyInto(out *AzureVMImage) { *out = *in @@ -879,6 +896,21 @@ func (in *ClusterVersionStatus) DeepCopy() *ClusterVersionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlaneManagedIdentities) DeepCopyInto(out *ControlPlaneManagedIdentities) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneManagedIdentities. +func (in *ControlPlaneManagedIdentities) DeepCopy() *ControlPlaneManagedIdentities { + if in == nil { + return nil + } + out := new(ControlPlaneManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSSpec) DeepCopyInto(out *DNSSpec) { *out = *in From a485b9af6419cbd701d1aa0b739dc3ff997f63b3 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 27 Sep 2024 10:33:14 -0400 Subject: [PATCH 3/5] Initialize the CP managed identities in the CLI Initialize the control plane managed identities in the CLI. These are initialized with the client ID of the Service Principal at the moment. Once the Microsoft Adapter sidecar containers support Managed Identities, the CLI will create a new managed identity for each of these fields. Signed-off-by: Bryan Cox --- cmd/cluster/azure/create.go | 16 ++++++++++++++++ ...luster_complicated_invocation_from_bryan.yaml | 11 +++++++++++ ...ster_create_with_a_ure_marketplace_image.yaml | 11 +++++++++++ ...luster_minimal_flags_necessary_to_render.yaml | 11 +++++++++++ 4 files changed, 49 insertions(+) diff --git a/cmd/cluster/azure/create.go b/cmd/cluster/azure/create.go index 7387b8bc581..2e2b07ee26a 100644 --- a/cmd/cluster/azure/create.go +++ b/cmd/cluster/azure/create.go @@ -201,6 +201,22 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e VnetID: o.infra.VNetID, SubnetID: o.infra.SubnetID, SecurityGroupID: o.infra.SecurityGroupID, + ManagedIdentities: hyperv1.AzureResourceManagedIdentities{ + ControlPlaneManagedIdentities: hyperv1.ControlPlaneManagedIdentities{ + // TODO these are initialized with the client ID of the Service Principal at the moment. Once the + // Microsoft Adapter sidecar containers support Managed Identities, the CLI will create a new + // managed identity for each of these fields. + AzureCloudProviderManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ClusterAPIAzureManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureKMSManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ControlPlaneManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + NetworkManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ImageRegistryManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + IngressManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureFileManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureDiskManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + }, + }, }, } diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml index cc074969181..bc3c19d5ca9 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml @@ -52,6 +52,17 @@ spec: credentials: name: bryans-cluster-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml index a9cdc403ebb..24cae45ddae 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml @@ -41,6 +41,17 @@ spec: credentials: name: bryans-cluster-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml index 736f6791100..34240261323 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml @@ -52,6 +52,17 @@ spec: credentials: name: example-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID From 372754b3e605dc5483b0eaff9506dbb92a29bf65 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 16 Aug 2024 14:46:41 -0400 Subject: [PATCH 4/5] Use Managed Identities for HCP Components This commit replaces the use of Service Principal for Managed Identities for authenticating with Azure API. This commit also adds the Microsoft adapter-init and adapter-server sidecar containers to the deployments of the following HCP components - CAPZ, Azure Cloud Provider (CCM), KMS, and CPO - when deploying on ARO HCP. Signed-off-by: Bryan Cox --- .../hostedcontrolplane/cloud/azure/params.go | 1 + .../cloud/azure/providerconfig.go | 14 ++- .../cloud/azure/reconcile.go | 14 ++- .../hostedcontrolplane_controller.go | 91 ++++++++++++++----- .../hostedcontrolplane/kas/deployment.go | 24 ++++- .../hostedcontrolplane/kas/deployment_test.go | 3 +- .../hostedcontrolplane/kas/kms/azure.go | 24 ++++- .../hostedcontrolplane/manifests/azure.go | 9 ++ .../hostedcluster/hostedcluster_controller.go | 58 +++++++++--- .../internal/platform/azure/azure.go | 33 +------ support/azureutil/azureutil.go | 50 +--------- 11 files changed, 200 insertions(+), 121 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go index 27073cd9b94..58d83e04ce9 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go @@ -42,6 +42,7 @@ func NewAzureParams(hcp *hyperv1.HostedControlPlane) *AzureParams { p.DeploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] } p.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) + p.DeploymentConfig.SetDefaultSecurityContext = false return p } diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go index 0cf682ebfe4..64d2e00ffab 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go @@ -17,7 +17,7 @@ const ( // ReconcileCloudConfig reconciles as expected by Nodes Kubelet. func ReconcileCloudConfig(cm *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { - cfg, err := azureConfigWithoutCredentials(hcp, credentialsSecret) + cfg, err := AzureConfigWithoutCredentials(hcp, credentialsSecret) if err != nil { return err } @@ -37,14 +37,13 @@ func ReconcileCloudConfig(cm *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, // ReconcileCloudConfigWithCredentials reconciles as expected by KAS/KCM. func ReconcileCloudConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { - cfg, err := azureConfigWithoutCredentials(hcp, credentialsSecret) + cfg, err := AzureConfigWithoutCredentials(hcp, credentialsSecret) if err != nil { return err } - cfg.AADClientID = string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - cfg.AADClientSecret = string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - cfg.UseManagedIdentityExtension = false + cfg.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureCloudProviderManagedIdentityClientID) + cfg.UseInstanceMetadata = false serializedConfig, err := json.MarshalIndent(cfg, "", " ") if err != nil { @@ -58,7 +57,7 @@ func ReconcileCloudConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.Hos return nil } -func azureConfigWithoutCredentials(hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) (AzureConfig, error) { +func AzureConfigWithoutCredentials(hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) (AzureConfig, error) { subnetName, err := azureutil.GetSubnetNameFromSubnetID(hcp.Spec.Platform.Azure.SubnetID) if err != nil { return AzureConfig{}, fmt.Errorf("failed to determine subnet name from SubnetID: %w", err) @@ -106,9 +105,8 @@ type AzureConfig struct { Cloud string `json:"cloud"` TenantID string `json:"tenantId"` UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` + UserAssignedIdentityID string `json:"userAssignedIdentityID"` SubscriptionID string `json:"subscriptionId"` - AADClientID string `json:"aadClientId"` - AADClientSecret string `json:"aadClientSecret"` ResourceGroup string `json:"resourceGroup"` Location string `json:"location"` VnetName string `json:"vnetName"` diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go index 69c33f0737f..df1280e96a1 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go @@ -1,12 +1,15 @@ package azure import ( + "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/proxy" "github.com/openshift/hypershift/support/util" @@ -22,7 +25,12 @@ func ReconcileCCMServiceAccount(sa *corev1.ServiceAccount, ownerRef config.Owner return nil } -func ReconcileDeployment(deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, p *AzureParams, serviceAccountName string, releaseImageProvider *imageprovider.ReleaseImageProvider) error { +func ReconcileDeployment(ctx context.Context, c client.Client, deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, p *AzureParams, serviceAccountName string, releaseImageProvider *imageprovider.ReleaseImageProvider) error { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + deployment.Spec = appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: ccmLabels(), @@ -35,8 +43,12 @@ func ReconcileDeployment(deployment *appsv1.Deployment, hcp *hyperv1.HostedContr Labels: ccmLabels(), }, Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + azureutil.AdapterInitContainer(), + }, Containers: []corev1.Container{ util.BuildContainer(ccmContainer(), buildCCMContainer(p, releaseImageProvider.GetImage("azure-cloud-controller-manager"), hcp.Namespace)), + azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"])), }, Volumes: []corev1.Volume{}, ServiceAccountName: serviceAccountName, diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 07e43a69603..fb42050c284 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -46,6 +46,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ingress" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ingressoperator" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" + hcpkms "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas/kms" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kcm" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/konnectivity" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/machineapprover" @@ -64,8 +65,9 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/snapshotcontroller" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage" pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" - sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" + "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" @@ -391,7 +393,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R Type: string(hyperv1.ValidHostedControlPlaneConfiguration), ObservedGeneration: hostedControlPlane.Generation, } - if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil { + if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil { condition.Status = metav1.ConditionFalse condition.Message = err.Error() condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason @@ -844,12 +846,19 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error { return nil } -func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error { - for _, svc := range hc.Spec.Services { +func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + for _, svc := range hcp.Spec.Services { if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) { return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service) } } + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + if err := verifyResourceGroupLocationsMatch(ctx, hcp); err != nil { + return err + } + } + return nil } @@ -2983,6 +2992,20 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex if hcp.Spec.SecretEncryption.KMS == nil { return fmt.Errorf("kms metadata not specified") } + if hcp.Spec.Platform.Type == hyperv1.AzurePlatform { + azureCreds, err := hyperazureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + // Reconcile KMS config secret + kmsConfigSecret := manifests.AzureKMSConfigSecret(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, kmsConfigSecret, func() error { + return hcpkms.ReconcileKMSConfigWithCredentials(kmsConfigSecret, hcp, azureCreds) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure cloud config with credentials: %w", err) + } + } if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error { return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS) }); err != nil { @@ -3065,7 +3088,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex } if _, err := createOrUpdate(ctx, r, kubeAPIServerDeployment, func() error { - return kas.ReconcileKubeAPIServerDeployment(kubeAPIServerDeployment, + return kas.ReconcileKubeAPIServerDeployment(ctx, r.Client, kubeAPIServerDeployment, hcp, p.OwnerRef, p.DeploymentConfig, @@ -4564,7 +4587,7 @@ func (r *HostedControlPlaneReconciler) reconcileCloudControllerManager(ctx conte p := azure.NewAzureParams(hcp) deployment := azure.CCMDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - return azure.ReconcileDeployment(deployment, hcp, p, sa.Name, releaseImageProvider) + return azure.ReconcileDeployment(ctx, r.Client, deployment, hcp, p, sa.Name, releaseImageProvider) }); err != nil { return fmt.Errorf("failed to reconcile %s cloud controller manager deployment: %w", hcp.Spec.Platform.Type, err) } @@ -5228,24 +5251,11 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex } azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} - if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - condition := metav1.Condition{ - Type: string(hyperv1.ValidAzureKMSConfig), - ObservedGeneration: hcp.Generation, - Status: metav1.ConditionUnknown, - Message: fmt.Sprintf("failed to get azure credentials secret: %v", err), - Reason: hyperv1.StatusUnknownReason, - } - meta.SetStatusCondition(&hcp.Status.Conditions, condition) - return + options := &azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureKMSManagedIdentityClientID), } - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - - cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) + cred, err := azidentity.NewManagedIdentityCredential(options) if err != nil { conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason, fmt.Sprintf("failed to obtain azure client credential: %v", err)) @@ -5347,3 +5357,40 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli } return false, nil } + +// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups +func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + options := &azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.ControlPlaneManagedIdentityClientID), + } + + creds, err := azidentity.NewManagedIdentityCredential(options) + if err != nil { + return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + } + + // Retrieve full vnet information from the VNET ID + vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + } + + // Retrieve full network security group information from the network security group ID + nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get network security group info to verify its location: %v", err) + } + + // Retrieve full resource group information from the resource group name + rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get resource group info to verify its location: %v", err) + } + + // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match + if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { + return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) + } + + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go index 0562511d123..854caba4ac9 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go @@ -2,7 +2,9 @@ package kas import ( "bytes" + "context" "fmt" + "os" "path" "strconv" "strings" @@ -14,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/aws" @@ -21,6 +24,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/proxy" @@ -99,7 +103,10 @@ func kasLabels() map[string]string { } } -func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment, +func ReconcileKubeAPIServerDeployment( + ctx context.Context, + c client.Client, + deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, @@ -319,6 +326,21 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment, if err := applyKMSConfig(&deployment.Spec.Template.Spec, secretEncryptionData, images); err != nil { return err } + + // Add the adapter-init and adapter-server containers for ARO HCP + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + + // ARO HCP needs elevated privileges in order to run the adapter-init container + deploymentConfig.SetDefaultSecurityContext = false + } case hyperv1.AESCBC: err := applyAESCBCKeyHashAnnotation(&deployment.Spec.Template, aesCBCActiveKey, aesCBCBackupKey) if err != nil { diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go index e616a598c4d..7eb437b7f1e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go @@ -1,6 +1,7 @@ package kas import ( + "golang.org/x/net/context" "testing" . "github.com/onsi/gomega" @@ -57,7 +58,7 @@ func TestReconcileKubeAPIServerDeploymentNoChanges(t *testing.T) { tc.config.Data = map[string]string{"config.json": "test-json"} tc.auditConfig.Data = map[string]string{"policy.yaml": "test-data"} tc.authConfig.Data = map[string]string{"auth.json": "test-data"} - err := ReconcileKubeAPIServerDeployment(kubeAPIDeployment, hcp, ownerRef, tc.deploymentConfig, tc.params.NamedCertificates(), tc.params.CloudProvider, + err := ReconcileKubeAPIServerDeployment(context.TODO(), nil, kubeAPIDeployment, hcp, ownerRef, tc.deploymentConfig, tc.params.NamedCertificates(), tc.params.CloudProvider, tc.params.CloudProviderConfig, tc.params.CloudProviderCreds, tc.params.Images, tc.config, tc.auditConfig, tc.authConfig, tc.params.AuditWebhookRef, tc.activeKey, tc.backupKey, 6443, "test-payload-version", tc.params.FeatureGate, nil, tc.params.CipherSuites()) g.Expect(err).To(BeNil()) g.Expect(expectedMinReadySeconds).To(Equal(kubeAPIDeployment.Spec.MinReadySeconds)) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go index 81ee7b4dacb..c0d1ed98b6a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go @@ -1,6 +1,7 @@ package kms import ( + "encoding/json" "fmt" "time" @@ -216,7 +217,7 @@ func kasVolumeAzureKMSCredentials() *corev1.Volume { func buildVolumeAzureKMSCredentials(v *corev1.Volume) { v.Secret = &corev1.SecretVolumeSource{ - SecretName: manifests.AzureProviderConfigWithCredentials("").Name, + SecretName: manifests.AzureKMSConfigSecret("").Name, Items: []corev1.KeyToPath{ { Key: azure.CloudConfigKey, @@ -225,3 +226,24 @@ func buildVolumeAzureKMSCredentials(v *corev1.Volume) { }, } } + +func ReconcileKMSConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { + cfg, err := azure.AzureConfigWithoutCredentials(hcp, credentialsSecret) + if err != nil { + return err + } + + cfg.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureKMSManagedIdentityClientID) + + cfg.UseInstanceMetadata = false + serializedConfig, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go index 39a373f72f4..bfb5491d91b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go @@ -25,3 +25,12 @@ func AzureProviderConfigWithCredentials(ns string) *corev1.Secret { }, } } + +func AzureKMSConfigSecret(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-kms-cloud-config", + Namespace: ns, + }, + } +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 317a031fe95..ceb6f2e2da8 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -1586,6 +1586,16 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques proxy.SetEnvVars(&capiProviderDeploymentSpec.Template.Spec.Containers[0].Env) } + // For ARO HCP, we need to add the Microsoft init and sidecar containers to the CAPI Provider deployment + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hcluster.Namespace, hcluster.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return ctrl.Result{}, err + } + + capiProviderDeploymentSpec.Template.Spec.Containers = append(capiProviderDeploymentSpec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + } + // Reconcile cluster prometheus RBAC resources if enabled if r.EnableOCPClusterMonitoring { if err := r.reconcileClusterPrometheusRBAC(ctx, createOrUpdate, hcp.Namespace); err != nil { @@ -2229,6 +2239,8 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont controlPlaneOperatorDeployment := controlplaneoperator.OperatorDeployment(controlPlaneNamespace.Name) _, err = createOrUpdate(ctx, r.Client, controlPlaneOperatorDeployment, func() error { return reconcileControlPlaneOperatorDeployment( + ctx, + r.Client, controlPlaneOperatorDeployment, openShiftTrustedCABundleConfigMapExists, hcluster, @@ -2478,6 +2490,8 @@ func GetControlPlaneOperatorImageLabels(ctx context.Context, hc *hyperv1.HostedC } func reconcileControlPlaneOperatorDeployment( + ctx context.Context, + c client.Client, deployment *appsv1.Deployment, openShiftTrustedCABundleConfigMapExists bool, hc *hyperv1.HostedCluster, @@ -2819,6 +2833,26 @@ func reconcileControlPlaneOperatorDeployment( if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" { deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] } + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + if deployment.Spec.Template.Spec.InitContainers == nil { + deployment.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + + // ARO HCP needs elevated privileges in order to run the adapter-init container + deployment.Spec.Template.Spec.SecurityContext = nil + deploymentConfig.SetDefaultSecurityContext = false + } + deploymentConfig.SetDefaults(hcp, nil, k8sutilspointer.Int(1)) deploymentConfig.SetRestartAnnotation(hc.ObjectMeta) deploymentConfig.ApplyTo(deployment) @@ -3188,11 +3222,18 @@ func reconcileCAPIProviderDeployment(deployment *appsv1.Deployment, capiProvider // Enforce ServiceAccount. deployment.Spec.Template.Spec.ServiceAccountName = sa.Name + defaultSecurityContext := setDefaultSecurityContext + + // For ARO HCP, the MI sidecar containers need privileged permissions to run + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + defaultSecurityContext = false + } + deploymentConfig := config.DeploymentConfig{ Scheduling: config.Scheduling{ PriorityClass: config.DefaultPriorityClass, }, - SetDefaultSecurityContext: setDefaultSecurityContext, + SetDefaultSecurityContext: defaultSecurityContext, AdditionalLabels: map[string]string{ config.NeedManagementKASAccessLabel: "true", }, @@ -4333,12 +4374,9 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } // Verify the credentials secret contains the data fields we expect - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ - Namespace: hc.Namespace, - Name: hc.Spec.Platform.Azure.Credentials.Name, - }} - if err := r.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - return fmt.Errorf("failed to get credentials secret for cluster: %w", err) + credentialsSecret, err := azureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hc.Namespace, hc.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err } var errs []error @@ -4348,12 +4386,6 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } } - // Verify the resource group locations match - err := azureutil.VerifyResourceGroupLocationsMatch(ctx, hc, credentialsSecret) - if err != nil { - errs = append(errs, err) - } - return utilerrors.NewAggregate(errs) } diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go index 21770c592a5..4db23d0dc2d 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go @@ -56,7 +56,7 @@ func (a Azure) ReconcileCAPIInfraCR( } if _, err := createOrUpdate(ctx, client, azureClusterIdentity, func() error { - return reconcileAzureClusterIdentity(ctx, client, hcluster, azureClusterIdentity, controlPlaneNamespace) + return reconcileAzureClusterIdentity(hcluster, azureClusterIdentity, controlPlaneNamespace) }); err != nil { return nil, fmt.Errorf("failed to reconcile Azure cluster identity: %w", err) } @@ -83,6 +83,7 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: k8sutilspointer.Int64(10), + InitContainers: []corev1.Container{azureutil.AdapterInitContainer()}, Containers: []corev1.Container{{ Name: "manager", Image: image, @@ -164,23 +165,6 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create return err } - // Sync Azure Client Secret in its own secret for since CAPZ needs it in a specific key value - // https://capz.sigs.k8s.io/topics/multitenancy#manual-service-principal-identity - azureClientSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "azure-client-secret", Namespace: controlPlaneNamespace}} - if _, err := createOrUpdate(ctx, c, azureClientSecret, func() error { - if azureClientSecret.Data == nil { - azureClientSecret.Data = map[string][]byte{} - } - for k, v := range source.Data { - if k == "AZURE_CLIENT_SECRET" { - azureClientSecret.Data["clientSecret"] = v - } - } - return nil - }); err != nil { - return err - } - // Sync CNCC secret cloudNetworkConfigCreds := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cloud-network-config-controller-creds"}} secretData := map[string][]byte{ @@ -247,17 +231,10 @@ func reconcileAzureCluster(azureCluster *capiazure.AzureCluster, hcluster *hyper return nil } -func reconcileAzureClusterIdentity(ctx context.Context, c client.Client, hcluster *hyperv1.HostedCluster, azureClusterIdentity *capiazure.AzureClusterIdentity, controlPlaneNamespace string) error { - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: hcluster.Spec.Platform.Azure.Credentials.Name, Namespace: controlPlaneNamespace}} - if err := c.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - return fmt.Errorf("failed to get secret %s: %w", credentialsSecret, err) - } - +func reconcileAzureClusterIdentity(hcluster *hyperv1.HostedCluster, azureClusterIdentity *capiazure.AzureClusterIdentity, controlPlaneNamespace string) error { azureClusterIdentity.Spec = capiazure.AzureClusterIdentitySpec{ - ClientID: string(credentialsSecret.Data["AZURE_CLIENT_ID"]), - ClientSecret: corev1.SecretReference{Name: "azure-client-secret", Namespace: controlPlaneNamespace}, - TenantID: string(credentialsSecret.Data["AZURE_TENANT_ID"]), - Type: capiazure.ServicePrincipal, + ClientID: string(hcluster.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.ClusterAPIAzureManagedIdentityClientID), + Type: capiazure.UserAssignedMSI, AllowedNamespaces: &capiazure.AllowedNamespaces{ NamespaceList: []string{ controlPlaneNamespace, diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index f1c2e4d0a25..aeec7778dbc 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -3,17 +3,13 @@ package azureutil import ( "context" "fmt" - "k8s.io/utils/ptr" "strings" - hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" - corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) @@ -154,8 +150,8 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro return vnet, nil } -// getNetworkSecurityGroupInfo gets the full information on a network security group based on its ID -func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { +// GetNetworkSecurityGroupInfo gets the full information on a network security group based on its ID +func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { partialNSGInfo, err := arm.ParseResourceID(nsgID) if err != nil { return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err) @@ -174,8 +170,8 @@ func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscription return nsg, nil } -// getResourceGroupInfo gets the full information on a resource group based on its name -func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { +// GetResourceGroupInfo gets the full information on a resource group based on its name +func GetResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { resourceGroupClient, err := armresources.NewResourceGroupsClient(subscriptionID, azureCreds, nil) if err != nil { return armresources.ResourceGroupsClientGetResponse{}, fmt.Errorf("failed to create new resource groups client: %w", err) @@ -189,44 +185,6 @@ func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID str return rg, nil } -// VerifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups -func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCluster, credentialsSecret *corev1.Secret) error { - // Setup azureCreds so we can retrieve the locations of the resource groups - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - - creds, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) - if err != nil { - return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) - } - - // Retrieve full vnet information from the VNET ID - vnet, err := GetVnetInfoFromVnetID(ctx, hc.Spec.Platform.Azure.VnetID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get vnet info to verify its location: %v", err) - } - - // Retrieve full network security group information from the network security group ID - nsg, err := getNetworkSecurityGroupInfo(ctx, hc.Spec.Platform.Azure.SecurityGroupID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get network security group info to verify its location: %v", err) - } - - // Retrieve full resource group information from the resource group name - rg, err := getResourceGroupInfo(ctx, hc.Spec.Platform.Azure.ResourceGroupName, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get resource group info to verify its location: %v", err) - } - - // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match - if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { - return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) - } - - return nil -} - // GetAzureCredentialsFromSecret gets the Service Principal client ID, client secret, and tenant ID from the credentials // secret. This function will be modified a bit once the Microsoft sidecar containers support Managed Identity are // delivered (expected Oct 2024). From 143e24197092270089dd1cb60d11c6ac44bf0a37 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 10 Sep 2024 20:17:36 -0400 Subject: [PATCH 5/5] Use Managed Identities for OpenShift Components This commit replaces the use of Service Principal for Managed Identities for authenticating with Azure API. This commit also adds the Microsoft adapter-init and adapter-server sidecar containers to the deployments of cluster-image-registry-operator (CIRO) and cluster-ingress-operator (CIO). This commit also passes the needed environment variables on to cluster-network-operator (CNO) so the operator can add the adapter-init and adapter-server sidecar containers to cloud network config controller (CNCC) and to cluster-storage-operator (CSO) for the azure-disk-csi-controller and azure-file-csi-controller. Signed-off-by: Bryan Cox --- .../cno/clusternetworkoperator.go | 38 ++++++++--- .../hostedcontrolplane_controller.go | 47 +++++++++++-- .../ingressoperator/ingressoperator.go | 47 +++++++++++-- .../hostedcontrolplane/manifests/storage.go | 19 ++++++ .../registryoperator/reconcile.go | 27 +++++++- .../registryoperator/reconcile_test.go | 5 +- .../hostedcontrolplane/storage/azure.go | 67 +++++++++++++++++++ .../hostedcontrolplane/storage/operator.go | 31 +++++++-- .../hostedcontrolplane/storage/params.go | 34 ++++++++-- 9 files changed, 285 insertions(+), 30 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/storage/azure.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go index f9d35a587dc..69125cd76d4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go @@ -6,19 +6,20 @@ import ( "os" "strconv" - "github.com/openshift/hypershift/support/proxy" - "github.com/openshift/hypershift/support/rhobsmonitoring" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/blang/semver" routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/proxy" + "github.com/openshift/hypershift/support/rhobsmonitoring" "github.com/openshift/hypershift/support/util" + + "github.com/blang/semver" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -27,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -74,9 +76,12 @@ type Params struct { DeploymentConfig config.DeploymentConfig IsPrivate bool DefaultIngressDomain string + NetworkManagedIdentity string + ClientIDSecret string + TenantID string } -func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool, defaultIngressDomain string) Params { +func NewParams(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool, defaultIngressDomain string) (Params, error) { p := Params{ Images: Images{ NetworkOperator: releaseImageProvider.GetImage("cluster-network-operator"), @@ -131,7 +136,18 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv p.APIServerPort = hcp.Status.ControlPlaneEndpoint.Port } - return p + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return p, fmt.Errorf("failed to get Azure credentials: %w", err) + } + + p.NetworkManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.NetworkManagedIdentityClientID) + p.ClientIDSecret = string(azureCredentials.Data["AZURE_CLIENT_SECRET"]) + p.TenantID = string(azureCredentials.Data["AZURE_TENANT_ID"]) + } + + return p, nil } func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, networkType hyperv1.NetworkType) error { @@ -473,8 +489,11 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat {Name: "CA_CONFIG_MAP", Value: params.CAConfigMap}, {Name: "CA_CONFIG_MAP_KEY", Value: params.CAConfigMapKey}, {Name: "TOKEN_AUDIENCE", Value: params.TokenAudience}, - - {Name: "RELEASE_VERSION", Value: params.ReleaseVersion}, + {Name: "ARO_HCP_MI_CLIENT_ID", Value: params.NetworkManagedIdentity}, + {Name: "AZURE_ADAPTER_INIT_IMAGE", Value: azureutil.AdapterInitImage}, + {Name: "AZURE_ADAPTER_SERVER_IMAGE", Value: azureutil.AdapterServerImage}, + {Name: "CLIENT_ID_SECRET", Value: params.ClientIDSecret}, + {Name: "TENANT_ID", Value: params.TenantID}, {Name: "APISERVER_OVERRIDE_HOST", Value: params.APIServerAddress}, // We need to pass this down to networking components on the nodes {Name: "APISERVER_OVERRIDE_PORT", Value: fmt.Sprint(params.APIServerPort)}, {Name: "OVN_NB_RAFT_ELECTION_TIMER", Value: "10"}, @@ -514,6 +533,7 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat {Name: "CLI_IMAGE", Value: params.Images.CLI}, {Name: "SOCKS5_PROXY_IMAGE", Value: params.Images.Socks5Proxy}, {Name: "OPENSHIFT_RELEASE_IMAGE", Value: params.DeploymentConfig.AdditionalAnnotations[hyperv1.ReleaseImageAnnotation]}, + {Name: "RELEASE_VERSION", Value: params.ReleaseVersion}, }...), Name: operatorName, Image: params.Images.NetworkOperator, diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index fb42050c284..989e787a345 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -22,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/kms" "github.com/go-logr/logr" + routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" @@ -81,7 +82,9 @@ import ( "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/reference" "github.com/openshift/hypershift/support/upsert" "github.com/openshift/hypershift/support/util" + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -3532,7 +3535,10 @@ func (r *HostedControlPlaneReconciler) reconcileClusterVersionOperator(ctx conte } func (r *HostedControlPlaneReconciler) reconcileClusterNetworkOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, hasRouteCap bool, createOrUpdate upsert.CreateOrUpdateFN) error { - p := cno.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext, r.DefaultIngressDomain) + p, err := cno.NewParams(ctx, r.Client, hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext, r.DefaultIngressDomain) + if err != nil { + return err + } sa := manifests.ClusterNetworkOperatorServiceAccount(hcp.Namespace) if _, err := createOrUpdate(ctx, r.Client, sa, func() error { @@ -3699,7 +3705,9 @@ func (r *HostedControlPlaneReconciler) reconcileIngressOperator(ctx context.Cont deployment := manifests.IngressOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - ingressoperator.ReconcileDeployment(deployment, p, hcp.Spec.Platform.Type) + if err := ingressoperator.ReconcileDeployment(ctx, r.Client, hcp, deployment, p, hcp.Spec.Platform.Type); err != nil { + return err + } return nil }); err != nil { return fmt.Errorf("failed to reconcile ingressoperator deployment: %w", err) @@ -4017,7 +4025,7 @@ func (r *HostedControlPlaneReconciler) reconcileImageRegistryOperator(ctx contex deployment := manifests.ImageRegistryOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - return registryoperator.ReconcileDeployment(deployment, params) + return registryoperator.ReconcileDeployment(ctx, r.Client, hcp, deployment, params) }); err != nil { return fmt.Errorf("failed to reconcile image registry operator deployment: %w", err) } @@ -4800,7 +4808,10 @@ func (r *HostedControlPlaneReconciler) reconcileCSISnapshotControllerOperator(ct } func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error { - params := storage.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext) + params, err := storage.NewParams(ctx, r.Client, hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext) + if err != nil { + return err + } deployment := manifests.ClusterStorageOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { @@ -4830,6 +4841,34 @@ func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx conte return fmt.Errorf("failed to reconcile cluster storage operator roleBinding: %w", err) } + // Reconcile azure-disk-csi-controller and azure-file-csi-controller configuration secrets for ARO HCP. This is + // needed so we can specify a unique managed identity for each controller to authenticate with the Managed Identity + // Azure API. + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + // Get the credentials secret so we can retrieve the tenant ID for the configuration + credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) + + // Reconcile the secret needed for azure-disk-csi-controller + azureDiskCSISecret := manifests.AzureDiskCSIConfig(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureDiskCSISecret(azureDiskCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure Disk CSI config: %w", err) + } + + // Reconcile the secret needed for azure-disk-csi-controller + azureFileCSISecret := manifests.AzureFileCSIConfig(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureFileCSISecret(azureFileCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure File CSI config: %w", err) + } + } + // TODO: create custom kubeconfig to the guest cluster + RBAC return nil diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go index e818ed1e815..482c039263f 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go @@ -1,30 +1,34 @@ package ingressoperator import ( + "context" "fmt" - configv1 "github.com/openshift/api/config/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/metrics" "github.com/openshift/hypershift/support/proxy" "github.com/openshift/hypershift/support/util" + "os" + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( operatorName = "ingress-operator" ingressOperatorContainerName = "ingress-operator" - metricsHostname = "ingress-operator" konnectivityProxyContainerName = "konnectivity-proxy" ingressOperatorMetricsPort = 60000 konnectivityProxyPort = 8090 @@ -70,14 +74,19 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv return p } -func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) { +func ReconcileDeployment(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) error { + // Determine if the deployment will be placed on ARO HCP + aroHCPDeployment := os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP + + // Initialize resource requests ingressOpResources := corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("80Mi"), corev1.ResourceCPU: resource.MustParse("10m"), }, } - // preserve existing resource requirements + + // Preserve existing resource requirements mainContainer := util.FindContainer(ingressOperatorContainerName, dep.Spec.Template.Spec.Containers) if mainContainer != nil { if len(mainContainer.Resources.Requests) > 0 || len(mainContainer.Resources.Limits) > 0 { @@ -85,6 +94,12 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp } } + // Initialize NO_PROXY environment variable and add the Azure Managed Identity API endpoint if ingress operator is being deployed on ARO HCP + noProxyValue := manifests.KubeAPIServerService("").Name + if aroHCPDeployment { + noProxyValue = noProxyValue + ",169.254.169.254" + } + dep.Spec.Replicas = ptr.To[int32](1) dep.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"name": operatorName}} dep.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType @@ -132,7 +147,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp }, { Name: "NO_PROXY", - Value: manifests.KubeAPIServerService("").Name, + Value: noProxyValue, }, }, Name: ingressOperatorContainerName, @@ -199,7 +214,29 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp }, ) + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "ARO_HCP_MI_CLIENT_ID", + Value: string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.IngressManagedIdentityClientID), + }) + + if dep.Spec.Template.Spec.InitContainers == nil { + dep.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + dep.Spec.Template.Spec.InitContainers = append(dep.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + params.DeploymentConfig.SetDefaultSecurityContext = false + } + params.DeploymentConfig.ApplyTo(dep) + return nil } func ingressOperatorKonnectivityProxyContainer(proxyImage string, proxyConfig *configv1.ProxySpec, noProxy string) corev1.Container { diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go index e09fbc8e570..fcc085100a8 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go @@ -4,6 +4,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func ClusterStorageOperatorDeployment(ns string) *appsv1.Deployment { @@ -33,3 +34,21 @@ func ClusterStorageOperatorServiceAccount(ns string) *corev1.ServiceAccount { sa.Namespace = ns return sa } + +func AzureDiskCSIConfig(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-disk-csi-config", + Namespace: ns, + }, + } +} + +func AzureFileCSIConfig(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-file-csi-config", + Namespace: ns, + }, + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go index 31f1a347447..ae769b38733 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go @@ -2,6 +2,8 @@ package registryoperator import ( "bytes" + "context" + "os" "path" "text/template" @@ -11,11 +13,13 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/metrics" @@ -152,7 +156,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv return params } -func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error { +func ReconcileDeployment(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, deployment *appsv1.Deployment, params Params) error { deployment.Spec = appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels(), @@ -194,6 +198,27 @@ func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error { ) } + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "ARO_HCP_MI_CLIENT_ID", + Value: "true", + }) + + if deployment.Spec.Template.Spec.InitContainers == nil { + deployment.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + params.deploymentConfig.SetDefaultSecurityContext = false + } + params.deploymentConfig.ApplyTo(deployment) return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go index 3255fb7a13e..5640db74d80 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go @@ -1,6 +1,8 @@ package registryoperator import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -35,8 +37,9 @@ func TestReconcileDeployment(t *testing.T) { } deployment := manifests.ImageRegistryOperatorDeployment("test-namespace") imageProvider := imageprovider.NewFromImages(images) + fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).Build() params := NewParams(hcp, "1.0.0", imageProvider, imageProvider, true) - if err := ReconcileDeployment(deployment, params); err != nil { + if err := ReconcileDeployment(context.TODO(), fakeClient, hcp, deployment, params); err != nil { t.Fatalf("unexpected error: %v", err) } deploymentYaml, err := util.SerializeResource(deployment, api.Scheme) diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go new file mode 100644 index 00000000000..fd8a6a514ac --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go @@ -0,0 +1,67 @@ +package storage + +import ( + "encoding/json" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" + + corev1 "k8s.io/api/core/v1" +) + +// initializeAzureCSIControllerConfig initializes an AzureConfig object which will be used to populate the secrets +// needed by azure-disk-csi-controller and azure-file-csi-controller. +// The source of truth for which fields are required by azure-disk is located here - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/deploy/example/azure.json. +// The source of truth for which fields are required by azure-file is located here - https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/deploy/example/azure.json. +// As of Sept 2024, the fields we need in HyperShift are the same between both of these sources of truth. +func initializeAzureCSIControllerConfig(hcp *hyperv1.HostedControlPlane, tenantID string) azure.AzureConfig { + azureConfig := azure.AzureConfig{ + // These fields are mandatory + Cloud: hcp.Spec.Platform.Azure.Cloud, + TenantID: tenantID, + SubscriptionID: hcp.Spec.Platform.Azure.SubscriptionID, + ResourceGroup: hcp.Spec.Platform.Azure.ResourceGroupName, + Location: hcp.Spec.Platform.Azure.Location, + + // These fields are mandatory when using managed identity; the user assigned identity ID is populated after this function call + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "", + } + + return azureConfig +} + +// ReconcileAzureDiskCSISecret reconciles the configuration for the secret as expected by azure-disk-csi-controller +func ReconcileAzureDiskCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureDiskManagedIdentityClientID) + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} + +// ReconcileAzureFileCSISecret reconciles the configuration for the secret as expected by azure-file-csi-controller +func ReconcileAzureFileCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureFileManagedIdentityClientID) + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go index 03f559846de..384c020cd38 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go @@ -1,18 +1,25 @@ package storage import ( + "fmt" + "os" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage/assets" assets2 "github.com/openshift/hypershift/support/assets" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/util" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) +const ClusterStorageOperatorContainerName = "cluster-storage-operator" + var ( operatorDeployment = assets2.MustDeployment(assets.ReadFile, "10_deployment-hypershift.yaml") operatorRole = assets2.MustRole(assets.ReadFile, "role.yaml") @@ -26,12 +33,26 @@ func ReconcileOperatorDeployment( params.OwnerRef.ApplyTo(deployment) deployment.Spec = operatorDeployment.DeepCopy().Spec - for i, container := range deployment.Spec.Template.Spec.Containers { - switch container.Name { - case "cluster-storage-operator": - deployment.Spec.Template.Spec.Containers[i].Image = params.StorageOperatorImage - params.ImageReplacer.replaceEnvVars(deployment.Spec.Template.Spec.Containers[i].Env) + + csoContainer := util.FindContainer(ClusterStorageOperatorContainerName, deployment.Spec.Template.Spec.Containers) + if csoContainer == nil { + return fmt.Errorf("could not find ClusterStorageOperator container for Deployment") + } + + csoContainer.Image = params.StorageOperatorImage + params.ImageReplacer.replaceEnvVars(csoContainer.Env) + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + aroHCPEnvs := []corev1.EnvVar{ + {Name: "AZURE_ADAPTER_INIT_IMAGE", Value: azureutil.AdapterInitImage}, + {Name: "AZURE_ADAPTER_SERVER_IMAGE", Value: azureutil.AdapterServerImage}, + {Name: "ARO_HCP_DISK_MI_CLIENT_ID", Value: params.AzureDiskManagedIdentity}, + {Name: "ARO_HCP_FILE_MI_CLIENT_ID", Value: params.AzureFileManagedIdentity}, + {Name: "CLIENT_ID_SECRET", Value: params.ClientIDSecret}, + {Name: "TENANT_ID", Value: params.TenantID}, } + + csoContainer.Env = append(csoContainer.Env, aroHCPEnvs...) } params.DeploymentConfig.ApplyTo(deployment) diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go index be4c3c6615d..3505962a383 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go @@ -1,11 +1,17 @@ package storage import ( + "context" + "fmt" + "os" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/util" utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -13,20 +19,26 @@ const ( ) type Params struct { - OwnerRef config.OwnerRef - StorageOperatorImage string - ImageReplacer *environmentReplacer + OwnerRef config.OwnerRef + StorageOperatorImage string + AzureDiskManagedIdentity string + AzureFileManagedIdentity string + ClientIDSecret string + TenantID string + ImageReplacer *environmentReplacer AvailabilityProberImage string config.DeploymentConfig } func NewParams( + ctx context.Context, + c client.Client, hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, - setDefaultSecurityContext bool) *Params { + setDefaultSecurityContext bool) (*Params, error) { ir := newEnvironmentReplacer() ir.setVersions(version) @@ -51,5 +63,17 @@ func NewParams( params.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.Int(1)) params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) - return ¶ms + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return nil, fmt.Errorf("failed to get Azure credentials: %w", err) + } + + params.AzureDiskManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureDiskManagedIdentityClientID) + params.AzureFileManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureFileManagedIdentityClientID) + params.ClientIDSecret = string(azureCredentials.Data["AZURE_CLIENT_SECRET"]) + params.TenantID = string(azureCredentials.Data["AZURE_TENANT_ID"]) + } + + return ¶ms, nil }