diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index 42a98277c51..2aa9d9960fc 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -1805,7 +1805,7 @@ type AzurePlatformSpec struct { // // Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. // - //Example: if your resource group ID is /subscriptions//resourceGroups/, your + // Example: if your resource group ID is /subscriptions//resourceGroups/, your // ResourceGroupName is . // // +kubebuilder:default:=default @@ -1857,8 +1857,98 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="SecurityGroupID is immutable" // +kubebuilder:validation:Required // +immutable - // +required SecurityGroupID string `json:"securityGroupID,omitempty"` + + // managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + // and data plane components that authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ManagedIdentities AzureResourceManagedIdentities `json:"managedIdentities,omitempty"` +} + +// AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +// and data plane components that authenticate with Azure's API. +type AzureResourceManagedIdentities struct { + // ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + // authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentities ControlPlaneManagedIdentities `json:"controlPlaneManagedIdentities"` + + // Future placeholder - DataPlaneMIs * DataPlaneManagedIdentities +} + +// ManagedIdentityClientID is a client ID of a managed identity +// +kubebuilder:validation:XValidation:rule="self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$')",message="the client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12." +type ManagedIdentityClientID string + +// ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +// to authenticate with Azure's API. +// Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +// The format a managed identity should be `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}`. +type ControlPlaneManagedIdentities struct { + // azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + // cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureCloudProviderManagedIdentityClientID ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID"` + + // clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + // azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ClusterAPIAzureManagedIdentityClientID ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID"` + + // controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + // operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentityClientID ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID"` + + // azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + // ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + // in the form 8-4-4-4-12. + // + // +optional + AzureKMSManagedIdentityClientID ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + + // imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + // of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ImageRegistryManagedIdentityClientID ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID"` + + // ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + IngressManagedIdentityClientID ManagedIdentityClientID `json:"ingressManagedIdentityClientID"` + + // networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + NetworkManagedIdentityClientID ManagedIdentityClientID `json:"networkManagedIdentityClientID"` + + // azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureDiskManagedIdentityClientID ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID"` + + // azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureFileManagedIdentityClientID ManagedIdentityClientID `json:"azureFileManagedIdentityClientID"` } // OpenStackPlatformSpec specifies configuration for clusters running on OpenStack. diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 14e4e51e549..ff6b0f7894b 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -564,6 +564,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in out.Credentials = in.Credentials + out.ManagedIdentities = in.ManagedIdentities } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzurePlatformSpec. @@ -576,6 +577,22 @@ func (in *AzurePlatformSpec) DeepCopy() *AzurePlatformSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureResourceManagedIdentities) DeepCopyInto(out *AzureResourceManagedIdentities) { + *out = *in + out.ControlPlaneManagedIdentities = in.ControlPlaneManagedIdentities +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureResourceManagedIdentities. +func (in *AzureResourceManagedIdentities) DeepCopy() *AzureResourceManagedIdentities { + if in == nil { + return nil + } + out := new(AzureResourceManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureVMImage) DeepCopyInto(out *AzureVMImage) { *out = *in @@ -879,6 +896,21 @@ func (in *ClusterVersionStatus) DeepCopy() *ClusterVersionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlaneManagedIdentities) DeepCopyInto(out *ControlPlaneManagedIdentities) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneManagedIdentities. +func (in *ControlPlaneManagedIdentities) DeepCopy() *ControlPlaneManagedIdentities { + if in == nil { + return nil + } + out := new(ControlPlaneManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSSpec) DeepCopyInto(out *DNSSpec) { *out = *in diff --git a/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go b/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go new file mode 100644 index 00000000000..c662e88f3e8 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/azureresourcemanagedidentities.go @@ -0,0 +1,38 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AzureResourceManagedIdentitiesApplyConfiguration represents an declarative configuration of the AzureResourceManagedIdentities type for use +// with apply. +type AzureResourceManagedIdentitiesApplyConfiguration struct { + ControlPlaneManagedIdentities *ControlPlaneManagedIdentitiesApplyConfiguration `json:"controlPlaneManagedIdentities,omitempty"` +} + +// AzureResourceManagedIdentitiesApplyConfiguration constructs an declarative configuration of the AzureResourceManagedIdentities type for use with +// apply. +func AzureResourceManagedIdentities() *AzureResourceManagedIdentitiesApplyConfiguration { + return &AzureResourceManagedIdentitiesApplyConfiguration{} +} + +// WithControlPlaneManagedIdentities sets the ControlPlaneManagedIdentities field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControlPlaneManagedIdentities field is set to the value of the last call. +func (b *AzureResourceManagedIdentitiesApplyConfiguration) WithControlPlaneManagedIdentities(value *ControlPlaneManagedIdentitiesApplyConfiguration) *AzureResourceManagedIdentitiesApplyConfiguration { + b.ControlPlaneManagedIdentities = value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go b/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go new file mode 100644 index 00000000000..1c7d273cb17 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/controlplanemanagedidentities.go @@ -0,0 +1,114 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" +) + +// ControlPlaneManagedIdentitiesApplyConfiguration represents an declarative configuration of the ControlPlaneManagedIdentities type for use +// with apply. +type ControlPlaneManagedIdentitiesApplyConfiguration struct { + AzureCloudProviderManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID,omitempty"` + ClusterAPIAzureManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID,omitempty"` + ControlPlaneManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID,omitempty"` + AzureKMSManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + ImageRegistryManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID,omitempty"` + IngressManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"ingressManagedIdentityClientID,omitempty"` + NetworkManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"networkManagedIdentityClientID,omitempty"` + AzureDiskManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID,omitempty"` + AzureFileManagedIdentityClientID *v1beta1.ManagedIdentityClientID `json:"azureFileManagedIdentityClientID,omitempty"` +} + +// ControlPlaneManagedIdentitiesApplyConfiguration constructs an declarative configuration of the ControlPlaneManagedIdentities type for use with +// apply. +func ControlPlaneManagedIdentities() *ControlPlaneManagedIdentitiesApplyConfiguration { + return &ControlPlaneManagedIdentitiesApplyConfiguration{} +} + +// WithAzureCloudProviderManagedIdentityClientID sets the AzureCloudProviderManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureCloudProviderManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureCloudProviderManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureCloudProviderManagedIdentityClientID = &value + return b +} + +// WithClusterAPIAzureManagedIdentityClientID sets the ClusterAPIAzureManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ClusterAPIAzureManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithClusterAPIAzureManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ClusterAPIAzureManagedIdentityClientID = &value + return b +} + +// WithControlPlaneManagedIdentityClientID sets the ControlPlaneManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControlPlaneManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithControlPlaneManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ControlPlaneManagedIdentityClientID = &value + return b +} + +// WithAzureKMSManagedIdentityClientID sets the AzureKMSManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureKMSManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureKMSManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureKMSManagedIdentityClientID = &value + return b +} + +// WithImageRegistryManagedIdentityClientID sets the ImageRegistryManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ImageRegistryManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithImageRegistryManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.ImageRegistryManagedIdentityClientID = &value + return b +} + +// WithIngressManagedIdentityClientID sets the IngressManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the IngressManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithIngressManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.IngressManagedIdentityClientID = &value + return b +} + +// WithNetworkManagedIdentityClientID sets the NetworkManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NetworkManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithNetworkManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.NetworkManagedIdentityClientID = &value + return b +} + +// WithAzureDiskManagedIdentityClientID sets the AzureDiskManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureDiskManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureDiskManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureDiskManagedIdentityClientID = &value + return b +} + +// WithAzureFileManagedIdentityClientID sets the AzureFileManagedIdentityClientID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AzureFileManagedIdentityClientID field is set to the value of the last call. +func (b *ControlPlaneManagedIdentitiesApplyConfiguration) WithAzureFileManagedIdentityClientID(value v1beta1.ManagedIdentityClientID) *ControlPlaneManagedIdentitiesApplyConfiguration { + b.AzureFileManagedIdentityClientID = &value + return b +} diff --git a/cmd/cluster/azure/create.go b/cmd/cluster/azure/create.go index 7387b8bc581..2e2b07ee26a 100644 --- a/cmd/cluster/azure/create.go +++ b/cmd/cluster/azure/create.go @@ -201,6 +201,22 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e VnetID: o.infra.VNetID, SubnetID: o.infra.SubnetID, SecurityGroupID: o.infra.SecurityGroupID, + ManagedIdentities: hyperv1.AzureResourceManagedIdentities{ + ControlPlaneManagedIdentities: hyperv1.ControlPlaneManagedIdentities{ + // TODO these are initialized with the client ID of the Service Principal at the moment. Once the + // Microsoft Adapter sidecar containers support Managed Identities, the CLI will create a new + // managed identity for each of these fields. + AzureCloudProviderManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ClusterAPIAzureManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureKMSManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ControlPlaneManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + NetworkManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + ImageRegistryManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + IngressManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureFileManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + AzureDiskManagedIdentityClientID: hyperv1.ManagedIdentityClientID(o.creds.ClientID), + }, + }, }, } diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml index cc074969181..bc3c19d5ca9 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_complicated_invocation_from_bryan.yaml @@ -52,6 +52,17 @@ spec: credentials: name: bryans-cluster-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml index a9cdc403ebb..24cae45ddae 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_create_with_a_ure_marketplace_image.yaml @@ -41,6 +41,17 @@ spec: credentials: name: bryans-cluster-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID diff --git a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml index 736f6791100..34240261323 100644 --- a/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml +++ b/cmd/cluster/azure/testdata/zz_fixture_TestCreateCluster_minimal_flags_necessary_to_render.yaml @@ -52,6 +52,17 @@ spec: credentials: name: example-cloud-credentials location: fakeLocation + managedIdentities: + controlPlaneManagedIdentities: + azureCloudProviderManagedIdentityClientID: fakeClientID + azureDiskManagedIdentityClientID: fakeClientID + azureFileManagedIdentityClientID: fakeClientID + azureKMSManagedIdentityClientID: fakeClientID + clusterAPIAzureManagedIdentityClientID: fakeClientID + controlPlaneManagedIdentityClientID: fakeClientID + imageRegistryManagedIdentityClientID: fakeClientID + ingressManagedIdentityClientID: fakeClientID + networkManagedIdentityClientID: fakeClientID resourceGroup: fakeResourceGroupName securityGroupID: fakeSecurityGroupID subnetID: fakeSubnetID diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml index 72c396dbcdc..94fd151abbc 100644 --- a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml +++ b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml @@ -8255,6 +8255,128 @@ spec: x-kubernetes-validations: - message: Location is immutable rule: self == oldSelf + managedIdentities: + description: |- + managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + and data plane components that authenticate with Azure's API. + properties: + controlPlaneManagedIdentities: + description: |- + ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + authenticate with Azure's API. + properties: + azureCloudProviderManagedIdentityClientID: + description: |- + azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureDiskManagedIdentityClientID: + description: |- + azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureFileManagedIdentityClientID: + description: |- + azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureKMSManagedIdentityClientID: + description: |- + azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + clusterAPIAzureManagedIdentityClientID: + description: |- + clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + controlPlaneManagedIdentityClientID: + description: |- + controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + imageRegistryManagedIdentityClientID: + description: |- + imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + ingressManagedIdentityClientID: + description: |- + ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + networkManagedIdentityClientID: + description: |- + networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + required: + - azureCloudProviderManagedIdentityClientID + - azureDiskManagedIdentityClientID + - azureFileManagedIdentityClientID + - clusterAPIAzureManagedIdentityClientID + - controlPlaneManagedIdentityClientID + - imageRegistryManagedIdentityClientID + - ingressManagedIdentityClientID + - networkManagedIdentityClientID + type: object + required: + - controlPlaneManagedIdentities + type: object resourceGroup: default: default description: |- diff --git a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml index 684cb311f3c..07bbf77c4f4 100644 --- a/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml +++ b/cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml @@ -8219,6 +8219,128 @@ spec: x-kubernetes-validations: - message: Location is immutable rule: self == oldSelf + managedIdentities: + description: |- + managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + and data plane components that authenticate with Azure's API. + properties: + controlPlaneManagedIdentities: + description: |- + ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + authenticate with Azure's API. + properties: + azureCloudProviderManagedIdentityClientID: + description: |- + azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureDiskManagedIdentityClientID: + description: |- + azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureFileManagedIdentityClientID: + description: |- + azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + azureKMSManagedIdentityClientID: + description: |- + azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + clusterAPIAzureManagedIdentityClientID: + description: |- + clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + controlPlaneManagedIdentityClientID: + description: |- + controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + imageRegistryManagedIdentityClientID: + description: |- + imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + ingressManagedIdentityClientID: + description: |- + ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + networkManagedIdentityClientID: + description: |- + networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + hyphen separated hexadecimal characters in the form 8-4-4-4-12. + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$') + required: + - azureCloudProviderManagedIdentityClientID + - azureDiskManagedIdentityClientID + - azureFileManagedIdentityClientID + - clusterAPIAzureManagedIdentityClientID + - controlPlaneManagedIdentityClientID + - imageRegistryManagedIdentityClientID + - ingressManagedIdentityClientID + - networkManagedIdentityClientID + type: object + required: + - controlPlaneManagedIdentities + type: object resourceGroup: default: default description: |- diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go index 27073cd9b94..58d83e04ce9 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/params.go @@ -42,6 +42,7 @@ func NewAzureParams(hcp *hyperv1.HostedControlPlane) *AzureParams { p.DeploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] } p.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) + p.DeploymentConfig.SetDefaultSecurityContext = false return p } diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go index 0cf682ebfe4..64d2e00ffab 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go @@ -17,7 +17,7 @@ const ( // ReconcileCloudConfig reconciles as expected by Nodes Kubelet. func ReconcileCloudConfig(cm *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { - cfg, err := azureConfigWithoutCredentials(hcp, credentialsSecret) + cfg, err := AzureConfigWithoutCredentials(hcp, credentialsSecret) if err != nil { return err } @@ -37,14 +37,13 @@ func ReconcileCloudConfig(cm *corev1.ConfigMap, hcp *hyperv1.HostedControlPlane, // ReconcileCloudConfigWithCredentials reconciles as expected by KAS/KCM. func ReconcileCloudConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { - cfg, err := azureConfigWithoutCredentials(hcp, credentialsSecret) + cfg, err := AzureConfigWithoutCredentials(hcp, credentialsSecret) if err != nil { return err } - cfg.AADClientID = string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - cfg.AADClientSecret = string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - cfg.UseManagedIdentityExtension = false + cfg.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureCloudProviderManagedIdentityClientID) + cfg.UseInstanceMetadata = false serializedConfig, err := json.MarshalIndent(cfg, "", " ") if err != nil { @@ -58,7 +57,7 @@ func ReconcileCloudConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.Hos return nil } -func azureConfigWithoutCredentials(hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) (AzureConfig, error) { +func AzureConfigWithoutCredentials(hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) (AzureConfig, error) { subnetName, err := azureutil.GetSubnetNameFromSubnetID(hcp.Spec.Platform.Azure.SubnetID) if err != nil { return AzureConfig{}, fmt.Errorf("failed to determine subnet name from SubnetID: %w", err) @@ -106,9 +105,8 @@ type AzureConfig struct { Cloud string `json:"cloud"` TenantID string `json:"tenantId"` UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` + UserAssignedIdentityID string `json:"userAssignedIdentityID"` SubscriptionID string `json:"subscriptionId"` - AADClientID string `json:"aadClientId"` - AADClientSecret string `json:"aadClientSecret"` ResourceGroup string `json:"resourceGroup"` Location string `json:"location"` VnetName string `json:"vnetName"` diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go index 69c33f0737f..df1280e96a1 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/reconcile.go @@ -1,12 +1,15 @@ package azure import ( + "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/proxy" "github.com/openshift/hypershift/support/util" @@ -22,7 +25,12 @@ func ReconcileCCMServiceAccount(sa *corev1.ServiceAccount, ownerRef config.Owner return nil } -func ReconcileDeployment(deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, p *AzureParams, serviceAccountName string, releaseImageProvider *imageprovider.ReleaseImageProvider) error { +func ReconcileDeployment(ctx context.Context, c client.Client, deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, p *AzureParams, serviceAccountName string, releaseImageProvider *imageprovider.ReleaseImageProvider) error { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + deployment.Spec = appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: ccmLabels(), @@ -35,8 +43,12 @@ func ReconcileDeployment(deployment *appsv1.Deployment, hcp *hyperv1.HostedContr Labels: ccmLabels(), }, Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + azureutil.AdapterInitContainer(), + }, Containers: []corev1.Container{ util.BuildContainer(ccmContainer(), buildCCMContainer(p, releaseImageProvider.GetImage("azure-cloud-controller-manager"), hcp.Namespace)), + azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"])), }, Volumes: []corev1.Volume{}, ServiceAccountName: serviceAccountName, diff --git a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go index f9d35a587dc..69125cd76d4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cno/clusternetworkoperator.go @@ -6,19 +6,20 @@ import ( "os" "strconv" - "github.com/openshift/hypershift/support/proxy" - "github.com/openshift/hypershift/support/rhobsmonitoring" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/blang/semver" routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/proxy" + "github.com/openshift/hypershift/support/rhobsmonitoring" "github.com/openshift/hypershift/support/util" + + "github.com/blang/semver" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -27,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -74,9 +76,12 @@ type Params struct { DeploymentConfig config.DeploymentConfig IsPrivate bool DefaultIngressDomain string + NetworkManagedIdentity string + ClientIDSecret string + TenantID string } -func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool, defaultIngressDomain string) Params { +func NewParams(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, setDefaultSecurityContext bool, defaultIngressDomain string) (Params, error) { p := Params{ Images: Images{ NetworkOperator: releaseImageProvider.GetImage("cluster-network-operator"), @@ -131,7 +136,18 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv p.APIServerPort = hcp.Status.ControlPlaneEndpoint.Port } - return p + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return p, fmt.Errorf("failed to get Azure credentials: %w", err) + } + + p.NetworkManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.NetworkManagedIdentityClientID) + p.ClientIDSecret = string(azureCredentials.Data["AZURE_CLIENT_SECRET"]) + p.TenantID = string(azureCredentials.Data["AZURE_TENANT_ID"]) + } + + return p, nil } func ReconcileRole(role *rbacv1.Role, ownerRef config.OwnerRef, networkType hyperv1.NetworkType) error { @@ -473,8 +489,11 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat {Name: "CA_CONFIG_MAP", Value: params.CAConfigMap}, {Name: "CA_CONFIG_MAP_KEY", Value: params.CAConfigMapKey}, {Name: "TOKEN_AUDIENCE", Value: params.TokenAudience}, - - {Name: "RELEASE_VERSION", Value: params.ReleaseVersion}, + {Name: "ARO_HCP_MI_CLIENT_ID", Value: params.NetworkManagedIdentity}, + {Name: "AZURE_ADAPTER_INIT_IMAGE", Value: azureutil.AdapterInitImage}, + {Name: "AZURE_ADAPTER_SERVER_IMAGE", Value: azureutil.AdapterServerImage}, + {Name: "CLIENT_ID_SECRET", Value: params.ClientIDSecret}, + {Name: "TENANT_ID", Value: params.TenantID}, {Name: "APISERVER_OVERRIDE_HOST", Value: params.APIServerAddress}, // We need to pass this down to networking components on the nodes {Name: "APISERVER_OVERRIDE_PORT", Value: fmt.Sprint(params.APIServerPort)}, {Name: "OVN_NB_RAFT_ELECTION_TIMER", Value: "10"}, @@ -514,6 +533,7 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat {Name: "CLI_IMAGE", Value: params.Images.CLI}, {Name: "SOCKS5_PROXY_IMAGE", Value: params.Images.Socks5Proxy}, {Name: "OPENSHIFT_RELEASE_IMAGE", Value: params.DeploymentConfig.AdditionalAnnotations[hyperv1.ReleaseImageAnnotation]}, + {Name: "RELEASE_VERSION", Value: params.ReleaseVersion}, }...), Name: operatorName, Image: params.Images.NetworkOperator, diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 07e43a69603..989e787a345 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -22,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/kms" "github.com/go-logr/logr" + routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" @@ -46,6 +47,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ingress" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ingressoperator" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" + hcpkms "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas/kms" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kcm" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/konnectivity" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/machineapprover" @@ -64,8 +66,9 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/snapshotcontroller" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage" pkimanifests "github.com/openshift/hypershift/control-plane-pki-operator/manifests" - sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" + "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" supportawsutil "github.com/openshift/hypershift/support/awsutil" + hyperazureutil "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/capabilities" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/conditions" @@ -79,7 +82,9 @@ import ( "github.com/openshift/hypershift/support/thirdparty/library-go/pkg/image/reference" "github.com/openshift/hypershift/support/upsert" "github.com/openshift/hypershift/support/util" + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -391,7 +396,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R Type: string(hyperv1.ValidHostedControlPlaneConfiguration), ObservedGeneration: hostedControlPlane.Generation, } - if err := r.validateConfigAndClusterCapabilities(hostedControlPlane); err != nil { + if err := r.validateConfigAndClusterCapabilities(ctx, hostedControlPlane); err != nil { condition.Status = metav1.ConditionFalse condition.Message = err.Error() condition.Reason = hyperv1.InsufficientClusterCapabilitiesReason @@ -844,12 +849,19 @@ func healthCheckKASEndpoint(ingressPoint string, port int) error { return nil } -func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(hc *hyperv1.HostedControlPlane) error { - for _, svc := range hc.Spec.Services { +func (r *HostedControlPlaneReconciler) validateConfigAndClusterCapabilities(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + for _, svc := range hcp.Spec.Services { if svc.Type == hyperv1.Route && !r.ManagementClusterCapabilities.Has(capabilities.CapabilityRoute) { return fmt.Errorf("cluster does not support Routes, but service %q is exposed via a Route", svc.Service) } } + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + if err := verifyResourceGroupLocationsMatch(ctx, hcp); err != nil { + return err + } + } + return nil } @@ -2983,6 +2995,20 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex if hcp.Spec.SecretEncryption.KMS == nil { return fmt.Errorf("kms metadata not specified") } + if hcp.Spec.Platform.Type == hyperv1.AzurePlatform { + azureCreds, err := hyperazureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + // Reconcile KMS config secret + kmsConfigSecret := manifests.AzureKMSConfigSecret(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, kmsConfigSecret, func() error { + return hcpkms.ReconcileKMSConfigWithCredentials(kmsConfigSecret, hcp, azureCreds) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure cloud config with credentials: %w", err) + } + } if _, err := createOrUpdate(ctx, r, encryptionConfigFile, func() error { return kas.ReconcileKMSEncryptionConfig(encryptionConfigFile, p.OwnerRef, hcp.Spec.SecretEncryption.KMS) }); err != nil { @@ -3065,7 +3091,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex } if _, err := createOrUpdate(ctx, r, kubeAPIServerDeployment, func() error { - return kas.ReconcileKubeAPIServerDeployment(kubeAPIServerDeployment, + return kas.ReconcileKubeAPIServerDeployment(ctx, r.Client, kubeAPIServerDeployment, hcp, p.OwnerRef, p.DeploymentConfig, @@ -3509,7 +3535,10 @@ func (r *HostedControlPlaneReconciler) reconcileClusterVersionOperator(ctx conte } func (r *HostedControlPlaneReconciler) reconcileClusterNetworkOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, hasRouteCap bool, createOrUpdate upsert.CreateOrUpdateFN) error { - p := cno.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext, r.DefaultIngressDomain) + p, err := cno.NewParams(ctx, r.Client, hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext, r.DefaultIngressDomain) + if err != nil { + return err + } sa := manifests.ClusterNetworkOperatorServiceAccount(hcp.Namespace) if _, err := createOrUpdate(ctx, r.Client, sa, func() error { @@ -3676,7 +3705,9 @@ func (r *HostedControlPlaneReconciler) reconcileIngressOperator(ctx context.Cont deployment := manifests.IngressOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - ingressoperator.ReconcileDeployment(deployment, p, hcp.Spec.Platform.Type) + if err := ingressoperator.ReconcileDeployment(ctx, r.Client, hcp, deployment, p, hcp.Spec.Platform.Type); err != nil { + return err + } return nil }); err != nil { return fmt.Errorf("failed to reconcile ingressoperator deployment: %w", err) @@ -3994,7 +4025,7 @@ func (r *HostedControlPlaneReconciler) reconcileImageRegistryOperator(ctx contex deployment := manifests.ImageRegistryOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - return registryoperator.ReconcileDeployment(deployment, params) + return registryoperator.ReconcileDeployment(ctx, r.Client, hcp, deployment, params) }); err != nil { return fmt.Errorf("failed to reconcile image registry operator deployment: %w", err) } @@ -4564,7 +4595,7 @@ func (r *HostedControlPlaneReconciler) reconcileCloudControllerManager(ctx conte p := azure.NewAzureParams(hcp) deployment := azure.CCMDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { - return azure.ReconcileDeployment(deployment, hcp, p, sa.Name, releaseImageProvider) + return azure.ReconcileDeployment(ctx, r.Client, deployment, hcp, p, sa.Name, releaseImageProvider) }); err != nil { return fmt.Errorf("failed to reconcile %s cloud controller manager deployment: %w", hcp.Spec.Platform.Type, err) } @@ -4777,7 +4808,10 @@ func (r *HostedControlPlaneReconciler) reconcileCSISnapshotControllerOperator(ct } func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx context.Context, hcp *hyperv1.HostedControlPlane, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, createOrUpdate upsert.CreateOrUpdateFN) error { - params := storage.NewParams(hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext) + params, err := storage.NewParams(ctx, r.Client, hcp, userReleaseImageProvider.Version(), releaseImageProvider, userReleaseImageProvider, r.SetDefaultSecurityContext) + if err != nil { + return err + } deployment := manifests.ClusterStorageOperatorDeployment(hcp.Namespace) if _, err := createOrUpdate(ctx, r, deployment, func() error { @@ -4807,6 +4841,34 @@ func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx conte return fmt.Errorf("failed to reconcile cluster storage operator roleBinding: %w", err) } + // Reconcile azure-disk-csi-controller and azure-file-csi-controller configuration secrets for ARO HCP. This is + // needed so we can specify a unique managed identity for each controller to authenticate with the Managed Identity + // Azure API. + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + // Get the credentials secret so we can retrieve the tenant ID for the configuration + credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { + return fmt.Errorf("failed to get Azure credentials secret: %w", err) + } + tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) + + // Reconcile the secret needed for azure-disk-csi-controller + azureDiskCSISecret := manifests.AzureDiskCSIConfig(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureDiskCSISecret(azureDiskCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure Disk CSI config: %w", err) + } + + // Reconcile the secret needed for azure-disk-csi-controller + azureFileCSISecret := manifests.AzureFileCSIConfig(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, azureDiskCSISecret, func() error { + return storage.ReconcileAzureFileCSISecret(azureFileCSISecret, hcp, tenantID) + }); err != nil { + return fmt.Errorf("failed to reconcile Azure File CSI config: %w", err) + } + } + // TODO: create custom kubeconfig to the guest cluster + RBAC return nil @@ -5228,24 +5290,11 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex } azureKmsSpec := hcp.Spec.SecretEncryption.KMS.Azure - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: hcp.Spec.Platform.Azure.Credentials.Name}} - if err := r.Client.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - condition := metav1.Condition{ - Type: string(hyperv1.ValidAzureKMSConfig), - ObservedGeneration: hcp.Generation, - Status: metav1.ConditionUnknown, - Message: fmt.Sprintf("failed to get azure credentials secret: %v", err), - Reason: hyperv1.StatusUnknownReason, - } - meta.SetStatusCondition(&hcp.Status.Conditions, condition) - return + options := &azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureKMSManagedIdentityClientID), } - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) - - cred, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) + cred, err := azidentity.NewManagedIdentityCredential(options) if err != nil { conditions.SetFalseCondition(hcp, hyperv1.ValidAzureKMSConfig, hyperv1.InvalidAzureCredentialsReason, fmt.Sprintf("failed to obtain azure client credential: %v", err)) @@ -5347,3 +5396,40 @@ func doesOpenShiftTrustedCABundleConfigMapForCPOExist(ctx context.Context, c cli } return false, nil } + +// verifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups +func verifyResourceGroupLocationsMatch(ctx context.Context, hcp *hyperv1.HostedControlPlane) error { + options := &azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.ControlPlaneManagedIdentityClientID), + } + + creds, err := azidentity.NewManagedIdentityCredential(options) + if err != nil { + return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + } + + // Retrieve full vnet information from the VNET ID + vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + } + + // Retrieve full network security group information from the network security group ID + nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get network security group info to verify its location: %v", err) + } + + // Retrieve full resource group information from the resource group name + rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds) + if err != nil { + return fmt.Errorf("failed to get resource group info to verify its location: %v", err) + } + + // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match + if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { + return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) + } + + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go index e818ed1e815..482c039263f 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ingressoperator/ingressoperator.go @@ -1,30 +1,34 @@ package ingressoperator import ( + "context" "fmt" - configv1 "github.com/openshift/api/config/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/metrics" "github.com/openshift/hypershift/support/proxy" "github.com/openshift/hypershift/support/util" + "os" + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( operatorName = "ingress-operator" ingressOperatorContainerName = "ingress-operator" - metricsHostname = "ingress-operator" konnectivityProxyContainerName = "konnectivity-proxy" ingressOperatorMetricsPort = 60000 konnectivityProxyPort = 8090 @@ -70,14 +74,19 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv return p } -func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) { +func ReconcileDeployment(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) error { + // Determine if the deployment will be placed on ARO HCP + aroHCPDeployment := os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP + + // Initialize resource requests ingressOpResources := corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("80Mi"), corev1.ResourceCPU: resource.MustParse("10m"), }, } - // preserve existing resource requirements + + // Preserve existing resource requirements mainContainer := util.FindContainer(ingressOperatorContainerName, dep.Spec.Template.Spec.Containers) if mainContainer != nil { if len(mainContainer.Resources.Requests) > 0 || len(mainContainer.Resources.Limits) > 0 { @@ -85,6 +94,12 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp } } + // Initialize NO_PROXY environment variable and add the Azure Managed Identity API endpoint if ingress operator is being deployed on ARO HCP + noProxyValue := manifests.KubeAPIServerService("").Name + if aroHCPDeployment { + noProxyValue = noProxyValue + ",169.254.169.254" + } + dep.Spec.Replicas = ptr.To[int32](1) dep.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"name": operatorName}} dep.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType @@ -132,7 +147,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp }, { Name: "NO_PROXY", - Value: manifests.KubeAPIServerService("").Name, + Value: noProxyValue, }, }, Name: ingressOperatorContainerName, @@ -199,7 +214,29 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyp }, ) + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "ARO_HCP_MI_CLIENT_ID", + Value: string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.IngressManagedIdentityClientID), + }) + + if dep.Spec.Template.Spec.InitContainers == nil { + dep.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + dep.Spec.Template.Spec.InitContainers = append(dep.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + params.DeploymentConfig.SetDefaultSecurityContext = false + } + params.DeploymentConfig.ApplyTo(dep) + return nil } func ingressOperatorKonnectivityProxyContainer(proxyImage string, proxyConfig *configv1.ProxySpec, noProxy string) corev1.Container { diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go index 0562511d123..854caba4ac9 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment.go @@ -2,7 +2,9 @@ package kas import ( "bytes" + "context" "fmt" + "os" "path" "strconv" "strings" @@ -14,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/aws" @@ -21,6 +24,7 @@ import ( "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/proxy" @@ -99,7 +103,10 @@ func kasLabels() map[string]string { } } -func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment, +func ReconcileKubeAPIServerDeployment( + ctx context.Context, + c client.Client, + deployment *appsv1.Deployment, hcp *hyperv1.HostedControlPlane, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, @@ -319,6 +326,21 @@ func ReconcileKubeAPIServerDeployment(deployment *appsv1.Deployment, if err := applyKMSConfig(&deployment.Spec.Template.Spec, secretEncryptionData, images); err != nil { return err } + + // Add the adapter-init and adapter-server containers for ARO HCP + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + + // ARO HCP needs elevated privileges in order to run the adapter-init container + deploymentConfig.SetDefaultSecurityContext = false + } case hyperv1.AESCBC: err := applyAESCBCKeyHashAnnotation(&deployment.Spec.Template, aesCBCActiveKey, aesCBCBackupKey) if err != nil { diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go index e616a598c4d..7eb437b7f1e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/deployment_test.go @@ -1,6 +1,7 @@ package kas import ( + "golang.org/x/net/context" "testing" . "github.com/onsi/gomega" @@ -57,7 +58,7 @@ func TestReconcileKubeAPIServerDeploymentNoChanges(t *testing.T) { tc.config.Data = map[string]string{"config.json": "test-json"} tc.auditConfig.Data = map[string]string{"policy.yaml": "test-data"} tc.authConfig.Data = map[string]string{"auth.json": "test-data"} - err := ReconcileKubeAPIServerDeployment(kubeAPIDeployment, hcp, ownerRef, tc.deploymentConfig, tc.params.NamedCertificates(), tc.params.CloudProvider, + err := ReconcileKubeAPIServerDeployment(context.TODO(), nil, kubeAPIDeployment, hcp, ownerRef, tc.deploymentConfig, tc.params.NamedCertificates(), tc.params.CloudProvider, tc.params.CloudProviderConfig, tc.params.CloudProviderCreds, tc.params.Images, tc.config, tc.auditConfig, tc.authConfig, tc.params.AuditWebhookRef, tc.activeKey, tc.backupKey, 6443, "test-payload-version", tc.params.FeatureGate, nil, tc.params.CipherSuites()) g.Expect(err).To(BeNil()) g.Expect(expectedMinReadySeconds).To(Equal(kubeAPIDeployment.Spec.MinReadySeconds)) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go index 81ee7b4dacb..c0d1ed98b6a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/kms/azure.go @@ -1,6 +1,7 @@ package kms import ( + "encoding/json" "fmt" "time" @@ -216,7 +217,7 @@ func kasVolumeAzureKMSCredentials() *corev1.Volume { func buildVolumeAzureKMSCredentials(v *corev1.Volume) { v.Secret = &corev1.SecretVolumeSource{ - SecretName: manifests.AzureProviderConfigWithCredentials("").Name, + SecretName: manifests.AzureKMSConfigSecret("").Name, Items: []corev1.KeyToPath{ { Key: azure.CloudConfigKey, @@ -225,3 +226,24 @@ func buildVolumeAzureKMSCredentials(v *corev1.Volume) { }, } } + +func ReconcileKMSConfigWithCredentials(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, credentialsSecret *corev1.Secret) error { + cfg, err := azure.AzureConfigWithoutCredentials(hcp, credentialsSecret) + if err != nil { + return err + } + + cfg.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureKMSManagedIdentityClientID) + + cfg.UseInstanceMetadata = false + serializedConfig, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go index 39a373f72f4..bfb5491d91b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/azure.go @@ -25,3 +25,12 @@ func AzureProviderConfigWithCredentials(ns string) *corev1.Secret { }, } } + +func AzureKMSConfigSecret(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-kms-cloud-config", + Namespace: ns, + }, + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go index e09fbc8e570..fcc085100a8 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/storage.go @@ -4,6 +4,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func ClusterStorageOperatorDeployment(ns string) *appsv1.Deployment { @@ -33,3 +34,21 @@ func ClusterStorageOperatorServiceAccount(ns string) *corev1.ServiceAccount { sa.Namespace = ns return sa } + +func AzureDiskCSIConfig(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-disk-csi-config", + Namespace: ns, + }, + } +} + +func AzureFileCSIConfig(ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azure-file-csi-config", + Namespace: ns, + }, + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go index 31f1a347447..ae769b38733 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go +++ b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile.go @@ -2,6 +2,8 @@ package registryoperator import ( "bytes" + "context" + "os" "path" "text/template" @@ -11,11 +13,13 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/certs" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/metrics" @@ -152,7 +156,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv return params } -func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error { +func ReconcileDeployment(ctx context.Context, c client.Client, hcp *hyperv1.HostedControlPlane, deployment *appsv1.Deployment, params Params) error { deployment.Spec = appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels(), @@ -194,6 +198,27 @@ func ReconcileDeployment(deployment *appsv1.Deployment, params Params) error { ) } + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "ARO_HCP_MI_CLIENT_ID", + Value: "true", + }) + + if deployment.Spec.Template.Spec.InitContainers == nil { + deployment.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + params.deploymentConfig.SetDefaultSecurityContext = false + } + params.deploymentConfig.ApplyTo(deployment) return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go index 3255fb7a13e..5640db74d80 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/registryoperator/reconcile_test.go @@ -1,6 +1,8 @@ package registryoperator import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -35,8 +37,9 @@ func TestReconcileDeployment(t *testing.T) { } deployment := manifests.ImageRegistryOperatorDeployment("test-namespace") imageProvider := imageprovider.NewFromImages(images) + fakeClient := fake.NewClientBuilder().WithScheme(api.Scheme).Build() params := NewParams(hcp, "1.0.0", imageProvider, imageProvider, true) - if err := ReconcileDeployment(deployment, params); err != nil { + if err := ReconcileDeployment(context.TODO(), fakeClient, hcp, deployment, params); err != nil { t.Fatalf("unexpected error: %v", err) } deploymentYaml, err := util.SerializeResource(deployment, api.Scheme) diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go new file mode 100644 index 00000000000..fd8a6a514ac --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/azure.go @@ -0,0 +1,67 @@ +package storage + +import ( + "encoding/json" + "fmt" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" + + corev1 "k8s.io/api/core/v1" +) + +// initializeAzureCSIControllerConfig initializes an AzureConfig object which will be used to populate the secrets +// needed by azure-disk-csi-controller and azure-file-csi-controller. +// The source of truth for which fields are required by azure-disk is located here - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/deploy/example/azure.json. +// The source of truth for which fields are required by azure-file is located here - https://github.com/kubernetes-sigs/azurefile-csi-driver/blob/master/deploy/example/azure.json. +// As of Sept 2024, the fields we need in HyperShift are the same between both of these sources of truth. +func initializeAzureCSIControllerConfig(hcp *hyperv1.HostedControlPlane, tenantID string) azure.AzureConfig { + azureConfig := azure.AzureConfig{ + // These fields are mandatory + Cloud: hcp.Spec.Platform.Azure.Cloud, + TenantID: tenantID, + SubscriptionID: hcp.Spec.Platform.Azure.SubscriptionID, + ResourceGroup: hcp.Spec.Platform.Azure.ResourceGroupName, + Location: hcp.Spec.Platform.Azure.Location, + + // These fields are mandatory when using managed identity; the user assigned identity ID is populated after this function call + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "", + } + + return azureConfig +} + +// ReconcileAzureDiskCSISecret reconciles the configuration for the secret as expected by azure-disk-csi-controller +func ReconcileAzureDiskCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureDiskManagedIdentityClientID) + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} + +// ReconcileAzureFileCSISecret reconciles the configuration for the secret as expected by azure-file-csi-controller +func ReconcileAzureFileCSISecret(secret *corev1.Secret, hcp *hyperv1.HostedControlPlane, tenantID string) error { + config := initializeAzureCSIControllerConfig(hcp, tenantID) + config.UserAssignedIdentityID = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureFileManagedIdentityClientID) + + serializedConfig, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to serialize cloudconfig: %w", err) + } + + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data[azure.CloudConfigKey] = serializedConfig + return nil +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go index 03f559846de..384c020cd38 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/operator.go @@ -1,18 +1,25 @@ package storage import ( + "fmt" + "os" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/storage/assets" assets2 "github.com/openshift/hypershift/support/assets" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/util" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) +const ClusterStorageOperatorContainerName = "cluster-storage-operator" + var ( operatorDeployment = assets2.MustDeployment(assets.ReadFile, "10_deployment-hypershift.yaml") operatorRole = assets2.MustRole(assets.ReadFile, "role.yaml") @@ -26,12 +33,26 @@ func ReconcileOperatorDeployment( params.OwnerRef.ApplyTo(deployment) deployment.Spec = operatorDeployment.DeepCopy().Spec - for i, container := range deployment.Spec.Template.Spec.Containers { - switch container.Name { - case "cluster-storage-operator": - deployment.Spec.Template.Spec.Containers[i].Image = params.StorageOperatorImage - params.ImageReplacer.replaceEnvVars(deployment.Spec.Template.Spec.Containers[i].Env) + + csoContainer := util.FindContainer(ClusterStorageOperatorContainerName, deployment.Spec.Template.Spec.Containers) + if csoContainer == nil { + return fmt.Errorf("could not find ClusterStorageOperator container for Deployment") + } + + csoContainer.Image = params.StorageOperatorImage + params.ImageReplacer.replaceEnvVars(csoContainer.Env) + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + aroHCPEnvs := []corev1.EnvVar{ + {Name: "AZURE_ADAPTER_INIT_IMAGE", Value: azureutil.AdapterInitImage}, + {Name: "AZURE_ADAPTER_SERVER_IMAGE", Value: azureutil.AdapterServerImage}, + {Name: "ARO_HCP_DISK_MI_CLIENT_ID", Value: params.AzureDiskManagedIdentity}, + {Name: "ARO_HCP_FILE_MI_CLIENT_ID", Value: params.AzureFileManagedIdentity}, + {Name: "CLIENT_ID_SECRET", Value: params.ClientIDSecret}, + {Name: "TENANT_ID", Value: params.TenantID}, } + + csoContainer.Env = append(csoContainer.Env, aroHCPEnvs...) } params.DeploymentConfig.ApplyTo(deployment) diff --git a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go index be4c3c6615d..3505962a383 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/storage/params.go +++ b/control-plane-operator/controllers/hostedcontrolplane/storage/params.go @@ -1,11 +1,17 @@ package storage import ( + "context" + "fmt" + "os" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/util" utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -13,20 +19,26 @@ const ( ) type Params struct { - OwnerRef config.OwnerRef - StorageOperatorImage string - ImageReplacer *environmentReplacer + OwnerRef config.OwnerRef + StorageOperatorImage string + AzureDiskManagedIdentity string + AzureFileManagedIdentity string + ClientIDSecret string + TenantID string + ImageReplacer *environmentReplacer AvailabilityProberImage string config.DeploymentConfig } func NewParams( + ctx context.Context, + c client.Client, hcp *hyperv1.HostedControlPlane, version string, releaseImageProvider *imageprovider.ReleaseImageProvider, userReleaseImageProvider *imageprovider.ReleaseImageProvider, - setDefaultSecurityContext bool) *Params { + setDefaultSecurityContext bool) (*Params, error) { ir := newEnvironmentReplacer() ir.setVersions(version) @@ -51,5 +63,17 @@ func NewParams( params.DeploymentConfig.SetDefaults(hcp, nil, utilpointer.Int(1)) params.DeploymentConfig.SetRestartAnnotation(hcp.ObjectMeta) - return ¶ms + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return nil, fmt.Errorf("failed to get Azure credentials: %w", err) + } + + params.AzureDiskManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureDiskManagedIdentityClientID) + params.AzureFileManagedIdentity = string(hcp.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.AzureFileManagedIdentityClientID) + params.ClientIDSecret = string(azureCredentials.Data["AZURE_CLIENT_SECRET"]) + params.TenantID = string(azureCredentials.Data["AZURE_TENANT_ID"]) + } + + return ¶ms, nil } diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 673584152de..626d16df52b 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -2737,6 +2737,53 @@ configuration for the Azure cloud provider, aka Azure cloud controller manager ( expected to exist under the same subscription as SubscriptionID.

+ + +managedIdentities
+ + +AzureResourceManagedIdentities + + + + +

managedIdentities contains the client IDs related to the managed identities needed for HCP control plane +and data plane components that authenticate with Azure’s API.

+ + + + +###AzureResourceManagedIdentities { #hypershift.openshift.io/v1beta1.AzureResourceManagedIdentities } +

+(Appears on: +AzurePlatformSpec) +

+

+

AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +and data plane components that authenticate with Azure’s API.

+

+ + + + + + + + + + + +
FieldDescription
+controlPlaneManagedIdentities
+ + +ControlPlaneManagedIdentities + + +		 +

ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to +authenticate with Azure’s API.

+
###AzureVMImage { #hypershift.openshift.io/v1beta1.AzureVMImage } @@ -3542,6 +3589,163 @@ and reports missing images if any.

+###ControlPlaneManagedIdentities { #hypershift.openshift.io/v1beta1.ControlPlaneManagedIdentities } +

+(Appears on: +AzureResourceManagedIdentities) +

+

+

ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +to authenticate with Azure’s API. +Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +The format a managed identity should be /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+azureCloudProviderManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure +cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+clusterAPIAzureManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api +azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated +hexadecimal characters in the form 8-4-4-4-12.

+
+controlPlaneManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane +operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated +hexadecimal characters in the form 8-4-4-4-12.

+
+azureKMSManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +(Optional) +

azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client +ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters +in the form 8-4-4-4-12.

+
+imageRegistryManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups +of hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+ingressManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+networkManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of +hyphen separated hexadecimal characters in the form 8-4-4-4-12.

+
+azureDiskManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen +separated hexadecimal characters in the form 8-4-4-4-12.

+
+azureFileManagedIdentityClientID
+ + +ManagedIdentityClientID + + +		 +

azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the +azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen +separated hexadecimal characters in the form 8-4-4-4-12.

+
###DNSSpec { #hypershift.openshift.io/v1beta1.DNSSpec }

(Appears on: @@ -6729,6 +6933,14 @@ is empty.

+###ManagedIdentityClientID { #hypershift.openshift.io/v1beta1.ManagedIdentityClientID } +

+(Appears on: +ControlPlaneManagedIdentities) +

+

+

ManagedIdentityClientID is a client ID of a managed identity

+

###MarketplaceImage { #hypershift.openshift.io/v1beta1.MarketplaceImage }

(Appears on: diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 317a031fe95..ceb6f2e2da8 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -1586,6 +1586,16 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques proxy.SetEnvVars(&capiProviderDeploymentSpec.Template.Spec.Containers[0].Env) } + // For ARO HCP, we need to add the Microsoft init and sidecar containers to the CAPI Provider deployment + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hcluster.Namespace, hcluster.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return ctrl.Result{}, err + } + + capiProviderDeploymentSpec.Template.Spec.Containers = append(capiProviderDeploymentSpec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + } + // Reconcile cluster prometheus RBAC resources if enabled if r.EnableOCPClusterMonitoring { if err := r.reconcileClusterPrometheusRBAC(ctx, createOrUpdate, hcp.Namespace); err != nil { @@ -2229,6 +2239,8 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont controlPlaneOperatorDeployment := controlplaneoperator.OperatorDeployment(controlPlaneNamespace.Name) _, err = createOrUpdate(ctx, r.Client, controlPlaneOperatorDeployment, func() error { return reconcileControlPlaneOperatorDeployment( + ctx, + r.Client, controlPlaneOperatorDeployment, openShiftTrustedCABundleConfigMapExists, hcluster, @@ -2478,6 +2490,8 @@ func GetControlPlaneOperatorImageLabels(ctx context.Context, hc *hyperv1.HostedC } func reconcileControlPlaneOperatorDeployment( + ctx context.Context, + c client.Client, deployment *appsv1.Deployment, openShiftTrustedCABundleConfigMapExists bool, hc *hyperv1.HostedCluster, @@ -2819,6 +2833,26 @@ func reconcileControlPlaneOperatorDeployment( if hcp.Annotations[hyperv1.ControlPlanePriorityClass] != "" { deploymentConfig.Scheduling.PriorityClass = hcp.Annotations[hyperv1.ControlPlanePriorityClass] } + + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + if deployment.Spec.Template.Spec.InitContainers == nil { + deployment.Spec.Template.Spec.InitContainers = []corev1.Container{} + } + + deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, azureutil.AdapterInitContainer()) + + azureCredentials, err := azureutil.GetAzureCredentialsFromSecret(ctx, c, hcp.Namespace, hcp.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err + } + + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, azureutil.AdapterServerContainer(string(azureCredentials.Data["AZURE_CLIENT_ID"]), string(azureCredentials.Data["AZURE_CLIENT_SECRET"]), string(azureCredentials.Data["AZURE_TENANT_ID"]))) + + // ARO HCP needs elevated privileges in order to run the adapter-init container + deployment.Spec.Template.Spec.SecurityContext = nil + deploymentConfig.SetDefaultSecurityContext = false + } + deploymentConfig.SetDefaults(hcp, nil, k8sutilspointer.Int(1)) deploymentConfig.SetRestartAnnotation(hc.ObjectMeta) deploymentConfig.ApplyTo(deployment) @@ -3188,11 +3222,18 @@ func reconcileCAPIProviderDeployment(deployment *appsv1.Deployment, capiProvider // Enforce ServiceAccount. deployment.Spec.Template.Spec.ServiceAccountName = sa.Name + defaultSecurityContext := setDefaultSecurityContext + + // For ARO HCP, the MI sidecar containers need privileged permissions to run + if os.Getenv("MANAGED_SERVICE") == hyperv1.AroHCP { + defaultSecurityContext = false + } + deploymentConfig := config.DeploymentConfig{ Scheduling: config.Scheduling{ PriorityClass: config.DefaultPriorityClass, }, - SetDefaultSecurityContext: setDefaultSecurityContext, + SetDefaultSecurityContext: defaultSecurityContext, AdditionalLabels: map[string]string{ config.NeedManagementKASAccessLabel: "true", }, @@ -4333,12 +4374,9 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } // Verify the credentials secret contains the data fields we expect - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ - Namespace: hc.Namespace, - Name: hc.Spec.Platform.Azure.Credentials.Name, - }} - if err := r.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - return fmt.Errorf("failed to get credentials secret for cluster: %w", err) + credentialsSecret, err := azureutil.GetAzureCredentialsFromSecret(ctx, r.Client, hc.Namespace, hc.Spec.Platform.Azure.Credentials.Name) + if err != nil { + return err } var errs []error @@ -4348,12 +4386,6 @@ func (r *HostedClusterReconciler) validateAzureConfig(ctx context.Context, hc *h } } - // Verify the resource group locations match - err := azureutil.VerifyResourceGroupLocationsMatch(ctx, hc, credentialsSecret) - if err != nil { - errs = append(errs, err) - } - return utilerrors.NewAggregate(errs) } diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go index 21770c592a5..4db23d0dc2d 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go @@ -56,7 +56,7 @@ func (a Azure) ReconcileCAPIInfraCR( } if _, err := createOrUpdate(ctx, client, azureClusterIdentity, func() error { - return reconcileAzureClusterIdentity(ctx, client, hcluster, azureClusterIdentity, controlPlaneNamespace) + return reconcileAzureClusterIdentity(hcluster, azureClusterIdentity, controlPlaneNamespace) }); err != nil { return nil, fmt.Errorf("failed to reconcile Azure cluster identity: %w", err) } @@ -83,6 +83,7 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: k8sutilspointer.Int64(10), + InitContainers: []corev1.Container{azureutil.AdapterInitContainer()}, Containers: []corev1.Container{{ Name: "manager", Image: image, @@ -164,23 +165,6 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create return err } - // Sync Azure Client Secret in its own secret for since CAPZ needs it in a specific key value - // https://capz.sigs.k8s.io/topics/multitenancy#manual-service-principal-identity - azureClientSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "azure-client-secret", Namespace: controlPlaneNamespace}} - if _, err := createOrUpdate(ctx, c, azureClientSecret, func() error { - if azureClientSecret.Data == nil { - azureClientSecret.Data = map[string][]byte{} - } - for k, v := range source.Data { - if k == "AZURE_CLIENT_SECRET" { - azureClientSecret.Data["clientSecret"] = v - } - } - return nil - }); err != nil { - return err - } - // Sync CNCC secret cloudNetworkConfigCreds := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cloud-network-config-controller-creds"}} secretData := map[string][]byte{ @@ -247,17 +231,10 @@ func reconcileAzureCluster(azureCluster *capiazure.AzureCluster, hcluster *hyper return nil } -func reconcileAzureClusterIdentity(ctx context.Context, c client.Client, hcluster *hyperv1.HostedCluster, azureClusterIdentity *capiazure.AzureClusterIdentity, controlPlaneNamespace string) error { - credentialsSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: hcluster.Spec.Platform.Azure.Credentials.Name, Namespace: controlPlaneNamespace}} - if err := c.Get(ctx, client.ObjectKeyFromObject(credentialsSecret), credentialsSecret); err != nil { - return fmt.Errorf("failed to get secret %s: %w", credentialsSecret, err) - } - +func reconcileAzureClusterIdentity(hcluster *hyperv1.HostedCluster, azureClusterIdentity *capiazure.AzureClusterIdentity, controlPlaneNamespace string) error { azureClusterIdentity.Spec = capiazure.AzureClusterIdentitySpec{ - ClientID: string(credentialsSecret.Data["AZURE_CLIENT_ID"]), - ClientSecret: corev1.SecretReference{Name: "azure-client-secret", Namespace: controlPlaneNamespace}, - TenantID: string(credentialsSecret.Data["AZURE_TENANT_ID"]), - Type: capiazure.ServicePrincipal, + ClientID: string(hcluster.Spec.Platform.Azure.ManagedIdentities.ControlPlaneManagedIdentities.ClusterAPIAzureManagedIdentityClientID), + Type: capiazure.UserAssignedMSI, AllowedNamespaces: &capiazure.AllowedNamespaces{ NamespaceList: []string{ controlPlaneNamespace, diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index 89ed85aa60d..aeec7778dbc 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -3,20 +3,25 @@ package azureutil import ( "context" "fmt" - "k8s.io/utils/ptr" "strings" - hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" - corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" ) +// We received this images directly from Microsoft; we should expect them to change as Microsoft continues development on both containers. +// We are scheduled to receive new updates to these containers in October 2024 to support Managed Identities. They currently only support Service Principal. +// TODO past October, will we receive new versions? +const ( + AdapterInitImage = "aromiwi.azurecr.io/artifact/b8e9ef87-cd63-4085-ab14-1c637806568c/buddy/adapter-init:20240905.9" + AdapterServerImage = "aromiwi.azurecr.io/artifact/b8e9ef87-cd63-4085-ab14-1c637806568c/buddy/adapter-server:20240905.5" +) + // GetSubnetNameFromSubnetID extracts the subnet name from a subnet ID // Example subnet ID: /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/ func GetSubnetNameFromSubnetID(subnetID string) (string, error) { @@ -145,8 +150,8 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro return vnet, nil } -// getNetworkSecurityGroupInfo gets the full information on a network security group based on its ID -func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { +// GetNetworkSecurityGroupInfo gets the full information on a network security group based on its ID +func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential) (armnetwork.SecurityGroupsClientGetResponse, error) { partialNSGInfo, err := arm.ParseResourceID(nsgID) if err != nil { return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err) @@ -165,8 +170,8 @@ func getNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscription return nsg, nil } -// getResourceGroupInfo gets the full information on a resource group based on its name -func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { +// GetResourceGroupInfo gets the full information on a resource group based on its name +func GetResourceGroupInfo(ctx context.Context, rgName string, subscriptionID string, azureCreds azcore.TokenCredential) (armresources.ResourceGroupsClientGetResponse, error) { resourceGroupClient, err := armresources.NewResourceGroupsClient(subscriptionID, azureCreds, nil) if err != nil { return armresources.ResourceGroupsClientGetResponse{}, fmt.Errorf("failed to create new resource groups client: %w", err) @@ -180,40 +185,63 @@ func getResourceGroupInfo(ctx context.Context, rgName string, subscriptionID str return rg, nil } -// VerifyResourceGroupLocationsMatch verifies the locations match for the VNET, network security group, and managed resource groups -func VerifyResourceGroupLocationsMatch(ctx context.Context, hc *hyperv1.HostedCluster, credentialsSecret *corev1.Secret) error { - // Setup azureCreds so we can retrieve the locations of the resource groups - tenantID := string(credentialsSecret.Data["AZURE_TENANT_ID"]) - clientID := string(credentialsSecret.Data["AZURE_CLIENT_ID"]) - clientSecret := string(credentialsSecret.Data["AZURE_CLIENT_SECRET"]) +// GetAzureCredentialsFromSecret gets the Service Principal client ID, client secret, and tenant ID from the credentials +// secret. This function will be modified a bit once the Microsoft sidecar containers support Managed Identity are +// delivered (expected Oct 2024). +func GetAzureCredentialsFromSecret(ctx context.Context, c client.Client, namespace, credsName string) (*corev1.Secret, error) { + var azureCredentials corev1.Secret - creds, err := azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, nil) - if err != nil { - return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + // Retrieve the Azure credentials secret to extract the needed fields for the managed identity containers + credentialsSecretName := client.ObjectKey{Namespace: namespace, Name: credsName} + if err := c.Get(ctx, credentialsSecretName, &azureCredentials); err != nil { + return nil, fmt.Errorf("failed to get secret %s: %w", credentialsSecretName, err) } - // Retrieve full vnet information from the VNET ID - vnet, err := GetVnetInfoFromVnetID(ctx, hc.Spec.Platform.Azure.VnetID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + for _, expectedKey := range []string{"AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", "AZURE_TENANT_ID"} { + if _, found := azureCredentials.Data[expectedKey]; !found { + return nil, fmt.Errorf("credentials secret for cluster doesn't have required key %s", expectedKey) + } } - // Retrieve full network security group information from the network security group ID - nsg, err := getNetworkSecurityGroupInfo(ctx, hc.Spec.Platform.Azure.SecurityGroupID, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get network security group info to verify its location: %v", err) - } - - // Retrieve full resource group information from the resource group name - rg, err := getResourceGroupInfo(ctx, hc.Spec.Platform.Azure.ResourceGroupName, hc.Spec.Platform.Azure.SubscriptionID, creds) - if err != nil { - return fmt.Errorf("failed to get resource group info to verify its location: %v", err) - } + return &azureCredentials, nil +} - // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match - if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { - return fmt.Errorf("the locations of the resource groups do not match - vnet location: %v; network security group location: %v; managed resource group location: %v", ptr.Deref(vnet.Location, ""), ptr.Deref(nsg.Location, ""), ptr.Deref(rg.Location, "")) - } +// AdapterInitContainer returns the Microsoft adapter-init init container. This container needs the NET_ADMIN permission +// so the adapter-server sidecar container can intercept the Managed Identity Azure API authentication calls. +func AdapterInitContainer() corev1.Container { + return corev1.Container{ + Name: "adapter-init", + Image: AdapterInitImage, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + }, + }, + }} +} - return nil +// AdapterServerContainer returns the Microsoft adapter-server sidecar container. Currently, this container mimics Azure +// Managed Identity approval and returns an authentication token. The container currently needs a Service Principal to +// do this. Future versions of this container will be able to take a Managed Identity instead. +func AdapterServerContainer(clientID, clientSecret, tenantID string) corev1.Container { + return corev1.Container{Name: "adapter-server", + Image: AdapterServerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"sp"}, + Env: []corev1.EnvVar{ + { + Name: "AZURE_CLIENT_ID", + Value: clientID, + }, + { + Name: "AZURE_CLIENT_SECRET", + Value: clientSecret, + }, + { + Name: "AZURE_TENANT_ID", + Value: tenantID, + }, + }} } diff --git a/support/azureutil/azureutil_test.go b/support/azureutil/azureutil_test.go index 6a5da3ad2da..9882749ef2d 100644 --- a/support/azureutil/azureutil_test.go +++ b/support/azureutil/azureutil_test.go @@ -1,9 +1,17 @@ package azureutil import ( + "context" "testing" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + . "github.com/onsi/gomega" + "github.com/openshift/hypershift/support/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestGetSubnetNameFromSubnetID(t *testing.T) { @@ -138,3 +146,116 @@ func TestGetVnetNameAndResourceGroupFromVnetID(t *testing.T) { }) } } + +func TestGetAzureCredentialsFromSecret(t *testing.T) { + tests := []struct { + testCaseName string + hc *hyperv1.HostedCluster + secret *corev1.Secret + expectedErr bool + }{ + { + testCaseName: "nominal test case", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentials"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_CLIENT_SECRET": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: false, + }, + { + testCaseName: "wrong secret name, err", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentialss"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_CLIENT_SECRET": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: true, + }, + { + testCaseName: "missing date from secret, err", + hc: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Credentials: corev1.LocalObjectReference{Name: "cloud-credentialss"}, + }, + }, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-credentials", + Namespace: "clusters", + }, + Data: map[string][]byte{ + "AZURE_CLIENT_ID": []byte("46fb37b5"), + "AZURE_TENANT_ID": []byte("46fb37b5"), + }, + }, + expectedErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.testCaseName, func(t *testing.T) { + g := NewGomegaWithT(t) + + objs := []crclient.Object{tc.hc, tc.secret} + + client := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(objs...).Build() + + creds, err := GetAzureCredentialsFromSecret(context.TODO(), client, tc.hc.Namespace, tc.hc.Spec.Platform.Azure.Credentials.Name) + if !tc.expectedErr { + g.Expect(err).To(BeNil()) + g.Expect(creds.Name).To(Equal(tc.hc.Spec.Platform.Azure.Credentials.Name)) + } else { + g.Expect(err).To(Not(BeNil())) + } + }) + } +} diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index 42a98277c51..2aa9d9960fc 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -1805,7 +1805,7 @@ type AzurePlatformSpec struct { // // Resource group naming requirements can be found here: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.ResourceGroup.Name/. // - //Example: if your resource group ID is /subscriptions//resourceGroups/, your + // Example: if your resource group ID is /subscriptions//resourceGroups/, your // ResourceGroupName is . // // +kubebuilder:default:=default @@ -1857,8 +1857,98 @@ type AzurePlatformSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="SecurityGroupID is immutable" // +kubebuilder:validation:Required // +immutable - // +required SecurityGroupID string `json:"securityGroupID,omitempty"` + + // managedIdentities contains the client IDs related to the managed identities needed for HCP control plane + // and data plane components that authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ManagedIdentities AzureResourceManagedIdentities `json:"managedIdentities,omitempty"` +} + +// AzureResourceManagedIdentities contains the client IDs related to the managed identities needed for HCP control plane +// and data plane components that authenticate with Azure's API. +type AzureResourceManagedIdentities struct { + // ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing to + // authenticate with Azure's API. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentities ControlPlaneManagedIdentities `json:"controlPlaneManagedIdentities"` + + // Future placeholder - DataPlaneMIs * DataPlaneManagedIdentities +} + +// ManagedIdentityClientID is a client ID of a managed identity +// +kubebuilder:validation:XValidation:rule="self.matches('^[{]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[}]?$')",message="the client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters in the form 8-4-4-4-12." +type ManagedIdentityClientID string + +// ControlPlaneManagedIdentities contains the client IDs of all the managed identities on the HCP control plane needing +// to authenticate with Azure's API. +// Managed identity regex pattern is from Microsoft here - https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity. +// The format a managed identity should be `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{managedIdentityName}`. +type ControlPlaneManagedIdentities struct { + // azureCloudProviderManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the azure + // cloud provider, aka ccm. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureCloudProviderManagedIdentityClientID ManagedIdentityClientID `json:"azureCloudProviderManagedIdentityClientID"` + + // clusterAPIAzureManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with cluster-api + // azure. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ClusterAPIAzureManagedIdentityClientID ManagedIdentityClientID `json:"clusterAPIAzureManagedIdentityClientID"` + + // controlPlaneManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the control plane + // operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated + // hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ControlPlaneManagedIdentityClientID ManagedIdentityClientID `json:"controlPlaneManagedIdentityClientID"` + + // azureKMSManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with Azure KMS. The client + // ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen separated hexadecimal characters + // in the form 8-4-4-4-12. + // + // +optional + AzureKMSManagedIdentityClientID ManagedIdentityClientID `json:"azureKMSManagedIdentityClientID,omitempty"` + + // imageRegistryManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-image-registry-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups + // of hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + ImageRegistryManagedIdentityClientID ManagedIdentityClientID `json:"imageRegistryManagedIdentityClientID"` + + // ingressManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-ingress-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + IngressManagedIdentityClientID ManagedIdentityClientID `json:"ingressManagedIdentityClientID"` + + // networkManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // cluster-network-operator. The client ID of a managed identity must be a valid UUID. It should be 5 groups of + // hyphen separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + NetworkManagedIdentityClientID ManagedIdentityClientID `json:"networkManagedIdentityClientID"` + + // azureDiskManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureDiskManagedIdentityClientID ManagedIdentityClientID `json:"azureDiskManagedIdentityClientID"` + + // azureFileManagedIdentityClientID is the client ID of a pre-existing managed identity ID associated with the + // azure-disk-controller. The client ID of a managed identity must be a valid UUID. It should be 5 groups of hyphen + // separated hexadecimal characters in the form 8-4-4-4-12. + // + // +kubebuilder:validation:Required + AzureFileManagedIdentityClientID ManagedIdentityClientID `json:"azureFileManagedIdentityClientID"` } // OpenStackPlatformSpec specifies configuration for clusters running on OpenStack. diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 14e4e51e549..ff6b0f7894b 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -564,6 +564,7 @@ func (in *AzureNodePoolPlatform) DeepCopy() *AzureNodePoolPlatform { func (in *AzurePlatformSpec) DeepCopyInto(out *AzurePlatformSpec) { *out = *in out.Credentials = in.Credentials + out.ManagedIdentities = in.ManagedIdentities } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzurePlatformSpec. @@ -576,6 +577,22 @@ func (in *AzurePlatformSpec) DeepCopy() *AzurePlatformSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureResourceManagedIdentities) DeepCopyInto(out *AzureResourceManagedIdentities) { + *out = *in + out.ControlPlaneManagedIdentities = in.ControlPlaneManagedIdentities +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureResourceManagedIdentities. +func (in *AzureResourceManagedIdentities) DeepCopy() *AzureResourceManagedIdentities { + if in == nil { + return nil + } + out := new(AzureResourceManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureVMImage) DeepCopyInto(out *AzureVMImage) { *out = *in @@ -879,6 +896,21 @@ func (in *ClusterVersionStatus) DeepCopy() *ClusterVersionStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControlPlaneManagedIdentities) DeepCopyInto(out *ControlPlaneManagedIdentities) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneManagedIdentities. +func (in *ControlPlaneManagedIdentities) DeepCopy() *ControlPlaneManagedIdentities { + if in == nil { + return nil + } + out := new(ControlPlaneManagedIdentities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSSpec) DeepCopyInto(out *DNSSpec) { *out = *in