From 3f801ea66dc591561adfd082ce577e437a9a4f1d Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:52:46 -0400 Subject: [PATCH 1/7] feat(api): support both managed and self-managed KMS authentication - Add AzureKMSSpec with mutually exclusive auth modes: `kms` (ManagedIdentity for ARO HCP) and `workloadIdentity` (WorkloadIdentity for self-managed via token-minter) - Add CEL XValidation rules enforcing mutual exclusivity, at-least-one, and immutability between the two authentication modes - Add AzureKeyVaultAccessType enum for Key Vault access mechanism selection - Add HostedCluster-level CEL rule requiring `selfManagedKMS` when using WorkloadIdentities authentication with Azure KMS - Reuse existing WorkloadIdentity type for KMS auth to maintain consistency with other Azure workload identity fields Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- api/hypershift/v1beta1/azure.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/api/hypershift/v1beta1/azure.go b/api/hypershift/v1beta1/azure.go index c99523ceedf..309b58e5a4e 100644 --- a/api/hypershift/v1beta1/azure.go +++ b/api/hypershift/v1beta1/azure.go @@ -810,6 +810,9 @@ const ( // AzureKMSSpec defines metadata about the configuration of the Azure KMS Secret Encryption provider using Azure key vault // // +kubebuilder:validation:XValidation:rule="!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName",message="backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault" +// +kubebuilder:validation:XValidation:rule="!(has(self.kms) && has(self.workloadIdentity))",message="kms and workloadIdentity are mutually exclusive" +// +kubebuilder:validation:XValidation:rule="has(self.kms) || has(self.workloadIdentity)",message="one of kms or workloadIdentity must be set" +// +kubebuilder:validation:XValidation:rule="has(self.kms) == has(oldSelf.kms)",message="the KMS authentication mode is immutable once set" type AzureKMSSpec struct { // activeKey defines the active key used to encrypt new secrets // @@ -821,9 +824,19 @@ type AzureKMSSpec struct { BackupKey *AzureKMSKey `json:"backupKey,omitempty"` // kms is a pre-existing managed identity used to authenticate with Azure KMS. + // This is used for managed Azure (ARO HCP) clusters. + // kms and workloadIdentity are mutually exclusive. // - // +required - KMS ManagedIdentity `json:"kms"` + // +optional + KMS ManagedIdentity `json:"kms,omitzero"` + + // workloadIdentity contains the workload identity used to authenticate + // with Azure Key Vault for KMS encryption via a token-minter sidecar. + // This identity must have "Key Vault Crypto User" role on the Key Vault. + // kms and workloadIdentity are mutually exclusive. + // + // +optional + WorkloadIdentity WorkloadIdentity `json:"workloadIdentity,omitzero"` // keyVaultAccess specifies how the Key Vault should be accessed. // When set to "Private", the control plane routes Key Vault traffic through From 9a21610152229313fba2a246083c11426e9f0f67 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:53:27 -0400 Subject: [PATCH 2/7] chore(api): regenerate CRDs, clients, deepcopy, and vendor Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- .../v1beta1/zz_generated.deepcopy.go | 1 + .../AAA_ungated.yaml | 35 +++++++++++++++++-- .../ClusterUpdateAcceptRisks.yaml | 35 +++++++++++++++++-- .../ClusterVersionOperatorConfiguration.yaml | 35 +++++++++++++++++-- .../ExternalOIDC.yaml | 35 +++++++++++++++++-- ...ernalOIDCWithUIDAndExtraClaimMappings.yaml | 35 +++++++++++++++++-- .../ExternalOIDCWithUpstreamParity.yaml | 35 +++++++++++++++++-- .../GCPPlatform.yaml | 35 +++++++++++++++++-- .../HCPEtcdBackup.yaml | 35 +++++++++++++++++-- ...perShiftOnlyDynamicResourceAllocation.yaml | 35 +++++++++++++++++-- .../ImageStreamImportMode.yaml | 35 +++++++++++++++++-- .../KMSEncryptionProvider.yaml | 35 +++++++++++++++++-- .../OpenStack.yaml | 35 +++++++++++++++++-- .../TLSAdherence.yaml | 35 +++++++++++++++++-- .../AAA_ungated.yaml | 35 +++++++++++++++++-- .../ClusterUpdateAcceptRisks.yaml | 35 +++++++++++++++++-- .../ClusterVersionOperatorConfiguration.yaml | 35 +++++++++++++++++-- .../ExternalOIDC.yaml | 35 +++++++++++++++++-- ...ernalOIDCWithUIDAndExtraClaimMappings.yaml | 35 +++++++++++++++++-- .../ExternalOIDCWithUpstreamParity.yaml | 35 +++++++++++++++++-- .../GCPPlatform.yaml | 35 +++++++++++++++++-- .../HCPEtcdBackup.yaml | 35 +++++++++++++++++-- ...perShiftOnlyDynamicResourceAllocation.yaml | 35 +++++++++++++++++-- .../ImageStreamImportMode.yaml | 35 +++++++++++++++++-- .../KMSEncryptionProvider.yaml | 35 +++++++++++++++++-- .../OpenStack.yaml | 35 +++++++++++++++++-- .../TLSAdherence.yaml | 35 +++++++++++++++++-- .../hypershift/v1beta1/azurekmsspec.go | 17 ++++++--- ...usters-Hypershift-CustomNoUpgrade.crd.yaml | 35 +++++++++++++++++-- ...hostedclusters-Hypershift-Default.crd.yaml | 35 +++++++++++++++++-- ...s-Hypershift-TechPreviewNoUpgrade.crd.yaml | 35 +++++++++++++++++-- ...planes-Hypershift-CustomNoUpgrade.crd.yaml | 35 +++++++++++++++++-- ...dcontrolplanes-Hypershift-Default.crd.yaml | 35 +++++++++++++++++-- ...s-Hypershift-TechPreviewNoUpgrade.crd.yaml | 35 +++++++++++++++++-- ...managed_azure_kms_secretproviderclass.yaml | 4 +-- .../api/hypershift/v1beta1/azure.go | 17 +++++++-- .../v1beta1/zz_generated.deepcopy.go | 1 + 37 files changed, 1056 insertions(+), 104 deletions(-) diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index d429305a30b..5954bdd73ad 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -651,6 +651,7 @@ func (in *AzureKMSSpec) DeepCopyInto(out *AzureKMSSpec) { **out = **in } out.KMS = in.KMS + out.WorkloadIdentity = in.WorkloadIdentity } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKMSSpec. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml index cd585082c85..fa952b03ab8 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml @@ -6046,8 +6046,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6096,15 +6098,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index 6c5b824f0b6..fa41eee8572 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -6029,8 +6029,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6079,15 +6081,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index d88d1cd27a9..d475e9fc5cc 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -6049,8 +6049,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6099,15 +6101,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml index 3a53c3c05c1..d94072cf9bd 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml @@ -6361,8 +6361,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6411,15 +6413,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index 850e21ea2a3..a2d16c89c5e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -6501,8 +6501,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6551,15 +6553,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index e6b9e20b742..e3f8d4956e6 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -6492,8 +6492,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6542,15 +6544,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml index 0e0d3a7117f..eb768e6b696 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml @@ -6475,8 +6475,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6525,15 +6527,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml index 1dcd56e78f2..13ef4e690e7 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -6094,8 +6094,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6144,15 +6146,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index 5d96096820b..a0027b0d74d 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -6051,8 +6051,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6101,15 +6103,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml index 01c3fac43d4..61c20aa0724 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -6047,8 +6047,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6097,15 +6099,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml index 44e48b304db..c9c6983f451 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -6105,8 +6105,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6155,15 +6157,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml index 7b0e49f3f99..7ba310f1c1e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml @@ -6580,8 +6580,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6630,15 +6632,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml index d18481f157f..871239aeb04 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml @@ -6069,8 +6069,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6119,15 +6121,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml index 873dd1d3b53..d1af5de7c6f 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml @@ -5901,8 +5901,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5951,15 +5953,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index 81e5c722d2d..849aee00c07 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -5884,8 +5884,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5934,15 +5936,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index 33eacd58b50..4f6507aeb1e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -5904,8 +5904,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5954,15 +5956,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml index 44ad9d8ed2b..1de483a48ec 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml @@ -6216,8 +6216,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6266,15 +6268,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index 968290ae2fa..40bad65502c 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -6356,8 +6356,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6406,15 +6408,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index 4b868e1a3ef..945cd0cc009 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -6347,8 +6347,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6397,15 +6399,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml index 53291008cc1..d11902b2fac 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml @@ -6330,8 +6330,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6380,15 +6382,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml index e4450913ffa..9d7f764d263 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -5949,8 +5949,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5999,15 +6001,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index 6fe328d94dd..b8c3890b21b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -5906,8 +5906,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5956,15 +5958,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml index 8760da61721..f6b4ae7f3a2 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -5902,8 +5902,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5952,15 +5954,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml index e5d2caea650..426e52a7dca 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -5960,8 +5960,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6010,15 +6012,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml index 22f92db2537..675c1d6f721 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml @@ -6435,8 +6435,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6485,15 +6487,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml index 30962b18779..bcb4a21223b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml @@ -5924,8 +5924,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -5974,15 +5976,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/client/applyconfiguration/hypershift/v1beta1/azurekmsspec.go b/client/applyconfiguration/hypershift/v1beta1/azurekmsspec.go index d9d2647bc3c..a6cfb9b0138 100644 --- a/client/applyconfiguration/hypershift/v1beta1/azurekmsspec.go +++ b/client/applyconfiguration/hypershift/v1beta1/azurekmsspec.go @@ -24,10 +24,11 @@ import ( // AzureKMSSpecApplyConfiguration represents a declarative configuration of the AzureKMSSpec type for use // with apply. type AzureKMSSpecApplyConfiguration struct { - ActiveKey *AzureKMSKeyApplyConfiguration `json:"activeKey,omitempty"` - BackupKey *AzureKMSKeyApplyConfiguration `json:"backupKey,omitempty"` - KMS *ManagedIdentityApplyConfiguration `json:"kms,omitempty"` - KeyVaultAccess *hypershiftv1beta1.AzureKeyVaultAccessType `json:"keyVaultAccess,omitempty"` + ActiveKey *AzureKMSKeyApplyConfiguration `json:"activeKey,omitempty"` + BackupKey *AzureKMSKeyApplyConfiguration `json:"backupKey,omitempty"` + KMS *ManagedIdentityApplyConfiguration `json:"kms,omitempty"` + WorkloadIdentity *WorkloadIdentityApplyConfiguration `json:"workloadIdentity,omitempty"` + KeyVaultAccess *hypershiftv1beta1.AzureKeyVaultAccessType `json:"keyVaultAccess,omitempty"` } // AzureKMSSpecApplyConfiguration constructs a declarative configuration of the AzureKMSSpec type for use with @@ -60,6 +61,14 @@ func (b *AzureKMSSpecApplyConfiguration) WithKMS(value *ManagedIdentityApplyConf return b } +// WithWorkloadIdentity sets the WorkloadIdentity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the WorkloadIdentity field is set to the value of the last call. +func (b *AzureKMSSpecApplyConfiguration) WithWorkloadIdentity(value *WorkloadIdentityApplyConfiguration) *AzureKMSSpecApplyConfiguration { + b.WorkloadIdentity = value + return b +} + // WithKeyVaultAccess sets the KeyVaultAccess field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the KeyVaultAccess field is set to the value of the last call. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml index 48c159bb721..9547f3cac23 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml @@ -7867,8 +7867,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -7917,15 +7919,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml index 80d34870899..5cf1224696a 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml @@ -6538,8 +6538,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6588,15 +6590,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml index 985515f0f51..19ce69379d9 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -7738,8 +7738,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -7788,15 +7790,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml index b3be64d7b3b..861af7174b2 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml @@ -7722,8 +7722,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -7772,15 +7774,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml index 6f530195fc0..14b2906319b 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml @@ -6393,8 +6393,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -6443,15 +6445,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml index 26ee13c9605..435b260f05f 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -7593,8 +7593,10 @@ spec: - "" type: string kms: - description: kms is a pre-existing managed identity used - to authenticate with Azure KMS. + description: |- + kms is a pre-existing managed identity used to authenticate with Azure KMS. + This is used for managed Azure (ARO HCP) clusters. + kms and workloadIdentity are mutually exclusive. properties: clientID: description: |- @@ -7643,15 +7645,42 @@ spec: - credentialsSecretName - objectEncoding type: object + workloadIdentity: + description: |- + workloadIdentity contains the workload identity used to authenticate + with Azure Key Vault for KMS encryption via a token-minter sidecar. + This identity must have "Key Vault Crypto User" role on the Key Vault. + kms and workloadIdentity are mutually exclusive. + properties: + clientID: + description: clientID is client ID of a federated + managed identity used in workload identity authentication + maxLength: 36 + minLength: 36 + pattern: ^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$ + type: string + x-kubernetes-validations: + - message: the client ID of a managed identity must + be a valid UUID. It should be 5 groups of hyphen + separated hexadecimal characters in the form 8-4-4-4-12. + rule: self.matches('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') + required: + - clientID + type: object required: - activeKey - - kms type: object x-kubernetes-validations: - message: backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault rule: '!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName' + - message: kms and workloadIdentity are mutually exclusive + rule: '!(has(self.kms) && has(self.workloadIdentity))' + - message: one of kms or workloadIdentity must be set + rule: has(self.kms) || has(self.workloadIdentity) + - message: the KMS authentication mode is immutable once set + rule: has(self.kms) == has(oldSelf.kms) ibmcloud: description: ibmcloud defines metadata for the IBM Cloud KMS encryption strategy diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_managed_azure_kms_secretproviderclass.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_managed_azure_kms_secretproviderclass.yaml index 99af047c92f..da463ee6784 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_managed_azure_kms_secretproviderclass.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_managed_azure_kms_secretproviderclass.yaml @@ -14,8 +14,8 @@ metadata: spec: parameters: keyvaultName: test-keyvault - objects: "\narray:\n - |\n objectName: \n objectEncoding: \n objectType: - secret\n" + objects: "\narray:\n - |\n objectName: test-kms-creds\n objectEncoding: + \n objectType: secret\n" tenantId: 00000000-0000-0000-0000-000000000000 usePodIdentity: "false" useVMManagedIdentity: "true" diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go index c99523ceedf..309b58e5a4e 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/azure.go @@ -810,6 +810,9 @@ const ( // AzureKMSSpec defines metadata about the configuration of the Azure KMS Secret Encryption provider using Azure key vault // // +kubebuilder:validation:XValidation:rule="!has(self.backupKey) || self.backupKey.keyVaultName == self.activeKey.keyVaultName",message="backupKey.keyVaultName must match activeKey.keyVaultName; both keys must reside in the same Key Vault" +// +kubebuilder:validation:XValidation:rule="!(has(self.kms) && has(self.workloadIdentity))",message="kms and workloadIdentity are mutually exclusive" +// +kubebuilder:validation:XValidation:rule="has(self.kms) || has(self.workloadIdentity)",message="one of kms or workloadIdentity must be set" +// +kubebuilder:validation:XValidation:rule="has(self.kms) == has(oldSelf.kms)",message="the KMS authentication mode is immutable once set" type AzureKMSSpec struct { // activeKey defines the active key used to encrypt new secrets // @@ -821,9 +824,19 @@ type AzureKMSSpec struct { BackupKey *AzureKMSKey `json:"backupKey,omitempty"` // kms is a pre-existing managed identity used to authenticate with Azure KMS. + // This is used for managed Azure (ARO HCP) clusters. + // kms and workloadIdentity are mutually exclusive. // - // +required - KMS ManagedIdentity `json:"kms"` + // +optional + KMS ManagedIdentity `json:"kms,omitzero"` + + // workloadIdentity contains the workload identity used to authenticate + // with Azure Key Vault for KMS encryption via a token-minter sidecar. + // This identity must have "Key Vault Crypto User" role on the Key Vault. + // kms and workloadIdentity are mutually exclusive. + // + // +optional + WorkloadIdentity WorkloadIdentity `json:"workloadIdentity,omitzero"` // keyVaultAccess specifies how the Key Vault should be accessed. // When set to "Private", the control plane routes Key Vault traffic through diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index d429305a30b..5954bdd73ad 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -651,6 +651,7 @@ func (in *AzureKMSSpec) DeepCopyInto(out *AzureKMSSpec) { **out = **in } out.KMS = in.KMS + out.WorkloadIdentity = in.WorkloadIdentity } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKMSSpec. From 76de18bf90a0744dbb64eb28b6972107909b8156 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:53:40 -0400 Subject: [PATCH 3/7] feat(cli): support KMS encryption on self-managed Azure cluster creation - Add --azure-kms-client-id and --azure-kms-tenant-id flags for workload identity-based KMS authentication - Add KMS client ID to IAM identity creation output - Add Azure flag descriptions for new KMS parameters - Update stable envtest validation test cases for Azure KMS Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- cmd/cluster/azure/create.go | 35 +++++++++----- cmd/infra/azure/create.go | 7 +++ cmd/infra/azure/create_iam.go | 10 ++-- cmd/infra/azure/identities.go | 75 +++++++++++++++++++++-------- cmd/infra/azure/identities_test.go | 48 +++++++++++++++--- cmd/infra/azure/types.go | 2 + cmd/util/azure_flag_descriptions.go | 1 + 7 files changed, 136 insertions(+), 42 deletions(-) diff --git a/cmd/cluster/azure/create.go b/cmd/cluster/azure/create.go index 75fb377606f..f1b210cba90 100644 --- a/cmd/cluster/azure/create.go +++ b/cmd/cluster/azure/create.go @@ -383,21 +383,34 @@ func (o *CreateOptions) ApplyPlatformSpecifics(cluster *hyperv1.HostedCluster) e } if o.encryptionKey != nil { + azureKMSSpec := &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: o.encryptionKey.KeyVaultName, + KeyName: o.encryptionKey.KeyName, + KeyVersion: o.encryptionKey.KeyVersion, + }, + } + + if o.infra.WorkloadIdentities != nil { + if o.infra.KMSClientID == "" { + return fmt.Errorf("self-managed Azure KMS requires a KMS workload identity; re-run 'hypershift create iam azure' with --enable-kms") + } + azureKMSSpec.WorkloadIdentity = hyperv1.WorkloadIdentity{ + ClientID: hyperv1.AzureClientID(o.infra.KMSClientID), + } + } else { + // Managed Azure (ARO HCP) with managed identities + azureKMSSpec.KMS = hyperv1.ManagedIdentity{ + CredentialsSecretName: o.KMSUserAssignedCredsSecretName, + ObjectEncoding: ObjectEncoding, + } + } + cluster.Spec.SecretEncryption = &hyperv1.SecretEncryptionSpec{ Type: hyperv1.KMS, KMS: &hyperv1.KMSSpec{ Provider: hyperv1.AZURE, - Azure: &hyperv1.AzureKMSSpec{ - ActiveKey: hyperv1.AzureKMSKey{ - KeyVaultName: o.encryptionKey.KeyVaultName, - KeyName: o.encryptionKey.KeyName, - KeyVersion: o.encryptionKey.KeyVersion, - }, - KMS: hyperv1.ManagedIdentity{ - CredentialsSecretName: o.KMSUserAssignedCredsSecretName, - ObjectEncoding: ObjectEncoding, - }, - }, + Azure: azureKMSSpec, }, } } diff --git a/cmd/infra/azure/create.go b/cmd/infra/azure/create.go index 191ed895423..95a84dedebd 100644 --- a/cmd/infra/azure/create.go +++ b/cmd/infra/azure/create.go @@ -274,6 +274,13 @@ func (o *CreateInfraOptions) handleIdentitiesAndRBAC(ctx context.Context, rbacMg if err := json.Unmarshal(workloadIdentitiesRaw, &result.WorkloadIdentities); err != nil { return fmt.Errorf("failed to unmarshal --workload-identities-file: %w", err) } + var iamExtra struct { + KMSClientID string `json:"kmsClientID,omitempty"` + } + if err := json.Unmarshal(workloadIdentitiesRaw, &iamExtra); err == nil { + result.KMSClientID = iamExtra.KMSClientID + } + if o.AssignServicePrincipalRoles { if err := rbacMgr.AssignWorkloadIdentities(ctx, o, result.WorkloadIdentities, resourceGroupName, nsgResourceGroupName, vnetResourceGroupName); err != nil { return err diff --git a/cmd/infra/azure/create_iam.go b/cmd/infra/azure/create_iam.go index ecb0f829ada..81b8ae2342c 100644 --- a/cmd/infra/azure/create_iam.go +++ b/cmd/infra/azure/create_iam.go @@ -42,6 +42,7 @@ func NewCreateIAMCommand() *cobra.Command { cmd.Flags().StringVar(&opts.OIDCIssuerURL, "oidc-issuer-url", opts.OIDCIssuerURL, util.OIDCIssuerURLDescription) cmd.Flags().StringVar(&opts.OutputFile, "output-file", opts.OutputFile, util.WorkloadIdentitiesOutputFileDescription) cmd.Flags().StringVar(&opts.Cloud, "cloud", opts.Cloud, util.CloudDescription) + cmd.Flags().BoolVar(&opts.EnableKMS, "enable-kms", opts.EnableKMS, util.EnableKMSDescription) _ = cmd.MarkFlagRequired("name") _ = cmd.MarkFlagRequired("infra-id") @@ -84,6 +85,7 @@ func BindCreateIAMProductFlags(opts *CreateIAMOptions, flags *pflag.FlagSet) { flags.StringVar(&opts.OIDCIssuerURL, "oidc-issuer-url", opts.OIDCIssuerURL, util.OIDCIssuerURLDescription) flags.StringVar(&opts.OutputFile, "output-file", opts.OutputFile, util.WorkloadIdentitiesOutputFileDescription) flags.StringVar(&opts.Cloud, "cloud", opts.Cloud, util.CloudDescription) + flags.BoolVar(&opts.EnableKMS, "enable-kms", opts.EnableKMS, util.EnableKMSDescription) } // Validate validates the CreateIAMOptions @@ -132,14 +134,14 @@ func (o *CreateIAMOptions) Run(ctx context.Context, l logr.Logger) error { identityManager := NewIdentityManager(subscriptionID, azureCreds, o.Cloud) // Create workload identities and federated credentials - workloadIdentities, err := identityManager.CreateWorkloadIdentitiesFromIAMOptions(ctx, l, o, o.ResourceGroupName) + iamOutput, err := identityManager.CreateWorkloadIdentitiesFromIAMOptions(ctx, l, o, o.ResourceGroupName) if err != nil { return fmt.Errorf("failed to create workload identities: %w", err) } - // Write output to file (directly as AzureWorkloadIdentities for compatibility - // with --workload-identities-file flag in create cluster/infra commands) - if err := o.writeOutput(workloadIdentities); err != nil { + // Write output to file. The format embeds AzureWorkloadIdentities fields at the top level + // for compatibility with --workload-identities-file, plus kmsClientID if KMS is enabled. + if err := o.writeOutput(iamOutput); err != nil { return err } diff --git a/cmd/infra/azure/identities.go b/cmd/infra/azure/identities.go index e9d6a815462..c7e9e061971 100644 --- a/cmd/infra/azure/identities.go +++ b/cmd/infra/azure/identities.go @@ -162,11 +162,22 @@ type WorkloadIdentityDefinition struct { FederatedCredentials []FederatedCredentialConfig } +// WorkloadIdentityOptions configures which optional workload identities to include. +type WorkloadIdentityOptions struct { + // Topology controls whether the controlPlaneOperator identity is included. + // Non-public topologies (Private, PublicAndPrivate) include the CPO identity. + // An empty string includes CPO (used during cleanup to ensure all identities are deleted). + Topology string + + // IncludeKMS controls whether the KMS workload identity is included. + // Set to true when KMS encryption is enabled for the cluster. + IncludeKMS bool +} + // GetWorkloadIdentityDefinitions returns all workload identity definitions for a cluster. // This is the single source of truth for identity names and their federated credentials, -// used by both create and destroy operations. The topology parameter controls whether -// private-topology-only identities (e.g., controlPlaneOperator) are included. -func GetWorkloadIdentityDefinitions(clusterName string, topology string) []WorkloadIdentityDefinition { +// used by both create and destroy operations. +func GetWorkloadIdentityDefinitions(clusterName string, opts WorkloadIdentityOptions) []WorkloadIdentityDefinition { definitions := []WorkloadIdentityDefinition{ { ComponentName: "disk", @@ -272,7 +283,7 @@ func GetWorkloadIdentityDefinitions(clusterName string, topology string) []Workl }, } - if topology != string(hyperv1.AzureTopologyPublic) { + if opts.Topology != string(hyperv1.AzureTopologyPublic) { definitions = append(definitions, WorkloadIdentityDefinition{ ComponentName: "controlPlaneOperator", IdentityNameSuffix: "-control-plane-operator", @@ -286,6 +297,20 @@ func GetWorkloadIdentityDefinitions(clusterName string, topology string) []Workl }) } + if opts.IncludeKMS { + definitions = append(definitions, WorkloadIdentityDefinition{ + ComponentName: "kms", + IdentityNameSuffix: "-kms", + FederatedCredentials: []FederatedCredentialConfig{ + { + CredentialName: clusterName + "-kms-fed-id", + Subject: "system:serviceaccount:kube-system:kms-provider", + Audience: "openshift", + }, + }, + }) + } + return definitions } @@ -332,11 +357,20 @@ func (i *IdentityManager) createFederatedIdentityCredential(ctx context.Context, return nil } +// IAMOutput wraps the workload identities and any additional identity outputs +// that are not part of AzureWorkloadIdentities (e.g., KMS for self-managed clusters). +type IAMOutput struct { + hyperv1.AzureWorkloadIdentities + KMSClientID string `json:"kmsClientID,omitempty"` +} + // CreateWorkloadIdentitiesFromIAMOptions creates all managed identities and federated credentials // for workload identity using CreateIAMOptions. This is used by the standalone IAM create command. -func (i *IdentityManager) CreateWorkloadIdentitiesFromIAMOptions(ctx context.Context, l logr.Logger, opts *CreateIAMOptions, resourceGroupName string) (*hyperv1.AzureWorkloadIdentities, error) { - workloadIdentities := &hyperv1.AzureWorkloadIdentities{} - definitions := GetWorkloadIdentityDefinitions(opts.Name, "") +func (i *IdentityManager) CreateWorkloadIdentitiesFromIAMOptions(ctx context.Context, l logr.Logger, opts *CreateIAMOptions, resourceGroupName string) (*IAMOutput, error) { + output := &IAMOutput{} + definitions := GetWorkloadIdentityDefinitions(opts.Name, WorkloadIdentityOptions{ + IncludeKMS: opts.EnableKMS, + }) for _, def := range definitions { identityName := opts.Name + def.IdentityNameSuffix @@ -348,26 +382,28 @@ func (i *IdentityManager) CreateWorkloadIdentitiesFromIAMOptions(ctx context.Con return nil, fmt.Errorf("failed to create %s managed identity: %w", def.ComponentName, err) } - // Set the client ID on the appropriate workload identity field + // Set the client ID on the appropriate field switch def.ComponentName { case "disk": - workloadIdentities.Disk.ClientID = hyperv1.AzureClientID(clientID) + output.Disk.ClientID = hyperv1.AzureClientID(clientID) case "file": - workloadIdentities.File.ClientID = hyperv1.AzureClientID(clientID) + output.File.ClientID = hyperv1.AzureClientID(clientID) case "imageRegistry": - workloadIdentities.ImageRegistry.ClientID = hyperv1.AzureClientID(clientID) + output.ImageRegistry.ClientID = hyperv1.AzureClientID(clientID) case "ingress": - workloadIdentities.Ingress.ClientID = hyperv1.AzureClientID(clientID) + output.Ingress.ClientID = hyperv1.AzureClientID(clientID) case "cloudProvider": - workloadIdentities.CloudProvider.ClientID = hyperv1.AzureClientID(clientID) + output.CloudProvider.ClientID = hyperv1.AzureClientID(clientID) case "nodePoolManagement": - workloadIdentities.NodePoolManagement.ClientID = hyperv1.AzureClientID(clientID) + output.NodePoolManagement.ClientID = hyperv1.AzureClientID(clientID) case "network": - workloadIdentities.Network.ClientID = hyperv1.AzureClientID(clientID) + output.Network.ClientID = hyperv1.AzureClientID(clientID) case "controlPlaneOperator": - workloadIdentities.ControlPlaneOperator = hyperv1.WorkloadIdentity{ + output.ControlPlaneOperator = hyperv1.WorkloadIdentity{ ClientID: hyperv1.AzureClientID(clientID), } + case "kms": + output.KMSClientID = clientID default: return nil, fmt.Errorf("unknown workload identity component: %s", def.ComponentName) } @@ -380,15 +416,16 @@ func (i *IdentityManager) CreateWorkloadIdentitiesFromIAMOptions(ctx context.Con } } - return workloadIdentities, nil + return output, nil } // DestroyWorkloadIdentities deletes all managed identities and their federated credentials for a cluster. // Federated credentials are explicitly deleted first, then the managed identity is deleted. // The method continues deleting remaining identities even if some fail, logging errors as it goes. func (i *IdentityManager) DestroyWorkloadIdentities(ctx context.Context, l logr.Logger, clusterName string, infraID string, resourceGroupName string) error { - // Pass empty topology to include all identities (including CPO) during cleanup - definitions := GetWorkloadIdentityDefinitions(clusterName, "") + // Pass empty topology and IncludeKMS: true to include all optional identities during cleanup; + // not-found errors are handled gracefully. + definitions := GetWorkloadIdentityDefinitions(clusterName, WorkloadIdentityOptions{IncludeKMS: true}) var errors []error for _, def := range definitions { diff --git a/cmd/infra/azure/identities_test.go b/cmd/infra/azure/identities_test.go index c824a20ca2d..3394c9ffd22 100644 --- a/cmd/infra/azure/identities_test.go +++ b/cmd/infra/azure/identities_test.go @@ -39,13 +39,13 @@ func TestNewIdentityManager(t *testing.T) { func TestGetWorkloadIdentityDefinitions(t *testing.T) { tests := map[string]struct { clusterName string - topology string + opts WorkloadIdentityOptions expectedCount int expectedComponent []string }{ - "When public topology it should return 7 identity definitions without controlPlaneOperator": { + "When public topology without KMS it should return 7 identity definitions": { clusterName: "test-cluster", - topology: "Public", + opts: WorkloadIdentityOptions{Topology: "Public"}, expectedCount: 7, expectedComponent: []string{ "disk", @@ -57,9 +57,9 @@ func TestGetWorkloadIdentityDefinitions(t *testing.T) { "network", }, }, - "When private topology it should return 8 identity definitions with controlPlaneOperator": { + "When private topology without KMS it should return 8 identity definitions with controlPlaneOperator": { clusterName: "test-cluster", - topology: "Private", + opts: WorkloadIdentityOptions{Topology: "Private"}, expectedCount: 8, expectedComponent: []string{ "disk", @@ -72,10 +72,41 @@ func TestGetWorkloadIdentityDefinitions(t *testing.T) { "controlPlaneOperator", }, }, - "When empty topology it should return 8 identity definitions for cleanup": { + "When public topology with KMS it should return 8 identity definitions including kms": { clusterName: "test-cluster", - topology: "", + opts: WorkloadIdentityOptions{Topology: "Public", IncludeKMS: true}, expectedCount: 8, + expectedComponent: []string{ + "disk", + "file", + "imageRegistry", + "ingress", + "cloudProvider", + "nodePoolManagement", + "network", + "kms", + }, + }, + "When private topology with KMS it should return 9 identity definitions": { + clusterName: "test-cluster", + opts: WorkloadIdentityOptions{Topology: "Private", IncludeKMS: true}, + expectedCount: 9, + expectedComponent: []string{ + "disk", + "file", + "imageRegistry", + "ingress", + "cloudProvider", + "nodePoolManagement", + "network", + "controlPlaneOperator", + "kms", + }, + }, + "When empty topology with KMS it should return 9 identity definitions for cleanup": { + clusterName: "test-cluster", + opts: WorkloadIdentityOptions{IncludeKMS: true}, + expectedCount: 9, expectedComponent: []string{ "disk", "file", @@ -85,6 +116,7 @@ func TestGetWorkloadIdentityDefinitions(t *testing.T) { "nodePoolManagement", "network", "controlPlaneOperator", + "kms", }, }, } @@ -93,7 +125,7 @@ func TestGetWorkloadIdentityDefinitions(t *testing.T) { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) - definitions := GetWorkloadIdentityDefinitions(test.clusterName, test.topology) + definitions := GetWorkloadIdentityDefinitions(test.clusterName, test.opts) // Verify count g.Expect(definitions).To(HaveLen(test.expectedCount)) diff --git a/cmd/infra/azure/types.go b/cmd/infra/azure/types.go index b4505d247f5..a62134d1ae2 100644 --- a/cmd/infra/azure/types.go +++ b/cmd/infra/azure/types.go @@ -43,6 +43,7 @@ type CreateInfraOutput struct { ControlPlaneMIs *hyperv1.AzureResourceManagedIdentities `json:"controlPlaneMIs"` DataPlaneIdentities hyperv1.DataPlaneManagedIdentities `json:"dataPlaneIdentities"` WorkloadIdentities *hyperv1.AzureWorkloadIdentities `json:"workloadIdentities"` + KMSClientID string `json:"kmsClientID,omitempty"` } // CreateIAMOptions holds options for creating Azure IAM resources (managed identities and federated credentials) @@ -56,6 +57,7 @@ type CreateIAMOptions struct { OIDCIssuerURL string OutputFile string Cloud string + EnableKMS bool } // DestroyIAMOptions holds options for destroying Azure IAM resources diff --git a/cmd/util/azure_flag_descriptions.go b/cmd/util/azure_flag_descriptions.go index 2d9b12d856c..771ae69db48 100644 --- a/cmd/util/azure_flag_descriptions.go +++ b/cmd/util/azure_flag_descriptions.go @@ -41,6 +41,7 @@ const ( EndpointAccessPrivateAdditionalAllowedSubscriptionsDescription = "Additional Azure subscription IDs permitted to create Private Endpoints (the guest cluster's own subscription is always automatically allowed)." // Encryption + EnableKMSDescription = "Create a KMS workload identity for Azure Key Vault KMS encryption. Use this when the cluster will be configured with --encryption-key-id." EncryptionKeyIDDescription = "Azure Key Vault key identifier used to encrypt etcd data via KMSv2 (format: https://.vault.azure.net/keys//)." EncryptionAtHostDescription = "Enable host-based encryption for VM disks and temp disks. Valid values: Enabled, Disabled." DiskEncryptionSetIDDescription = "Full resource ID of an Azure Disk Encryption Set used to encrypt NodePool OS disks with customer-managed keys." From bdd7f5d9f9442dbd81b8926b34585439e9b10f64 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:53:48 -0400 Subject: [PATCH 4/7] feat(hypershift-operator): support Azure KMS encryption on self-managed clusters - Add KMS secret provider class reconciliation for self-managed Azure using workload identity credentials - Add IsSelfManagedAzure helper to azureutil for platform detection - Add KMS-related constants for config paths and identifiers - Add token-minter sidecar support for KMS workload identity authentication - Update HO azure platform controller to handle both managed identity and workload identity KMS modes Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- .../internal/platform/azure/azure.go | 27 ++-- .../internal/platform/azure/azure_test.go | 116 ++++++++++++++++++ support/azureutil/azureutil.go | 8 ++ support/azureutil/azureutil_test.go | 66 ++++++++++ support/config/constants.go | 4 + .../token-minter-container.go | 10 +- .../token-minter-container_test.go | 19 +-- 7 files changed, 226 insertions(+), 24 deletions(-) diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go index a83bbb88978..c93c3417c58 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "os" + "path" "strings" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -203,13 +204,12 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy } // For self-managed Azure with workload identity, instruct Azure SDK to use the minted SA token - if azureutil.IsSelfManagedAzure(hcluster.Spec.Platform.Type) && - hcluster.Spec.Platform.Azure.AzureAuthenticationConfig.WorkloadIdentities != nil { + if azureutil.IsSelfManagedAzureWithWorkloadIdentity(hcluster.Spec.Platform.Type, hcluster.Spec.Platform.Azure) { deploymentSpec.Template.Spec.Containers[0].Env = append( deploymentSpec.Template.Spec.Containers[0].Env, corev1.EnvVar{ Name: "AZURE_FEDERATED_TOKEN_FILE", - Value: "/var/run/secrets/openshift/serviceaccount/token", + Value: path.Join(config.CloudTokenMountPath, "token"), }, corev1.EnvVar{ Name: "AZURE_CLIENT_ID", @@ -232,7 +232,7 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy deploymentSpec.Template.Spec.Containers[0].VolumeMounts = append(deploymentSpec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ Name: tokenVolume.Name, - MountPath: "/var/run/secrets/openshift/serviceaccount", + MountPath: config.CloudTokenMountPath, }) deploymentSpec.Template.Spec.Containers = append(deploymentSpec.Template.Spec.Containers, corev1.Container{ @@ -243,7 +243,7 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy "--token-audience=openshift", "--service-account-namespace=kube-system", "--service-account-name=capi-provider", - "--token-file=/var/run/secrets/openshift/serviceaccount/token", + fmt.Sprintf("--token-file=%s", path.Join(config.CloudTokenMountPath, "token")), "--kubeconfig=/etc/kubernetes/kubeconfig", }, ImagePullPolicy: corev1.PullIfNotPresent, @@ -256,7 +256,7 @@ func (a Azure) CAPIProviderDeploymentSpec(hcluster *hyperv1.HostedCluster, _ *hy VolumeMounts: []corev1.VolumeMount{ { Name: tokenVolume.Name, - MountPath: "/var/run/secrets/openshift/serviceaccount", + MountPath: config.CloudTokenMountPath, }, { Name: "svc-kubeconfig", @@ -280,9 +280,9 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create } // For self-managed Azure, use workload identity credentials; for managed Azure, use the existing approach - if azureutil.IsSelfManagedAzure(hcluster.Spec.Platform.Type) && hcluster.Spec.Platform.Azure.AzureAuthenticationConfig.WorkloadIdentities != nil { + if azureutil.IsSelfManagedAzureWithWorkloadIdentity(hcluster.Spec.Platform.Type, hcluster.Spec.Platform.Azure) { // Add federated token file for workload identity authentication - baseSecretData["azure_federated_token_file"] = []byte("/var/run/secrets/openshift/serviceaccount/token") + baseSecretData["azure_federated_token_file"] = []byte(path.Join(config.CloudTokenMountPath, "token")) // Create credentials for each control plane operator using workload identity client IDs workloadIdentities := hcluster.Spec.Platform.Azure.AzureAuthenticationConfig.WorkloadIdentities @@ -324,7 +324,7 @@ func (a Azure) ReconcileCredentials(ctx context.Context, c client.Client, create if _, err := createOrUpdate(ctx, c, cloudNetworkConfigCreds, func() error { secretData := maps.Clone(baseSecretData) // For self-managed Azure with workload identities, add the network client ID - if azureutil.IsSelfManagedAzure(hcluster.Spec.Platform.Type) && hcluster.Spec.Platform.Azure.AzureAuthenticationConfig.WorkloadIdentities != nil { + if azureutil.IsSelfManagedAzureWithWorkloadIdentity(hcluster.Spec.Platform.Type, hcluster.Spec.Platform.Azure) { secretData["azure_client_id"] = []byte(hcluster.Spec.Platform.Azure.AzureAuthenticationConfig.WorkloadIdentities.Network.ClientID) } cloudNetworkConfigCreds.Data = secretData @@ -464,7 +464,14 @@ func reconcileKMSConfigSecret(secret *corev1.Secret, hc *hyperv1.HostedCluster) UseInstanceMetadata: false, LoadBalancerSku: "standard", DisableOutboundSNAT: true, - AADMSIDataPlaneIdentityPath: config.ManagedAzureCertificatePath + hc.Spec.SecretEncryption.KMS.Azure.KMS.CredentialsSecretName, + } + + if azureutil.IsAroHCP() && hc.Spec.SecretEncryption.KMS.Azure.KMS.CredentialsSecretName != "" { + azureConfig.AADMSIDataPlaneIdentityPath = config.ManagedAzureCertificatePath + hc.Spec.SecretEncryption.KMS.Azure.KMS.CredentialsSecretName + } else if hc.Spec.SecretEncryption.KMS.Azure.WorkloadIdentity.ClientID != "" { + azureConfig.UseWorkloadIdentityExtension = true + } else { + return fmt.Errorf("azure KMS configured but neither managed identity (kms) nor workload identity (workloadIdentity) credentials are set") } serializedConfig, err := json.MarshalIndent(azureConfig, "", " ") diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure_test.go b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure_test.go index 40b75d76b35..b71468ea237 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure_test.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/azure/azure_test.go @@ -2,12 +2,14 @@ package azure import ( "context" + "encoding/json" "fmt" "testing" . "github.com/onsi/gomega" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + azurecloud "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/cloud/azure" "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/config" @@ -411,3 +413,117 @@ func TestReconcileCredentials(t *testing.T) { }) } } + +func TestReconcileKMSConfigSecret(t *testing.T) { + baseHC := func() *hyperv1.HostedCluster { + return &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, + Spec: hyperv1.HostedClusterSpec{ + InfraID: "test-infra", + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + Cloud: "AzurePublicCloud", + TenantID: "test-tenant-id", + SubscriptionID: "test-sub-id", + ResourceGroupName: "test-rg", + Location: "eastus", + }, + }, + SecretEncryption: &hyperv1.SecretEncryptionSpec{ + Type: hyperv1.KMS, + KMS: &hyperv1.KMSSpec{ + Provider: hyperv1.AZURE, + Azure: &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "v1", + }, + }, + }, + }, + }, + } + } + + testCases := []struct { + name string + managedService string + hc *hyperv1.HostedCluster + expectErr bool + validate func(g Gomega, cfg azurecloud.AzureConfig) + }{ + { + name: "When ARO HCP it should set AADMSIDataPlaneIdentityPath", + managedService: hyperv1.AroHCP, + hc: func() *hyperv1.HostedCluster { + hc := baseHC() + hc.Spec.SecretEncryption.KMS.Azure.KMS = hyperv1.ManagedIdentity{ + CredentialsSecretName: "kms-creds", + } + return hc + }(), + validate: func(g Gomega, cfg azurecloud.AzureConfig) { + g.Expect(cfg.AADMSIDataPlaneIdentityPath).To(Equal(config.ManagedAzureCertificatePath + "kms-creds")) + g.Expect(cfg.UseWorkloadIdentityExtension).To(BeFalse()) + g.Expect(cfg.AADClientID).To(BeEmpty()) + }, + }, + { + name: "When self-managed Azure with workload identities it should set federated identity fields", + hc: func() *hyperv1.HostedCluster { + hc := baseHC() + hc.Spec.SecretEncryption.KMS.Azure.WorkloadIdentity = hyperv1.WorkloadIdentity{ + ClientID: "kms-client-id", + } + return hc + }(), + validate: func(g Gomega, cfg azurecloud.AzureConfig) { + g.Expect(cfg.UseWorkloadIdentityExtension).To(BeTrue()) + g.Expect(cfg.AADClientID).To(BeEmpty()) + g.Expect(cfg.AADMSIDataPlaneIdentityPath).To(BeEmpty()) + }, + }, + { + name: "When Azure KMS without any credentials it should return an error", + hc: baseHC(), + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + if tc.managedService != "" { + t.Setenv("MANAGED_SERVICE", tc.managedService) + } + + secret := &corev1.Secret{} + err := reconcileKMSConfigSecret(secret, tc.hc) + if tc.expectErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(secret.Data).To(HaveKey(azurecloud.CloudConfigKey)) + + var cfg azurecloud.AzureConfig + err = json.Unmarshal(secret.Data[azurecloud.CloudConfigKey], &cfg) + g.Expect(err).ToNot(HaveOccurred()) + + // Verify common base fields + g.Expect(cfg.Cloud).To(Equal("AzurePublicCloud")) + g.Expect(cfg.TenantID).To(Equal("test-tenant-id")) + g.Expect(cfg.SubscriptionID).To(Equal("test-sub-id")) + g.Expect(cfg.ResourceGroup).To(Equal("test-rg")) + g.Expect(cfg.Location).To(Equal("eastus")) + g.Expect(cfg.LoadBalancerName).To(Equal("test-infra")) + g.Expect(cfg.CloudProviderBackoff).To(BeTrue()) + g.Expect(cfg.LoadBalancerSku).To(Equal("standard")) + + tc.validate(g, cfg) + }) + } +} diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index e11580efe81..07836f93d3a 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -270,6 +270,14 @@ func IsSelfManagedAzure(platform hyperv1.PlatformType) bool { return platform == hyperv1.AzurePlatform && !IsAroHCP() } +// IsSelfManagedAzureWithWorkloadIdentity returns true if the platform is self-managed Azure +// and workload identities are configured. +func IsSelfManagedAzureWithWorkloadIdentity(platformType hyperv1.PlatformType, azure *hyperv1.AzurePlatformSpec) bool { + return IsSelfManagedAzure(platformType) && + azure != nil && + azure.AzureAuthenticationConfig.WorkloadIdentities != nil +} + // SetAsAroHCPTest sets the proper environment variable for the test, designating this is an ARO-HCP environment func SetAsAroHCPTest(t *testing.T) { t.Setenv("MANAGED_SERVICE", hyperv1.AroHCP) diff --git a/support/azureutil/azureutil_test.go b/support/azureutil/azureutil_test.go index e81deea5b4a..e0dc89d3777 100644 --- a/support/azureutil/azureutil_test.go +++ b/support/azureutil/azureutil_test.go @@ -741,6 +741,72 @@ func isCapabilityDisabled(capabilities *hyperv1.Capabilities, capability hyperv1 return false } +func TestIsSelfManagedAzureWithWorkloadIdentity(t *testing.T) { + tests := []struct { + name string + platformType hyperv1.PlatformType + azure *hyperv1.AzurePlatformSpec + managedSvc string + expected bool + }{ + { + name: "When Azure platform with workload identities configured it should return true", + platformType: hyperv1.AzurePlatform, + azure: &hyperv1.AzurePlatformSpec{ + AzureAuthenticationConfig: hyperv1.AzureAuthenticationConfiguration{ + WorkloadIdentities: &hyperv1.AzureWorkloadIdentities{}, + }, + }, + expected: true, + }, + { + name: "When Azure platform with nil workload identities it should return false", + platformType: hyperv1.AzurePlatform, + azure: &hyperv1.AzurePlatformSpec{ + AzureAuthenticationConfig: hyperv1.AzureAuthenticationConfiguration{}, + }, + expected: false, + }, + { + name: "When Azure platform with nil azure spec it should return false", + platformType: hyperv1.AzurePlatform, + azure: nil, + expected: false, + }, + { + name: "When non-Azure platform with workload identities it should return false", + platformType: hyperv1.AWSPlatform, + azure: &hyperv1.AzurePlatformSpec{ + AzureAuthenticationConfig: hyperv1.AzureAuthenticationConfiguration{ + WorkloadIdentities: &hyperv1.AzureWorkloadIdentities{}, + }, + }, + expected: false, + }, + { + name: "When ARO-HCP managed service with workload identities it should return false", + platformType: hyperv1.AzurePlatform, + azure: &hyperv1.AzurePlatformSpec{ + AzureAuthenticationConfig: hyperv1.AzureAuthenticationConfiguration{ + WorkloadIdentities: &hyperv1.AzureWorkloadIdentities{}, + }, + }, + managedSvc: hyperv1.AroHCP, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewGomegaWithT(t) + if tc.managedSvc != "" { + t.Setenv("MANAGED_SERVICE", tc.managedSvc) + } + g.Expect(IsSelfManagedAzureWithWorkloadIdentity(tc.platformType, tc.azure)).To(Equal(tc.expected)) + }) + } +} + func TestNewARMClientOptions(t *testing.T) { tests := []struct { name string diff --git a/support/config/constants.go b/support/config/constants.go index f1cb399706d..3eac77aee49 100644 --- a/support/config/constants.go +++ b/support/config/constants.go @@ -66,6 +66,10 @@ const ( HypershiftImageTag = "latest" ) +// CloudTokenMountPath is the mount path for federated workload identity tokens +// used by cloud provider components in self-managed Azure clusters. +const CloudTokenMountPath = "/var/run/secrets/openshift/serviceaccount" + // Azure Default Values const ( // DefaultAzureLocation is the default Azure region for resource creation diff --git a/support/controlplane-component/token-minter-container.go b/support/controlplane-component/token-minter-container.go index 92f1f514057..7a1d2aa6bd2 100644 --- a/support/controlplane-component/token-minter-container.go +++ b/support/controlplane-component/token-minter-container.go @@ -6,6 +6,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/azureutil" + "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/podspec" corev1 "k8s.io/api/core/v1" @@ -13,7 +14,6 @@ import ( ) const ( - cloudTokenFileMountPath = "/var/run/secrets/openshift/serviceaccount" kubeAPITokenFileMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" ) @@ -62,7 +62,7 @@ func (opts TokenMinterContainerOptions) injectTokenMinterContainer(cpContext Con podSpec.Volumes = append(podSpec.Volumes, tokenVolume) container := opts.buildContainer(cpContext.HCP, CloudToken, image, tokenVolume) - opts.injectContainer(cpContext.NativeSidecarContainersEnabled, podSpec, container, cloudTokenFileMountPath, tokenVolume.Name) + opts.injectContainer(cpContext.NativeSidecarContainersEnabled, podSpec, container, config.CloudTokenMountPath, tokenVolume.Name) } if opts.TokenType == KubeAPIServerToken || opts.TokenType == CloudAndAPIServerToken { @@ -90,8 +90,8 @@ func (opts TokenMinterContainerOptions) injectContainer(nativeSidecarsEnabled bo container.StartupProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - // The token-minter always writes to cloudTokenFileMountPath regardless of token type. - Command: []string{"test", "-f", path.Join(cloudTokenFileMountPath, "token")}, + // The token-minter always writes to CloudTokenMountPath regardless of token type. + Command: []string{"test", "-f", path.Join(config.CloudTokenMountPath, "token")}, }, }, PeriodSeconds: 1, @@ -112,7 +112,7 @@ func (opts TokenMinterContainerOptions) injectContainer(nativeSidecarsEnabled bo } func (opts TokenMinterContainerOptions) buildContainer(hcp *hyperv1.HostedControlPlane, tokenType TokenType, image string, tokenVolume corev1.Volume) corev1.Container { - tokenFileMountPath := "/var/run/secrets/openshift/serviceaccount" + tokenFileMountPath := config.CloudTokenMountPath var audience string switch tokenType { diff --git a/support/controlplane-component/token-minter-container_test.go b/support/controlplane-component/token-minter-container_test.go index c5e3afe4258..9ef666f2a4f 100644 --- a/support/controlplane-component/token-minter-container_test.go +++ b/support/controlplane-component/token-minter-container_test.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/gomega" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/config" corev1 "k8s.io/api/core/v1" ) @@ -35,7 +36,7 @@ func TestInjectContainer(t *testing.T) { g := NewGomegaWithT(t) podSpec := basePodSpec() - baseOpts.injectContainer(true, podSpec, baseContainer, cloudTokenFileMountPath, "cloud-token") + baseOpts.injectContainer(true, podSpec, baseContainer, config.CloudTokenMountPath, "cloud-token") g.Expect(podSpec.InitContainers).To(HaveLen(1)) g.Expect(podSpec.Containers).To(HaveLen(1), "should not add to regular containers") @@ -50,19 +51,19 @@ func TestInjectContainer(t *testing.T) { g := NewGomegaWithT(t) podSpec := basePodSpec() - baseOpts.injectContainer(true, podSpec, baseContainer, cloudTokenFileMountPath, "cloud-token") + baseOpts.injectContainer(true, podSpec, baseContainer, config.CloudTokenMountPath, "cloud-token") initContainer := podSpec.InitContainers[0] g.Expect(initContainer.StartupProbe).ToNot(BeNil()) g.Expect(initContainer.StartupProbe.Exec).ToNot(BeNil()) g.Expect(initContainer.StartupProbe.Exec.Command).To(Equal( - []string{"test", "-f", path.Join(cloudTokenFileMountPath, "token")}, + []string{"test", "-f", path.Join(config.CloudTokenMountPath, "token")}, )) g.Expect(initContainer.StartupProbe.PeriodSeconds).To(Equal(int32(1))) g.Expect(initContainer.StartupProbe.FailureThreshold).To(Equal(int32(30))) }) - t.Run("When native sidecars are enabled with KubeAPIServerToken it should use cloudTokenFileMountPath for the startup probe", func(t *testing.T) { + t.Run("When native sidecars are enabled with KubeAPIServerToken it should use config.CloudTokenMountPath for the startup probe", func(t *testing.T) { g := NewGomegaWithT(t) podSpec := basePodSpec() @@ -70,7 +71,7 @@ func TestInjectContainer(t *testing.T) { initContainer := podSpec.InitContainers[0] g.Expect(initContainer.StartupProbe.Exec.Command).To(Equal( - []string{"test", "-f", path.Join(cloudTokenFileMountPath, "token")}, + []string{"test", "-f", path.Join(config.CloudTokenMountPath, "token")}, ), "probe must check the token-minter's own mount path, not the main container's") }) @@ -78,7 +79,7 @@ func TestInjectContainer(t *testing.T) { g := NewGomegaWithT(t) podSpec := basePodSpec() - baseOpts.injectContainer(false, podSpec, baseContainer, cloudTokenFileMountPath, "cloud-token") + baseOpts.injectContainer(false, podSpec, baseContainer, config.CloudTokenMountPath, "cloud-token") g.Expect(podSpec.InitContainers).To(BeEmpty()) g.Expect(podSpec.Containers).To(HaveLen(2)) @@ -101,7 +102,7 @@ func TestInjectContainer(t *testing.T) { for _, nativeSidecars := range []bool{true, false} { podSpec := basePodSpec() - oneShotOpts.injectContainer(nativeSidecars, podSpec, baseContainer, cloudTokenFileMountPath, "cloud-token") + oneShotOpts.injectContainer(nativeSidecars, podSpec, baseContainer, config.CloudTokenMountPath, "cloud-token") g.Expect(podSpec.InitContainers).To(HaveLen(1), "oneshot minters should be injected as init containers") g.Expect(podSpec.Containers).To(HaveLen(1), "oneshot minters should not be added to regular containers") @@ -115,12 +116,12 @@ func TestInjectContainer(t *testing.T) { for _, nativeSidecars := range []bool{true, false} { podSpec := basePodSpec() - baseOpts.injectContainer(nativeSidecars, podSpec, baseContainer, cloudTokenFileMountPath, "cloud-token") + baseOpts.injectContainer(nativeSidecars, podSpec, baseContainer, config.CloudTokenMountPath, "cloud-token") mainContainer := podSpec.Containers[0] g.Expect(mainContainer.VolumeMounts).To(ContainElement(corev1.VolumeMount{ Name: "cloud-token", - MountPath: cloudTokenFileMountPath, + MountPath: config.CloudTokenMountPath, })) } }) From c71936fe29e37e7a9dac49a3f65c2bd85de22eb5 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:53:59 -0400 Subject: [PATCH 5/7] feat(control-plane-operator): support Azure KMS encryption on self-managed clusters - Add Azure KMS provider with self-managed mode support using token-minter sidecar for workload identity authentication - Add KMS pod configuration for both managed identity and workload identity authentication modes - Add encryption config generation for Azure KMS with key hashing - Update KAS deployment to mount KMS-specific volumes and containers - Add self-managed Azure KMS unit tests for encryption config and pod configuration Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- .../cloud/azure/providerconfig.go | 1 + .../hostedcontrolplane_controller.go | 16 +- .../hostedcontrolplane_controller_test.go | 3 + .../hostedcontrolplane/manifests/kas.go | 2 +- .../hostedcontrolplane/v2/kas/deployment.go | 2 +- .../hostedcontrolplane/v2/kas/kms.go | 24 +- .../hostedcontrolplane/v2/kas/kms/aws.go | 4 +- .../hostedcontrolplane/v2/kas/kms/azure.go | 127 ++- .../v2/kas/kms/azure_test.go | 965 ++++++++++++++++++ .../v2/kas/secretencryption_test.go | 240 +++++ 10 files changed, 1363 insertions(+), 21 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure_test.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go index c4e36d904c4..052153b9d16 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/cloud/azure/providerconfig.go @@ -14,6 +14,7 @@ type AzureConfig struct { Cloud string `json:"cloud"` TenantID string `json:"tenantId"` UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` + UseWorkloadIdentityExtension bool `json:"useWorkloadIdentityExtension"` SubscriptionID string `json:"subscriptionId"` AADClientID string `json:"aadClientId"` AADClientCertPath string `json:"aadClientCertPath"` diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index d21ee116ffd..82a1498a7bb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -2913,7 +2913,7 @@ func (r *HostedControlPlaneReconciler) validateAWSKMSConfig(ctx context.Context, return } - sa := manifests.KASContainerAWSKMSProviderServiceAccount() + sa := manifests.KASContainerKMSProviderServiceAccount() token, err := k8sutil.CreateTokenForServiceAccount(ctx, sa, k8sutil.ServiceAccountClient(guestClient, sa.Namespace)) if err != nil { // service account might not be created in the guest cluster or KAS is not operational. @@ -3031,6 +3031,20 @@ func (r *HostedControlPlaneReconciler) validateAzureKMSConfig(ctx context.Contex } log.Info("Reusing existing UserAssignedManagedIdentity credentials for KMS to authenticate to Azure") } + } else if hyperazureutil.IsSelfManagedAzure(hcp.Spec.Platform.Type) { + // For self-managed Azure, the KMS provider uses workload identity with a federated token + // minted inside the KAS pod. The CPO cannot validate Key Vault access directly because + // it does not have the KMS workload identity credentials. Key Vault access is validated + // at runtime by the azure-kms-provider container. + condition := metav1.Condition{ + Type: string(hyperv1.ValidAzureKMSConfig), + ObservedGeneration: hcp.Generation, + Status: metav1.ConditionTrue, + Message: "KMS configuration accepted; Key Vault access is validated at runtime by the KMS provider", + Reason: hyperv1.AsExpectedReason, + } + meta.SetStatusCondition(&hcp.Status.Conditions, condition) + return } azureKeyVaultDNSSuffix, err := hyperazureutil.GetKeyVaultDNSSuffixFromCloudType(hcp.Spec.Platform.Azure.Cloud) diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go index 3ddf48cd6be..f6c060f3e05 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go @@ -1087,6 +1087,9 @@ func TestControlPlaneComponents(t *testing.T) { KeyName: "test-key", KeyVersion: "1", }, + KMS: hyperv1.ManagedIdentity{ + CredentialsSecretName: "test-kms-creds", + }, KeyVaultAccess: hyperv1.AzureKeyVaultPrivate, }, }, diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go index e070b5ff558..c27a2f1913f 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/kas.go @@ -105,7 +105,7 @@ func KASDeployment(controlPlaneNamespace string) *appsv1.Deployment { } } -func KASContainerAWSKMSProviderServiceAccount() *corev1.ServiceAccount { +func KASContainerKMSProviderServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: "kms-provider", diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go index 9664f02d7da..cedcc84af82 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go @@ -127,7 +127,7 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep applyGenericSecretEncryptionConfig(&deployment.Spec.Template.Spec) switch secretEncryption.Type { case hyperv1.KMS: - if err := applyKMSConfig(&deployment.Spec.Template.Spec, secretEncryption, newKMSImages(hcp)); err != nil { + if err := applyKMSConfig(&deployment.Spec.Template.Spec, secretEncryption, newKMSImages(hcp), hcp); err != nil { return err } } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms.go index 5ae3bffe5f8..993e3976836 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms.go @@ -7,17 +7,18 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms" "github.com/openshift/hypershift/support/api" + "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/podspec" corev1 "k8s.io/api/core/v1" ) -func applyKMSConfig(podSpec *corev1.PodSpec, secretEncryptionData *hyperv1.SecretEncryptionSpec, images kmsImages) error { +func applyKMSConfig(podSpec *corev1.PodSpec, secretEncryptionData *hyperv1.SecretEncryptionSpec, images kmsImages, hcp *hyperv1.HostedControlPlane) error { if secretEncryptionData.KMS == nil { return fmt.Errorf("kms metadata not specified") } - provider, err := getKMSProvider(secretEncryptionData.KMS, images) + provider, err := getKMSProvider(secretEncryptionData.KMS, images, hcp) if err != nil { return err } @@ -34,7 +35,7 @@ func applyKMSConfig(podSpec *corev1.PodSpec, secretEncryptionData *hyperv1.Secre } func generateKMSEncryptionConfig(kmsSpec *hyperv1.KMSSpec, apiVersion string) ([]byte, error) { - provider, err := getKMSProvider(kmsSpec, kmsImages{}) + provider, err := getKMSProvider(kmsSpec, kmsImages{}, nil) if err != nil { return nil, err } @@ -52,14 +53,27 @@ func generateKMSEncryptionConfig(kmsSpec *hyperv1.KMSSpec, apiVersion string) ([ return bufferInstance.Bytes(), nil } -func getKMSProvider(kmsSpec *hyperv1.KMSSpec, images kmsImages) (kms.KMSProvider, error) { +// getKMSProvider returns a KMS provider for the given spec. When hcp is nil (called from +// generateKMSEncryptionConfig), the provider is always created as "managed" because encryption +// config generation only produces the EncryptionConfiguration resource and does not need +// platform-specific pod/volume configuration. +func getKMSProvider(kmsSpec *hyperv1.KMSSpec, images kmsImages, hcp *hyperv1.HostedControlPlane) (kms.KMSProvider, error) { switch kmsSpec.Provider { case hyperv1.IBMCloud: return kms.NewIBMCloudKMSProvider(kmsSpec.IBMCloud, images.IBMCloudKMS) case hyperv1.AWS: return kms.NewAWSKMSProvider(kmsSpec.AWS, images.AWSKMS, images.TokenMinterImage) case hyperv1.AZURE: - return kms.NewAzureKMSProvider(kmsSpec.Azure, images.AzureKMS) + isSelfManaged := hcp != nil && azureutil.IsSelfManagedAzure(hcp.Spec.Platform.Type) + opts := kms.AzureKMSProviderOptions{ + IsSelfManaged: isSelfManaged, + TokenMinterImage: images.TokenMinterImage, + } + if isSelfManaged && kmsSpec.Azure.WorkloadIdentity.ClientID != "" { + opts.KMSClientID = string(kmsSpec.Azure.WorkloadIdentity.ClientID) + opts.TenantID = hcp.Spec.Platform.Azure.TenantID + } + return kms.NewAzureKMSProvider(kmsSpec.Azure, images.AzureKMS, opts) default: return nil, fmt.Errorf("unrecognized kms provider %s", kmsSpec.Provider) } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/aws.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/aws.go index 8696f37bdaf..40725bc04ea 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/aws.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/aws.go @@ -253,8 +253,8 @@ func buildKASContainerAWSKMSTokenMinter(image string) func(*corev1.Container) { c.Command = []string{"/usr/bin/control-plane-operator", "token-minter"} c.Args = []string{ "--token-audience=openshift", - fmt.Sprintf("--service-account-namespace=%s", manifests.KASContainerAWSKMSProviderServiceAccount().Namespace), - fmt.Sprintf("--service-account-name=%s", manifests.KASContainerAWSKMSProviderServiceAccount().Name), + fmt.Sprintf("--service-account-namespace=%s", manifests.KASContainerKMSProviderServiceAccount().Namespace), + fmt.Sprintf("--service-account-name=%s", manifests.KASContainerKMSProviderServiceAccount().Name), fmt.Sprintf("--token-file=%s", path.Join(awsKMSVolumeMounts.Path(c.Name, kasVolumeAWSKMSCloudProviderToken().Name), "token")), fmt.Sprintf("--kubeconfig=%s", path.Join(awsKMSVolumeMounts.Path(c.Name, kasVolumeLocalhostKubeconfig), podspec.KubeconfigKey)), } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure.go index f7e8db3a219..8f9cbb83539 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure.go @@ -2,6 +2,7 @@ package kms import ( "fmt" + "path" "time" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -58,17 +59,41 @@ var ( var _ KMSProvider = &azureKMSProvider{} type azureKMSProvider struct { - kmsSpec *hyperv1.AzureKMSSpec - kmsImage string + kmsSpec *hyperv1.AzureKMSSpec + kmsImage string + isSelfManaged bool + kmsClientID string + tenantID string + tokenMinterImage string } -func NewAzureKMSProvider(kmsSpec *hyperv1.AzureKMSSpec, image string) (*azureKMSProvider, error) { +// AzureKMSProviderOptions contains optional configuration for Azure KMS providers. +type AzureKMSProviderOptions struct { + IsSelfManaged bool + KMSClientID string + TenantID string + TokenMinterImage string +} + +func NewAzureKMSProvider(kmsSpec *hyperv1.AzureKMSSpec, image string, opts AzureKMSProviderOptions) (*azureKMSProvider, error) { if kmsSpec == nil { return nil, fmt.Errorf("azure kms metadata not specified") } + if opts.IsSelfManaged { + if opts.KMSClientID == "" || opts.TenantID == "" { + return nil, fmt.Errorf("kmsClientID and tenantID are required for self-managed Azure KMS") + } + if opts.TokenMinterImage == "" { + return nil, fmt.Errorf("tokenMinterImage is required for self-managed Azure KMS") + } + } return &azureKMSProvider{ - kmsSpec: kmsSpec, - kmsImage: image, + kmsSpec: kmsSpec, + kmsImage: image, + isSelfManaged: opts.IsSelfManaged, + kmsClientID: opts.KMSClientID, + tenantID: opts.TenantID, + tokenMinterImage: opts.TokenMinterImage, }, nil } @@ -126,9 +151,18 @@ func (p *azureKMSProvider) GenerateKMSPodConfig() (*KMSPodConfig, error) { podConfig.Volumes = append(podConfig.Volumes, podspec.BuildVolume(kasVolumeAzureKMSCredentials(), buildVolumeAzureKMSCredentials), podspec.BuildVolume(kasVolumeKMSSocket(), buildVolumeKMSSocket), - podspec.BuildVolume(kasVolumeKMSSecretStore(), buildVolumeKMSSecretStore), ) + if p.isSelfManaged { + podConfig.Volumes = append(podConfig.Volumes, + podspec.BuildVolume(kasVolumeAzureKMSCloudToken(), buildVolumeAzureKMSCloudToken), + ) + } else { + podConfig.Volumes = append(podConfig.Volumes, + podspec.BuildVolume(kasVolumeKMSSecretStore(), buildVolumeKMSSecretStore), + ) + } + podConfig.Containers = append(podConfig.Containers, podspec.BuildContainer( kasContainerAzureKMSActive(), @@ -142,6 +176,12 @@ func (p *azureKMSProvider) GenerateKMSPodConfig() (*KMSPodConfig, error) { ) } + if p.isSelfManaged { + podConfig.Containers = append(podConfig.Containers, + podspec.BuildContainer(kasContainerAzureKMSTokenMinter(), p.buildKASContainerAzureKMSTokenMinter()), + ) + } + podConfig.KASContainerMutate = func(c *corev1.Container) { c.VolumeMounts = append(c.VolumeMounts, azureKMSVolumeMounts.ContainerMounts(KasMainContainerName)...) } @@ -172,11 +212,26 @@ func (p *azureKMSProvider) buildKASContainerAzureKMS(kmsKey hyperv1.AzureKMSKey, "-v=1", } c.VolumeMounts = azureKMSVolumeMounts.ContainerMounts(c.Name) - c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ - Name: config.ManagedAzureKMSSecretStoreVolumeName, - MountPath: config.ManagedAzureCertificateMountPath, - ReadOnly: true, - }) + + if p.isSelfManaged { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: kasVolumeAzureKMSCloudToken().Name, + MountPath: config.CloudTokenMountPath, + ReadOnly: true, + }) + c.Env = append(c.Env, + corev1.EnvVar{Name: "AZURE_CLIENT_ID", Value: p.kmsClientID}, + corev1.EnvVar{Name: "AZURE_TENANT_ID", Value: p.tenantID}, + corev1.EnvVar{Name: "AZURE_FEDERATED_TOKEN_FILE", Value: path.Join(config.CloudTokenMountPath, "token")}, + ) + } else { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: config.ManagedAzureKMSSecretStoreVolumeName, + MountPath: config.ManagedAzureCertificateMountPath, + ReadOnly: true, + }) + } + c.LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -200,6 +255,37 @@ func (p *azureKMSProvider) buildKASContainerAzureKMS(kmsKey hyperv1.AzureKMSKey, } } +func (p *azureKMSProvider) buildKASContainerAzureKMSTokenMinter() func(*corev1.Container) { + return func(c *corev1.Container) { + c.Image = p.tokenMinterImage + c.ImagePullPolicy = corev1.PullIfNotPresent + c.Command = []string{"/usr/bin/control-plane-operator", "token-minter"} + c.Args = []string{ + "--token-audience=openshift", + fmt.Sprintf("--service-account-namespace=%s", manifests.KASContainerKMSProviderServiceAccount().Namespace), + fmt.Sprintf("--service-account-name=%s", manifests.KASContainerKMSProviderServiceAccount().Name), + fmt.Sprintf("--token-file=%s", path.Join(config.CloudTokenMountPath, "token")), + fmt.Sprintf("--kubeconfig=%s", path.Join("/etc/kubernetes", podspec.KubeconfigKey)), + } + c.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("30Mi"), + }, + } + c.VolumeMounts = []corev1.VolumeMount{ + { + Name: kasVolumeAzureKMSCloudToken().Name, + MountPath: config.CloudTokenMountPath, + }, + { + Name: kasVolumeLocalhostKubeconfig, + MountPath: "/etc/kubernetes", + }, + } + } +} + func kasContainerAzureKMSActive() *corev1.Container { return &corev1.Container{ Name: "azure-kms-provider-active", @@ -212,6 +298,12 @@ func kasContainerAzureKMSBackup() *corev1.Container { } } +func kasContainerAzureKMSTokenMinter() *corev1.Container { + return &corev1.Container{ + Name: "azure-kms-token-minter", + } +} + func kasVolumeAzureKMSCredentials() *corev1.Volume { return &corev1.Volume{ Name: "azure-kms-credentials", @@ -248,8 +340,21 @@ func buildVolumeKMSSecretStore(v *corev1.Volume) { } } +func kasVolumeAzureKMSCloudToken() *corev1.Volume { + return &corev1.Volume{ + Name: "azure-kms-cloud-token", + } +} + +func buildVolumeAzureKMSCloudToken(v *corev1.Volume) { + v.EmptyDir = &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory} +} + func AdaptAzureSecretProvider(cpContext component.WorkloadContext, secretProvider *secretsstorev1.SecretProviderClass) error { managedIdentity := cpContext.HCP.Spec.SecretEncryption.KMS.Azure.KMS + if managedIdentity.CredentialsSecretName == "" { + return fmt.Errorf("managed identity credentials secret name is required for Azure KMS secret provider") + } secretproviderclass.ReconcileManagedAzureSecretProviderClass(secretProvider, cpContext.HCP, managedIdentity) return nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure_test.go new file mode 100644 index 00000000000..0b2bc35ec58 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kms/azure_test.go @@ -0,0 +1,965 @@ +package kms + +import ( + "fmt" + "path" + "testing" + "time" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/config" + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/podspec" + "github.com/openshift/hypershift/support/util" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" +) + +func validAzureKMSSpec() *hyperv1.AzureKMSSpec { + return &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "1", + }, + } +} + +func validAzureKMSSpecWithBackup() *hyperv1.AzureKMSSpec { + spec := validAzureKMSSpec() + spec.BackupKey = &hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "backup-key", + KeyVersion: "1", + } + return spec +} + +func TestNewAzureKMSProvider(t *testing.T) { + tests := []struct { + name string + kmsSpec *hyperv1.AzureKMSSpec + image string + opts AzureKMSProviderOptions + expectError bool + errContains string + }{ + { + name: "When kmsSpec is nil it should return an error", + kmsSpec: nil, + image: "test-image:latest", + opts: AzureKMSProviderOptions{}, + expectError: true, + errContains: "azure kms metadata not specified", + }, + { + name: "When self-managed with empty kmsClientID it should return an error", + kmsSpec: validAzureKMSSpec(), + image: "test-image:latest", + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "", + TenantID: "test-tenant-id", + }, + expectError: true, + errContains: "kmsClientID and tenantID are required", + }, + { + name: "When self-managed with empty tenantID it should return an error", + kmsSpec: validAzureKMSSpec(), + image: "test-image:latest", + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "", + }, + expectError: true, + errContains: "kmsClientID and tenantID are required", + }, + { + name: "When self-managed with empty tokenMinterImage it should return an error", + kmsSpec: validAzureKMSSpec(), + image: "test-image:latest", + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "", + }, + expectError: true, + errContains: "tokenMinterImage is required", + }, + { + name: "When managed Azure it should create provider successfully", + kmsSpec: validAzureKMSSpec(), + image: "test-image:latest", + opts: AzureKMSProviderOptions{ + IsSelfManaged: false, + }, + expectError: false, + }, + { + name: "When self-managed Azure with valid options it should create provider successfully", + kmsSpec: validAzureKMSSpec(), + image: "test-image:latest", + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }, + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(tc.kmsSpec, tc.image, tc.opts) + if tc.expectError { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.errContains)) + g.Expect(provider).To(BeNil()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(provider).NotTo(BeNil()) + } + }) + } +} + +func TestGenerateKMSPodConfig_SelfManaged(t *testing.T) { + tests := []struct { + name string + check func(g Gomega, podConfig *KMSPodConfig) + }{ + { + name: "When self-managed it should include token minter container with correct config", + check: func(g Gomega, podConfig *KMSPodConfig) { + var tokenMinter *containerInfo + for i, c := range podConfig.Containers { + if c.Name == "azure-kms-token-minter" { + tokenMinter = &containerInfo{idx: i, container: c} + break + } + } + g.Expect(tokenMinter).NotTo(BeNil(), "expected token minter container to be present") + g.Expect(tokenMinter.container.Command).To(Equal([]string{"/usr/bin/control-plane-operator", "token-minter"})) + g.Expect(tokenMinter.container.Args).To(ContainElement("--token-audience=openshift")) + g.Expect(tokenMinter.container.Args).To(ContainElement("--service-account-namespace=kube-system")) + g.Expect(tokenMinter.container.Args).To(ContainElement("--service-account-name=kms-provider")) + g.Expect(tokenMinter.container.Args).To(ContainElement( + ContainSubstring("--token-file=" + path.Join(config.CloudTokenMountPath, "token")), + )) + g.Expect(tokenMinter.container.Args).To(ContainElement( + ContainSubstring("--kubeconfig=" + path.Join("/etc/kubernetes", podspec.KubeconfigKey)), + )) + + // Verify volume mounts + volumeNames := make([]string, 0, len(tokenMinter.container.VolumeMounts)) + for _, vm := range tokenMinter.container.VolumeMounts { + volumeNames = append(volumeNames, vm.Name) + } + g.Expect(volumeNames).To(ContainElement("azure-kms-cloud-token")) + g.Expect(volumeNames).To(ContainElement(kasVolumeLocalhostKubeconfig)) + }, + }, + { + name: "When self-managed it should include cloud-token emptyDir volume", + check: func(g Gomega, podConfig *KMSPodConfig) { + found := false + for _, v := range podConfig.Volumes { + if v.Name == "azure-kms-cloud-token" { + found = true + g.Expect(v.EmptyDir).NotTo(BeNil(), "expected cloud-token volume to be an emptyDir") + break + } + } + g.Expect(found).To(BeTrue(), "expected cloud-token volume to be present") + }, + }, + { + name: "When self-managed it should NOT include secret-store CSI volume", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, v := range podConfig.Volumes { + g.Expect(v.Name).NotTo(Equal(config.ManagedAzureKMSSecretStoreVolumeName), + "expected secret-store CSI volume to NOT be present in self-managed mode") + } + }, + }, + { + name: "When self-managed the KMS container should mount cloud-token volume", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, c := range podConfig.Containers { + if c.Name == "azure-kms-provider-active" { + found := false + for _, vm := range c.VolumeMounts { + if vm.Name == "azure-kms-cloud-token" { + found = true + g.Expect(vm.MountPath).To(Equal(config.CloudTokenMountPath)) + g.Expect(vm.ReadOnly).To(BeTrue()) + break + } + } + g.Expect(found).To(BeTrue(), "expected active KMS container to mount cloud-token volume") + return + } + } + g.Expect(false).To(BeTrue(), "active KMS container not found") + }, + }, + { + name: "When self-managed the KMS container should have workload identity env vars", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, c := range podConfig.Containers { + if c.Name == "azure-kms-provider-active" { + envMap := map[string]string{} + for _, e := range c.Env { + envMap[e.Name] = e.Value + } + g.Expect(envMap).To(HaveKeyWithValue("AZURE_CLIENT_ID", "test-client-id")) + g.Expect(envMap).To(HaveKeyWithValue("AZURE_TENANT_ID", "test-tenant-id")) + g.Expect(envMap).To(HaveKeyWithValue("AZURE_FEDERATED_TOKEN_FILE", + path.Join(config.CloudTokenMountPath, "token"))) + return + } + } + g.Expect(false).To(BeTrue(), "active KMS container not found") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(podConfig).NotTo(BeNil()) + + tc.check(g, podConfig) + }) + } +} + +type containerInfo struct { + idx int + container corev1.Container +} + +func TestGenerateKMSPodConfig_Managed(t *testing.T) { + tests := []struct { + name string + check func(g Gomega, podConfig *KMSPodConfig) + }{ + { + name: "When managed it should NOT include token minter container", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, c := range podConfig.Containers { + g.Expect(c.Name).NotTo(Equal("azure-kms-token-minter"), + "expected token minter container to NOT be present in managed mode") + } + }, + }, + { + name: "When managed it should include secret-store CSI volume", + check: func(g Gomega, podConfig *KMSPodConfig) { + found := false + for _, v := range podConfig.Volumes { + if v.Name == config.ManagedAzureKMSSecretStoreVolumeName { + found = true + g.Expect(v.CSI).NotTo(BeNil(), "expected secret-store volume to use CSI driver") + g.Expect(v.CSI.Driver).To(Equal(config.ManagedAzureSecretsStoreCSIDriver)) + break + } + } + g.Expect(found).To(BeTrue(), "expected secret-store CSI volume to be present") + }, + }, + { + name: "When managed it should NOT include cloud-token volume", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, v := range podConfig.Volumes { + g.Expect(v.Name).NotTo(Equal("azure-kms-cloud-token"), + "expected cloud-token volume to NOT be present in managed mode") + } + }, + }, + { + name: "When managed the KMS container should NOT have workload identity env vars", + check: func(g Gomega, podConfig *KMSPodConfig) { + for _, c := range podConfig.Containers { + if c.Name == "azure-kms-provider-active" { + for _, e := range c.Env { + g.Expect(e.Name).NotTo(Equal("AZURE_CLIENT_ID"), + "expected managed KMS container to NOT have AZURE_CLIENT_ID env var") + } + return + } + } + g.Expect(false).To(BeTrue(), "active KMS container not found") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(podConfig).NotTo(BeNil()) + + tc.check(g, podConfig) + }) + } +} + +func TestGenerateKMSPodConfig_BackupKey(t *testing.T) { + t.Run("When self-managed backup key is specified it should include backup KMS container with env vars", func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpecWithBackup(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(podConfig).NotTo(BeNil()) + + found := false + for _, c := range podConfig.Containers { + if c.Name == "azure-kms-provider-backup" { + found = true + envMap := map[string]string{} + for _, e := range c.Env { + envMap[e.Name] = e.Value + } + g.Expect(envMap).To(HaveKeyWithValue("AZURE_CLIENT_ID", "test-client-id")) + g.Expect(envMap).To(HaveKeyWithValue("AZURE_TENANT_ID", "test-tenant-id")) + g.Expect(envMap).To(HaveKeyWithValue("AZURE_FEDERATED_TOKEN_FILE", path.Join(config.CloudTokenMountPath, "token"))) + break + } + } + g.Expect(found).To(BeTrue(), "expected backup KMS container to be present when backup key is specified") + }) + + t.Run("When managed backup key is specified it should include backup container with secret-store mount", func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpecWithBackup(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(podConfig).NotTo(BeNil()) + + found := false + for _, c := range podConfig.Containers { + if c.Name == "azure-kms-provider-backup" { + found = true + hasSecretStore := false + hasCloudToken := false + for _, vm := range c.VolumeMounts { + if vm.Name == config.ManagedAzureKMSSecretStoreVolumeName { + hasSecretStore = true + } + if vm.Name == "azure-kms-cloud-token" { + hasCloudToken = true + } + } + g.Expect(hasSecretStore).To(BeTrue(), "expected backup KMS container to have secret-store mount") + g.Expect(hasCloudToken).To(BeFalse(), "expected backup KMS container to NOT have cloud-token mount") + break + } + } + g.Expect(found).To(BeTrue(), "expected backup KMS container to be present when backup key is specified") + }) +} + +func findContainer(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + +func TestGenerateKMSPodConfig_ActiveContainerArgs(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + active := findContainer(podConfig.Containers, "azure-kms-provider-active") + g.Expect(active).NotTo(BeNil(), "expected active KMS container to exist") + + tests := []struct { + name string + expected string + }{ + { + name: "When active KMS container is created it should pass the key vault name", + expected: "--keyvault-name=test-vault", + }, + { + name: "When active KMS container is created it should pass the key name", + expected: "--key-name=test-key", + }, + { + name: "When active KMS container is created it should pass the key version", + expected: "--key-version=1", + }, + { + name: "When active KMS container is created it should listen on the active unix socket", + expected: fmt.Sprintf("--listen-addr=unix:///opt/%s", azureActiveKMSUnixSocketFileName), + }, + { + name: "When active KMS container is created it should use port 8787 for health checks", + expected: fmt.Sprintf("--healthz-port=%d", azureActiveKMSHealthPort), + }, + { + name: "When active KMS container is created it should expose metrics on port 8095", + expected: fmt.Sprintf("--metrics-addr=%s", azureActiveKMSMetricsAddr), + }, + { + name: "When active KMS container is created it should set the healthz path", + expected: "--healthz-path=/healthz", + }, + { + name: "When active KMS container is created it should point to the azure.json config file", + expected: "--config-file-path=/etc/kubernetes/azure.json", + }, + { + name: "When active KMS container is created it should enable verbose logging", + expected: "-v=1", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(active.Args).To(ContainElement(tc.expected)) + }) + } +} + +func TestGenerateKMSPodConfig_BackupContainerArgs(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpecWithBackup(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + backup := findContainer(podConfig.Containers, "azure-kms-provider-backup") + g.Expect(backup).NotTo(BeNil(), "expected backup KMS container to exist") + + tests := []struct { + name string + expected string + }{ + { + name: "When backup KMS container is created it should pass the backup key vault name", + expected: "--keyvault-name=test-vault", + }, + { + name: "When backup KMS container is created it should pass the backup key name", + expected: "--key-name=backup-key", + }, + { + name: "When backup KMS container is created it should listen on the backup unix socket", + expected: fmt.Sprintf("--listen-addr=unix:///opt/%s", azureBackupKMSUnixSocketFileName), + }, + { + name: "When backup KMS container is created it should use port 8788 for health checks", + expected: fmt.Sprintf("--healthz-port=%d", azureBackupKMSHealthPort), + }, + { + name: "When backup KMS container is created it should expose metrics on port 8096", + expected: fmt.Sprintf("--metrics-addr=%s", azureBackupKMSMetricsAddr), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(backup.Args).To(ContainElement(tc.expected)) + }) + } +} + +func TestGenerateKMSPodConfig_LivenessProbe(t *testing.T) { + tests := []struct { + name string + containerFn func(podConfig *KMSPodConfig) *corev1.Container + healthPort int + }{ + { + name: "active KMS container", + containerFn: func(podConfig *KMSPodConfig) *corev1.Container { + return findContainer(podConfig.Containers, "azure-kms-provider-active") + }, + healthPort: azureActiveKMSHealthPort, + }, + { + name: "backup KMS container", + containerFn: func(podConfig *KMSPodConfig) *corev1.Container { + return findContainer(podConfig.Containers, "azure-kms-provider-backup") + }, + healthPort: azureBackupKMSHealthPort, + }, + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("When %s is created it should have a correctly configured liveness probe", tc.name), func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpecWithBackup(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + c := tc.containerFn(podConfig) + g.Expect(c).NotTo(BeNil()) + g.Expect(c.LivenessProbe).NotTo(BeNil(), "expected liveness probe to be configured") + g.Expect(c.LivenessProbe.HTTPGet).NotTo(BeNil(), "expected HTTP GET probe handler") + g.Expect(c.LivenessProbe.HTTPGet.Path).To(Equal("/healthz")) + g.Expect(c.LivenessProbe.HTTPGet.Port.IntValue()).To(Equal(tc.healthPort)) + g.Expect(c.LivenessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTP)) + g.Expect(c.LivenessProbe.InitialDelaySeconds).To(Equal(int32(120))) + g.Expect(c.LivenessProbe.PeriodSeconds).To(Equal(int32(300))) + g.Expect(c.LivenessProbe.TimeoutSeconds).To(Equal(int32(160))) + g.Expect(c.LivenessProbe.FailureThreshold).To(Equal(int32(3))) + g.Expect(c.LivenessProbe.SuccessThreshold).To(Equal(int32(1))) + }) + } +} + +func TestGenerateKMSPodConfig_ResourceRequests(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + tests := []struct { + name string + container string + cpu string + memory string + }{ + { + name: "When active KMS container is created it should request 10m CPU and 10Mi memory", + container: "azure-kms-provider-active", + cpu: "10m", + memory: "10Mi", + }, + { + name: "When token minter container is created it should request 10m CPU and 30Mi memory", + container: "azure-kms-token-minter", + cpu: "10m", + memory: "30Mi", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c := findContainer(podConfig.Containers, tc.container) + g.Expect(c).NotTo(BeNil()) + g.Expect(c.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse(tc.cpu))) + g.Expect(c.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse(tc.memory))) + }) + } +} + +func TestGenerateKMSPodConfig_VolumeMountPaths(t *testing.T) { + tests := []struct { + name string + isSelfManaged bool + opts AzureKMSProviderOptions + checks func(g Gomega, podConfig *KMSPodConfig) + }{ + { + name: "When self-managed KMS container is created it should mount credentials at /etc/kubernetes and socket at /opt", + isSelfManaged: true, + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }, + checks: func(g Gomega, podConfig *KMSPodConfig) { + c := findContainer(podConfig.Containers, "azure-kms-provider-active") + g.Expect(c).NotTo(BeNil()) + mountMap := map[string]string{} + for _, vm := range c.VolumeMounts { + mountMap[vm.Name] = vm.MountPath + } + g.Expect(mountMap).To(HaveKeyWithValue("azure-kms-credentials", "/etc/kubernetes")) + g.Expect(mountMap).To(HaveKeyWithValue("kms-socket", "/opt")) + g.Expect(mountMap).To(HaveKeyWithValue("azure-kms-cloud-token", config.CloudTokenMountPath)) + }, + }, + { + name: "When managed KMS container is created it should mount secret-store CSI volume at the certificate path", + isSelfManaged: false, + opts: AzureKMSProviderOptions{ + IsSelfManaged: false, + }, + checks: func(g Gomega, podConfig *KMSPodConfig) { + c := findContainer(podConfig.Containers, "azure-kms-provider-active") + g.Expect(c).NotTo(BeNil()) + mountMap := map[string]string{} + for _, vm := range c.VolumeMounts { + mountMap[vm.Name] = vm.MountPath + } + g.Expect(mountMap).To(HaveKeyWithValue("azure-kms-credentials", "/etc/kubernetes")) + g.Expect(mountMap).To(HaveKeyWithValue("kms-socket", "/opt")) + g.Expect(mountMap).To(HaveKeyWithValue(config.ManagedAzureKMSSecretStoreVolumeName, config.ManagedAzureCertificateMountPath)) + }, + }, + { + name: "When token minter is created it should mount cloud-token and kubeconfig volumes", + isSelfManaged: true, + opts: AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }, + checks: func(g Gomega, podConfig *KMSPodConfig) { + c := findContainer(podConfig.Containers, "azure-kms-token-minter") + g.Expect(c).NotTo(BeNil()) + mountMap := map[string]string{} + for _, vm := range c.VolumeMounts { + mountMap[vm.Name] = vm.MountPath + } + g.Expect(mountMap).To(HaveKeyWithValue("azure-kms-cloud-token", config.CloudTokenMountPath)) + g.Expect(mountMap).To(HaveKeyWithValue(kasVolumeLocalhostKubeconfig, "/etc/kubernetes")) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", tc.opts) + g.Expect(err).NotTo(HaveOccurred()) + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + tc.checks(g, podConfig) + }) + } +} + +func TestGenerateKMSPodConfig_KASContainerMutation(t *testing.T) { + t.Run("When KAS container mutation is applied it should mount the KMS socket volume at /opt", func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + kasContainer := &corev1.Container{Name: KasMainContainerName} + podConfig.KASContainerMutate(kasContainer) + + mountMap := map[string]string{} + for _, vm := range kasContainer.VolumeMounts { + mountMap[vm.Name] = vm.MountPath + } + g.Expect(mountMap).To(HaveKeyWithValue("kms-socket", "/opt")) + }) +} + +func TestGenerateKMSPodConfig_ContainerPorts(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpecWithBackup(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + tests := []struct { + name string + containerName string + expectedPort int32 + }{ + { + name: "When active KMS container is created it should expose health port 8787", + containerName: "azure-kms-provider-active", + expectedPort: int32(azureActiveKMSHealthPort), + }, + { + name: "When backup KMS container is created it should expose health port 8788", + containerName: "azure-kms-provider-backup", + expectedPort: int32(azureBackupKMSHealthPort), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c := findContainer(podConfig.Containers, tc.containerName) + g.Expect(c).NotTo(BeNil()) + g.Expect(c.Ports).To(HaveLen(1)) + g.Expect(c.Ports[0].Name).To(Equal("http")) + g.Expect(c.Ports[0].ContainerPort).To(Equal(tc.expectedPort)) + g.Expect(c.Ports[0].Protocol).To(Equal(corev1.ProtocolTCP)) + }) + } +} + +func TestGenerateKMSPodConfig_ImagePullPolicy(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: true, + KMSClientID: "test-client-id", + TenantID: "test-tenant-id", + TokenMinterImage: "test-token-minter:latest", + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + tests := []struct { + name string + containerName string + }{ + { + name: "When active KMS container is created it should use IfNotPresent pull policy", + containerName: "azure-kms-provider-active", + }, + { + name: "When token minter container is created it should use IfNotPresent pull policy", + containerName: "azure-kms-token-minter", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + c := findContainer(podConfig.Containers, tc.containerName) + g.Expect(c).NotTo(BeNil()) + g.Expect(c.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + }) + } +} + +func TestGenerateKMSPodConfig_NoBackupContainerWithoutBackupKey(t *testing.T) { + t.Run("When no backup key is specified it should not create a backup container", func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(validAzureKMSSpec(), "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + podConfig, err := provider.GenerateKMSPodConfig() + g.Expect(err).NotTo(HaveOccurred()) + + backup := findContainer(podConfig.Containers, "azure-kms-provider-backup") + g.Expect(backup).To(BeNil(), "expected no backup container when backup key is not specified") + }) +} + +func TestGenerateKMSEncryptionConfig_Azure(t *testing.T) { + tests := []struct { + name string + spec *hyperv1.AzureKMSSpec + checks func(g Gomega, encConfig interface{}) + }{ + { + name: "When only active key is configured it should create encryption config with one KMS provider and Identity fallback", + spec: validAzureKMSSpec(), + }, + { + name: "When backup key is also configured it should create encryption config with two KMS providers and Identity fallback", + spec: validAzureKMSSpecWithBackup(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + provider, err := NewAzureKMSProvider(tc.spec, "test-kms-image:latest", AzureKMSProviderOptions{ + IsSelfManaged: false, + }) + g.Expect(err).NotTo(HaveOccurred()) + + encConfig, err := provider.GenerateKMSEncryptionConfig("v2") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(encConfig).NotTo(BeNil()) + + g.Expect(encConfig.Kind).To(Equal(encryptionConfigurationKind)) + g.Expect(encConfig.Resources).To(HaveLen(1)) + g.Expect(encConfig.Resources[0].Resources).To(Equal(config.KMSEncryptedObjects())) + + providers := encConfig.Resources[0].Providers + if tc.spec.BackupKey != nil { + g.Expect(providers).To(HaveLen(3), "expected active KMS + backup KMS + Identity") + } else { + g.Expect(providers).To(HaveLen(2), "expected active KMS + Identity") + } + + // First provider is always the active KMS + g.Expect(providers[0].KMS).NotTo(BeNil()) + activeKeyHash, err := util.HashStruct(tc.spec.ActiveKey) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(providers[0].KMS.Name).To(Equal(fmt.Sprintf("azure-%s", activeKeyHash))) + g.Expect(providers[0].KMS.APIVersion).To(Equal("v2")) + g.Expect(providers[0].KMS.Endpoint).To(Equal(azureActiveKMSUnixSocket)) + g.Expect(providers[0].KMS.Timeout).To(Equal(&metav1.Duration{Duration: 35 * time.Second})) + + if tc.spec.BackupKey != nil { + g.Expect(providers[1].KMS).NotTo(BeNil()) + backupKeyHash, err := util.HashStruct(tc.spec.BackupKey) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(providers[1].KMS.Name).To(Equal(fmt.Sprintf("azure-%s", backupKeyHash))) + g.Expect(providers[1].KMS.Endpoint).To(Equal(azureBackupKMSUnixSocket)) + g.Expect(providers[1].KMS.Timeout).To(Equal(&metav1.Duration{Duration: 35 * time.Second})) + // Last provider is Identity + g.Expect(providers[2].Identity).NotTo(BeNil()) + } else { + // Last provider is Identity + g.Expect(providers[1].Identity).NotTo(BeNil()) + } + }) + } +} + +func TestAdaptAzureSecretProvider(t *testing.T) { + tests := []struct { + name string + credSecret string + expectError bool + errContains string + }{ + { + name: "When managed identity credentials secret name is empty it should return an error", + credSecret: "", + expectError: true, + errContains: "managed identity credentials secret name is required", + }, + { + name: "When managed identity credentials secret name is set it should succeed", + credSecret: "kms-identity-creds", + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: "test-ns", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + SecretEncryption: &hyperv1.SecretEncryptionSpec{ + KMS: &hyperv1.KMSSpec{ + Provider: hyperv1.AZURE, + Azure: &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "1", + }, + KMS: hyperv1.ManagedIdentity{ + CredentialsSecretName: tc.credSecret, + }, + }, + }, + }, + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + AzureAuthenticationConfig: hyperv1.AzureAuthenticationConfiguration{ + AzureAuthenticationConfigType: hyperv1.AzureAuthenticationTypeManagedIdentities, + ManagedIdentities: &hyperv1.AzureResourceManagedIdentities{ + ControlPlane: hyperv1.ControlPlaneManagedIdentities{ + ManagedIdentitiesKeyVault: hyperv1.ManagedAzureKeyVault{ + Name: "test-kv", + TenantID: "test-tenant-id", + }, + }, + }, + }, + }, + }, + }, + } + + cpContext := component.WorkloadContext{ + HCP: hcp, + } + secretProvider := &secretsstorev1.SecretProviderClass{} + + err := AdaptAzureSecretProvider(cpContext, secretProvider) + if tc.expectError { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.errContains)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go index 0c8a9bccd1b..82d2ea68d32 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go @@ -9,6 +9,7 @@ import ( "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/config" controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -154,6 +155,245 @@ func TestReconcileKMSEncryptionConfigAWS(t *testing.T) { } } +func TestReconcileKMSEncryptionConfigAzure(t *testing.T) { + encryptionSpec := &hyperv1.KMSSpec{Provider: hyperv1.AZURE, Azure: &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "test-version", + }, + }} + + testCases := []struct { + name string + config *v1.EncryptionConfiguration + expectedConfig *v1.EncryptionConfiguration + }{ + { + name: "When no existing encryption config it should generate v2 config", + expectedConfig: generateExpectedAzureEncryptionConfig(t, kmsAPIVersionV2), + }, + { + name: "When existing KMS v1 config it should preserve v1", + config: &v1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"}, + Resources: []v1.ResourceConfiguration{ + { + Resources: config.KMSEncryptedObjects(), + Providers: []v1.ProviderConfiguration{ + {KMS: &v1.KMSConfiguration{APIVersion: "v1"}}, + }, + }, + }, + }, + expectedConfig: generateExpectedAzureEncryptionConfig(t, kmsAPIVersionV1), + }, + { + name: "When existing KMS v2 config it should preserve v2", + config: &v1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"}, + Resources: []v1.ResourceConfiguration{ + { + Resources: config.KMSEncryptedObjects(), + Providers: []v1.ProviderConfiguration{ + {KMS: &v1.KMSConfiguration{APIVersion: "v2"}}, + }, + }, + }, + }, + expectedConfig: generateExpectedAzureEncryptionConfig(t, kmsAPIVersionV2), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + encryptionConfigFile := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-encryption-config", + Namespace: "test-namespace", + }, + Data: make(map[string][]byte), + } + + clientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme) + if tc.config != nil { + buff := bytes.NewBuffer([]byte{}) + err := api.YamlSerializer.Encode(tc.config, buff) + if err != nil { + t.Errorf("failed to encode encryption config: %v", err) + } + encryptionConfigFile.Data[secretEncryptionConfigurationKey] = buff.Bytes() + clientBuilder.WithObjects(encryptionConfigFile) + } + + cpContext := controlplanecomponent.WorkloadContext{ + HCP: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + SecretEncryption: &hyperv1.SecretEncryptionSpec{ + Type: hyperv1.KMS, + KMS: encryptionSpec, + }, + }, + }, + Client: clientBuilder.Build(), + } + err := adaptSecretEncryptionConfig(cpContext, encryptionConfigFile) + if err != nil { + t.Errorf("failed to reconcile KMS encryption config: %v", err) + } + + encryptionConfigBytes := encryptionConfigFile.Data[secretEncryptionConfigurationKey] + if len(encryptionConfigBytes) == 0 { + t.Error("reconciled empty encryption config") + } + encConfig := v1.EncryptionConfiguration{} + gvks, _, err := api.Scheme.ObjectKinds(&encConfig) + if err != nil || len(gvks) == 0 { + t.Errorf("cannot determine gvk of resource: %v", err) + } + if _, _, err = api.YamlSerializer.Decode(encryptionConfigBytes, &gvks[0], &encConfig); err != nil { + t.Errorf("cannot decode resource: %v", err) + } + + if diff := cmp.Diff(encConfig, *tc.expectedConfig); diff != "" { + t.Errorf("reconciled encryption config differs from expected: %s", diff) + } + }) + } +} + +func TestReconcileKMSEncryptionConfigAzureSelfManaged(t *testing.T) { + encryptionSpec := &hyperv1.KMSSpec{Provider: hyperv1.AZURE, Azure: &hyperv1.AzureKMSSpec{ + ActiveKey: hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "test-version", + }, + WorkloadIdentity: hyperv1.WorkloadIdentity{ + ClientID: "kms-client-id", + }, + }} + + testCases := []struct { + name string + config *v1.EncryptionConfiguration + expectedConfig *v1.EncryptionConfiguration + }{ + { + name: "When self-managed Azure with no existing config it should generate v2 encryption config", + expectedConfig: generateExpectedAzureEncryptionConfig(t, kmsAPIVersionV2), + }, + { + name: "When self-managed Azure with existing v1 config it should preserve v1", + config: &v1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"}, + Resources: []v1.ResourceConfiguration{ + { + Resources: config.KMSEncryptedObjects(), + Providers: []v1.ProviderConfiguration{ + {KMS: &v1.KMSConfiguration{APIVersion: "v1"}}, + }, + }, + }, + }, + expectedConfig: generateExpectedAzureEncryptionConfig(t, kmsAPIVersionV1), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + encryptionConfigFile := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-encryption-config", + Namespace: "test-namespace", + }, + Data: make(map[string][]byte), + } + + clientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme) + if tc.config != nil { + buff := bytes.NewBuffer([]byte{}) + err := api.YamlSerializer.Encode(tc.config, buff) + if err != nil { + t.Fatalf("failed to encode encryption config: %v", err) + } + encryptionConfigFile.Data[secretEncryptionConfigurationKey] = buff.Bytes() + clientBuilder.WithObjects(encryptionConfigFile) + } + + cpContext := controlplanecomponent.WorkloadContext{ + HCP: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + Azure: &hyperv1.AzurePlatformSpec{ + TenantID: "test-tenant-id", + }, + }, + SecretEncryption: &hyperv1.SecretEncryptionSpec{ + Type: hyperv1.KMS, + KMS: encryptionSpec, + }, + }, + }, + Client: clientBuilder.Build(), + } + err := adaptSecretEncryptionConfig(cpContext, encryptionConfigFile) + if err != nil { + t.Fatalf("failed to reconcile KMS encryption config: %v", err) + } + + encryptionConfigBytes := encryptionConfigFile.Data[secretEncryptionConfigurationKey] + if len(encryptionConfigBytes) == 0 { + t.Fatal("reconciled empty encryption config") + } + encConfig := v1.EncryptionConfiguration{} + gvks, _, err := api.Scheme.ObjectKinds(&encConfig) + if err != nil || len(gvks) == 0 { + t.Fatalf("cannot determine gvk of resource: %v", err) + } + if _, _, err = api.YamlSerializer.Decode(encryptionConfigBytes, &gvks[0], &encConfig); err != nil { + t.Fatalf("cannot decode resource: %v", err) + } + + if diff := cmp.Diff(encConfig, *tc.expectedConfig); diff != "" { + t.Errorf("reconciled encryption config differs from expected: %s", diff) + } + }) + } +} + +func generateExpectedAzureEncryptionConfig(t testing.TB, apiVersion string) *v1.EncryptionConfiguration { + t.Helper() + activeKeyHash, err := util.HashStruct(hyperv1.AzureKMSKey{ + KeyVaultName: "test-vault", + KeyName: "test-key", + KeyVersion: "test-version", + }) + if err != nil { + t.Fatalf("failed to hash Azure KMS key: %v", err) + } + return &v1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"}, + Resources: []v1.ResourceConfiguration{ + { + Resources: config.KMSEncryptedObjects(), + Providers: []v1.ProviderConfiguration{ + { + KMS: &v1.KMSConfiguration{ + APIVersion: apiVersion, + Name: "azure-" + activeKeyHash, + Endpoint: "unix:///opt/azurekmsactive.socket", + Timeout: &metav1.Duration{Duration: 35 * time.Second}, + }, + }, + {Identity: &v1.IdentityConfiguration{}}, + }, + }, + }, + } +} + func generateExpectedEncryptionConfig(apiVersion string) *v1.EncryptionConfiguration { config := &v1.EncryptionConfiguration{ TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"}, From 165c790a203b2eedc5b1a82aee0186d71794f547 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:54:15 -0400 Subject: [PATCH 6/7] test(e2e): add Azure KMS tests for self-managed clusters - Add e2e test for Azure KMS encryption on self-managed clusters - Add envtest validation test cases for Azure KMS mutual exclusivity, immutability (both directions), and key version update scenarios - Add reverse immutability test: switching from workloadIdentity to kms must also fail Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- .../stable.hostedclusters.kms.testsuite.yaml | 611 ++++++++++++++++++ test/e2e/create_cluster_test.go | 13 +- 2 files changed, 619 insertions(+), 5 deletions(-) create mode 100644 cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.kms.testsuite.yaml diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.kms.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.kms.testsuite.yaml new file mode 100644 index 00000000000..4981e7e8c34 --- /dev/null +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.kms.testsuite.yaml @@ -0,0 +1,611 @@ +apiVersion: apiextensions.k8s.io/v1 +name: "HostedCluster Azure KMS validation" +crdName: hostedclusters.hypershift.openshift.io +version: v1beta1 +tests: + onCreate: + - name: When both kms and workloadIdentity are set it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + kms: + credentialsSecretName: kms-creds + objectEncoding: utf-8 + workloadIdentity: + clientID: "22222222-2222-2222-2222-222222222222" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "kms and workloadIdentity are mutually exclusive" + + - name: When neither kms nor workloadIdentity is set it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "one of kms or workloadIdentity must be set" + + - name: When only kms is set it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + kms: + credentialsSecretName: kms-creds + objectEncoding: utf-8 + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + onUpdate: + - name: When switching from kms to workloadIdentity it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + kms: + credentialsSecretName: kms-creds + objectEncoding: utf-8 + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + workloadIdentity: + clientID: "22222222-2222-2222-2222-222222222222" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "the KMS authentication mode is immutable once set" + + - name: When switching from workloadIdentity to kms it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + workloadIdentity: + clientID: "22222222-2222-2222-2222-222222222222" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + kms: + credentialsSecretName: kms-creds + objectEncoding: utf-8 + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "the KMS authentication mode is immutable once set" + + - name: When only workloadIdentity is set and key version is updated it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "1" + workloadIdentity: + clientID: "22222222-2222-2222-2222-222222222222" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: Azure + azure: + location: eastus + resourceGroupName: test-rg + vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" + subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" + subscriptionID: "12345678-1234-5678-9012-123456789012" + securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" + tenantID: "87654321-4321-8765-2109-876543210987" + azureAuthenticationConfig: + azureAuthenticationConfigType: WorkloadIdentities + workloadIdentities: + imageRegistry: + clientID: "11111111-1111-1111-1111-111111111111" + ingress: + clientID: "11111111-1111-1111-1111-111111111111" + file: + clientID: "11111111-1111-1111-1111-111111111111" + disk: + clientID: "11111111-1111-1111-1111-111111111111" + nodePoolManagement: + clientID: "11111111-1111-1111-1111-111111111111" + cloudProvider: + clientID: "11111111-1111-1111-1111-111111111111" + network: + clientID: "11111111-1111-1111-1111-111111111111" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + type: kms + kms: + provider: Azure + azure: + activeKey: + keyVaultName: test-vault + keyName: test-key + keyVersion: "2" + workloadIdentity: + clientID: "22222222-2222-2222-2222-222222222222" + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} diff --git a/test/e2e/create_cluster_test.go b/test/e2e/create_cluster_test.go index 81b1124d1e2..7e5d6563b9f 100644 --- a/test/e2e/create_cluster_test.go +++ b/test/e2e/create_cluster_test.go @@ -377,14 +377,15 @@ func TestCreateClusterCustomConfig(t *testing.T) { if globalOpts.ConfigurableClusterOptions.AzureEncryptionKeyID == "" { t.Fatal("azure encryption key id is required") } - if globalOpts.ConfigurableClusterOptions.AzureKMSUserAssignedCredsSecretName == "" { - t.Fatal("azure kms user assigned creds secret name is required") + // KMS user assigned creds secret name is only required for managed Azure (ARO HCP) + if azureutil.IsAroHCP() && globalOpts.ConfigurableClusterOptions.AzureKMSUserAssignedCredsSecretName == "" { + t.Fatal("azure kms user assigned creds secret name is required for managed Azure") } kmsUserAssignedCredsSecretName = globalOpts.ConfigurableClusterOptions.AzureKMSUserAssignedCredsSecretName kmsKeyInfo, err = azureutil.GetAzureEncryptionKeyInfo(globalOpts.ConfigurableClusterOptions.AzureEncryptionKeyID) if err != nil { - t.Fatal("failed to get azure encryption key info: %w", err) + t.Fatalf("failed to get azure encryption key info: %v", err) } } @@ -404,8 +405,10 @@ func TestCreateClusterCustomConfig(t *testing.T) { g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.ActiveKey.KeyVaultName).To(Equal(kmsKeyInfo.KeyVaultName)) g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.ActiveKey.KeyName).To(Equal(kmsKeyInfo.KeyName)) g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.ActiveKey.KeyVersion).To(Equal(kmsKeyInfo.KeyVersion)) - g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.KMS.CredentialsSecretName).To(Equal(kmsUserAssignedCredsSecretName)) - g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.KMS.ObjectEncoding).To(Equal(hyperv1.ObjectEncodingFormat("utf-8"))) + if azureutil.IsAroHCP() { + g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.KMS.CredentialsSecretName).To(Equal(kmsUserAssignedCredsSecretName)) + g.Expect(hostedCluster.Spec.SecretEncryption.KMS.Azure.KMS.ObjectEncoding).To(Equal(hyperv1.ObjectEncodingFormat("utf-8"))) + } } guestClient := e2eutil.WaitForGuestClient(t, testContext, mgtClient, hostedCluster) From c3138250b4e48b22c6f9f7b30537604c9f9d14e5 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Tue, 19 May 2026 06:54:21 -0400 Subject: [PATCH 7/7] docs: add KMS encryption documentation for self-managed Azure - Add Azure KMS setup guide for self-managed clusters including workload identity configuration and Key Vault access - Update azure-workload-identity-setup with KMS identity creation - Regenerate API reference and aggregated docs Signed-off-by: Bryan Cox Commit-Message-Assisted-by: Claude (via Claude Code) --- .../azure/azure-workload-identity-setup.md | 18 ++ .../create-self-managed-azure-cluster.md | 128 +++++++++++++ docs/content/reference/aggregated-docs.md | 171 +++++++++++++++++- docs/content/reference/api.md | 25 ++- 4 files changed, 338 insertions(+), 4 deletions(-) diff --git a/docs/content/how-to/azure/azure-workload-identity-setup.md b/docs/content/how-to/azure/azure-workload-identity-setup.md index af75d932da2..c7523d6b119 100644 --- a/docs/content/how-to/azure/azure-workload-identity-setup.md +++ b/docs/content/how-to/azure/azure-workload-identity-setup.md @@ -62,6 +62,24 @@ This creates 7 managed identities with federated credentials for: - NodePool Management - Network Operator +To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: + +```bash +hypershift create iam azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ + --output-file workload-identities.json \ + --enable-kms +``` + +!!! warning "KMS Key Vault Role Assignment" + + If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See [Enabling KMS Encryption](create-self-managed-azure-cluster.md#enabling-kms-encryption-etcd-encryption-at-rest) for the role assignment commands. + For complete documentation on the IAM commands, see [Create Azure IAM Resources Separately](create-iam-separately.md). ## Configure OIDC Issuer diff --git a/docs/content/how-to/azure/create-self-managed-azure-cluster.md b/docs/content/how-to/azure/create-self-managed-azure-cluster.md index 6d213260b89..f1b337508e8 100644 --- a/docs/content/how-to/azure/create-self-managed-azure-cluster.md +++ b/docs/content/how-to/azure/create-self-managed-azure-cluster.md @@ -267,6 +267,134 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ - `--diagnostics-storage-account-type Managed`: Use Azure managed storage for diagnostics - `--control-plane-operator-image`: Custom HyperShift operator image (optional) +## Enabling KMS Encryption (etcd Encryption at Rest) + +Self-managed Azure HostedClusters support encrypting etcd data at rest using [Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/general/overview) with the KMSv2 protocol. This requires: + +1. An Azure Key Vault with a cryptographic key +2. A workload identity with `Key Vault Crypto User` role on the Key Vault + +### Prerequisites + +Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity (using the `INFRA_ID` set during [Azure Workload Identity Setup](azure-workload-identity-setup.md)): + +```bash +hypershift create iam azure \ + --name "$CLUSTER_NAME" \ + --infra-id "$INFRA_ID" \ + --azure-creds "$AZURE_CREDS" \ + --location "$LOCATION" \ + --resource-group-name "$PERSISTENT_RG_NAME" \ + --oidc-issuer-url "$OIDC_ISSUER_URL" \ + --output-file ./workload-identities.json \ + --enable-kms +``` + +### Create a Key Vault and Key + +!!! note "RBAC Key Vault Permissions" + + The Key Vault is created with `--enable-rbac-authorization`, which means the creator does **not** automatically get data plane access. You must have the `Key Vault Crypto Officer` role (or equivalent) on the Key Vault to create and manage keys. If the key creation step fails with a `Forbidden` error, assign yourself the role: + + ```bash + MY_OBJECT_ID=$(az ad signed-in-user show --query id -o tsv) + KV_ID=$(az keyvault show --name "${KV_NAME}" --query id -o tsv) + az role assignment create \ + --assignee-object-id "${MY_OBJECT_ID}" \ + --assignee-principal-type User \ + --role "Key Vault Crypto Officer" \ + --scope "${KV_ID}" + ``` + +```bash +# Create Key Vault +KV_NAME="${PREFIX}-kv" +az keyvault create \ + --name "${KV_NAME}" \ + --resource-group "${MANAGED_RG_NAME}" \ + --location "${LOCATION}" \ + --enable-rbac-authorization + +# Create encryption key +KEY_NAME="${PREFIX}-etcd-key" +az keyvault key create \ + --vault-name "${KV_NAME}" \ + --name "${KEY_NAME}" \ + --kty RSA \ + --size 2048 + +# Get the key ID (used as --encryption-key-id) +ENCRYPTION_KEY_ID=$(az keyvault key show \ + --vault-name "${KV_NAME}" \ + --name "${KEY_NAME}" \ + --query key.kid -o tsv) +``` + +### Assign Key Vault Crypto User Role to the KMS Identity + +!!! warning "Manual Step Required" + + The `--auto-assign-roles` / `--assign-service-principal-roles` flag does **not** assign the Key Vault role because the Key Vault scope is user-provided and not known to the CLI at role-assignment time. You must perform this role assignment manually. + +Grant the KMS workload identity the `Key Vault Crypto User` role on your Key Vault so it can encrypt and decrypt etcd data: + +```bash +# Get the principal ID of the KMS managed identity +# The identity name follows the pattern: {clusterName}-kms-{infraID} +# List identities in the resource group to find the exact name: +# az identity list --resource-group "${PERSISTENT_RG_NAME}" --query "[?contains(name, 'kms')]" -o table +KMS_MI_NAME=$(az identity list \ + --resource-group "${PERSISTENT_RG_NAME}" \ + --query "[?contains(name, '${CLUSTER_NAME}-kms')].name" -o tsv) +KMS_PRINCIPAL_ID=$(az identity show \ + --name "${KMS_MI_NAME}" \ + --resource-group "${PERSISTENT_RG_NAME}" \ + --query principalId -o tsv) + +# Get the Key Vault resource ID +KV_ID=$(az keyvault show --name "${KV_NAME}" --query id -o tsv) + +# Assign Key Vault Crypto User role to the KMS identity +az role assignment create \ + --assignee-object-id "${KMS_PRINCIPAL_ID}" \ + --assignee-principal-type ServicePrincipal \ + --role "Key Vault Crypto User" \ + --scope "${KV_ID}" +``` + +### Create the Cluster with KMS + +Add the `--encryption-key-id` flag to your cluster creation command: + +```bash +${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ + --name "$CLUSTER_NAME" \ + --namespace "$CLUSTER_NAMESPACE" \ + --azure-creds $AZURE_CREDS \ + --location ${LOCATION} \ + --node-pool-replicas 2 \ + --base-domain $PARENT_DNS_ZONE \ + --pull-secret $PULL_SECRET \ + --generate-ssh \ + --release-image ${RELEASE_IMAGE} \ + --external-dns-domain ${DNS_ZONE_NAME} \ + --resource-group-name "${MANAGED_RG_NAME}" \ + --vnet-id "${GetVnetID}" \ + --subnet-id "${GetSubnetID}" \ + --network-security-group-id "${GetNsgID}" \ + --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ + --oidc-issuer-url "${OIDC_ISSUER_URL}" \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --assign-service-principal-roles \ + --workload-identities-file ./workload-identities.json \ + --encryption-key-id "${ENCRYPTION_KEY_ID}" \ + --diagnostics-storage-account-type Managed +``` + +!!! note "KMS Authentication" + + For self-managed Azure, the KMS provider authenticates using the `kms` workload identity specified in your `workload-identities.json`. This is different from managed Azure (ARO HCP), which uses managed identities with CSI secret store volumes. The `--kms-credentials-secret-name` flag is not needed for self-managed clusters. + ## Verification Check the cluster status and access: diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index fd8f2099e95..1f9d4d35ed6 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -8110,6 +8110,24 @@ This creates 7 managed identities with federated credentials for: - NodePool Management - Network Operator +To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: + +```bash +hypershift create iam azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ + --output-file workload-identities.json \ + --enable-kms +``` + +!!! warning "KMS Key Vault Role Assignment" + + If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See Enabling KMS Encryption for the role assignment commands. + For complete documentation on the IAM commands, see Create Azure IAM Resources Separately. ## Configure OIDC Issuer @@ -9528,6 +9546,134 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ - `--diagnostics-storage-account-type Managed`: Use Azure managed storage for diagnostics - `--control-plane-operator-image`: Custom HyperShift operator image (optional) +## Enabling KMS Encryption (etcd Encryption at Rest) + +Self-managed Azure HostedClusters support encrypting etcd data at rest using Azure Key Vault with the KMSv2 protocol. This requires: + +1. An Azure Key Vault with a cryptographic key +2. A workload identity with `Key Vault Crypto User` role on the Key Vault + +### Prerequisites + +Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity (using the `INFRA_ID` set during Azure Workload Identity Setup): + +```bash +hypershift create iam azure \ + --name "$CLUSTER_NAME" \ + --infra-id "$INFRA_ID" \ + --azure-creds "$AZURE_CREDS" \ + --location "$LOCATION" \ + --resource-group-name "$PERSISTENT_RG_NAME" \ + --oidc-issuer-url "$OIDC_ISSUER_URL" \ + --output-file ./workload-identities.json \ + --enable-kms +``` + +### Create a Key Vault and Key + +!!! note "RBAC Key Vault Permissions" + + The Key Vault is created with `--enable-rbac-authorization`, which means the creator does **not** automatically get data plane access. You must have the `Key Vault Crypto Officer` role (or equivalent) on the Key Vault to create and manage keys. If the key creation step fails with a `Forbidden` error, assign yourself the role: + + ```bash + MY_OBJECT_ID=$(az ad signed-in-user show --query id -o tsv) + KV_ID=$(az keyvault show --name "${KV_NAME}" --query id -o tsv) + az role assignment create \ + --assignee-object-id "${MY_OBJECT_ID}" \ + --assignee-principal-type User \ + --role "Key Vault Crypto Officer" \ + --scope "${KV_ID}" + ``` + +```bash +# Create Key Vault +KV_NAME="${PREFIX}-kv" +az keyvault create \ + --name "${KV_NAME}" \ + --resource-group "${MANAGED_RG_NAME}" \ + --location "${LOCATION}" \ + --enable-rbac-authorization + +# Create encryption key +KEY_NAME="${PREFIX}-etcd-key" +az keyvault key create \ + --vault-name "${KV_NAME}" \ + --name "${KEY_NAME}" \ + --kty RSA \ + --size 2048 + +# Get the key ID (used as --encryption-key-id) +ENCRYPTION_KEY_ID=$(az keyvault key show \ + --vault-name "${KV_NAME}" \ + --name "${KEY_NAME}" \ + --query key.kid -o tsv) +``` + +### Assign Key Vault Crypto User Role to the KMS Identity + +!!! warning "Manual Step Required" + + The `--auto-assign-roles` / `--assign-service-principal-roles` flag does **not** assign the Key Vault role because the Key Vault scope is user-provided and not known to the CLI at role-assignment time. You must perform this role assignment manually. + +Grant the KMS workload identity the `Key Vault Crypto User` role on your Key Vault so it can encrypt and decrypt etcd data: + +```bash +# Get the principal ID of the KMS managed identity +# The identity name follows the pattern: {clusterName}-kms-{infraID} +# List identities in the resource group to find the exact name: +# az identity list --resource-group "${PERSISTENT_RG_NAME}" --query "[?contains(name, 'kms')]" -o table +KMS_MI_NAME=$(az identity list \ + --resource-group "${PERSISTENT_RG_NAME}" \ + --query "[?contains(name, '${CLUSTER_NAME}-kms')].name" -o tsv) +KMS_PRINCIPAL_ID=$(az identity show \ + --name "${KMS_MI_NAME}" \ + --resource-group "${PERSISTENT_RG_NAME}" \ + --query principalId -o tsv) + +# Get the Key Vault resource ID +KV_ID=$(az keyvault show --name "${KV_NAME}" --query id -o tsv) + +# Assign Key Vault Crypto User role to the KMS identity +az role assignment create \ + --assignee-object-id "${KMS_PRINCIPAL_ID}" \ + --assignee-principal-type ServicePrincipal \ + --role "Key Vault Crypto User" \ + --scope "${KV_ID}" +``` + +### Create the Cluster with KMS + +Add the `--encryption-key-id` flag to your cluster creation command: + +```bash +${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ + --name "$CLUSTER_NAME" \ + --namespace "$CLUSTER_NAMESPACE" \ + --azure-creds $AZURE_CREDS \ + --location ${LOCATION} \ + --node-pool-replicas 2 \ + --base-domain $PARENT_DNS_ZONE \ + --pull-secret $PULL_SECRET \ + --generate-ssh \ + --release-image ${RELEASE_IMAGE} \ + --external-dns-domain ${DNS_ZONE_NAME} \ + --resource-group-name "${MANAGED_RG_NAME}" \ + --vnet-id "${GetVnetID}" \ + --subnet-id "${GetSubnetID}" \ + --network-security-group-id "${GetNsgID}" \ + --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ + --oidc-issuer-url "${OIDC_ISSUER_URL}" \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --assign-service-principal-roles \ + --workload-identities-file ./workload-identities.json \ + --encryption-key-id "${ENCRYPTION_KEY_ID}" \ + --diagnostics-storage-account-type Managed +``` + +!!! note "KMS Authentication" + + For self-managed Azure, the KMS provider authenticates using the `kms` workload identity specified in your `workload-identities.json`. This is different from managed Azure (ARO HCP), which uses managed identities with CSI secret store volumes. The `--kms-credentials-secret-name` flag is not needed for self-managed clusters. + ## Verification Check the cluster status and access: @@ -35833,7 +35979,7 @@ secrets can continue to be decrypted until they are all re-encrypted with the ac -kms
+kms,omitzero
ManagedIdentity @@ -35841,7 +35987,27 @@ ManagedIdentity -

kms is a pre-existing managed identity used to authenticate with Azure KMS.

+(Optional) +

kms is a pre-existing managed identity used to authenticate with Azure KMS. +This is used for managed Azure (ARO HCP) clusters. +kms and workloadIdentity are mutually exclusive.

+ + + + +workloadIdentity,omitzero
+ + +WorkloadIdentity + + + + +(Optional) +

workloadIdentity contains the workload identity used to authenticate +with Azure Key Vault for KMS encryption via a token-minter sidecar. +This identity must have “Key Vault Crypto User” role on the Key Vault. +kms and workloadIdentity are mutually exclusive.

@@ -48933,6 +49099,7 @@ string ###WorkloadIdentity { #hypershift.openshift.io/v1beta1.WorkloadIdentity }

(Appears on: +AzureKMSSpec, AzureWorkloadIdentities)

diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 5de01ff9124..81f244bc303 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -3419,7 +3419,7 @@ secrets can continue to be decrypted until they are all re-encrypted with the ac -kms
+kms,omitzero
ManagedIdentity @@ -3427,7 +3427,27 @@ ManagedIdentity -

kms is a pre-existing managed identity used to authenticate with Azure KMS.

+(Optional) +

kms is a pre-existing managed identity used to authenticate with Azure KMS. +This is used for managed Azure (ARO HCP) clusters. +kms and workloadIdentity are mutually exclusive.

+ + + + +workloadIdentity,omitzero
+ + +WorkloadIdentity + + + + +(Optional) +

workloadIdentity contains the workload identity used to authenticate +with Azure Key Vault for KMS encryption via a token-minter sidecar. +This identity must have “Key Vault Crypto User” role on the Key Vault. +kms and workloadIdentity are mutually exclusive.

@@ -16519,6 +16539,7 @@ string ###WorkloadIdentity { #hypershift.openshift.io/v1beta1.WorkloadIdentity }

(Appears on: +AzureKMSSpec, AzureWorkloadIdentities)