From c7272bc8288d983f95dac70eebecfa7cb5604545 Mon Sep 17 00:00:00 2001 From: Bavneet Singh Date: Tue, 30 Sep 2025 09:55:49 -0700 Subject: [PATCH 1/4] pester tests for gateway-workload identity scenarios --- testing/pipeline/k8s-custom-pipelines.yml | 4 + .../GatewayWorkloadIdentity.Tests.ps1 | 165 ++++++++++++++++++ testing/test/helper/Constants.ps1 | 2 +- 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 diff --git a/testing/pipeline/k8s-custom-pipelines.yml b/testing/pipeline/k8s-custom-pipelines.yml index 2cfa227661d..cda957ac09b 100644 --- a/testing/pipeline/k8s-custom-pipelines.yml +++ b/testing/pipeline/k8s-custom-pipelines.yml @@ -48,6 +48,10 @@ stages: parameters: jobName: WorkloadIdentityTest path: ./test/configurations/WorkloadIdentity.Tests.ps1 + - template: ./templates/run-test.yml + parameters: + jobName: GatewayWorkloadIdentityTest + path: ./test/configurations/GatewayWorkloadIdentity.Tests.ps1 - template: ./templates/run-test.yml parameters: jobName: TroubleshootTest diff --git a/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 new file mode 100644 index 00000000000..40afd496cc8 --- /dev/null +++ b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 @@ -0,0 +1,165 @@ +Describe 'Onboarding with Gateway Scenario' { + BeforeAll { + . $PSScriptRoot/../helper/Constants.ps1 + + $gatewayResourceId = "/subscriptions/15c06b1b-01d6-407b-bb21-740b8617dea3/resourceGroups/connectedk8sCLITestResources/providers/Microsoft.HybridCompute/gateways/gateway-test-cli" + } + + It 'Check if onboarding works with gateway enabled' { + az connectedk8s connect -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup -l $ARC_LOCATION --gateway-resource-id $gatewayResourceId --enable-oidc-issuer --enable-workload-identity + $? | Should -BeTrue + Start-Sleep -Seconds 10 + + # Loop and retry until the configuration installs + $n = 0 + do + { + $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output) + $provisioningState = ($output | ConvertFrom-Json).provisioningState + $gatewayStatus = $jsonOutput.RootElement.GetProperty("gateway").GetProperty("enabled").GetBoolean() + $securityProfile = $jsonOutput.RootElement.GetProperty("securityProfile").GetProperty("workloadIdentity").GetProperty("enabled").GetBoolean() + $oidcIssuerProfile = $jsonOutput.RootElement.GetProperty("oidcIssuerProfile").GetProperty("enabled").GetBoolean() + $issuerUrl = $jsonOutput.RootElement.GetProperty("oidcIssuerProfile").GetProperty("issuerUrl").GetString() + $selfHostedIssuerUrl = $jsonOutput.RootElement.GetProperty("oidcIssuerProfile").GetProperty("selfHostedIssuerUrl").GetString() + $agentState = $jsonOutput.RootElement.GetProperty("arcAgentProfile").GetProperty("agentState").GetString() + Write-Host "Provisioning State: $provisioningState" + Write-Host "Gateway Status: $gatewayStatus" + Write-Host "Security Profile Status: $securityProfile" + Write-Host "OIDC Issuer Profile Status: $oidcIssuerProfile" + Write-Host "Issuer Url: $issuerUrl" + Write-Host "Self Hosted Issuer Url: $selfHostedIssuerUrl" + Write-Host "Agent State: $agentState" + if ($provisioningState -eq $SUCCEEDED -and $gatewayStatus -eq $true-and + $securityProfile -eq $true -and + $oidcIssuerProfile -eq $true -and + ![string]::IsNullOrEmpty($issuerUrl) -and + $issuerUrl -like "*unitedkingdom*" -and + [string]::IsNullOrEmpty($selfHostedIssuerUrl) -and + $agentState -eq $SUCCEEDED) { + break + } + Start-Sleep -Seconds 30 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It 'Disable the gateway' { + az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --disable-gateway --disable-workload-identity + $? | Should -BeTrue + Start-Sleep -Seconds 10 + + # Loop and retry until the configuration installs + $n = 0 + do + { + $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output) + $provisioningState = ($output | ConvertFrom-Json).provisioningState + $gatewayStatus = $jsonOutput.RootElement.GetProperty("gateway").GetProperty("enabled").GetBoolean() + $securityProfile = $jsonOutput.RootElement.GetProperty("securityProfile").GetProperty("workloadIdentity").GetProperty("enabled").GetBoolean() + $agentState = $jsonOutput.RootElement.GetProperty("arcAgentProfile").GetProperty("agentState").GetString() + Write-Host "Provisioning State: $provisioningState" + Write-Host "Gateway Status: $gatewayStatus" + Write-Host "Security Profile Status: $securityProfile" + Write-Host "Agent State: $agentState" + if ($provisioningState -eq $SUCCEEDED -and $gatewayStatus -eq $false -and $securityProfile -eq $false -and $agentState -eq $SUCCEEDED) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It 'Update the cluster to use gateway again using update cmd' { + az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --gateway-resource-id $gatewayResourceId --enable-workload-identity + $? | Should -BeTrue + Start-Sleep -Seconds 10 + + # Loop and retry until the configuration installs + $n = 0 + do + { + $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output) + $provisioningState = ($output | ConvertFrom-Json).provisioningState + $gatewayStatus = $jsonOutput.RootElement.GetProperty("gateway").GetProperty("enabled").GetBoolean() + $securityProfile = $jsonOutput.RootElement.GetProperty("securityProfile").GetProperty("workloadIdentity").GetProperty("enabled").GetBoolean() + $agentState = $jsonOutput.RootElement.GetProperty("arcAgentProfile").GetProperty("agentState").GetString() + Write-Host "Provisioning State: $provisioningState" + Write-Host "Gateway Status: $gatewayStatus" + Write-Host "Security Profile Status: $securityProfile" + Write-Host "Agent State: $agentState" + if ($provisioningState -eq $SUCCEEDED -and + $gatewayStatus -eq $true -and + $securityProfile -eq $true -and + $agentState -eq $SUCCEEDED) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It 'Disable the gateway' { + az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --disable-gateway + $? | Should -BeTrue + Start-Sleep -Seconds 10 + + # Loop and retry until the configuration installs + $n = 0 + do + { + $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output) + $provisioningState = ($output | ConvertFrom-Json).provisioningState + $gatewayStatus = $jsonOutput.RootElement.GetProperty("gateway").GetProperty("enabled").GetBoolean() + Write-Host "Provisioning State: $provisioningState" + Write-Host "Gateway Status: $gatewayStatus" + if ($provisioningState -eq $SUCCEEDED -and $gatewayStatus -eq $false) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It 'Disable workload identity' { + az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --disable-workload-identity + $? | Should -BeTrue + Start-Sleep -Seconds 10 + + # Loop and retry until the configuration installs + $n = 0 + do + { + $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output) + $provisioningState = ($output | ConvertFrom-Json).provisioningState + $securityProfile = $jsonOutput.RootElement.GetProperty("securityProfile").GetProperty("workloadIdentity").GetProperty("enabled").GetBoolean() + $agentState = $jsonOutput.RootElement.GetProperty("arcAgentProfile").GetProperty("agentState").GetString() + Write-Host "Provisioning State: $provisioningState" + Write-Host "Security Profile Status: $securityProfile" + Write-Host "Agent State: $agentState" + if ($provisioningState -eq $SUCCEEDED -and $securityProfile -eq $false -and $agentState -eq $SUCCEEDED) { + break + } + Start-Sleep -Seconds 10 + $n += 1 + } while ($n -le $MAX_RETRY_ATTEMPTS) + $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + } + + It "Delete the connected instance" { + az connectedk8s delete -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --force -y + $? | Should -BeTrue + + # Configuration should be removed from the resource model + az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup + $? | Should -BeFalse + } +} \ No newline at end of file diff --git a/testing/test/helper/Constants.ps1 b/testing/test/helper/Constants.ps1 index 43006f78a69..0b41d9dc7cc 100644 --- a/testing/test/helper/Constants.ps1 +++ b/testing/test/helper/Constants.ps1 @@ -1,5 +1,5 @@ $ENVCONFIG = Get-Content -Path $PSScriptRoot/../../settings.json | ConvertFrom-Json $MAX_RETRY_ATTEMPTS = 30 -$ARC_LOCATION = "uksouth" +$ARC_LOCATION = "centraluseuap" $SUCCEEDED = "Succeeded" \ No newline at end of file From 0615cb74e2ba79045fa28ab8afcfbe70e21a835b Mon Sep 17 00:00:00 2001 From: Bavneet Singh Date: Tue, 30 Sep 2025 09:58:19 -0700 Subject: [PATCH 2/4] update the description of the tests --- .../test/configurations/GatewayWorkloadIdentity.Tests.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 index 40afd496cc8..33cd70ec58f 100644 --- a/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 +++ b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 @@ -5,7 +5,7 @@ Describe 'Onboarding with Gateway Scenario' { $gatewayResourceId = "/subscriptions/15c06b1b-01d6-407b-bb21-740b8617dea3/resourceGroups/connectedk8sCLITestResources/providers/Microsoft.HybridCompute/gateways/gateway-test-cli" } - It 'Check if onboarding works with gateway enabled' { + It 'Check if onboarding works with gateway and workload identity enabled' { az connectedk8s connect -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup -l $ARC_LOCATION --gateway-resource-id $gatewayResourceId --enable-oidc-issuer --enable-workload-identity $? | Should -BeTrue Start-Sleep -Seconds 10 @@ -45,7 +45,7 @@ Describe 'Onboarding with Gateway Scenario' { $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS } - It 'Disable the gateway' { + It 'Disable the gateway and workload identity' { az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --disable-gateway --disable-workload-identity $? | Should -BeTrue Start-Sleep -Seconds 10 @@ -73,7 +73,7 @@ Describe 'Onboarding with Gateway Scenario' { $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS } - It 'Update the cluster to use gateway again using update cmd' { + It 'Update the cluster to use gateway and workload identity again using update cmd' { az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --gateway-resource-id $gatewayResourceId --enable-workload-identity $? | Should -BeTrue Start-Sleep -Seconds 10 From e8198a24ebc512810d8a0853b50f5099fc554d4a Mon Sep 17 00:00:00 2001 From: Bavneet Singh Date: Tue, 30 Sep 2025 10:53:03 -0700 Subject: [PATCH 3/4] update the description of the test --- testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 index 33cd70ec58f..40533444059 100644 --- a/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 +++ b/testing/test/configurations/GatewayWorkloadIdentity.Tests.ps1 @@ -1,4 +1,4 @@ -Describe 'Onboarding with Gateway Scenario' { +Describe 'Onboarding with Gateway and Workload Identity Scenario' { BeforeAll { . $PSScriptRoot/../helper/Constants.ps1 From 694b14396b4cc374ddab4509d7fbca636bdee8ec Mon Sep 17 00:00:00 2001 From: Bavneet Singh Date: Wed, 1 Oct 2025 17:46:23 -0700 Subject: [PATCH 4/4] temporary location add to run tests --- src/connectedk8s/azext_connectedk8s/_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connectedk8s/azext_connectedk8s/_constants.py b/src/connectedk8s/azext_connectedk8s/_constants.py index bb327ad3812..53ab6498f10 100644 --- a/src/connectedk8s/azext_connectedk8s/_constants.py +++ b/src/connectedk8s/azext_connectedk8s/_constants.py @@ -530,4 +530,4 @@ Gateway_Cluster_Resource_Update_Failed_Fault_Type = ( "Gateway-Cluster-Resource-Update-Failed" ) -GATEWAY_ASSOCIATE_URL = "https://management.azure.com/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Kubernetes/connectedClusters/{cluster_name}/providers/Microsoft.HybridCompute/settings/Default?api-version={api_version}" +GATEWAY_ASSOCIATE_URL = "https://uksouth.management.azure.com/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Kubernetes/connectedClusters/{cluster_name}/providers/Microsoft.HybridCompute/settings/Default?api-version={api_version}"